Unclear reason for the backup

backups/ocr-rename-silent-20260909-142627/redmine_ticket_attachments.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Attachment helpers for the power-user Redmine ticket client.
+
+List (cache-first), download to local dir, upload via Redmine REST:
+  POST /uploads.json?filename=…  (raw body)
+  PUT  /issues/:id.json with issue.uploads token
+"""
+
+from __future__ import annotations
+
+import json
+import mimetypes
+import re
+from pathlib import Path
+from typing import Any
+from urllib import error as urllib_error
+from urllib import parse as urllib_parse
+from urllib import request as urllib_request
+
+
+def safe_filename(name: str) -> str:
+    name = (name or "attachment.bin").strip().replace("\\", "_").replace("/", "_")
+    name = re.sub(r"[^\w.\- ()\[\]+@]+", "_", name)
+    return name or "attachment.bin"
+
+
+def attachments_from_issue(issue: dict[str, Any]) -> list[dict[str, Any]]:
+    atts = issue.get("attachments")
+    if isinstance(atts, list):
+        return [a for a in atts if isinstance(a, dict)]
+    return []
+
+
+def attachment_dir(cache_dir: Path, project: str, issue_id: int) -> Path:
+    path = Path(cache_dir) / project / "attachments" / str(int(issue_id))
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def pick_attachment(attachments: list[dict[str, Any]], attachment_id: int | None) -> dict[str, Any]:
+    if not attachments:
+        raise LookupError("Issue has no attachments.")
+    if attachment_id is None:
+        if len(attachments) == 1:
+            return attachments[0]
+        raise LookupError(
+            "Multiple attachments; pass --attachment-id. ids="
+            + ",".join(str(a.get("id")) for a in attachments)
+        )
+    for att in attachments:
+        if int(att.get("id") or 0) == int(attachment_id):
+            return att
+    raise LookupError(f"Attachment id {attachment_id} not found on issue.")
+
+
+def download_url_for_attachment(att: dict[str, Any], base_url: str) -> str:
+    urls = download_url_candidates(att, base_url)
+    return urls[0]
+
+
+def download_url_candidates(att: dict[str, Any], base_url: str) -> list[str]:
+    """Ordered download URL candidates (host rewrite + path variants)."""
+    base = base_url.rstrip("/")
+    aid = att.get("id")
+    filename = str(att.get("filename") or "").strip()
+    candidates: list[str] = []
+
+    def add(url: str) -> None:
+        if url and url not in candidates:
+            candidates.append(url)
+
+    for key in ("content_url", "contentUrl"):
+        raw = att.get(key)
+        if not raw:
+            continue
+        raw = str(raw)
+        add(raw)
+        # rewrite host to active base_url (prod content_url on LAN client, etc.)
+        try:
+            from urllib.parse import urlparse, urlunparse
+
+            parsed = urlparse(raw)
+            base_p = urlparse(base if "://" in base else f"http://{base}")
+            rewritten = urlunparse(
+                (
+                    base_p.scheme or parsed.scheme,
+                    base_p.netloc or parsed.netloc,
+                    parsed.path,
+                    "",
+                    parsed.query,
+                    "",
+                )
+            )
+            add(rewritten)
+        except Exception:
+            pass
+
+    if aid is not None:
+        add(f"{base}/attachments/download/{int(aid)}")
+        if filename:
+            from urllib.parse import quote
+
+            add(f"{base}/attachments/download/{int(aid)}/{quote(filename)}")
+    if not candidates:
+        raise LookupError("Attachment missing id and content_url.")
+    return candidates
+
+
+def download_attachment_bytes(
+    *,
+    url: str,
+    api_key: str,
+    impersonate_user: str | None = None,
+    timeout: int = 120,
+    fallback_urls: list[str] | None = None,
+) -> bytes:
+    headers = {
+        "X-Redmine-API-Key": api_key,
+        "Accept": "*/*",
+    }
+    if impersonate_user:
+        headers["X-Redmine-Switch-User"] = impersonate_user
+    urls = [url] + [u for u in (fallback_urls or []) if u != url]
+    last_err: Exception | None = None
+    for u in urls:
+        req = urllib_request.Request(u, headers=headers, method="GET")
+        try:
+            with urllib_request.urlopen(req, timeout=timeout) as resp:
+                return resp.read()
+        except urllib_error.HTTPError as exc:
+            detail = exc.read().decode("utf-8", errors="replace")[:200]
+            last_err = RuntimeError(f"HTTP {exc.code} downloading attachment ({u}): {detail}")
+            continue
+        except urllib_error.URLError as exc:
+            last_err = RuntimeError(f"Could not download attachment ({u}): {exc.reason}")
+            continue
+    raise last_err or RuntimeError("Attachment download failed")
+
+
+def upload_file_token(
+    *,
+    base_url: str,
+    api_key: str,
+    path: Path,
+    filename: str | None = None,
+    impersonate_user: str | None = None,
+    timeout: int = 120,
+) -> dict[str, Any]:
+    """POST raw file to /uploads.json?filename=… Returns upload object with token."""
+    path = Path(path)
+    if not path.is_file():
+        raise FileNotFoundError(f"File not found: {path}")
+    name = filename or path.name
+    data = path.read_bytes()
+    q = urllib_parse.urlencode({"filename": name})
+    url = f"{base_url.rstrip('/')}/uploads.json?{q}"
+    headers = {
+        "X-Redmine-API-Key": api_key,
+        "Content-Type": "application/octet-stream",
+        "Accept": "application/json",
+    }
+    if impersonate_user:
+        headers["X-Redmine-Switch-User"] = impersonate_user
+    req = urllib_request.Request(url, data=data, headers=headers, method="POST")
+    try:
+        with urllib_request.urlopen(req, timeout=timeout) as resp:
+            raw = resp.read()
+    except urllib_error.HTTPError as exc:
+        detail = exc.read().decode("utf-8", errors="replace")[:500]
+        raise RuntimeError(f"HTTP {exc.code} uploading file: {detail}") from exc
+    except urllib_error.URLError as exc:
+        raise RuntimeError(f"Could not upload file: {exc.reason}") from exc
+    if not raw:
+        raise RuntimeError("Empty response from /uploads.json")
+    payload = json.loads(raw.decode("utf-8"))
+    upload = payload.get("upload") if isinstance(payload, dict) else None
+    if not isinstance(upload, dict) or not upload.get("token"):
+        raise RuntimeError(f"Unexpected upload response: {payload!r}")
+    content_type = mimetypes.guess_type(name)[0] or "application/octet-stream"
+    upload.setdefault("filename", name)
+    upload.setdefault("content_type", content_type)
+    return upload

backups/ocr-rename-silent-20260909-142627/redmine_ticket_ocr.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""OCR / text extract helpers for scanned ticket attachments.
+
+Primary path: pdftotext (fast, works on digital PDFs).
+Fallback: pdftoppm + tesseract for image-heavy scans.
+Also handles common image attachments via tesseract.
+
+Does not write to Redmine — callers queue subject/notes drafts.
+"""
+
+from __future__ import annotations
+
+import re
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+def _run(cmd: list[str], *, timeout: int = 120) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        cmd,
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+        check=False,
+    )
+
+
+def extract_text_from_pdf(path: Path, *, max_chars: int = 12000) -> tuple[str, str]:
+    """Return (text, method). method is pdftotext|tesseract|empty."""
+    path = Path(path)
+    text = ""
+    pdftotext = shutil.which("pdftotext")
+    if pdftotext:
+        proc = _run([pdftotext, "-layout", "-nopgbrk", str(path), "-"], timeout=60)
+        text = (proc.stdout or "").strip()
+        if len(re.sub(r"\s+", "", text)) >= 40:
+            return text[:max_chars], "pdftotext"
+
+    # OCR fallback: first 2 pages as PNG
+    pdftoppm = shutil.which("pdftoppm")
+    tesseract = shutil.which("tesseract")
+    if not pdftoppm or not tesseract:
+        return text[:max_chars] if text else "", "empty" if not text else "pdftotext-thin"
+
+    with tempfile.TemporaryDirectory(prefix="rtl-ocr-") as td:
+        out_prefix = Path(td) / "page"
+        _run([pdftoppm, "-png", "-f", "1", "-l", "2", "-r", "200", str(path), str(out_prefix)], timeout=90)
+        chunks: list[str] = []
+        for png in sorted(Path(td).glob("page*.png")):
+            proc = _run([tesseract, str(png), "stdout", "-l", "eng"], timeout=90)
+            if proc.stdout:
+                chunks.append(proc.stdout.strip())
+        ocr_text = "\n\n".join(chunks).strip()
+        if ocr_text:
+            return ocr_text[:max_chars], "tesseract"
+    return text[:max_chars] if text else "", "empty"
+
+
+def extract_text_from_image(path: Path, *, max_chars: int = 12000) -> tuple[str, str]:
+    tesseract = shutil.which("tesseract")
+    if not tesseract:
+        return "", "empty"
+    proc = _run([tesseract, str(path), "stdout", "-l", "eng"], timeout=90)
+    text = (proc.stdout or "").strip()
+    return (text[:max_chars], "tesseract") if text else ("", "empty")
+
+
+def extract_text_from_file(path: Path, *, max_chars: int = 12000) -> tuple[str, str]:
+    path = Path(path)
+    suffix = path.suffix.lower()
+    if suffix == ".pdf":
+        return extract_text_from_pdf(path, max_chars=max_chars)
+    if suffix in {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".webp", ".gif"}:
+        return extract_text_from_image(path, max_chars=max_chars)
+    if suffix in {".txt", ".md", ".csv"}:
+        try:
+            return path.read_text(encoding="utf-8", errors="replace")[:max_chars], "read"
+        except OSError:
+            return "", "empty"
+    return "", "empty"
+
+
+_PERSON_LINE = re.compile(
+    r"^(?:name|employee|applicant|patient|to|for)\s*[:\-]\s*(.+)$",
+    re.I | re.M,
+)
+_COMPANY_LINE = re.compile(
+    r"^(?:company|organization|employer|vendor|client)\s*[:\-]\s*(.+)$",
+    re.I | re.M,
+)
+_DATE_LINE = re.compile(
+    r"\b((?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4}|\d{1,2}[/-]\d{1,2}[/-]\d{2,4})\b",
+    re.I,
+)
+_DOC_HINTS = (
+    (r"\bw-?2\b", "W-2"),
+    (r"\bw-?4\b", "W-4"),
+    (r"\bi-?9\b", "I-9"),
+    (r"\bdriver.?s?\s+license\b|\bdl\b", "Driver License"),
+    (r"\bpassport\b", "Passport"),
+    (r"\binvoice\b", "Invoice"),
+    (r"\bpacking\s+slip\b", "Packing Slip"),
+    (r"\bbill\s+of\s+lading\b|\bbol\b", "Bill of Lading"),
+    (r"\bresume\b|\bcv\b", "Resume"),
+    (r"\boffer\s+letter\b", "Offer Letter"),
+    (r"\bnda\b|non-?disclosure", "NDA"),
+    (r"\bhandbook\b", "Handbook"),
+    (r"\bdirect\s+deposit\b", "Direct Deposit Form"),
+)
+
+
+def suggest_subject_from_text(
+    text: str,
+    *,
+    original_subject: str = "",
+    max_len: int = 120,
+) -> str:
+    """Heuristic subject for scanned-doc tickets."""
+    clean = re.sub(r"[ \t]+", " ", text or "")
+    lines = [ln.strip() for ln in clean.splitlines() if ln.strip()]
+    # Drop very short / noise lines
+    content_lines = [ln for ln in lines if len(ln) >= 4 and not re.fullmatch(r"[\d\W]+", ln)][:40]
+
+    doc_type = "Scanned document"
+    joined = "\n".join(content_lines[:30])
+    for pat, label in _DOC_HINTS:
+        if re.search(pat, joined, re.I):
+            doc_type = label
+            break
+
+    person = None
+    m = _PERSON_LINE.search(joined)
+    if m:
+        person = re.sub(r"\s+", " ", m.group(1)).strip(" .,-")[:60]
+    company = None
+    m = _COMPANY_LINE.search(joined)
+    if m:
+        company = re.sub(r"\s+", " ", m.group(1)).strip(" .,-")[:60]
+
+    # Fallback: first substantial title-ish line
+    title_line = None
+    for ln in content_lines[:8]:
+        if len(ln) < 8 or len(ln) > 80:
+            continue
+        if re.search(r"https?://|@|page\s+\d", ln, re.I):
+            continue
+        title_line = ln
+        break
+
+    date = None
+    dm = _DATE_LINE.search(joined)
+    if dm:
+        date = dm.group(1)
+
+    parts: list[str] = [doc_type]
+    if person:
+        parts.append(f"— {person}")
+    elif company:
+        parts.append(f"— {company}")
+    elif title_line and title_line.lower() not in doc_type.lower():
+        parts.append(f"— {title_line[:50]}")
+    if date and date not in " ".join(parts):
+        parts.append(f"({date})")
+
+    subject = " ".join(parts)
+    subject = re.sub(r"\s+", " ", subject).strip()
+    if len(subject) > max_len:
+        subject = subject[: max_len - 1].rstrip() + "…"
+    if not subject or subject == "Scanned document":
+        # last resort: original if not generic
+        orig = (original_subject or "").strip()
+        if orig and not re.search(r"scanned\s+document", orig, re.I):
+            return orig[:max_len]
+        if title_line:
+            return f"Scanned document — {title_line[:80]}"[:max_len]
+    return subject
+
+
+def attachment_is_ocr_candidate(att: dict[str, Any]) -> bool:
+    name = str(att.get("filename") or att.get("name") or "").lower()
+    ctype = str(att.get("content_type") or "").lower()
+    if name.endswith(".pdf") or "pdf" in ctype:
+        return True
+    if any(name.endswith(ext) for ext in (".png", ".jpg", ".jpeg", ".tif", ".tiff", ".webp")):
+        return True
+    if ctype.startswith("image/"):
+        return True
+    return False
+
+
+def looks_like_generic_scan_subject(subject: str) -> bool:
+    s = (subject or "").strip().lower()
+    if not s:
+        return True
+    return bool(
+        re.search(
+            r"scanned\s+document|scan\s+from|hp\s+scan|scanner|untitled|document\s+from\s+scan",
+            s,
+        )
+    )
+
+
+def build_ocr_note(
+    *,
+    issue_id: int,
+    files: list[dict[str, str]],
+    proposed_subject: str,
+) -> str:
+    lines = [
+        "=== OCR / attachment review (agent) ===",
+        f"issue_id: {issue_id}",
+        f"proposed_subject: {proposed_subject}",
+        "",
+    ]
+    for f in files:
+        lines.append(f"## {f.get('filename')} ({f.get('method')})")
+        lines.append("")
+        snippet = (f.get("text") or "")[:2500].strip()
+        lines.append(snippet or "_(no extractable text)_")
+        lines.append("")
+    lines.append("_Generated by redmine_ticket_ocr; verify before customer-facing use._")
+    return "\n".join(lines) + "\n"

backups/ocr-rename-silent-20260909-142627/redmine_ticket_view.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""Markdown rendering + pager helpers for cached Redmine tickets."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any
+
+
+def _nested_name(issue: dict[str, Any], key: str) -> str:
+    value = issue.get(key)
+    if isinstance(value, dict):
+        name = value.get("name")
+        return str(name) if name is not None else ""
+    return "" if value is None else str(value)
+
+
+def _helpdesk(issue: dict[str, Any]) -> dict[str, Any]:
+    hd = issue.get("helpdesk_ticket") or {}
+    return hd if isinstance(hd, dict) else {}
+
+
+def _contact(issue: dict[str, Any]) -> dict[str, Any]:
+    contact = _helpdesk(issue).get("contact") or {}
+    return contact if isinstance(contact, dict) else {}
+
+
+def render_ticket_markdown(issue: dict[str, Any]) -> str:
+    issue_id = issue.get("id")
+    subject = str(issue.get("subject") or "").strip()
+    status = _nested_name(issue, "status")
+    project = _nested_name(issue, "project")
+    category = _nested_name(issue, "category")
+    assigned = _nested_name(issue, "assigned_to") or "(unassigned)"
+    author = _nested_name(issue, "author")
+    priority = _nested_name(issue, "priority")
+    created = issue.get("created_on") or ""
+    updated = issue.get("updated_on") or ""
+    hd = _helpdesk(issue)
+    contact = _contact(issue)
+    from_addr = hd.get("from_address") or contact.get("email") or ""
+    contact_id = contact.get("id")
+    company = contact.get("company") or ""
+    contact_name = contact.get("name") or ""
+    description = str(issue.get("description") or "").strip()
+    redmine_url = f"https://rm.ldrprep.com/issues/{issue_id}" if issue_id else ""
+
+    lines = [
+        f"# #{issue_id} — {subject}",
+        "",
+        "## Meta",
+        "",
+        f"- **Project:** {project}",
+        f"- **Status:** {status}",
+        f"- **Priority:** {priority}",
+        f"- **Category:** {category or '(none)'}",
+        f"- **Assignee:** {assigned}",
+        f"- **Author:** {author}",
+        f"- **Created:** {created}",
+        f"- **Updated:** {updated}",
+        f"- **Redmine:** {redmine_url}",
+        "",
+        "## Helpdesk / contact",
+        "",
+        f"- **From:** {from_addr}",
+        f"- **Contact ID:** {contact_id if contact_id is not None else '(none)'}",
+        f"- **Contact name:** {contact_name or '(none)'}",
+        f"- **Company:** {company or '(none)'}",
+        "",
+        "## Detected entities (heuristic)",
+        "",
+    ]
+    try:
+        from redmine_ticket_entities import extract_entities_from_issue, format_entities_line
+
+        ents = extract_entities_from_issue(issue)
+        if ents:
+            for e in ents:
+                conf = e.get("confidence") or "high"
+                lines.append(f"- **{e.get('type')}:** `{e.get('value')}` ({conf})")
+        else:
+            lines.append("_(none)_")
+    except Exception:
+        lines.append("_(entity extractor unavailable)_")
+    lines.extend(
+        [
+            "",
+            "## Description",
+            "",
+            description or "_(empty)_",
+            "",
+            "## Journals",
+            "",
+        ]
+    )
+
+    journals = issue.get("journals") or []
+    if not journals:
+        lines.append("_(no journals)_")
+    else:
+        for journal in journals:
+            if not isinstance(journal, dict):
+                continue
+            jid = journal.get("id")
+            jcreated = journal.get("created_on") or ""
+            user = journal.get("user") or {}
+            uname = user.get("name") if isinstance(user, dict) else ""
+            notes = str(journal.get("notes") or "").strip()
+            details = journal.get("details") or []
+            lines.append(f"### Journal #{jid} — {jcreated} — {uname}")
+            lines.append("")
+            if details:
+                for detail in details:
+                    if not isinstance(detail, dict):
+                        continue
+                    lines.append(
+                        f"- change: {detail.get('property')}/{detail.get('name')}: "
+                        f"{detail.get('old_value')!r} → {detail.get('new_value')!r}"
+                    )
+                lines.append("")
+            if notes:
+                lines.append(notes)
+            else:
+                lines.append("_(no notes)_")
+            lines.append("")
+
+    lines.append("---")
+    lines.append(f"_Rendered from cache; source updated_on={updated}_")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def markdown_cache_path(cache_dir: Path, project: str, issue_id: int | str) -> Path:
+    return Path(cache_dir) / project / "markdown" / f"{issue_id}.md"
+
+
+def write_ticket_markdown(
+    issue: dict[str, Any],
+    cache_dir: Path,
+    project: str,
+    *,
+    force: bool = False,
+) -> Path:
+    """Write markdown cache if missing or ticket updated_on changed (via sidecar stamp)."""
+    issue_id = issue["id"]
+    path = markdown_cache_path(cache_dir, project, issue_id)
+    stamp_path = path.with_suffix(".md.updated_on")
+    updated_on = str(issue.get("updated_on") or "")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if (
+        not force
+        and path.is_file()
+        and stamp_path.is_file()
+        and stamp_path.read_text(encoding="utf-8").strip() == updated_on
+    ):
+        return path
+    path.write_text(render_ticket_markdown(issue), encoding="utf-8")
+    stamp_path.write_text(updated_on + "\n", encoding="utf-8")
+    return path
+
+
+def resolve_pager(preferred: str = "auto") -> list[str]:
+    """Return argv prefix to display markdown (pager program + flags)."""
+    preferred = (preferred or "auto").lower()
+    if preferred == "cat":
+        return ["cat"]
+    if preferred in {"glow", "bat", "batcat"}:
+        path = shutil.which(preferred)
+        if not path:
+            raise FileNotFoundError(f"Pager {preferred!r} not found on PATH")
+        if preferred == "glow":
+            return [path]
+        return [path, "--language", "markdown", "--style", "plain"]
+    # auto
+    for name, args in (
+        ("glow", []),
+        ("bat", ["--language", "markdown", "--style", "plain"]),
+        ("batcat", ["--language", "markdown", "--style", "plain"]),
+    ):
+        path = shutil.which(name)
+        if path:
+            return [path, *args]
+    return ["cat"]
+
+
+def display_markdown_file(path: Path, pager: str = "auto") -> int:
+    cmd = resolve_pager(pager) + [str(path)]
+    # Prefer streaming to user terminal
+    env = os.environ.copy()
+    try:
+        completed = subprocess.run(cmd, check=False, env=env)
+        return int(completed.returncode)
+    except FileNotFoundError:
+        print(path.read_text(encoding="utf-8"))
+        return 0