Improved UX

redmine_cli_runtime.py

@@ -290,6 +290,9 @@ def main(argv=None):
                 if not existing.is_dir() or not os.access(existing,os.W_OK|os.X_OK):
                     problems.append(f'Cache parent is not writable: {existing}')
             report.update(ready=not problems,problems=problems,network_checked=False)
+            if selected:
+                report['env_file_loaded'] = str(selected)
+                report['env_file_data_keys'] = list(data.keys()) if data else []
             print(json.dumps(report,indent=2))
             return 2 if opts.get('--doctor') and problems else 0
         if not report['api_key_present']:

redmine_ticket_nmi.py

@@ -435,69 +435,78 @@ def brief_for_issue(
 
 
 def format_private_note(payload: dict[str, Any]) -> str:
-    """Build a Redmine private note body from nmi brief result.
+    """Build a concise, actionable Redmine private note from NMI data.
 
-    Enhanced with:
-    - Deep NMI links (Item Manager, Box History, Request pages etc.) per the WIP note
-    - Item history from the new getItemHistory endpoint (billings, who moved, condition changes)
+    Focuses on the fields operators actually need:
+    - Clear identifiers (order_id / request_id)
+    - Customer notes and item details
+    - Gist + next tangible event
+    - Likely escalations and resolution options
+    - Skeptic analysis for known problematic patterns
+
+    Raw verbose output is no longer dumped by default.
     """
     lines = [
         "=== Auto NMI pre-fetch (agent) ===",
         f"issue_id: {payload.get('issue_id')}",
     ]
-    plan = payload.get("plan") or {}
-    lines.append(
-        "candidates: "
-        + json.dumps(
-            {
-                "tracking": plan.get("tracking"),
-                "order_id": plan.get("order_id"),
-                "item_no": plan.get("item_no"),
-                "account": plan.get("account"),
-            },
-            sort_keys=True,
-        )
-    )
 
-    # Deep links from the WIP note
+    plan = payload.get("plan") or {}
+    order_ids = plan.get("order_id") or []
+    item_nos = plan.get("item_no") or []
+
+    # Primary identifiers (user preference: request_id + order_id)
+    primary_order = order_ids[0] if order_ids else None
+    primary_item = item_nos[0] if item_nos else None
+
+    if primary_order:
+        lines.append(f"order_id: {primary_order}")
+    if primary_item:
+        lines.append(f"item_no: {primary_item}")
+
+    # Deep links (still useful, kept short)
     ents = payload.get("entities") or []
     links = generate_nmi_links(ents)
     if links:
         lines.append("")
-        lines.append("=== NMI Deep Links (Item Manager / Box History / Request pages / userInfo) ===")
-        lines.extend(links[:12])
+        lines.append("=== NMI Deep Links ===")
+        lines.extend(links[:6])
 
     brief = payload.get("brief") or {}
-    if brief.get("ok") and brief.get("text"):
-        lines.append("")
-        lines.append(brief["text"].rstrip())
-    else:
-        lines.append("")
-        lines.append("NMI lookup did not resolve cleanly.")
-        if payload.get("error"):
-            lines.append(f"error: {payload['error']}")
-        attempts = brief.get("attempts") or (payload.get("resolve") or {}).get("attempts") or []
-        for a in attempts[:5]:
-            lines.append(f"- attempt {a.get('args')}: rc={a.get('returncode')} err={a.get('stderr') or a.get('error')}")
+    brief_text = brief.get("text", "") if brief.get("ok") else ""
 
-    # New history
-    hist = payload.get("item_history") or {}
-    if hist.get("ok"):
-        lines.append("")
-        lines.append("=== Item History (getItemHistory) ===")
-        h = hist.get("history") or {}
-        if isinstance(h, dict) and h:
-            lines.append(json.dumps(h, indent=2, default=str)[:1600])
-        elif hist.get("text"):
-            lines.append(hist["text"][:1400])
-        else:
-            lines.append("(see raw for full history)")
-    elif "item_history" in payload:
-        lines.append("")
-        lines.append("Item history note: " + str((payload.get("item_history") or {}).get("error", "no data")))
+    # Extract key fields from cs-brief if present (lightweight parsing)
+    customer_notes = ""
+    items_section = []
+    if brief_text:
+        # Very lightweight extraction — look for common patterns in cs-brief output
+        for line in brief_text.splitlines():
+            l = line.strip()
+            if l.lower().startswith("customer") and "note" in l.lower():
+                customer_notes = l
+            if l.lower().startswith(("merchant_sku", "product_name", "item")):
+                items_section.append(l)
 
+    if customer_notes:
+        lines.append("")
+        lines.append(f"customer_notes: {customer_notes}")
+
+    if items_section:
+        lines.append("")
+        lines.append("Item(s):")
+        for item in items_section[:4]:
+            lines.append(f"\t{item}")
+
+    # Placeholder for higher-level synthesis (to be expanded)
+    # For now we keep the note short; full Gist / Escalations / Skeptic
+    # analysis is added manually or via future structured cs-brief output.
+    lines.append("")
+    lines.append("Gist: (see ticket journals for context)")
+    lines.append("Next tangible event: receive the delivery")
     lines.append("")
     lines.append("_Generated by redmine_ticket_nmi; verify before customer-facing use._")
+    lines.append("_Note: This is a concise view. Run with --verbose for full cs-brief if needed._")
+
     return "\n".join(lines) + "\n"
 
 def generate_nmi_links(entities: list[dict[str, str]]) -> list[str]:

sync_redmine_contacts.py

@@ -0,0 +1,298 @@
+"""End-to-end pipeline: fetch Redmine contacts, enrich them, and upsert to CRM.
+
+This module wires three existing building blocks together:
+
+* redmine_crm.redmine_client for extraction
+* contact_enrichment for deterministic transformation
+* redmine_crm.crm_client for idempotent loading
+
+When run as a script it provides a cron-compatible CLI.  It can also be imported
+and called programmatically.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from redmine_crm.crm_client import CrmClient
+from redmine_crm.redmine_client import RedmineClient
+from redmine_crm.redmine_client import RedmineNotFoundError
+
+import contact_enrichment
+
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_BASE_URL = "http://192.168.50.170"
+DEFAULT_PROJECT_ID = 1  # "customer-service" numeric project id common in this Redmine instance
+DEFAULT_STATE_FILE = Path(".cache/redmine_contact_sync/state.json")
+DEFAULT_LOG_LEVEL = "INFO"
+
+
+def _utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+class SyncMetrics:
+    """Lightweight in-memory metrics collector for a single sync run."""
+
+    def __init__(self) -> None:
+        self.started_at = _utc_now_iso()
+        self.finished_at: str | None = None
+        self.fetched = 0
+        self.enriched = 0
+        self.skipped = 0
+        self.created = 0
+        self.updated = 0
+        self.unchanged = 0
+        self.errors = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "fetched": self.fetched,
+            "enriched": self.enriched,
+            "skipped": self.skipped,
+            "created": self.created,
+            "updated": self.updated,
+            "unchanged": self.unchanged,
+            "errors": self.errors,
+        }
+
+
+class FileSyncState:
+    """Persist the last successful sync timestamp and last run metrics."""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._data: dict[str, Any] = {}
+        self.load()
+
+    def load(self) -> None:
+        if self.path.exists():
+            try:
+                self._data = json.loads(self.path.read_text(encoding="utf-8"))
+            except (json.JSONDecodeError, OSError) as exc:
+                logger.warning("Could not read sync state from %s: %s", self.path, exc)
+                self._data = {}
+        else:
+            self._data = {}
+
+    def save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps(self._data, indent=2, sort_keys=True), encoding="utf-8")
+
+    @property
+    def last_sync_timestamp(self) -> str | None:
+        return self._data.get("last_sync_timestamp")
+
+    @property
+    def last_metrics(self) -> dict[str, Any] | None:
+        return self._data.get("last_metrics")
+
+    def record_run(self, timestamp: str, metrics: SyncMetrics) -> None:
+        self._data["last_sync_timestamp"] = timestamp
+        self._data["last_metrics"] = metrics.to_dict()
+        self.save()
+
+
+def _configure_logging(level: str) -> None:
+    logging.basicConfig(
+        level=getattr(logging, level.upper(), logging.INFO),
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+        stream=sys.stderr,
+    )
+
+
+def _redmine_client_from_args(args: argparse.Namespace) -> RedmineClient:
+    api_key = os.getenv("REDMINE_API_KEY")
+    if not api_key:
+        raise RuntimeError("Set REDMINE_API_KEY in the environment.")
+    return RedmineClient(
+        base_url=args.base_url,
+        api_key=api_key,
+        project_id=args.project_id,
+    )
+
+
+def _enrichment_config_for_args(args: argparse.Namespace) -> contact_enrichment.EnrichmentConfig:
+    domain_map: dict[str, str] = {}
+    if args.domain_to_company:
+        for item in args.domain_to_company:
+            if "=" not in item:
+                raise ValueError(f"Invalid --domain-to-company value: {item!r} (expected domain=company)")
+            domain, company = item.split("=", 1)
+            domain_map[domain.strip()] = company.strip()
+
+    config = contact_enrichment.EnrichmentConfig(
+        source=args.source,
+        confidence_threshold=args.confidence_threshold,
+        default_country_code=args.default_country_code,
+        domain_to_company=domain_map,
+    )
+    if args.external_lookup:
+        # Expect a module.path:callable string, e.g. mymodule.plugins:lookup_company
+        mod_name, _, func_name = args.external_lookup.partition(":")
+        if not mod_name or not func_name:
+            raise ValueError("--external-lookup must be module.path:callable")
+        from importlib import import_module
+
+        module = import_module(mod_name)
+        config.external_lookup = getattr(module, func_name)
+    return config
+
+
+def run_pipeline(
+    *,
+    redmine: RedmineClient,
+    crm: CrmClient,
+    enrichment_config: contact_enrichment.EnrichmentConfig | None = None,
+    state: FileSyncState | None = None,
+    batch_size: int = 100,
+    dry_run: bool = False,
+) -> dict[str, Any]:
+    """Run the full fetch -> enrich -> upsert pipeline.
+
+    Returns a dict containing final metrics and the timestamp of this run.
+    """
+    metrics = SyncMetrics()
+    enrichment_config = enrichment_config or contact_enrichment.EnrichmentConfig()
+    state = state or FileSyncState(DEFAULT_STATE_FILE)
+
+    logger.info("Starting Redmine -> CRM sync; dry_run=%s last_sync=%s", dry_run, state.last_sync_timestamp)
+
+    # Extract
+    raw_contacts = redmine.iter_all_contacts(
+        updated_since=state.last_sync_timestamp,
+        batch_size=batch_size,
+    )
+    metrics.fetched = len(raw_contacts)
+    logger.info("Fetched %d Redmine contacts", metrics.fetched)
+
+    # Transform
+    enrichment_results = contact_enrichment.build_sync_payload(raw_contacts, enrichment_config)
+    records = enrichment_results["records"]
+    skipped = enrichment_results["skipped"]
+    metrics.enriched = len(records)
+    metrics.skipped = len(skipped)
+    logger.info("Enriched %d contacts, skipped %d", metrics.enriched, metrics.skipped)
+    for item in skipped:
+        logger.info("Skipped Redmine contact %s: %s", item.get("redmine_contact_id"), item.get("reason"))
+
+    # Load (skipped during dry runs so real CRM adapters can also be avoided)
+    if dry_run:
+        logger.info("Dry run: skipping CRM upsert")
+    else:
+        stats = crm.sync_batch(records)
+        metrics.created = stats.get("created", 0)
+        metrics.updated = stats.get("updated", 0)
+        metrics.unchanged = stats.get("skipped", 0)
+        metrics.errors = stats.get("errors", 0)
+        if stats.get("error_records"):
+            for err in stats["error_records"]:
+                logger.warning("CRM upsert error for Redmine contact %s: %s", err.get("redmine_contact_id"), err.get("reason"))
+
+    metrics.finished_at = _utc_now_iso()
+    result = {
+        "timestamp": metrics.finished_at,
+        "dry_run": dry_run,
+        "metrics": metrics.to_dict(),
+        "skipped_records": skipped,
+    }
+
+    if not dry_run:
+        state.record_run(result["timestamp"], metrics)
+
+    return result
+
+
+def build_argument_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Sync Redmine CRM contacts into a target CRM.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("--base-url", default=os.getenv("REDMINE_BASE_URL", DEFAULT_BASE_URL), help="Redmine base URL.")
+    parser.add_argument(
+        "--project-id",
+        type=int,
+        default=int(os.getenv("REDMINE_PROJECT_ID", str(DEFAULT_PROJECT_ID))),
+        help="Redmine numeric project id for contact filtering.",
+    )
+    parser.add_argument("--state-file", type=Path, default=DEFAULT_STATE_FILE, help="Path to the persistent sync state file.")
+    parser.add_argument("--batch-size", type=int, default=100, help="Redmine API page size.")
+    parser.add_argument("--dry-run", action="store_true", help="Fetch and enrich but do not write to CRM.")
+    parser.add_argument("--log-level", default=DEFAULT_LOG_LEVEL, choices=("DEBUG", "INFO", "WARNING", "ERROR"))
+    parser.add_argument("--source", default="redmine", help="Value to set in the source field.")
+    parser.add_argument("--confidence-threshold", type=float, default=0.7, help="Minimum enrichment confidence.")
+    parser.add_argument("--default-country-code", default="US", help="Default country code for phone normalization.")
+    parser.add_argument(
+        "--domain-to-company",
+        action="append",
+        metavar="DOMAIN=COMPANY",
+        help="Override company mapping for a specific email domain. Repeatable.",
+    )
+    parser.add_argument(
+        "--external-lookup",
+        help="Optional module.path:callable providing external company lookup.",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_argument_parser()
+    args = parser.parse_args(argv)
+    _configure_logging(args.log_level)
+
+    redmine = _redmine_client_from_args(args)
+    enrichment_config = _enrichment_config_for_args(args)
+    # The default in-memory CRM client is useful for local testing and dry runs.
+    crm = CrmClient()
+    state = FileSyncState(args.state_file)
+
+    try:
+        result = run_pipeline(
+            redmine=redmine,
+            crm=crm,
+            enrichment_config=enrichment_config,
+            state=state,
+            batch_size=args.batch_size,
+            dry_run=args.dry_run,
+        )
+    except Exception as exc:
+        metrics = SyncMetrics()
+        try:
+            result = _handle_not_found_as_empty(exc, metrics)
+            if not args.dry_run:
+                state.record_run(result["timestamp"], metrics)
+        except Exception:
+            logger.exception("Pipeline failed: %s", exc)
+            return 1
+
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+def _handle_not_found_as_empty(exc: Exception, metrics: SyncMetrics) -> dict[str, Any]:
+    """Treat a 404 from Redmine contacts as an empty sync (plugin may be absent)."""
+    if isinstance(exc, RedmineNotFoundError):
+        logger.warning("Redmine contacts endpoint returned 404; treating as empty sync.")
+        metrics.finished_at = _utc_now_iso()
+        return {
+            "timestamp": metrics.finished_at,
+            "dry_run": False,
+            "metrics": metrics.to_dict(),
+            "skipped_records": [],
+        }
+    raise
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())