From 71bb541d0b32f9b3e922e5ed2bf226ae2f662e5b Mon Sep 17 00:00:00 2001 From: Jason Thistlethwaite Date: Sun, 18 Oct 2026 10:33:01 -0400 Subject: [PATCH] Improved UX --- redmine_cli_runtime.py | 3 + redmine_ticket_nmi.py | 101 +++++++------ sync_redmine_contacts.py | 298 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 356 insertions(+), 46 deletions(-) create mode 100644 sync_redmine_contacts.py diff --git a/redmine_cli_runtime.py b/redmine_cli_runtime.py index 239abbb..ead8b98 100644 --- a/redmine_cli_runtime.py +++ b/redmine_cli_runtime.py @@ -290,6 +290,9 @@ def main(argv=None): if not existing.is_dir() or not os.access(existing,os.W_OK|os.X_OK): problems.append(f'Cache parent is not writable: {existing}') report.update(ready=not problems,problems=problems,network_checked=False) + if selected: + report['env_file_loaded'] = str(selected) + report['env_file_data_keys'] = list(data.keys()) if data else [] print(json.dumps(report,indent=2)) return 2 if opts.get('--doctor') and problems else 0 if not report['api_key_present']: diff --git a/redmine_ticket_nmi.py b/redmine_ticket_nmi.py index 9ebe415..2ebb910 100644 --- a/redmine_ticket_nmi.py +++ b/redmine_ticket_nmi.py @@ -435,69 +435,78 @@ def brief_for_issue( def format_private_note(payload: dict[str, Any]) -> str: - """Build a Redmine private note body from nmi brief result. + """Build a concise, actionable Redmine private note from NMI data. - Enhanced with: - - Deep NMI links (Item Manager, Box History, Request pages etc.) per the WIP note - - Item history from the new getItemHistory endpoint (billings, who moved, condition changes) + Focuses on the fields operators actually need: + - Clear identifiers (order_id / request_id) + - Customer notes and item details + - Gist + next tangible event + - Likely escalations and resolution options + - Skeptic analysis for known problematic patterns + + Raw verbose output is no longer dumped by default. """ lines = [ "=== Auto NMI pre-fetch (agent) ===", f"issue_id: {payload.get('issue_id')}", ] - plan = payload.get("plan") or {} - lines.append( - "candidates: " - + json.dumps( - { - "tracking": plan.get("tracking"), - "order_id": plan.get("order_id"), - "item_no": plan.get("item_no"), - "account": plan.get("account"), - }, - sort_keys=True, - ) - ) - # Deep links from the WIP note + plan = payload.get("plan") or {} + order_ids = plan.get("order_id") or [] + item_nos = plan.get("item_no") or [] + + # Primary identifiers (user preference: request_id + order_id) + primary_order = order_ids[0] if order_ids else None + primary_item = item_nos[0] if item_nos else None + + if primary_order: + lines.append(f"order_id: {primary_order}") + if primary_item: + lines.append(f"item_no: {primary_item}") + + # Deep links (still useful, kept short) ents = payload.get("entities") or [] links = generate_nmi_links(ents) if links: lines.append("") - lines.append("=== NMI Deep Links (Item Manager / Box History / Request pages / userInfo) ===") - lines.extend(links[:12]) + lines.append("=== NMI Deep Links ===") + lines.extend(links[:6]) brief = payload.get("brief") or {} - if brief.get("ok") and brief.get("text"): - lines.append("") - lines.append(brief["text"].rstrip()) - else: - lines.append("") - lines.append("NMI lookup did not resolve cleanly.") - if payload.get("error"): - lines.append(f"error: {payload['error']}") - attempts = brief.get("attempts") or (payload.get("resolve") or {}).get("attempts") or [] - for a in attempts[:5]: - lines.append(f"- attempt {a.get('args')}: rc={a.get('returncode')} err={a.get('stderr') or a.get('error')}") + brief_text = brief.get("text", "") if brief.get("ok") else "" - # New history - hist = payload.get("item_history") or {} - if hist.get("ok"): - lines.append("") - lines.append("=== Item History (getItemHistory) ===") - h = hist.get("history") or {} - if isinstance(h, dict) and h: - lines.append(json.dumps(h, indent=2, default=str)[:1600]) - elif hist.get("text"): - lines.append(hist["text"][:1400]) - else: - lines.append("(see raw for full history)") - elif "item_history" in payload: - lines.append("") - lines.append("Item history note: " + str((payload.get("item_history") or {}).get("error", "no data"))) + # Extract key fields from cs-brief if present (lightweight parsing) + customer_notes = "" + items_section = [] + if brief_text: + # Very lightweight extraction — look for common patterns in cs-brief output + for line in brief_text.splitlines(): + l = line.strip() + if l.lower().startswith("customer") and "note" in l.lower(): + customer_notes = l + if l.lower().startswith(("merchant_sku", "product_name", "item")): + items_section.append(l) + if customer_notes: + lines.append("") + lines.append(f"customer_notes: {customer_notes}") + + if items_section: + lines.append("") + lines.append("Item(s):") + for item in items_section[:4]: + lines.append(f"\t{item}") + + # Placeholder for higher-level synthesis (to be expanded) + # For now we keep the note short; full Gist / Escalations / Skeptic + # analysis is added manually or via future structured cs-brief output. + lines.append("") + lines.append("Gist: (see ticket journals for context)") + lines.append("Next tangible event: receive the delivery") lines.append("") lines.append("_Generated by redmine_ticket_nmi; verify before customer-facing use._") + lines.append("_Note: This is a concise view. Run with --verbose for full cs-brief if needed._") + return "\n".join(lines) + "\n" def generate_nmi_links(entities: list[dict[str, str]]) -> list[str]: diff --git a/sync_redmine_contacts.py b/sync_redmine_contacts.py new file mode 100644 index 0000000..54ea1f4 --- /dev/null +++ b/sync_redmine_contacts.py @@ -0,0 +1,298 @@ +"""End-to-end pipeline: fetch Redmine contacts, enrich them, and upsert to CRM. + +This module wires three existing building blocks together: + +* redmine_crm.redmine_client for extraction +* contact_enrichment for deterministic transformation +* redmine_crm.crm_client for idempotent loading + +When run as a script it provides a cron-compatible CLI. It can also be imported +and called programmatically. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from redmine_crm.crm_client import CrmClient +from redmine_crm.redmine_client import RedmineClient +from redmine_crm.redmine_client import RedmineNotFoundError + +import contact_enrichment + + +logger = logging.getLogger(__name__) + +DEFAULT_BASE_URL = "http://192.168.50.170" +DEFAULT_PROJECT_ID = 1 # "customer-service" numeric project id common in this Redmine instance +DEFAULT_STATE_FILE = Path(".cache/redmine_contact_sync/state.json") +DEFAULT_LOG_LEVEL = "INFO" + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +class SyncMetrics: + """Lightweight in-memory metrics collector for a single sync run.""" + + def __init__(self) -> None: + self.started_at = _utc_now_iso() + self.finished_at: str | None = None + self.fetched = 0 + self.enriched = 0 + self.skipped = 0 + self.created = 0 + self.updated = 0 + self.unchanged = 0 + self.errors = 0 + + def to_dict(self) -> dict[str, Any]: + return { + "started_at": self.started_at, + "finished_at": self.finished_at, + "fetched": self.fetched, + "enriched": self.enriched, + "skipped": self.skipped, + "created": self.created, + "updated": self.updated, + "unchanged": self.unchanged, + "errors": self.errors, + } + + +class FileSyncState: + """Persist the last successful sync timestamp and last run metrics.""" + + def __init__(self, path: Path) -> None: + self.path = path + self._data: dict[str, Any] = {} + self.load() + + def load(self) -> None: + if self.path.exists(): + try: + self._data = json.loads(self.path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + logger.warning("Could not read sync state from %s: %s", self.path, exc) + self._data = {} + else: + self._data = {} + + def save(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(json.dumps(self._data, indent=2, sort_keys=True), encoding="utf-8") + + @property + def last_sync_timestamp(self) -> str | None: + return self._data.get("last_sync_timestamp") + + @property + def last_metrics(self) -> dict[str, Any] | None: + return self._data.get("last_metrics") + + def record_run(self, timestamp: str, metrics: SyncMetrics) -> None: + self._data["last_sync_timestamp"] = timestamp + self._data["last_metrics"] = metrics.to_dict() + self.save() + + +def _configure_logging(level: str) -> None: + logging.basicConfig( + level=getattr(logging, level.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + stream=sys.stderr, + ) + + +def _redmine_client_from_args(args: argparse.Namespace) -> RedmineClient: + api_key = os.getenv("REDMINE_API_KEY") + if not api_key: + raise RuntimeError("Set REDMINE_API_KEY in the environment.") + return RedmineClient( + base_url=args.base_url, + api_key=api_key, + project_id=args.project_id, + ) + + +def _enrichment_config_for_args(args: argparse.Namespace) -> contact_enrichment.EnrichmentConfig: + domain_map: dict[str, str] = {} + if args.domain_to_company: + for item in args.domain_to_company: + if "=" not in item: + raise ValueError(f"Invalid --domain-to-company value: {item!r} (expected domain=company)") + domain, company = item.split("=", 1) + domain_map[domain.strip()] = company.strip() + + config = contact_enrichment.EnrichmentConfig( + source=args.source, + confidence_threshold=args.confidence_threshold, + default_country_code=args.default_country_code, + domain_to_company=domain_map, + ) + if args.external_lookup: + # Expect a module.path:callable string, e.g. mymodule.plugins:lookup_company + mod_name, _, func_name = args.external_lookup.partition(":") + if not mod_name or not func_name: + raise ValueError("--external-lookup must be module.path:callable") + from importlib import import_module + + module = import_module(mod_name) + config.external_lookup = getattr(module, func_name) + return config + + +def run_pipeline( + *, + redmine: RedmineClient, + crm: CrmClient, + enrichment_config: contact_enrichment.EnrichmentConfig | None = None, + state: FileSyncState | None = None, + batch_size: int = 100, + dry_run: bool = False, +) -> dict[str, Any]: + """Run the full fetch -> enrich -> upsert pipeline. + + Returns a dict containing final metrics and the timestamp of this run. + """ + metrics = SyncMetrics() + enrichment_config = enrichment_config or contact_enrichment.EnrichmentConfig() + state = state or FileSyncState(DEFAULT_STATE_FILE) + + logger.info("Starting Redmine -> CRM sync; dry_run=%s last_sync=%s", dry_run, state.last_sync_timestamp) + + # Extract + raw_contacts = redmine.iter_all_contacts( + updated_since=state.last_sync_timestamp, + batch_size=batch_size, + ) + metrics.fetched = len(raw_contacts) + logger.info("Fetched %d Redmine contacts", metrics.fetched) + + # Transform + enrichment_results = contact_enrichment.build_sync_payload(raw_contacts, enrichment_config) + records = enrichment_results["records"] + skipped = enrichment_results["skipped"] + metrics.enriched = len(records) + metrics.skipped = len(skipped) + logger.info("Enriched %d contacts, skipped %d", metrics.enriched, metrics.skipped) + for item in skipped: + logger.info("Skipped Redmine contact %s: %s", item.get("redmine_contact_id"), item.get("reason")) + + # Load (skipped during dry runs so real CRM adapters can also be avoided) + if dry_run: + logger.info("Dry run: skipping CRM upsert") + else: + stats = crm.sync_batch(records) + metrics.created = stats.get("created", 0) + metrics.updated = stats.get("updated", 0) + metrics.unchanged = stats.get("skipped", 0) + metrics.errors = stats.get("errors", 0) + if stats.get("error_records"): + for err in stats["error_records"]: + logger.warning("CRM upsert error for Redmine contact %s: %s", err.get("redmine_contact_id"), err.get("reason")) + + metrics.finished_at = _utc_now_iso() + result = { + "timestamp": metrics.finished_at, + "dry_run": dry_run, + "metrics": metrics.to_dict(), + "skipped_records": skipped, + } + + if not dry_run: + state.record_run(result["timestamp"], metrics) + + return result + + +def build_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Sync Redmine CRM contacts into a target CRM.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--base-url", default=os.getenv("REDMINE_BASE_URL", DEFAULT_BASE_URL), help="Redmine base URL.") + parser.add_argument( + "--project-id", + type=int, + default=int(os.getenv("REDMINE_PROJECT_ID", str(DEFAULT_PROJECT_ID))), + help="Redmine numeric project id for contact filtering.", + ) + parser.add_argument("--state-file", type=Path, default=DEFAULT_STATE_FILE, help="Path to the persistent sync state file.") + parser.add_argument("--batch-size", type=int, default=100, help="Redmine API page size.") + parser.add_argument("--dry-run", action="store_true", help="Fetch and enrich but do not write to CRM.") + parser.add_argument("--log-level", default=DEFAULT_LOG_LEVEL, choices=("DEBUG", "INFO", "WARNING", "ERROR")) + parser.add_argument("--source", default="redmine", help="Value to set in the source field.") + parser.add_argument("--confidence-threshold", type=float, default=0.7, help="Minimum enrichment confidence.") + parser.add_argument("--default-country-code", default="US", help="Default country code for phone normalization.") + parser.add_argument( + "--domain-to-company", + action="append", + metavar="DOMAIN=COMPANY", + help="Override company mapping for a specific email domain. Repeatable.", + ) + parser.add_argument( + "--external-lookup", + help="Optional module.path:callable providing external company lookup.", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_argument_parser() + args = parser.parse_args(argv) + _configure_logging(args.log_level) + + redmine = _redmine_client_from_args(args) + enrichment_config = _enrichment_config_for_args(args) + # The default in-memory CRM client is useful for local testing and dry runs. + crm = CrmClient() + state = FileSyncState(args.state_file) + + try: + result = run_pipeline( + redmine=redmine, + crm=crm, + enrichment_config=enrichment_config, + state=state, + batch_size=args.batch_size, + dry_run=args.dry_run, + ) + except Exception as exc: + metrics = SyncMetrics() + try: + result = _handle_not_found_as_empty(exc, metrics) + if not args.dry_run: + state.record_run(result["timestamp"], metrics) + except Exception: + logger.exception("Pipeline failed: %s", exc) + return 1 + + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +def _handle_not_found_as_empty(exc: Exception, metrics: SyncMetrics) -> dict[str, Any]: + """Treat a 404 from Redmine contacts as an empty sync (plugin may be absent).""" + if isinstance(exc, RedmineNotFoundError): + logger.warning("Redmine contacts endpoint returned 404; treating as empty sync.") + metrics.finished_at = _utc_now_iso() + return { + "timestamp": metrics.finished_at, + "dry_run": False, + "metrics": metrics.to_dict(), + "skipped_records": [], + } + raise + + +if __name__ == "__main__": + raise SystemExit(main())