Improved UX
This commit is contained in:
Jason Thistlethwaite
@@ -0,0 +1,298 @@
|
||||
"""End-to-end pipeline: fetch Redmine contacts, enrich them, and upsert to CRM.
|
||||
|
||||
This module wires three existing building blocks together:
|
||||
|
||||
* redmine_crm.redmine_client for extraction
|
||||
* contact_enrichment for deterministic transformation
|
||||
* redmine_crm.crm_client for idempotent loading
|
||||
|
||||
When run as a script it provides a cron-compatible CLI. It can also be imported
|
||||
and called programmatically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from redmine_crm.crm_client import CrmClient
|
||||
from redmine_crm.redmine_client import RedmineClient
|
||||
from redmine_crm.redmine_client import RedmineNotFoundError
|
||||
|
||||
import contact_enrichment
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_BASE_URL = "http://192.168.50.170"
|
||||
DEFAULT_PROJECT_ID = 1 # "customer-service" numeric project id common in this Redmine instance
|
||||
DEFAULT_STATE_FILE = Path(".cache/redmine_contact_sync/state.json")
|
||||
DEFAULT_LOG_LEVEL = "INFO"
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
class SyncMetrics:
|
||||
"""Lightweight in-memory metrics collector for a single sync run."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.started_at = _utc_now_iso()
|
||||
self.finished_at: str | None = None
|
||||
self.fetched = 0
|
||||
self.enriched = 0
|
||||
self.skipped = 0
|
||||
self.created = 0
|
||||
self.updated = 0
|
||||
self.unchanged = 0
|
||||
self.errors = 0
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"fetched": self.fetched,
|
||||
"enriched": self.enriched,
|
||||
"skipped": self.skipped,
|
||||
"created": self.created,
|
||||
"updated": self.updated,
|
||||
"unchanged": self.unchanged,
|
||||
"errors": self.errors,
|
||||
}
|
||||
|
||||
|
||||
class FileSyncState:
|
||||
"""Persist the last successful sync timestamp and last run metrics."""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self._data: dict[str, Any] = {}
|
||||
self.load()
|
||||
|
||||
def load(self) -> None:
|
||||
if self.path.exists():
|
||||
try:
|
||||
self._data = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
logger.warning("Could not read sync state from %s: %s", self.path, exc)
|
||||
self._data = {}
|
||||
else:
|
||||
self._data = {}
|
||||
|
||||
def save(self) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text(json.dumps(self._data, indent=2, sort_keys=True), encoding="utf-8")
|
||||
|
||||
@property
|
||||
def last_sync_timestamp(self) -> str | None:
|
||||
return self._data.get("last_sync_timestamp")
|
||||
|
||||
@property
|
||||
def last_metrics(self) -> dict[str, Any] | None:
|
||||
return self._data.get("last_metrics")
|
||||
|
||||
def record_run(self, timestamp: str, metrics: SyncMetrics) -> None:
|
||||
self._data["last_sync_timestamp"] = timestamp
|
||||
self._data["last_metrics"] = metrics.to_dict()
|
||||
self.save()
|
||||
|
||||
|
||||
def _configure_logging(level: str) -> None:
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, level.upper(), logging.INFO),
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
|
||||
|
||||
def _redmine_client_from_args(args: argparse.Namespace) -> RedmineClient:
|
||||
api_key = os.getenv("REDMINE_API_KEY")
|
||||
if not api_key:
|
||||
raise RuntimeError("Set REDMINE_API_KEY in the environment.")
|
||||
return RedmineClient(
|
||||
base_url=args.base_url,
|
||||
api_key=api_key,
|
||||
project_id=args.project_id,
|
||||
)
|
||||
|
||||
|
||||
def _enrichment_config_for_args(args: argparse.Namespace) -> contact_enrichment.EnrichmentConfig:
|
||||
domain_map: dict[str, str] = {}
|
||||
if args.domain_to_company:
|
||||
for item in args.domain_to_company:
|
||||
if "=" not in item:
|
||||
raise ValueError(f"Invalid --domain-to-company value: {item!r} (expected domain=company)")
|
||||
domain, company = item.split("=", 1)
|
||||
domain_map[domain.strip()] = company.strip()
|
||||
|
||||
config = contact_enrichment.EnrichmentConfig(
|
||||
source=args.source,
|
||||
confidence_threshold=args.confidence_threshold,
|
||||
default_country_code=args.default_country_code,
|
||||
domain_to_company=domain_map,
|
||||
)
|
||||
if args.external_lookup:
|
||||
# Expect a module.path:callable string, e.g. mymodule.plugins:lookup_company
|
||||
mod_name, _, func_name = args.external_lookup.partition(":")
|
||||
if not mod_name or not func_name:
|
||||
raise ValueError("--external-lookup must be module.path:callable")
|
||||
from importlib import import_module
|
||||
|
||||
module = import_module(mod_name)
|
||||
config.external_lookup = getattr(module, func_name)
|
||||
return config
|
||||
|
||||
|
||||
def run_pipeline(
|
||||
*,
|
||||
redmine: RedmineClient,
|
||||
crm: CrmClient,
|
||||
enrichment_config: contact_enrichment.EnrichmentConfig | None = None,
|
||||
state: FileSyncState | None = None,
|
||||
batch_size: int = 100,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the full fetch -> enrich -> upsert pipeline.
|
||||
|
||||
Returns a dict containing final metrics and the timestamp of this run.
|
||||
"""
|
||||
metrics = SyncMetrics()
|
||||
enrichment_config = enrichment_config or contact_enrichment.EnrichmentConfig()
|
||||
state = state or FileSyncState(DEFAULT_STATE_FILE)
|
||||
|
||||
logger.info("Starting Redmine -> CRM sync; dry_run=%s last_sync=%s", dry_run, state.last_sync_timestamp)
|
||||
|
||||
# Extract
|
||||
raw_contacts = redmine.iter_all_contacts(
|
||||
updated_since=state.last_sync_timestamp,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
metrics.fetched = len(raw_contacts)
|
||||
logger.info("Fetched %d Redmine contacts", metrics.fetched)
|
||||
|
||||
# Transform
|
||||
enrichment_results = contact_enrichment.build_sync_payload(raw_contacts, enrichment_config)
|
||||
records = enrichment_results["records"]
|
||||
skipped = enrichment_results["skipped"]
|
||||
metrics.enriched = len(records)
|
||||
metrics.skipped = len(skipped)
|
||||
logger.info("Enriched %d contacts, skipped %d", metrics.enriched, metrics.skipped)
|
||||
for item in skipped:
|
||||
logger.info("Skipped Redmine contact %s: %s", item.get("redmine_contact_id"), item.get("reason"))
|
||||
|
||||
# Load (skipped during dry runs so real CRM adapters can also be avoided)
|
||||
if dry_run:
|
||||
logger.info("Dry run: skipping CRM upsert")
|
||||
else:
|
||||
stats = crm.sync_batch(records)
|
||||
metrics.created = stats.get("created", 0)
|
||||
metrics.updated = stats.get("updated", 0)
|
||||
metrics.unchanged = stats.get("skipped", 0)
|
||||
metrics.errors = stats.get("errors", 0)
|
||||
if stats.get("error_records"):
|
||||
for err in stats["error_records"]:
|
||||
logger.warning("CRM upsert error for Redmine contact %s: %s", err.get("redmine_contact_id"), err.get("reason"))
|
||||
|
||||
metrics.finished_at = _utc_now_iso()
|
||||
result = {
|
||||
"timestamp": metrics.finished_at,
|
||||
"dry_run": dry_run,
|
||||
"metrics": metrics.to_dict(),
|
||||
"skipped_records": skipped,
|
||||
}
|
||||
|
||||
if not dry_run:
|
||||
state.record_run(result["timestamp"], metrics)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def build_argument_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Sync Redmine CRM contacts into a target CRM.",
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--base-url", default=os.getenv("REDMINE_BASE_URL", DEFAULT_BASE_URL), help="Redmine base URL.")
|
||||
parser.add_argument(
|
||||
"--project-id",
|
||||
type=int,
|
||||
default=int(os.getenv("REDMINE_PROJECT_ID", str(DEFAULT_PROJECT_ID))),
|
||||
help="Redmine numeric project id for contact filtering.",
|
||||
)
|
||||
parser.add_argument("--state-file", type=Path, default=DEFAULT_STATE_FILE, help="Path to the persistent sync state file.")
|
||||
parser.add_argument("--batch-size", type=int, default=100, help="Redmine API page size.")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Fetch and enrich but do not write to CRM.")
|
||||
parser.add_argument("--log-level", default=DEFAULT_LOG_LEVEL, choices=("DEBUG", "INFO", "WARNING", "ERROR"))
|
||||
parser.add_argument("--source", default="redmine", help="Value to set in the source field.")
|
||||
parser.add_argument("--confidence-threshold", type=float, default=0.7, help="Minimum enrichment confidence.")
|
||||
parser.add_argument("--default-country-code", default="US", help="Default country code for phone normalization.")
|
||||
parser.add_argument(
|
||||
"--domain-to-company",
|
||||
action="append",
|
||||
metavar="DOMAIN=COMPANY",
|
||||
help="Override company mapping for a specific email domain. Repeatable.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--external-lookup",
|
||||
help="Optional module.path:callable providing external company lookup.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = build_argument_parser()
|
||||
args = parser.parse_args(argv)
|
||||
_configure_logging(args.log_level)
|
||||
|
||||
redmine = _redmine_client_from_args(args)
|
||||
enrichment_config = _enrichment_config_for_args(args)
|
||||
# The default in-memory CRM client is useful for local testing and dry runs.
|
||||
crm = CrmClient()
|
||||
state = FileSyncState(args.state_file)
|
||||
|
||||
try:
|
||||
result = run_pipeline(
|
||||
redmine=redmine,
|
||||
crm=crm,
|
||||
enrichment_config=enrichment_config,
|
||||
state=state,
|
||||
batch_size=args.batch_size,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
except Exception as exc:
|
||||
metrics = SyncMetrics()
|
||||
try:
|
||||
result = _handle_not_found_as_empty(exc, metrics)
|
||||
if not args.dry_run:
|
||||
state.record_run(result["timestamp"], metrics)
|
||||
except Exception:
|
||||
logger.exception("Pipeline failed: %s", exc)
|
||||
return 1
|
||||
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def _handle_not_found_as_empty(exc: Exception, metrics: SyncMetrics) -> dict[str, Any]:
|
||||
"""Treat a 404 from Redmine contacts as an empty sync (plugin may be absent)."""
|
||||
if isinstance(exc, RedmineNotFoundError):
|
||||
logger.warning("Redmine contacts endpoint returned 404; treating as empty sync.")
|
||||
metrics.finished_at = _utc_now_iso()
|
||||
return {
|
||||
"timestamp": metrics.finished_at,
|
||||
"dry_run": False,
|
||||
"metrics": metrics.to_dict(),
|
||||
"skipped_records": [],
|
||||
}
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user