redmine/sync_redmine_contacts.py

"""End-to-end pipeline: fetch Redmine contacts, enrich them, and upsert to CRM.

This module wires three existing building blocks together:

* redmine_crm.redmine_client for extraction
* contact_enrichment for deterministic transformation
* redmine_crm.crm_client for idempotent loading

When run as a script it provides a cron-compatible CLI.  It can also be imported
and called programmatically.
"""

from __future__ import annotations

import argparse
import json
import logging
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

from redmine_crm.crm_client import CrmClient
from redmine_crm.redmine_client import RedmineClient
from redmine_crm.redmine_client import RedmineNotFoundError

import contact_enrichment


logger = logging.getLogger(__name__)

DEFAULT_BASE_URL = "http://192.168.50.170"
DEFAULT_PROJECT_ID = 1  # "customer-service" numeric project id common in this Redmine instance
DEFAULT_STATE_FILE = Path(".cache/redmine_contact_sync/state.json")
DEFAULT_LOG_LEVEL = "INFO"


def _utc_now_iso() -> str:
    return datetime.now(timezone.utc).isoformat()


class SyncMetrics:
    """Lightweight in-memory metrics collector for a single sync run."""

    def __init__(self) -> None:
        self.started_at = _utc_now_iso()
        self.finished_at: str | None = None
        self.fetched = 0
        self.enriched = 0
        self.skipped = 0
        self.created = 0
        self.updated = 0
        self.unchanged = 0
        self.errors = 0

    def to_dict(self) -> dict[str, Any]:
        return {
            "started_at": self.started_at,
            "finished_at": self.finished_at,
            "fetched": self.fetched,
            "enriched": self.enriched,
            "skipped": self.skipped,
            "created": self.created,
            "updated": self.updated,
            "unchanged": self.unchanged,
            "errors": self.errors,
        }


class FileSyncState:
    """Persist the last successful sync timestamp and last run metrics."""

    def __init__(self, path: Path) -> None:
        self.path = path
        self._data: dict[str, Any] = {}
        self.load()

    def load(self) -> None:
        if self.path.exists():
            try:
                self._data = json.loads(self.path.read_text(encoding="utf-8"))
            except (json.JSONDecodeError, OSError) as exc:
                logger.warning("Could not read sync state from %s: %s", self.path, exc)
                self._data = {}
        else:
            self._data = {}

    def save(self) -> None:
        self.path.parent.mkdir(parents=True, exist_ok=True)
        self.path.write_text(json.dumps(self._data, indent=2, sort_keys=True), encoding="utf-8")

    @property
    def last_sync_timestamp(self) -> str | None:
        return self._data.get("last_sync_timestamp")

    @property
    def last_metrics(self) -> dict[str, Any] | None:
        return self._data.get("last_metrics")

    def record_run(self, timestamp: str, metrics: SyncMetrics) -> None:
        self._data["last_sync_timestamp"] = timestamp
        self._data["last_metrics"] = metrics.to_dict()
        self.save()


def _configure_logging(level: str) -> None:
    logging.basicConfig(
        level=getattr(logging, level.upper(), logging.INFO),
        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
        stream=sys.stderr,
    )


def _redmine_client_from_args(args: argparse.Namespace) -> RedmineClient:
    api_key = os.getenv("REDMINE_API_KEY")
    if not api_key:
        raise RuntimeError("Set REDMINE_API_KEY in the environment.")
    return RedmineClient(
        base_url=args.base_url,
        api_key=api_key,
        project_id=args.project_id,
    )


def _enrichment_config_for_args(args: argparse.Namespace) -> contact_enrichment.EnrichmentConfig:
    domain_map: dict[str, str] = {}
    if args.domain_to_company:
        for item in args.domain_to_company:
            if "=" not in item:
                raise ValueError(f"Invalid --domain-to-company value: {item!r} (expected domain=company)")
            domain, company = item.split("=", 1)
            domain_map[domain.strip()] = company.strip()

    config = contact_enrichment.EnrichmentConfig(
        source=args.source,
        confidence_threshold=args.confidence_threshold,
        default_country_code=args.default_country_code,
        domain_to_company=domain_map,
    )
    if args.external_lookup:
        # Expect a module.path:callable string, e.g. mymodule.plugins:lookup_company
        mod_name, _, func_name = args.external_lookup.partition(":")
        if not mod_name or not func_name:
            raise ValueError("--external-lookup must be module.path:callable")
        from importlib import import_module

        module = import_module(mod_name)
        config.external_lookup = getattr(module, func_name)
    return config


def run_pipeline(
    *,
    redmine: RedmineClient,
    crm: CrmClient,
    enrichment_config: contact_enrichment.EnrichmentConfig | None = None,
    state: FileSyncState | None = None,
    batch_size: int = 100,
    dry_run: bool = False,
) -> dict[str, Any]:
    """Run the full fetch -> enrich -> upsert pipeline.

    Returns a dict containing final metrics and the timestamp of this run.
    """
    metrics = SyncMetrics()
    enrichment_config = enrichment_config or contact_enrichment.EnrichmentConfig()
    state = state or FileSyncState(DEFAULT_STATE_FILE)

    logger.info("Starting Redmine -> CRM sync; dry_run=%s last_sync=%s", dry_run, state.last_sync_timestamp)

    # Extract
    raw_contacts = redmine.iter_all_contacts(
        updated_since=state.last_sync_timestamp,
        batch_size=batch_size,
    )
    metrics.fetched = len(raw_contacts)
    logger.info("Fetched %d Redmine contacts", metrics.fetched)

    # Transform
    enrichment_results = contact_enrichment.build_sync_payload(raw_contacts, enrichment_config)
    records = enrichment_results["records"]
    skipped = enrichment_results["skipped"]
    metrics.enriched = len(records)
    metrics.skipped = len(skipped)
    logger.info("Enriched %d contacts, skipped %d", metrics.enriched, metrics.skipped)
    for item in skipped:
        logger.info("Skipped Redmine contact %s: %s", item.get("redmine_contact_id"), item.get("reason"))

    # Load (skipped during dry runs so real CRM adapters can also be avoided)
    if dry_run:
        logger.info("Dry run: skipping CRM upsert")
    else:
        stats = crm.sync_batch(records)
        metrics.created = stats.get("created", 0)
        metrics.updated = stats.get("updated", 0)
        metrics.unchanged = stats.get("skipped", 0)
        metrics.errors = stats.get("errors", 0)
        if stats.get("error_records"):
            for err in stats["error_records"]:
                logger.warning("CRM upsert error for Redmine contact %s: %s", err.get("redmine_contact_id"), err.get("reason"))

    metrics.finished_at = _utc_now_iso()
    result = {
        "timestamp": metrics.finished_at,
        "dry_run": dry_run,
        "metrics": metrics.to_dict(),
        "skipped_records": skipped,
    }

    if not dry_run:
        state.record_run(result["timestamp"], metrics)

    return result


def build_argument_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="Sync Redmine CRM contacts into a target CRM.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument("--base-url", default=os.getenv("REDMINE_BASE_URL", DEFAULT_BASE_URL), help="Redmine base URL.")
    parser.add_argument(
        "--project-id",
        type=int,
        default=int(os.getenv("REDMINE_PROJECT_ID", str(DEFAULT_PROJECT_ID))),
        help="Redmine numeric project id for contact filtering.",
    )
    parser.add_argument("--state-file", type=Path, default=DEFAULT_STATE_FILE, help="Path to the persistent sync state file.")
    parser.add_argument("--batch-size", type=int, default=100, help="Redmine API page size.")
    parser.add_argument("--dry-run", action="store_true", help="Fetch and enrich but do not write to CRM.")
    parser.add_argument("--log-level", default=DEFAULT_LOG_LEVEL, choices=("DEBUG", "INFO", "WARNING", "ERROR"))
    parser.add_argument("--source", default="redmine", help="Value to set in the source field.")
    parser.add_argument("--confidence-threshold", type=float, default=0.7, help="Minimum enrichment confidence.")
    parser.add_argument("--default-country-code", default="US", help="Default country code for phone normalization.")
    parser.add_argument(
        "--domain-to-company",
        action="append",
        metavar="DOMAIN=COMPANY",
        help="Override company mapping for a specific email domain. Repeatable.",
    )
    parser.add_argument(
        "--external-lookup",
        help="Optional module.path:callable providing external company lookup.",
    )
    return parser


def main(argv: list[str] | None = None) -> int:
    parser = build_argument_parser()
    args = parser.parse_args(argv)
    _configure_logging(args.log_level)

    redmine = _redmine_client_from_args(args)
    enrichment_config = _enrichment_config_for_args(args)
    # The default in-memory CRM client is useful for local testing and dry runs.
    crm = CrmClient()
    state = FileSyncState(args.state_file)

    try:
        result = run_pipeline(
            redmine=redmine,
            crm=crm,
            enrichment_config=enrichment_config,
            state=state,
            batch_size=args.batch_size,
            dry_run=args.dry_run,
        )
    except Exception as exc:
        metrics = SyncMetrics()
        try:
            result = _handle_not_found_as_empty(exc, metrics)
            if not args.dry_run:
                state.record_run(result["timestamp"], metrics)
        except Exception:
            logger.exception("Pipeline failed: %s", exc)
            return 1

    print(json.dumps(result, indent=2, sort_keys=True))
    return 0


def _handle_not_found_as_empty(exc: Exception, metrics: SyncMetrics) -> dict[str, Any]:
    """Treat a 404 from Redmine contacts as an empty sync (plugin may be absent)."""
    if isinstance(exc, RedmineNotFoundError):
        logger.warning("Redmine contacts endpoint returned 404; treating as empty sync.")
        metrics.finished_at = _utc_now_iso()
        return {
            "timestamp": metrics.finished_at,
            "dry_run": False,
            "metrics": metrics.to_dict(),
            "skipped_records": [],
        }
    raise


if __name__ == "__main__":
    raise SystemExit(main())