#!/usr/bin/env python3 """Contact enrichment: map Redmine CRM contacts to a target CRM schema. This module is intentionally decoupled from ``redmine_contacts.py`` so the sync module can import it without pulling in CLI / cache logic. All enrichment is deterministic and side-effect free; external lookups are accepted via a configurable provider so callers can inject rate-limited clients. """ from __future__ import annotations import json import re from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any, Callable EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") PHONE_RE = re.compile(r"[\d\s\-().+]{7,}") URL_RE = re.compile(r"https?://[^\s<>\"']+|www\.[^\s<>\"']+") def _utc_now() -> datetime: return datetime.now(timezone.utc) @dataclass class EnrichmentResult: """Output of enriching a single Redmine contact.""" contact: dict[str, Any] enriched: dict[str, Any] changed_fields: list[str] warnings: list[str] = field(default_factory=list) skipped: bool = False reason: str = "" @dataclass class EnrichmentConfig: """Configuration for the enrichment pipeline.""" source: str = "redmine" confidence_threshold: float = 0.7 default_country_code: str = "US" tags_from_redmine: list[str] = field(default_factory=lambda: ["from-redmine"]) external_lookup: Callable[[str], dict[str, Any]] | None = None domain_to_company: dict[str, str] = field(default_factory=dict) class EnrichmentError(RuntimeError): pass class RateLimitError(EnrichmentError): """Raised when an external lookup exhausts retries or hits a hard limit.""" pass def enrich_contact( redmine_contact: dict[str, Any], config: EnrichmentConfig | None = None, ) -> EnrichmentResult: """Map a Redmine contact to the target CRM schema and fill missing fields.""" config = config or EnrichmentConfig() contact_id = redmine_contact.get("id") if contact_id is None: return EnrichmentResult( contact=redmine_contact, enriched={}, changed_fields=[], skipped=True, reason="Missing Redmine contact id", ) warnings: list[str] = [] emails = _extract_emails(redmine_contact, warnings) phones = _extract_phones(redmine_contact, config.default_country_code, warnings) company = _enrich_company(redmine_contact, emails, config, warnings) name_parts = _name_parts(redmine_contact) address = _extract_address(redmine_contact) notes = _build_notes(redmine_contact) tags = _build_tags(redmine_contact, config) enriched: dict[str, Any] = { "crm_id": None, "redmine_contact_id": contact_id, "full_name": name_parts["full_name"], "first_name": name_parts["first_name"], "middle_name": name_parts.get("middle_name"), "last_name": name_parts["last_name"], "company": company, "title": _clean_str(redmine_contact.get("job_title")), "emails": emails, "phones": phones, "website": _clean_str(redmine_contact.get("website")), "birthday": _format_birthday(redmine_contact.get("birthday")), "address": address, "notes": notes, "tags": tags, "source": config.source, "synced_at": _utc_now().isoformat(), "enriched_at": _utc_now().isoformat(), "last_redmine_update": _format_timestamp(redmine_contact.get("updated_at", redmine_contact.get("updated_on"))), } changed_fields = [key for key, value in enriched.items() if value not in (None, "", [], {})] if not emails and not company: warnings.append("No email or company; record may be unmatchable in CRM") return EnrichmentResult( contact=redmine_contact, enriched=enriched, changed_fields=changed_fields, warnings=warnings, ) def _clean_str(value: Any) -> str | None: if value is None: return None cleaned = str(value).strip() return cleaned or None def _extract_emails(redmine_contact: dict[str, Any], warnings: list[str]) -> list[dict[str, Any]]: emails: list[dict[str, Any]] = [] primary = _clean_str(redmine_contact.get("email")) if primary: emails.append({"value": primary.lower(), "type": "work"}) # Search background and custom fields for additional emails when primary missing. if not emails: haystack = _contact_text_haystack(redmine_contact) for match in EMAIL_RE.findall(haystack): candidate = match.lower() if not any(existing["value"] == candidate for existing in emails): emails.append({"value": candidate, "type": "work"}) if not emails: warnings.append("No email found") return emails def _extract_phones( redmine_contact: dict[str, Any], default_country_code: str, warnings: list[str], ) -> list[dict[str, Any]]: phones: list[dict[str, Any]] = [] raw_phone = _clean_str(redmine_contact.get("phone")) if raw_phone: normalized = _normalize_phone(raw_phone, default_country_code) if normalized: phones.append({"value": normalized, "type": _classify_phone_type(raw_phone)}) # Do not mine background for phones unless the explicit field is empty. if not phones: haystack = _contact_text_haystack(redmine_contact) for match in PHONE_RE.findall(haystack): normalized = _normalize_phone(match, default_country_code) if normalized and not any(existing["value"] == normalized for existing in phones): phones.append({"value": normalized, "type": _classify_phone_type(match)}) if len(phones) >= 3: break if not phones: warnings.append("No phone found") return phones def _normalize_phone(raw: str, default_country_code: str) -> str | None: digits = re.sub(r"\D", "", raw) if len(digits) < 7: return None if len(digits) == 10 and default_country_code == "US": digits = "1" + digits if digits.startswith("1") and len(digits) == 11: return f"+{digits}" return f"+{digits}" def _classify_phone_type(raw: str) -> str: lowered = raw.lower() if "mobile" in lowered or "cell" in lowered or "m:" in lowered: return "mobile" if "home" in lowered or "h:" in lowered: return "home" return "work" def _enrich_company( redmine_contact: dict[str, Any], emails: list[dict[str, Any]], config: EnrichmentConfig, warnings: list[str], ) -> str | None: company = _clean_str(redmine_contact.get("company")) if company: return company if config.external_lookup is not None: try: lookup = config.external_lookup(str(redmine_contact.get("id"))) company = _clean_str(lookup.get("company")) if company: return company except RateLimitError: warnings.append("External company lookup hit rate limit") except Exception as exc: # noqa: BLE001 warnings.append(f"External company lookup failed: {exc}") if emails: domain = emails[0]["value"].split("@", 1)[-1] company = config.domain_to_company.get(domain) if company: return company company = _company_from_domain(domain) if company: return company warnings.append("Company could not be enriched") return None def _company_from_domain(domain: str) -> str | None: # Simple deterministic heuristic; keeps component dependency-free. if not domain or "." not in domain: return None name = domain.rsplit(".", 1)[0] name = re.sub(r"[^a-zA-Z0-9]+", " ", name).strip() if not name: return None return name.capitalize() + " Inc" def _name_parts(redmine_contact: dict[str, Any]) -> dict[str, Any]: first = _clean_str(redmine_contact.get("first_name")) middle = _clean_str(redmine_contact.get("middle_name")) last = _clean_str(redmine_contact.get("last_name")) if redmine_contact.get("is_company"): company = _clean_str(redmine_contact.get("company")) or first or "" return {"full_name": company, "first_name": company, "middle_name": None, "last_name": None} parts = [part for part in (first, middle, last) if part] full_name = " ".join(parts) # If no structured name, try to infer from display-style strings. if not full_name: full_name = _clean_str(redmine_contact.get("name")) or "" if full_name: split = full_name.split() first = split[0] if split else None last = " ".join(split[1:]) if len(split) > 1 else None return {"full_name": full_name or None, "first_name": first, "middle_name": middle, "last_name": last} def _extract_address(redmine_contact: dict[str, Any]) -> dict[str, Any]: attrs = redmine_contact.get("address_attributes") or {} if not attrs: return {} return { "street": _clean_str(attrs.get("street")), "city": _clean_str(attrs.get("city")), "region": _clean_str(attrs.get("state")), "country": _clean_str(attrs.get("country")), "postal_code": _clean_str(attrs.get("zip")), } def _build_notes(redmine_contact: dict[str, Any]) -> str: parts: list[str] = [] background = _clean_str(redmine_contact.get("background")) if background: parts.append(background) custom_fields = redmine_contact.get("custom_fields") or [] for field in custom_fields: name = field.get("name") value = field.get("value") if name and value: parts.append(f"{name}: {value}") contact_id = redmine_contact.get("id") if contact_id is not None: parts.append(f"Redmine contact: {contact_id}") return "\n".join(parts) def _build_tags(redmine_contact: dict[str, Any], config: EnrichmentConfig) -> list[str]: raw_tags = redmine_contact.get("tag_list") or [] if isinstance(raw_tags, str): tags = {part.strip().lower() for part in raw_tags.split(",") if part.strip()} elif isinstance(raw_tags, list): tags = {str(tag).strip().lower() for tag in raw_tags if str(tag).strip()} else: tags = set() for tag in config.tags_from_redmine: tags.add(tag.lower()) project_id = redmine_contact.get("project_id") if project_id is not None: tags.add(f"project-{project_id}") return sorted(tags) def _contact_text_haystack(redmine_contact: dict[str, Any]) -> str: pieces: list[str] = [] background = _clean_str(redmine_contact.get("background")) if background: pieces.append(background) for field in redmine_contact.get("custom_fields") or []: value = field.get("value") if value: pieces.append(str(value)) return "\n".join(pieces) def _format_birthday(value: Any) -> str | None: cleaned = _clean_str(value) if not cleaned: return None for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%m/%d/%Y", "%d/%m/%Y"): try: return datetime.strptime(cleaned, fmt).date().isoformat() except ValueError: continue return None def _format_timestamp(value: Any) -> str | None: if value is None: return None raw = str(value).strip() if not raw: return None normalized = raw.replace("Z", "+00:00") try: dt = datetime.fromisoformat(normalized) except ValueError: return None if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) return dt.astimezone(timezone.utc).isoformat() def build_sync_payload( redmine_contacts_iter: list[dict[str, Any]], config: EnrichmentConfig | None = None, ) -> dict[str, Any]: """Enrich a batch and return a summary suitable for a sync module.""" config = config or EnrichmentConfig() results = [enrich_contact(contact, config) for contact in redmine_contacts_iter] return { "enriched_at": _utc_now().isoformat(), "count": len([r for r in results if not r.skipped]), "records": [result.enriched for result in results if not result.skipped], "skipped": [ {"redmine_contact_id": result.contact.get("id"), "reason": result.reason} for result in results if result.skipped ], "warnings": [ {"redmine_contact_id": result.contact.get("id"), "warning": warning} for result in results for warning in result.warnings ], } def with_rate_limit( func: Callable[..., Any], *, max_calls: int = 10, per_seconds: float = 1.0, max_retries: int = 5, backoff_base: float = 1.0, backoff_cap: float = 30.0, ) -> Callable[..., Any]: """Wrap a callable with a simple token-bucket rate limit and retry policy. The wrapper is stateful; create one per external provider session. It sleeps using ``time.sleep`` so it is safe for sync scripts but should not be used inside async loops. """ import time tokens = float(max_calls) last_refill = time.monotonic() lock_retried = 0 def _refill() -> None: nonlocal tokens, last_refill now = time.monotonic() elapsed = now - last_refill tokens = min(float(max_calls), tokens + elapsed * (max_calls / per_seconds)) last_refill = now def wrapper(*args: Any, **kwargs: Any) -> Any: nonlocal tokens, lock_retried for attempt in range(max_retries + 1): _refill() if tokens < 1: sleep_time = (1 - tokens) * (per_seconds / max_calls) time.sleep(max(0.0, sleep_time)) _refill() tokens -= 1 try: return func(*args, **kwargs) except RateLimitError: raise except Exception as exc: # noqa: BLE001 if attempt == max_retries: raise wait = min(backoff_cap, backoff_base * (2 ** attempt)) time.sleep(wait) raise EnrichmentError("Rate-limited wrapper exhausted retries") return wrapper if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description="Enrich Redmine CRM contacts for sync.") parser.add_argument("contacts", type=json.loads, help="JSON-encoded Redmine contact or list of contacts.") parser.add_argument("--source", default="redmine") parser.add_argument("--confidence-threshold", type=float, default=0.7) args = parser.parse_args() raw = args.contacts contacts = raw if isinstance(raw, list) else [raw] config = EnrichmentConfig(source=args.source, confidence_threshold=args.confidence_threshold) print(json.dumps(build_sync_payload(contacts, config), indent=2, sort_keys=True))