Organized
This commit is contained in:
Jason Thistlethwaite
@@ -0,0 +1,441 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Contact enrichment: map Redmine CRM contacts to a target CRM schema.
|
||||
|
||||
This module is intentionally decoupled from ``redmine_contacts.py`` so the sync
|
||||
module can import it without pulling in CLI / cache logic. All enrichment is
|
||||
deterministic and side-effect free; external lookups are accepted via a
|
||||
configurable provider so callers can inject rate-limited clients.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
PHONE_RE = re.compile(r"[\d\s\-().+]{7,}")
|
||||
URL_RE = re.compile(r"https?://[^\s<>\"']+|www\.[^\s<>\"']+")
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichmentResult:
|
||||
"""Output of enriching a single Redmine contact."""
|
||||
|
||||
contact: dict[str, Any]
|
||||
enriched: dict[str, Any]
|
||||
changed_fields: list[str]
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
skipped: bool = False
|
||||
reason: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichmentConfig:
|
||||
"""Configuration for the enrichment pipeline."""
|
||||
|
||||
source: str = "redmine"
|
||||
confidence_threshold: float = 0.7
|
||||
default_country_code: str = "US"
|
||||
tags_from_redmine: list[str] = field(default_factory=lambda: ["from-redmine"])
|
||||
external_lookup: Callable[[str], dict[str, Any]] | None = None
|
||||
domain_to_company: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
class EnrichmentError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class RateLimitError(EnrichmentError):
|
||||
"""Raised when an external lookup exhausts retries or hits a hard limit."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def enrich_contact(
|
||||
redmine_contact: dict[str, Any],
|
||||
config: EnrichmentConfig | None = None,
|
||||
) -> EnrichmentResult:
|
||||
"""Map a Redmine contact to the target CRM schema and fill missing fields."""
|
||||
config = config or EnrichmentConfig()
|
||||
contact_id = redmine_contact.get("id")
|
||||
if contact_id is None:
|
||||
return EnrichmentResult(
|
||||
contact=redmine_contact,
|
||||
enriched={},
|
||||
changed_fields=[],
|
||||
skipped=True,
|
||||
reason="Missing Redmine contact id",
|
||||
)
|
||||
|
||||
warnings: list[str] = []
|
||||
|
||||
emails = _extract_emails(redmine_contact, warnings)
|
||||
phones = _extract_phones(redmine_contact, config.default_country_code, warnings)
|
||||
company = _enrich_company(redmine_contact, emails, config, warnings)
|
||||
name_parts = _name_parts(redmine_contact)
|
||||
address = _extract_address(redmine_contact)
|
||||
notes = _build_notes(redmine_contact)
|
||||
tags = _build_tags(redmine_contact, config)
|
||||
|
||||
enriched: dict[str, Any] = {
|
||||
"crm_id": None,
|
||||
"redmine_contact_id": contact_id,
|
||||
"full_name": name_parts["full_name"],
|
||||
"first_name": name_parts["first_name"],
|
||||
"middle_name": name_parts.get("middle_name"),
|
||||
"last_name": name_parts["last_name"],
|
||||
"company": company,
|
||||
"title": _clean_str(redmine_contact.get("job_title")),
|
||||
"emails": emails,
|
||||
"phones": phones,
|
||||
"website": _clean_str(redmine_contact.get("website")),
|
||||
"birthday": _format_birthday(redmine_contact.get("birthday")),
|
||||
"address": address,
|
||||
"notes": notes,
|
||||
"tags": tags,
|
||||
"source": config.source,
|
||||
"synced_at": _utc_now().isoformat(),
|
||||
"enriched_at": _utc_now().isoformat(),
|
||||
"last_redmine_update": _format_timestamp(redmine_contact.get("updated_at", redmine_contact.get("updated_on"))),
|
||||
}
|
||||
|
||||
changed_fields = [key for key, value in enriched.items() if value not in (None, "", [], {})]
|
||||
|
||||
if not emails and not company:
|
||||
warnings.append("No email or company; record may be unmatchable in CRM")
|
||||
|
||||
return EnrichmentResult(
|
||||
contact=redmine_contact,
|
||||
enriched=enriched,
|
||||
changed_fields=changed_fields,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
def _clean_str(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
cleaned = str(value).strip()
|
||||
return cleaned or None
|
||||
|
||||
|
||||
def _extract_emails(redmine_contact: dict[str, Any], warnings: list[str]) -> list[dict[str, Any]]:
|
||||
emails: list[dict[str, Any]] = []
|
||||
primary = _clean_str(redmine_contact.get("email"))
|
||||
if primary:
|
||||
emails.append({"value": primary.lower(), "type": "work"})
|
||||
|
||||
# Search background and custom fields for additional emails when primary missing.
|
||||
if not emails:
|
||||
haystack = _contact_text_haystack(redmine_contact)
|
||||
for match in EMAIL_RE.findall(haystack):
|
||||
candidate = match.lower()
|
||||
if not any(existing["value"] == candidate for existing in emails):
|
||||
emails.append({"value": candidate, "type": "work"})
|
||||
if not emails:
|
||||
warnings.append("No email found")
|
||||
return emails
|
||||
|
||||
|
||||
def _extract_phones(
|
||||
redmine_contact: dict[str, Any],
|
||||
default_country_code: str,
|
||||
warnings: list[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
phones: list[dict[str, Any]] = []
|
||||
raw_phone = _clean_str(redmine_contact.get("phone"))
|
||||
if raw_phone:
|
||||
normalized = _normalize_phone(raw_phone, default_country_code)
|
||||
if normalized:
|
||||
phones.append({"value": normalized, "type": _classify_phone_type(raw_phone)})
|
||||
|
||||
# Do not mine background for phones unless the explicit field is empty.
|
||||
if not phones:
|
||||
haystack = _contact_text_haystack(redmine_contact)
|
||||
for match in PHONE_RE.findall(haystack):
|
||||
normalized = _normalize_phone(match, default_country_code)
|
||||
if normalized and not any(existing["value"] == normalized for existing in phones):
|
||||
phones.append({"value": normalized, "type": _classify_phone_type(match)})
|
||||
if len(phones) >= 3:
|
||||
break
|
||||
if not phones:
|
||||
warnings.append("No phone found")
|
||||
return phones
|
||||
|
||||
|
||||
def _normalize_phone(raw: str, default_country_code: str) -> str | None:
|
||||
digits = re.sub(r"\D", "", raw)
|
||||
if len(digits) < 7:
|
||||
return None
|
||||
if len(digits) == 10 and default_country_code == "US":
|
||||
digits = "1" + digits
|
||||
if digits.startswith("1") and len(digits) == 11:
|
||||
return f"+{digits}"
|
||||
return f"+{digits}"
|
||||
|
||||
|
||||
def _classify_phone_type(raw: str) -> str:
|
||||
lowered = raw.lower()
|
||||
if "mobile" in lowered or "cell" in lowered or "m:" in lowered:
|
||||
return "mobile"
|
||||
if "home" in lowered or "h:" in lowered:
|
||||
return "home"
|
||||
return "work"
|
||||
|
||||
|
||||
def _enrich_company(
|
||||
redmine_contact: dict[str, Any],
|
||||
emails: list[dict[str, Any]],
|
||||
config: EnrichmentConfig,
|
||||
warnings: list[str],
|
||||
) -> str | None:
|
||||
company = _clean_str(redmine_contact.get("company"))
|
||||
if company:
|
||||
return company
|
||||
|
||||
if config.external_lookup is not None:
|
||||
try:
|
||||
lookup = config.external_lookup(str(redmine_contact.get("id")))
|
||||
company = _clean_str(lookup.get("company"))
|
||||
if company:
|
||||
return company
|
||||
except RateLimitError:
|
||||
warnings.append("External company lookup hit rate limit")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
warnings.append(f"External company lookup failed: {exc}")
|
||||
|
||||
if emails:
|
||||
domain = emails[0]["value"].split("@", 1)[-1]
|
||||
company = config.domain_to_company.get(domain)
|
||||
if company:
|
||||
return company
|
||||
company = _company_from_domain(domain)
|
||||
if company:
|
||||
return company
|
||||
|
||||
warnings.append("Company could not be enriched")
|
||||
return None
|
||||
|
||||
|
||||
def _company_from_domain(domain: str) -> str | None:
|
||||
# Simple deterministic heuristic; keeps component dependency-free.
|
||||
if not domain or "." not in domain:
|
||||
return None
|
||||
name = domain.rsplit(".", 1)[0]
|
||||
name = re.sub(r"[^a-zA-Z0-9]+", " ", name).strip()
|
||||
if not name:
|
||||
return None
|
||||
return name.capitalize() + " Inc"
|
||||
|
||||
|
||||
def _name_parts(redmine_contact: dict[str, Any]) -> dict[str, Any]:
|
||||
first = _clean_str(redmine_contact.get("first_name"))
|
||||
middle = _clean_str(redmine_contact.get("middle_name"))
|
||||
last = _clean_str(redmine_contact.get("last_name"))
|
||||
|
||||
if redmine_contact.get("is_company"):
|
||||
company = _clean_str(redmine_contact.get("company")) or first or ""
|
||||
return {"full_name": company, "first_name": company, "middle_name": None, "last_name": None}
|
||||
|
||||
parts = [part for part in (first, middle, last) if part]
|
||||
full_name = " ".join(parts)
|
||||
|
||||
# If no structured name, try to infer from display-style strings.
|
||||
if not full_name:
|
||||
full_name = _clean_str(redmine_contact.get("name")) or ""
|
||||
if full_name:
|
||||
split = full_name.split()
|
||||
first = split[0] if split else None
|
||||
last = " ".join(split[1:]) if len(split) > 1 else None
|
||||
|
||||
return {"full_name": full_name or None, "first_name": first, "middle_name": middle, "last_name": last}
|
||||
|
||||
|
||||
def _extract_address(redmine_contact: dict[str, Any]) -> dict[str, Any]:
|
||||
attrs = redmine_contact.get("address_attributes") or {}
|
||||
if not attrs:
|
||||
return {}
|
||||
return {
|
||||
"street": _clean_str(attrs.get("street")),
|
||||
"city": _clean_str(attrs.get("city")),
|
||||
"region": _clean_str(attrs.get("state")),
|
||||
"country": _clean_str(attrs.get("country")),
|
||||
"postal_code": _clean_str(attrs.get("zip")),
|
||||
}
|
||||
|
||||
|
||||
def _build_notes(redmine_contact: dict[str, Any]) -> str:
|
||||
parts: list[str] = []
|
||||
background = _clean_str(redmine_contact.get("background"))
|
||||
if background:
|
||||
parts.append(background)
|
||||
|
||||
custom_fields = redmine_contact.get("custom_fields") or []
|
||||
for field in custom_fields:
|
||||
name = field.get("name")
|
||||
value = field.get("value")
|
||||
if name and value:
|
||||
parts.append(f"{name}: {value}")
|
||||
|
||||
contact_id = redmine_contact.get("id")
|
||||
if contact_id is not None:
|
||||
parts.append(f"Redmine contact: {contact_id}")
|
||||
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _build_tags(redmine_contact: dict[str, Any], config: EnrichmentConfig) -> list[str]:
|
||||
raw_tags = redmine_contact.get("tag_list") or []
|
||||
if isinstance(raw_tags, str):
|
||||
tags = {part.strip().lower() for part in raw_tags.split(",") if part.strip()}
|
||||
elif isinstance(raw_tags, list):
|
||||
tags = {str(tag).strip().lower() for tag in raw_tags if str(tag).strip()}
|
||||
else:
|
||||
tags = set()
|
||||
|
||||
for tag in config.tags_from_redmine:
|
||||
tags.add(tag.lower())
|
||||
|
||||
project_id = redmine_contact.get("project_id")
|
||||
if project_id is not None:
|
||||
tags.add(f"project-{project_id}")
|
||||
|
||||
return sorted(tags)
|
||||
|
||||
|
||||
def _contact_text_haystack(redmine_contact: dict[str, Any]) -> str:
|
||||
pieces: list[str] = []
|
||||
background = _clean_str(redmine_contact.get("background"))
|
||||
if background:
|
||||
pieces.append(background)
|
||||
for field in redmine_contact.get("custom_fields") or []:
|
||||
value = field.get("value")
|
||||
if value:
|
||||
pieces.append(str(value))
|
||||
return "\n".join(pieces)
|
||||
|
||||
|
||||
def _format_birthday(value: Any) -> str | None:
|
||||
cleaned = _clean_str(value)
|
||||
if not cleaned:
|
||||
return None
|
||||
for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%m/%d/%Y", "%d/%m/%Y"):
|
||||
try:
|
||||
return datetime.strptime(cleaned, fmt).date().isoformat()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _format_timestamp(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
raw = str(value).strip()
|
||||
if not raw:
|
||||
return None
|
||||
normalized = raw.replace("Z", "+00:00")
|
||||
try:
|
||||
dt = datetime.fromisoformat(normalized)
|
||||
except ValueError:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def build_sync_payload(
|
||||
redmine_contacts_iter: list[dict[str, Any]],
|
||||
config: EnrichmentConfig | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Enrich a batch and return a summary suitable for a sync module."""
|
||||
config = config or EnrichmentConfig()
|
||||
results = [enrich_contact(contact, config) for contact in redmine_contacts_iter]
|
||||
return {
|
||||
"enriched_at": _utc_now().isoformat(),
|
||||
"count": len([r for r in results if not r.skipped]),
|
||||
"records": [result.enriched for result in results if not result.skipped],
|
||||
"skipped": [
|
||||
{"redmine_contact_id": result.contact.get("id"), "reason": result.reason}
|
||||
for result in results
|
||||
if result.skipped
|
||||
],
|
||||
"warnings": [
|
||||
{"redmine_contact_id": result.contact.get("id"), "warning": warning}
|
||||
for result in results
|
||||
for warning in result.warnings
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def with_rate_limit(
|
||||
func: Callable[..., Any],
|
||||
*,
|
||||
max_calls: int = 10,
|
||||
per_seconds: float = 1.0,
|
||||
max_retries: int = 5,
|
||||
backoff_base: float = 1.0,
|
||||
backoff_cap: float = 30.0,
|
||||
) -> Callable[..., Any]:
|
||||
"""Wrap a callable with a simple token-bucket rate limit and retry policy.
|
||||
|
||||
The wrapper is stateful; create one per external provider session. It sleeps
|
||||
using ``time.sleep`` so it is safe for sync scripts but should not be used
|
||||
inside async loops.
|
||||
"""
|
||||
import time
|
||||
|
||||
tokens = float(max_calls)
|
||||
last_refill = time.monotonic()
|
||||
lock_retried = 0
|
||||
|
||||
def _refill() -> None:
|
||||
nonlocal tokens, last_refill
|
||||
now = time.monotonic()
|
||||
elapsed = now - last_refill
|
||||
tokens = min(float(max_calls), tokens + elapsed * (max_calls / per_seconds))
|
||||
last_refill = now
|
||||
|
||||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
nonlocal tokens, lock_retried
|
||||
for attempt in range(max_retries + 1):
|
||||
_refill()
|
||||
if tokens < 1:
|
||||
sleep_time = (1 - tokens) * (per_seconds / max_calls)
|
||||
time.sleep(max(0.0, sleep_time))
|
||||
_refill()
|
||||
tokens -= 1
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except RateLimitError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if attempt == max_retries:
|
||||
raise
|
||||
wait = min(backoff_cap, backoff_base * (2 ** attempt))
|
||||
time.sleep(wait)
|
||||
raise EnrichmentError("Rate-limited wrapper exhausted retries")
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Enrich Redmine CRM contacts for sync.")
|
||||
parser.add_argument("contacts", type=json.loads, help="JSON-encoded Redmine contact or list of contacts.")
|
||||
parser.add_argument("--source", default="redmine")
|
||||
parser.add_argument("--confidence-threshold", type=float, default=0.7)
|
||||
args = parser.parse_args()
|
||||
|
||||
raw = args.contacts
|
||||
contacts = raw if isinstance(raw, list) else [raw]
|
||||
config = EnrichmentConfig(source=args.source, confidence_threshold=args.confidence_threshold)
|
||||
print(json.dumps(build_sync_payload(contacts, config), indent=2, sort_keys=True))
|
||||
Reference in New Issue
Block a user