Jason Thistlethwaite
311 lines
9.9 KiB
Python
311 lines
9.9 KiB
Python
"""Map Redmine contact payloads to the target CRM schema and enrich missing data."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
|
|
_PHONE_RE = re.compile(r"[\d+\-().\s]{7,}")
|
|
|
|
|
|
def _coalesce(*values: Any) -> Any:
|
|
"""Return the first non-empty value."""
|
|
for value in values:
|
|
if value:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _to_iso(ts: Optional[str]) -> Optional[str]:
|
|
"""Normalize Redmine timestamps to ISO-8601 in UTC."""
|
|
if not ts:
|
|
return None
|
|
try:
|
|
# Redmine uses ISO-ish strings; this accepts several common formats.
|
|
parsed = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
parsed = parsed.astimezone(timezone.utc)
|
|
return parsed.isoformat()
|
|
except ValueError as exc:
|
|
logger.debug("Could not parse timestamp %r: %s", ts, exc)
|
|
return ts
|
|
|
|
|
|
def _normalize_name(
|
|
first: Optional[str],
|
|
last: Optional[str],
|
|
middle: Optional[str],
|
|
) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
|
first = (first or "").strip() or None
|
|
middle = (middle or "").strip() or None
|
|
last = (last or "").strip() or None
|
|
return first, middle, last
|
|
|
|
|
|
def _build_full_name(
|
|
first: Optional[str],
|
|
last: Optional[str],
|
|
middle: Optional[str],
|
|
company: Optional[str],
|
|
is_company: bool,
|
|
) -> str:
|
|
if is_company and company:
|
|
return company
|
|
|
|
parts = [p for p in (first, middle, last) if p]
|
|
if parts:
|
|
return " ".join(parts)
|
|
if company:
|
|
return company
|
|
return "Unknown"
|
|
|
|
|
|
def _classify_phone(raw: str) -> str:
|
|
raw = re.sub(r"[^\d+]", "", raw)
|
|
# Heuristic: mobile numbers are often longer and start with a country code or specific prefixes.
|
|
if raw.startswith("+") and len(raw) >= 11:
|
|
return "mobile"
|
|
# North American mobile indicators are unreliable; default to work.
|
|
return "work"
|
|
|
|
|
|
def _normalize_phone(raw: Optional[str]) -> Optional[str]:
|
|
if not raw:
|
|
return None
|
|
digits = re.sub(r"[^\d+]", "", raw)
|
|
return digits if digits else None
|
|
|
|
|
|
def _extract_emails_from_text(text: Optional[str]) -> List[str]:
|
|
if not text:
|
|
return []
|
|
return list(dict.fromkeys(_EMAIL_RE.findall(text)))
|
|
|
|
|
|
def _company_from_email(email: Optional[str]) -> Optional[str]:
|
|
"""Heuristic: derive a company display name from an email domain."""
|
|
if not email or "@" not in email:
|
|
return None
|
|
domain = email.split("@")[1].strip().lower()
|
|
if not domain:
|
|
return None
|
|
# Strip common TLDs and subdomain noise.
|
|
name = domain.split(":")[0]
|
|
tokens = name.split(".")
|
|
if tokens[0] in ("www", "mail", "smtp", "contact", "info"):
|
|
tokens = tokens[1:]
|
|
if len(tokens) >= 2:
|
|
tokens = tokens[:-1]
|
|
core = tokens[0] if tokens else domain.split(".")[0]
|
|
title = core.replace("-", " ").title()
|
|
if title.lower() in ("gmail", "yahoo", "hotmail", "outlook", "aol"):
|
|
return None
|
|
return f"{title} Inc"
|
|
|
|
|
|
def _parse_address(addr: Optional[Dict[str, Any]]) -> Dict[str, Optional[str]]:
|
|
if not addr:
|
|
return {
|
|
"street": None,
|
|
"city": None,
|
|
"region": None,
|
|
"country": None,
|
|
"postal_code": None,
|
|
}
|
|
return {
|
|
"street": _coalesce(addr.get("street")),
|
|
"city": _coalesce(addr.get("city")),
|
|
"region": _coalesce(addr.get("state")),
|
|
"country": _coalesce(addr.get("country")),
|
|
"postal_code": _coalesce(addr.get("zip")),
|
|
}
|
|
|
|
|
|
def _normalize_tags(tag_list: Any) -> List[str]:
|
|
tags: List[str] = []
|
|
if isinstance(tag_list, str):
|
|
tag_list = [t.strip() for t in tag_list.split(",")]
|
|
if isinstance(tag_list, list):
|
|
for tag in tag_list:
|
|
if tag:
|
|
normalized = str(tag).strip().lower()
|
|
if normalized and normalized not in tags:
|
|
tags.append(normalized)
|
|
# Always mark origin.
|
|
if "from-redmine" not in tags:
|
|
tags.append("from-redmine")
|
|
return sorted(tags)
|
|
|
|
|
|
def _extract_custom(
|
|
custom_fields: Optional[List[Dict[str, Any]]],
|
|
keys: List[str],
|
|
) -> Optional[str]:
|
|
"""Return the value of the first custom field matching one of keys (case-insensitive)."""
|
|
if not custom_fields:
|
|
return None
|
|
key_set = {k.lower() for k in keys}
|
|
for field in custom_fields:
|
|
name = (field.get("name") or "").strip().lower()
|
|
if name in key_set:
|
|
value = field.get("value")
|
|
if value:
|
|
return str(value).strip()
|
|
return None
|
|
|
|
|
|
def _merge_notes(
|
|
background: Optional[str],
|
|
custom_fields: Optional[List[Dict[str, Any]]],
|
|
contact_url: str,
|
|
) -> str:
|
|
parts = []
|
|
if background:
|
|
parts.append(str(background).strip())
|
|
if custom_fields:
|
|
for field in custom_fields:
|
|
name = field.get("name")
|
|
value = field.get("value")
|
|
if name and value:
|
|
# Skip fields already promoted to typed CRM fields (e.g. LinkedIn).
|
|
if name.lower() in {"linkedin"}:
|
|
continue
|
|
parts.append(f"{name}: {value}")
|
|
parts.append(f"Redmine source: {contact_url}")
|
|
return "\n\n".join(parts)
|
|
|
|
|
|
class EnrichmentResult:
|
|
"""Result of enrichment: data plus metadata."""
|
|
|
|
def __init__(self) -> None:
|
|
self.company_source: Optional[str] = None
|
|
self.email_source: Optional[str] = None
|
|
self.confidence_issues: List[str] = []
|
|
|
|
|
|
def enrich(
|
|
mapped: Dict[str, Any],
|
|
raw: Dict[str, Any],
|
|
*,
|
|
enable_domain_lookup: bool = True,
|
|
confidence_threshold: float = 0.7,
|
|
) -> Tuple[Dict[str, Any], EnrichmentResult]:
|
|
"""Apply enrichment rules to a mapped CRM record."""
|
|
result = EnrichmentResult()
|
|
|
|
# Email fallback: scan background and custom fields.
|
|
if not mapped.get("emails"):
|
|
candidates = _extract_emails_from_text(raw.get("background"))
|
|
if candidates:
|
|
mapped["emails"] = [{"value": candidates[0], "type": "work"}]
|
|
result.email_source = "background"
|
|
|
|
if not mapped.get("emails"):
|
|
result.confidence_issues.append("No email found; record may not be upsertable")
|
|
|
|
# Company enrichment.
|
|
if not mapped.get("company") and mapped.get("emails"):
|
|
email = mapped["emails"][0]["value"]
|
|
if enable_domain_lookup:
|
|
company = _company_from_email(email)
|
|
if company:
|
|
mapped["company"] = company
|
|
result.company_source = "domain_heuristic"
|
|
if not mapped.get("company"):
|
|
result.confidence_issues.append("Company unknown")
|
|
|
|
if mapped.get("company") and not mapped.get("emails"):
|
|
# Company but no email: confidence low.
|
|
result.confidence_issues.append("Company available but no email")
|
|
|
|
# Phone normalization.
|
|
phones = mapped.get("phones") or []
|
|
normalized_phones: List[Dict[str, str]] = []
|
|
seen: set[str] = set()
|
|
for phone in phones:
|
|
value = _normalize_phone(phone.get("value"))
|
|
if value and value not in seen:
|
|
seen.add(value)
|
|
normalized_phones.append({
|
|
"value": value,
|
|
"type": phone.get("type", "work"),
|
|
})
|
|
mapped["phones"] = normalized_phones
|
|
|
|
# Confidence gate: not a hard failure, but logged.
|
|
if result.confidence_issues:
|
|
logger.info(
|
|
"Enrichment confidence issues for Redmine contact %s: %s",
|
|
mapped.get("redmine_contact_id"),
|
|
"; ".join(result.confidence_issues),
|
|
)
|
|
|
|
mapped["enriched_at"] = datetime.now(timezone.utc).isoformat()
|
|
return mapped, result
|
|
|
|
|
|
def map_contact(raw: Dict[str, Any], *, base_url: str) -> Dict[str, Any]:
|
|
"""Map a Redmine contact payload to the normalized CRM schema."""
|
|
contact_id = raw.get("id")
|
|
first_name, middle_name, last_name = _normalize_name(
|
|
raw.get("first_name"),
|
|
raw.get("last_name"),
|
|
raw.get("middle_name"),
|
|
)
|
|
company = _coalesce(raw.get("company"))
|
|
is_company = bool(raw.get("is_company"))
|
|
full_name = _build_full_name(
|
|
first_name, last_name, middle_name, company, is_company
|
|
)
|
|
|
|
emails: List[Dict[str, str]] = []
|
|
primary_email = _coalesce(raw.get("email"))
|
|
if primary_email:
|
|
emails.append({"value": primary_email, "type": "work"})
|
|
|
|
phones: List[Dict[str, str]] = []
|
|
raw_phone = _coalesce(raw.get("phone"))
|
|
if raw_phone:
|
|
phones.append({"value": raw_phone, "type": _classify_phone(raw_phone)})
|
|
|
|
urls: List[Dict[str, str]] = []
|
|
website = _coalesce(raw.get("website"))
|
|
if website:
|
|
urls.append({"value": website, "type": "work"})
|
|
linkedin = _extract_custom(raw.get("custom_fields"), ["linkedin", "linked_in"])
|
|
if linkedin:
|
|
urls.append({"value": linkedin, "type": "linkedin"})
|
|
|
|
contact_url = f"{base_url.rstrip('/')}/contacts/{contact_id}"
|
|
|
|
return {
|
|
"crm_id": None,
|
|
"redmine_contact_id": contact_id,
|
|
"full_name": full_name,
|
|
"first_name": first_name,
|
|
"middle_name": middle_name,
|
|
"last_name": last_name,
|
|
"company": company,
|
|
"title": _coalesce(raw.get("job_title")),
|
|
"emails": emails,
|
|
"phones": phones,
|
|
"urls": urls,
|
|
"website": website,
|
|
"birthday": _to_iso(raw.get("birthday")),
|
|
"address": _parse_address(raw.get("address_attributes")),
|
|
"notes": _merge_notes(raw.get("background"), raw.get("custom_fields"), contact_url),
|
|
"tags": _normalize_tags(raw.get("tag_list")),
|
|
"source": "redmine",
|
|
"synced_at": datetime.now(timezone.utc).isoformat(),
|
|
"enriched_at": None,
|
|
"last_redmine_update": _to_iso(raw.get("updated_at")),
|
|
"is_company": is_company,
|
|
"project_id": raw.get("project_id"),
|
|
"assigned_to_id": raw.get("assigned_to_id"),
|
|
}
|