Organized
This commit is contained in:
Jason Thistlethwaite
@@ -0,0 +1,310 @@
|
||||
"""Map Redmine contact payloads to the target CRM schema and enrich missing data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
|
||||
_PHONE_RE = re.compile(r"[\d+\-().\s]{7,}")
|
||||
|
||||
|
||||
def _coalesce(*values: Any) -> Any:
|
||||
"""Return the first non-empty value."""
|
||||
for value in values:
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _to_iso(ts: Optional[str]) -> Optional[str]:
|
||||
"""Normalize Redmine timestamps to ISO-8601 in UTC."""
|
||||
if not ts:
|
||||
return None
|
||||
try:
|
||||
# Redmine uses ISO-ish strings; this accepts several common formats.
|
||||
parsed = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
parsed = parsed.astimezone(timezone.utc)
|
||||
return parsed.isoformat()
|
||||
except ValueError as exc:
|
||||
logger.debug("Could not parse timestamp %r: %s", ts, exc)
|
||||
return ts
|
||||
|
||||
|
||||
def _normalize_name(
|
||||
first: Optional[str],
|
||||
last: Optional[str],
|
||||
middle: Optional[str],
|
||||
) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
||||
first = (first or "").strip() or None
|
||||
middle = (middle or "").strip() or None
|
||||
last = (last or "").strip() or None
|
||||
return first, middle, last
|
||||
|
||||
|
||||
def _build_full_name(
|
||||
first: Optional[str],
|
||||
last: Optional[str],
|
||||
middle: Optional[str],
|
||||
company: Optional[str],
|
||||
is_company: bool,
|
||||
) -> str:
|
||||
if is_company and company:
|
||||
return company
|
||||
|
||||
parts = [p for p in (first, middle, last) if p]
|
||||
if parts:
|
||||
return " ".join(parts)
|
||||
if company:
|
||||
return company
|
||||
return "Unknown"
|
||||
|
||||
|
||||
def _classify_phone(raw: str) -> str:
|
||||
raw = re.sub(r"[^\d+]", "", raw)
|
||||
# Heuristic: mobile numbers are often longer and start with a country code or specific prefixes.
|
||||
if raw.startswith("+") and len(raw) >= 11:
|
||||
return "mobile"
|
||||
# North American mobile indicators are unreliable; default to work.
|
||||
return "work"
|
||||
|
||||
|
||||
def _normalize_phone(raw: Optional[str]) -> Optional[str]:
|
||||
if not raw:
|
||||
return None
|
||||
digits = re.sub(r"[^\d+]", "", raw)
|
||||
return digits if digits else None
|
||||
|
||||
|
||||
def _extract_emails_from_text(text: Optional[str]) -> List[str]:
|
||||
if not text:
|
||||
return []
|
||||
return list(dict.fromkeys(_EMAIL_RE.findall(text)))
|
||||
|
||||
|
||||
def _company_from_email(email: Optional[str]) -> Optional[str]:
|
||||
"""Heuristic: derive a company display name from an email domain."""
|
||||
if not email or "@" not in email:
|
||||
return None
|
||||
domain = email.split("@")[1].strip().lower()
|
||||
if not domain:
|
||||
return None
|
||||
# Strip common TLDs and subdomain noise.
|
||||
name = domain.split(":")[0]
|
||||
tokens = name.split(".")
|
||||
if tokens[0] in ("www", "mail", "smtp", "contact", "info"):
|
||||
tokens = tokens[1:]
|
||||
if len(tokens) >= 2:
|
||||
tokens = tokens[:-1]
|
||||
core = tokens[0] if tokens else domain.split(".")[0]
|
||||
title = core.replace("-", " ").title()
|
||||
if title.lower() in ("gmail", "yahoo", "hotmail", "outlook", "aol"):
|
||||
return None
|
||||
return f"{title} Inc"
|
||||
|
||||
|
||||
def _parse_address(addr: Optional[Dict[str, Any]]) -> Dict[str, Optional[str]]:
|
||||
if not addr:
|
||||
return {
|
||||
"street": None,
|
||||
"city": None,
|
||||
"region": None,
|
||||
"country": None,
|
||||
"postal_code": None,
|
||||
}
|
||||
return {
|
||||
"street": _coalesce(addr.get("street")),
|
||||
"city": _coalesce(addr.get("city")),
|
||||
"region": _coalesce(addr.get("state")),
|
||||
"country": _coalesce(addr.get("country")),
|
||||
"postal_code": _coalesce(addr.get("zip")),
|
||||
}
|
||||
|
||||
|
||||
def _normalize_tags(tag_list: Any) -> List[str]:
|
||||
tags: List[str] = []
|
||||
if isinstance(tag_list, str):
|
||||
tag_list = [t.strip() for t in tag_list.split(",")]
|
||||
if isinstance(tag_list, list):
|
||||
for tag in tag_list:
|
||||
if tag:
|
||||
normalized = str(tag).strip().lower()
|
||||
if normalized and normalized not in tags:
|
||||
tags.append(normalized)
|
||||
# Always mark origin.
|
||||
if "from-redmine" not in tags:
|
||||
tags.append("from-redmine")
|
||||
return sorted(tags)
|
||||
|
||||
|
||||
def _extract_custom(
|
||||
custom_fields: Optional[List[Dict[str, Any]]],
|
||||
keys: List[str],
|
||||
) -> Optional[str]:
|
||||
"""Return the value of the first custom field matching one of keys (case-insensitive)."""
|
||||
if not custom_fields:
|
||||
return None
|
||||
key_set = {k.lower() for k in keys}
|
||||
for field in custom_fields:
|
||||
name = (field.get("name") or "").strip().lower()
|
||||
if name in key_set:
|
||||
value = field.get("value")
|
||||
if value:
|
||||
return str(value).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _merge_notes(
|
||||
background: Optional[str],
|
||||
custom_fields: Optional[List[Dict[str, Any]]],
|
||||
contact_url: str,
|
||||
) -> str:
|
||||
parts = []
|
||||
if background:
|
||||
parts.append(str(background).strip())
|
||||
if custom_fields:
|
||||
for field in custom_fields:
|
||||
name = field.get("name")
|
||||
value = field.get("value")
|
||||
if name and value:
|
||||
# Skip fields already promoted to typed CRM fields (e.g. LinkedIn).
|
||||
if name.lower() in {"linkedin"}:
|
||||
continue
|
||||
parts.append(f"{name}: {value}")
|
||||
parts.append(f"Redmine source: {contact_url}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
class EnrichmentResult:
|
||||
"""Result of enrichment: data plus metadata."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.company_source: Optional[str] = None
|
||||
self.email_source: Optional[str] = None
|
||||
self.confidence_issues: List[str] = []
|
||||
|
||||
|
||||
def enrich(
|
||||
mapped: Dict[str, Any],
|
||||
raw: Dict[str, Any],
|
||||
*,
|
||||
enable_domain_lookup: bool = True,
|
||||
confidence_threshold: float = 0.7,
|
||||
) -> Tuple[Dict[str, Any], EnrichmentResult]:
|
||||
"""Apply enrichment rules to a mapped CRM record."""
|
||||
result = EnrichmentResult()
|
||||
|
||||
# Email fallback: scan background and custom fields.
|
||||
if not mapped.get("emails"):
|
||||
candidates = _extract_emails_from_text(raw.get("background"))
|
||||
if candidates:
|
||||
mapped["emails"] = [{"value": candidates[0], "type": "work"}]
|
||||
result.email_source = "background"
|
||||
|
||||
if not mapped.get("emails"):
|
||||
result.confidence_issues.append("No email found; record may not be upsertable")
|
||||
|
||||
# Company enrichment.
|
||||
if not mapped.get("company") and mapped.get("emails"):
|
||||
email = mapped["emails"][0]["value"]
|
||||
if enable_domain_lookup:
|
||||
company = _company_from_email(email)
|
||||
if company:
|
||||
mapped["company"] = company
|
||||
result.company_source = "domain_heuristic"
|
||||
if not mapped.get("company"):
|
||||
result.confidence_issues.append("Company unknown")
|
||||
|
||||
if mapped.get("company") and not mapped.get("emails"):
|
||||
# Company but no email: confidence low.
|
||||
result.confidence_issues.append("Company available but no email")
|
||||
|
||||
# Phone normalization.
|
||||
phones = mapped.get("phones") or []
|
||||
normalized_phones: List[Dict[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for phone in phones:
|
||||
value = _normalize_phone(phone.get("value"))
|
||||
if value and value not in seen:
|
||||
seen.add(value)
|
||||
normalized_phones.append({
|
||||
"value": value,
|
||||
"type": phone.get("type", "work"),
|
||||
})
|
||||
mapped["phones"] = normalized_phones
|
||||
|
||||
# Confidence gate: not a hard failure, but logged.
|
||||
if result.confidence_issues:
|
||||
logger.info(
|
||||
"Enrichment confidence issues for Redmine contact %s: %s",
|
||||
mapped.get("redmine_contact_id"),
|
||||
"; ".join(result.confidence_issues),
|
||||
)
|
||||
|
||||
mapped["enriched_at"] = datetime.now(timezone.utc).isoformat()
|
||||
return mapped, result
|
||||
|
||||
|
||||
def map_contact(raw: Dict[str, Any], *, base_url: str) -> Dict[str, Any]:
|
||||
"""Map a Redmine contact payload to the normalized CRM schema."""
|
||||
contact_id = raw.get("id")
|
||||
first_name, middle_name, last_name = _normalize_name(
|
||||
raw.get("first_name"),
|
||||
raw.get("last_name"),
|
||||
raw.get("middle_name"),
|
||||
)
|
||||
company = _coalesce(raw.get("company"))
|
||||
is_company = bool(raw.get("is_company"))
|
||||
full_name = _build_full_name(
|
||||
first_name, last_name, middle_name, company, is_company
|
||||
)
|
||||
|
||||
emails: List[Dict[str, str]] = []
|
||||
primary_email = _coalesce(raw.get("email"))
|
||||
if primary_email:
|
||||
emails.append({"value": primary_email, "type": "work"})
|
||||
|
||||
phones: List[Dict[str, str]] = []
|
||||
raw_phone = _coalesce(raw.get("phone"))
|
||||
if raw_phone:
|
||||
phones.append({"value": raw_phone, "type": _classify_phone(raw_phone)})
|
||||
|
||||
urls: List[Dict[str, str]] = []
|
||||
website = _coalesce(raw.get("website"))
|
||||
if website:
|
||||
urls.append({"value": website, "type": "work"})
|
||||
linkedin = _extract_custom(raw.get("custom_fields"), ["linkedin", "linked_in"])
|
||||
if linkedin:
|
||||
urls.append({"value": linkedin, "type": "linkedin"})
|
||||
|
||||
contact_url = f"{base_url.rstrip('/')}/contacts/{contact_id}"
|
||||
|
||||
return {
|
||||
"crm_id": None,
|
||||
"redmine_contact_id": contact_id,
|
||||
"full_name": full_name,
|
||||
"first_name": first_name,
|
||||
"middle_name": middle_name,
|
||||
"last_name": last_name,
|
||||
"company": company,
|
||||
"title": _coalesce(raw.get("job_title")),
|
||||
"emails": emails,
|
||||
"phones": phones,
|
||||
"urls": urls,
|
||||
"website": website,
|
||||
"birthday": _to_iso(raw.get("birthday")),
|
||||
"address": _parse_address(raw.get("address_attributes")),
|
||||
"notes": _merge_notes(raw.get("background"), raw.get("custom_fields"), contact_url),
|
||||
"tags": _normalize_tags(raw.get("tag_list")),
|
||||
"source": "redmine",
|
||||
"synced_at": datetime.now(timezone.utc).isoformat(),
|
||||
"enriched_at": None,
|
||||
"last_redmine_update": _to_iso(raw.get("updated_at")),
|
||||
"is_company": is_company,
|
||||
"project_id": raw.get("project_id"),
|
||||
"assigned_to_id": raw.get("assigned_to_id"),
|
||||
}
|
||||
Reference in New Issue
Block a user