Organized
This commit is contained in:
Jason Thistlethwaite
@@ -0,0 +1,230 @@
|
||||
"""CRM upsert client supporting idempotency by Redmine ID and email fallback."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CrmError(Exception):
|
||||
"""Base exception for CRM errors."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
redmine_id: Optional[int] = None,
|
||||
crm_id: Optional[str] = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.redmine_id = redmine_id
|
||||
self.crm_id = crm_id
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UpsertResult:
|
||||
crm_id: str
|
||||
created: bool
|
||||
updated: bool
|
||||
skipped_unchanged: bool
|
||||
redmine_contact_id: int
|
||||
|
||||
|
||||
class CrmClient:
|
||||
"""Generic CRM upsert wrapper.
|
||||
|
||||
Provides primitives for local testing; real CRM adapters should subclass
|
||||
and implement _find_by_redmine_id, _find_by_email, _create, and _update.
|
||||
"""
|
||||
|
||||
# Shared simulated CRM store for the in-memory adapter.
|
||||
_store: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
find_by_redmine_id: Optional[Callable[[int], Optional[Dict[str, Any]]]] = None,
|
||||
find_by_email: Optional[Callable[[str], Optional[Dict[str, Any]]]] = None,
|
||||
create: Optional[Callable[[Dict[str, Any]], str]] = None,
|
||||
update: Optional[Callable[[str, Dict[str, Any]], None]] = None,
|
||||
) -> None:
|
||||
self.api_key = api_key
|
||||
self.base_url = base_url
|
||||
self._find_by_redmine_id_fn = find_by_redmine_id
|
||||
self._find_by_email_fn = find_by_email
|
||||
self._create_fn = create
|
||||
self._update_fn = update
|
||||
|
||||
# --- Extensibility hooks (override in subclasses) -----------------------
|
||||
|
||||
def _find_by_redmine_id(self, redmine_id: int) -> Optional[Dict[str, Any]]:
|
||||
if self._find_by_redmine_id_fn:
|
||||
return self._find_by_redmine_id_fn(redmine_id)
|
||||
for record in self._store.values():
|
||||
if record.get("redmine_contact_id") == redmine_id:
|
||||
return record
|
||||
return None
|
||||
|
||||
def _find_by_email(self, email: str) -> Optional[Dict[str, Any]]:
|
||||
if self._find_by_email_fn:
|
||||
return self._find_by_email_fn(email)
|
||||
normalized = email.lower().strip()
|
||||
for record in self._store.values():
|
||||
for entry in record.get("emails", []):
|
||||
if entry.get("value", "").lower().strip() == normalized:
|
||||
return record
|
||||
return None
|
||||
|
||||
def _create(self, record: Dict[str, Any]) -> str:
|
||||
if self._create_fn:
|
||||
return self._create_fn(record)
|
||||
crm_id = f"crm::{record['redmine_contact_id']}"
|
||||
self._store[crm_id] = record.copy()
|
||||
return crm_id
|
||||
|
||||
def _update(self, crm_id: str, record: Dict[str, Any]) -> None:
|
||||
if self._update_fn:
|
||||
self._update_fn(crm_id, record)
|
||||
return
|
||||
self._store[crm_id] = record.copy()
|
||||
|
||||
# --- Public API ---------------------------------------------------------
|
||||
|
||||
def _normalize_email(self, email: Optional[str]) -> Optional[str]:
|
||||
if not email:
|
||||
return None
|
||||
return email.lower().strip()
|
||||
|
||||
def _has_changed(
|
||||
self,
|
||||
existing: Dict[str, Any],
|
||||
candidate: Dict[str, Any],
|
||||
) -> bool:
|
||||
"""Return True if meaningful Redmine data changed."""
|
||||
if not existing:
|
||||
return True
|
||||
candidate_ts = candidate.get("last_redmine_update")
|
||||
existing_ts = existing.get("last_redmine_update")
|
||||
if candidate_ts and existing_ts:
|
||||
return candidate_ts != existing_ts
|
||||
# Fallback to structural comparison for fields we care about.
|
||||
keys = {
|
||||
"full_name",
|
||||
"company",
|
||||
"title",
|
||||
"emails",
|
||||
"phones",
|
||||
"urls",
|
||||
"address",
|
||||
"notes",
|
||||
"tags",
|
||||
"birthday",
|
||||
}
|
||||
for key in keys:
|
||||
if existing.get(key) != candidate.get(key):
|
||||
return True
|
||||
return False
|
||||
|
||||
def upsert(self, record: Dict[str, Any]) -> UpsertResult:
|
||||
"""Idempotently upsert a mapped Redmine contact into the CRM."""
|
||||
redmine_id = record.get("redmine_contact_id")
|
||||
if redmine_id is None:
|
||||
raise CrmError("Record missing redmine_contact_id")
|
||||
|
||||
existing = self._find_by_redmine_id(redmine_id)
|
||||
fallback_email: Optional[str] = None
|
||||
if not existing and record.get("emails"):
|
||||
fallback_email = self._normalize_email(record["emails"][0]["value"])
|
||||
if fallback_email:
|
||||
existing = self._find_by_email(fallback_email)
|
||||
|
||||
if existing:
|
||||
crm_id = existing["crm_id"]
|
||||
if not crm_id:
|
||||
raise CrmError(
|
||||
"Existing CRM record missing crm_id",
|
||||
redmine_id=redmine_id,
|
||||
)
|
||||
if not self._has_changed(existing, record):
|
||||
logger.info(
|
||||
"CRM record %s unchanged for Redmine contact %s; skipping",
|
||||
crm_id,
|
||||
redmine_id,
|
||||
)
|
||||
return UpsertResult(
|
||||
crm_id=crm_id,
|
||||
created=False,
|
||||
updated=False,
|
||||
skipped_unchanged=True,
|
||||
redmine_contact_id=redmine_id,
|
||||
)
|
||||
|
||||
record["crm_id"] = crm_id
|
||||
if fallback_email and not existing.get("redmine_contact_id"):
|
||||
# If matched by email, attach Redmine ID to close the loop.
|
||||
logger.info(
|
||||
"Matched Redmine contact %s by email %s; updating CRM record %s",
|
||||
redmine_id,
|
||||
fallback_email,
|
||||
crm_id,
|
||||
)
|
||||
self._update(crm_id, record)
|
||||
return UpsertResult(
|
||||
crm_id=crm_id,
|
||||
created=False,
|
||||
updated=True,
|
||||
skipped_unchanged=False,
|
||||
redmine_contact_id=redmine_id,
|
||||
)
|
||||
|
||||
crm_id = self._create(record)
|
||||
record["crm_id"] = crm_id
|
||||
self._store[crm_id] = record.copy()
|
||||
return UpsertResult(
|
||||
crm_id=crm_id,
|
||||
created=True,
|
||||
updated=False,
|
||||
skipped_unchanged=False,
|
||||
redmine_contact_id=redmine_id,
|
||||
)
|
||||
|
||||
def sync_batch(
|
||||
self,
|
||||
records: List[Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
"""Sync a batch and return aggregated stats."""
|
||||
stats = {
|
||||
"processed": 0,
|
||||
"created": 0,
|
||||
"updated": 0,
|
||||
"skipped": 0,
|
||||
"errors": 0,
|
||||
"error_records": [],
|
||||
}
|
||||
for record in records:
|
||||
stats["processed"] += 1
|
||||
try:
|
||||
result = self.upsert(record)
|
||||
if result.created:
|
||||
stats["created"] += 1
|
||||
elif result.updated:
|
||||
stats["updated"] += 1
|
||||
else:
|
||||
stats["skipped"] += 1
|
||||
except CrmError as exc:
|
||||
stats["errors"] += 1
|
||||
stats["error_records"].append(
|
||||
{
|
||||
"redmine_contact_id": exc.redmine_id,
|
||||
"crm_id": exc.crm_id,
|
||||
"reason": str(exc),
|
||||
}
|
||||
)
|
||||
logger.warning("CRM upsert failed: %s", exc)
|
||||
logger.info("CRM sync batch stats: %s", stats)
|
||||
return stats
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Map Redmine contact payloads to the target CRM schema and enrich missing data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
|
||||
_PHONE_RE = re.compile(r"[\d+\-().\s]{7,}")
|
||||
|
||||
|
||||
def _coalesce(*values: Any) -> Any:
|
||||
"""Return the first non-empty value."""
|
||||
for value in values:
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _to_iso(ts: Optional[str]) -> Optional[str]:
|
||||
"""Normalize Redmine timestamps to ISO-8601 in UTC."""
|
||||
if not ts:
|
||||
return None
|
||||
try:
|
||||
# Redmine uses ISO-ish strings; this accepts several common formats.
|
||||
parsed = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
parsed = parsed.astimezone(timezone.utc)
|
||||
return parsed.isoformat()
|
||||
except ValueError as exc:
|
||||
logger.debug("Could not parse timestamp %r: %s", ts, exc)
|
||||
return ts
|
||||
|
||||
|
||||
def _normalize_name(
|
||||
first: Optional[str],
|
||||
last: Optional[str],
|
||||
middle: Optional[str],
|
||||
) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
||||
first = (first or "").strip() or None
|
||||
middle = (middle or "").strip() or None
|
||||
last = (last or "").strip() or None
|
||||
return first, middle, last
|
||||
|
||||
|
||||
def _build_full_name(
|
||||
first: Optional[str],
|
||||
last: Optional[str],
|
||||
middle: Optional[str],
|
||||
company: Optional[str],
|
||||
is_company: bool,
|
||||
) -> str:
|
||||
if is_company and company:
|
||||
return company
|
||||
|
||||
parts = [p for p in (first, middle, last) if p]
|
||||
if parts:
|
||||
return " ".join(parts)
|
||||
if company:
|
||||
return company
|
||||
return "Unknown"
|
||||
|
||||
|
||||
def _classify_phone(raw: str) -> str:
|
||||
raw = re.sub(r"[^\d+]", "", raw)
|
||||
# Heuristic: mobile numbers are often longer and start with a country code or specific prefixes.
|
||||
if raw.startswith("+") and len(raw) >= 11:
|
||||
return "mobile"
|
||||
# North American mobile indicators are unreliable; default to work.
|
||||
return "work"
|
||||
|
||||
|
||||
def _normalize_phone(raw: Optional[str]) -> Optional[str]:
|
||||
if not raw:
|
||||
return None
|
||||
digits = re.sub(r"[^\d+]", "", raw)
|
||||
return digits if digits else None
|
||||
|
||||
|
||||
def _extract_emails_from_text(text: Optional[str]) -> List[str]:
|
||||
if not text:
|
||||
return []
|
||||
return list(dict.fromkeys(_EMAIL_RE.findall(text)))
|
||||
|
||||
|
||||
def _company_from_email(email: Optional[str]) -> Optional[str]:
|
||||
"""Heuristic: derive a company display name from an email domain."""
|
||||
if not email or "@" not in email:
|
||||
return None
|
||||
domain = email.split("@")[1].strip().lower()
|
||||
if not domain:
|
||||
return None
|
||||
# Strip common TLDs and subdomain noise.
|
||||
name = domain.split(":")[0]
|
||||
tokens = name.split(".")
|
||||
if tokens[0] in ("www", "mail", "smtp", "contact", "info"):
|
||||
tokens = tokens[1:]
|
||||
if len(tokens) >= 2:
|
||||
tokens = tokens[:-1]
|
||||
core = tokens[0] if tokens else domain.split(".")[0]
|
||||
title = core.replace("-", " ").title()
|
||||
if title.lower() in ("gmail", "yahoo", "hotmail", "outlook", "aol"):
|
||||
return None
|
||||
return f"{title} Inc"
|
||||
|
||||
|
||||
def _parse_address(addr: Optional[Dict[str, Any]]) -> Dict[str, Optional[str]]:
|
||||
if not addr:
|
||||
return {
|
||||
"street": None,
|
||||
"city": None,
|
||||
"region": None,
|
||||
"country": None,
|
||||
"postal_code": None,
|
||||
}
|
||||
return {
|
||||
"street": _coalesce(addr.get("street")),
|
||||
"city": _coalesce(addr.get("city")),
|
||||
"region": _coalesce(addr.get("state")),
|
||||
"country": _coalesce(addr.get("country")),
|
||||
"postal_code": _coalesce(addr.get("zip")),
|
||||
}
|
||||
|
||||
|
||||
def _normalize_tags(tag_list: Any) -> List[str]:
|
||||
tags: List[str] = []
|
||||
if isinstance(tag_list, str):
|
||||
tag_list = [t.strip() for t in tag_list.split(",")]
|
||||
if isinstance(tag_list, list):
|
||||
for tag in tag_list:
|
||||
if tag:
|
||||
normalized = str(tag).strip().lower()
|
||||
if normalized and normalized not in tags:
|
||||
tags.append(normalized)
|
||||
# Always mark origin.
|
||||
if "from-redmine" not in tags:
|
||||
tags.append("from-redmine")
|
||||
return sorted(tags)
|
||||
|
||||
|
||||
def _extract_custom(
|
||||
custom_fields: Optional[List[Dict[str, Any]]],
|
||||
keys: List[str],
|
||||
) -> Optional[str]:
|
||||
"""Return the value of the first custom field matching one of keys (case-insensitive)."""
|
||||
if not custom_fields:
|
||||
return None
|
||||
key_set = {k.lower() for k in keys}
|
||||
for field in custom_fields:
|
||||
name = (field.get("name") or "").strip().lower()
|
||||
if name in key_set:
|
||||
value = field.get("value")
|
||||
if value:
|
||||
return str(value).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _merge_notes(
|
||||
background: Optional[str],
|
||||
custom_fields: Optional[List[Dict[str, Any]]],
|
||||
contact_url: str,
|
||||
) -> str:
|
||||
parts = []
|
||||
if background:
|
||||
parts.append(str(background).strip())
|
||||
if custom_fields:
|
||||
for field in custom_fields:
|
||||
name = field.get("name")
|
||||
value = field.get("value")
|
||||
if name and value:
|
||||
# Skip fields already promoted to typed CRM fields (e.g. LinkedIn).
|
||||
if name.lower() in {"linkedin"}:
|
||||
continue
|
||||
parts.append(f"{name}: {value}")
|
||||
parts.append(f"Redmine source: {contact_url}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
class EnrichmentResult:
|
||||
"""Result of enrichment: data plus metadata."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.company_source: Optional[str] = None
|
||||
self.email_source: Optional[str] = None
|
||||
self.confidence_issues: List[str] = []
|
||||
|
||||
|
||||
def enrich(
|
||||
mapped: Dict[str, Any],
|
||||
raw: Dict[str, Any],
|
||||
*,
|
||||
enable_domain_lookup: bool = True,
|
||||
confidence_threshold: float = 0.7,
|
||||
) -> Tuple[Dict[str, Any], EnrichmentResult]:
|
||||
"""Apply enrichment rules to a mapped CRM record."""
|
||||
result = EnrichmentResult()
|
||||
|
||||
# Email fallback: scan background and custom fields.
|
||||
if not mapped.get("emails"):
|
||||
candidates = _extract_emails_from_text(raw.get("background"))
|
||||
if candidates:
|
||||
mapped["emails"] = [{"value": candidates[0], "type": "work"}]
|
||||
result.email_source = "background"
|
||||
|
||||
if not mapped.get("emails"):
|
||||
result.confidence_issues.append("No email found; record may not be upsertable")
|
||||
|
||||
# Company enrichment.
|
||||
if not mapped.get("company") and mapped.get("emails"):
|
||||
email = mapped["emails"][0]["value"]
|
||||
if enable_domain_lookup:
|
||||
company = _company_from_email(email)
|
||||
if company:
|
||||
mapped["company"] = company
|
||||
result.company_source = "domain_heuristic"
|
||||
if not mapped.get("company"):
|
||||
result.confidence_issues.append("Company unknown")
|
||||
|
||||
if mapped.get("company") and not mapped.get("emails"):
|
||||
# Company but no email: confidence low.
|
||||
result.confidence_issues.append("Company available but no email")
|
||||
|
||||
# Phone normalization.
|
||||
phones = mapped.get("phones") or []
|
||||
normalized_phones: List[Dict[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for phone in phones:
|
||||
value = _normalize_phone(phone.get("value"))
|
||||
if value and value not in seen:
|
||||
seen.add(value)
|
||||
normalized_phones.append({
|
||||
"value": value,
|
||||
"type": phone.get("type", "work"),
|
||||
})
|
||||
mapped["phones"] = normalized_phones
|
||||
|
||||
# Confidence gate: not a hard failure, but logged.
|
||||
if result.confidence_issues:
|
||||
logger.info(
|
||||
"Enrichment confidence issues for Redmine contact %s: %s",
|
||||
mapped.get("redmine_contact_id"),
|
||||
"; ".join(result.confidence_issues),
|
||||
)
|
||||
|
||||
mapped["enriched_at"] = datetime.now(timezone.utc).isoformat()
|
||||
return mapped, result
|
||||
|
||||
|
||||
def map_contact(raw: Dict[str, Any], *, base_url: str) -> Dict[str, Any]:
|
||||
"""Map a Redmine contact payload to the normalized CRM schema."""
|
||||
contact_id = raw.get("id")
|
||||
first_name, middle_name, last_name = _normalize_name(
|
||||
raw.get("first_name"),
|
||||
raw.get("last_name"),
|
||||
raw.get("middle_name"),
|
||||
)
|
||||
company = _coalesce(raw.get("company"))
|
||||
is_company = bool(raw.get("is_company"))
|
||||
full_name = _build_full_name(
|
||||
first_name, last_name, middle_name, company, is_company
|
||||
)
|
||||
|
||||
emails: List[Dict[str, str]] = []
|
||||
primary_email = _coalesce(raw.get("email"))
|
||||
if primary_email:
|
||||
emails.append({"value": primary_email, "type": "work"})
|
||||
|
||||
phones: List[Dict[str, str]] = []
|
||||
raw_phone = _coalesce(raw.get("phone"))
|
||||
if raw_phone:
|
||||
phones.append({"value": raw_phone, "type": _classify_phone(raw_phone)})
|
||||
|
||||
urls: List[Dict[str, str]] = []
|
||||
website = _coalesce(raw.get("website"))
|
||||
if website:
|
||||
urls.append({"value": website, "type": "work"})
|
||||
linkedin = _extract_custom(raw.get("custom_fields"), ["linkedin", "linked_in"])
|
||||
if linkedin:
|
||||
urls.append({"value": linkedin, "type": "linkedin"})
|
||||
|
||||
contact_url = f"{base_url.rstrip('/')}/contacts/{contact_id}"
|
||||
|
||||
return {
|
||||
"crm_id": None,
|
||||
"redmine_contact_id": contact_id,
|
||||
"full_name": full_name,
|
||||
"first_name": first_name,
|
||||
"middle_name": middle_name,
|
||||
"last_name": last_name,
|
||||
"company": company,
|
||||
"title": _coalesce(raw.get("job_title")),
|
||||
"emails": emails,
|
||||
"phones": phones,
|
||||
"urls": urls,
|
||||
"website": website,
|
||||
"birthday": _to_iso(raw.get("birthday")),
|
||||
"address": _parse_address(raw.get("address_attributes")),
|
||||
"notes": _merge_notes(raw.get("background"), raw.get("custom_fields"), contact_url),
|
||||
"tags": _normalize_tags(raw.get("tag_list")),
|
||||
"source": "redmine",
|
||||
"synced_at": datetime.now(timezone.utc).isoformat(),
|
||||
"enriched_at": None,
|
||||
"last_redmine_update": _to_iso(raw.get("updated_at")),
|
||||
"is_company": is_company,
|
||||
"project_id": raw.get("project_id"),
|
||||
"assigned_to_id": raw.get("assigned_to_id"),
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Redmine Contacts REST API client with retry, auth, and logging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RedmineError(Exception):
|
||||
"""Base exception for Redmine client errors."""
|
||||
|
||||
def __init__(self, message: str, *, redmine_id: Optional[int] = None) -> None:
|
||||
super().__init__(message)
|
||||
self.redmine_id = redmine_id
|
||||
|
||||
|
||||
class RedmineAuthError(RedmineError):
|
||||
"""Permanent auth failure (401, 403)."""
|
||||
|
||||
|
||||
class RedmineTransientError(RedmineError):
|
||||
"""Retryable error (network, 5xx, rate limit)."""
|
||||
|
||||
|
||||
class RedmineNotFoundError(RedmineError):
|
||||
"""Resource not found (404)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RetryConfig:
|
||||
max_attempts: int = 5
|
||||
base_delay_seconds: float = 1.0
|
||||
max_delay_seconds: float = 30.0
|
||||
|
||||
|
||||
class RedmineClient:
|
||||
"""Thin, retry-aware wrapper around Redmine's REST API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
*,
|
||||
project_id: Optional[int] = None,
|
||||
timeout: float = 30.0,
|
||||
retry: Optional[RetryConfig] = None,
|
||||
) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
self.project_id = project_id
|
||||
self.timeout = timeout
|
||||
self.retry = retry or RetryConfig()
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
"X-Redmine-API-Key": self.api_key,
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
})
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
json_body: Optional[Dict[str, Any]] = None,
|
||||
redmine_id: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
url = f"{self.base_url}{path}"
|
||||
last_error: Optional[Exception] = None
|
||||
|
||||
for attempt in range(1, self.retry.max_attempts + 1):
|
||||
transient_error: Optional[RedmineTransientError] = None
|
||||
|
||||
try:
|
||||
response = self.session.request(
|
||||
method,
|
||||
url,
|
||||
params=params,
|
||||
json=json_body,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
last_error = exc
|
||||
logger.warning(
|
||||
"Redmine network error on attempt %d/%d: %s",
|
||||
attempt,
|
||||
self.retry.max_attempts,
|
||||
exc,
|
||||
)
|
||||
if attempt == self.retry.max_attempts:
|
||||
raise RedmineTransientError(
|
||||
f"Network error after {self.retry.max_attempts} attempts: {exc}",
|
||||
redmine_id=redmine_id,
|
||||
) from exc
|
||||
else:
|
||||
logger.debug(
|
||||
"Redmine request %s %s attempt %d/%d status=%d",
|
||||
method,
|
||||
url,
|
||||
attempt,
|
||||
self.retry.max_attempts,
|
||||
response.status_code,
|
||||
)
|
||||
|
||||
if response.status_code == HTTPStatus.OK.value:
|
||||
if not response.content:
|
||||
return {}
|
||||
return response.json()
|
||||
|
||||
if response.status_code in (
|
||||
HTTPStatus.UNAUTHORIZED.value,
|
||||
HTTPStatus.FORBIDDEN.value,
|
||||
):
|
||||
raise RedmineAuthError(
|
||||
f"Authentication failed ({response.status_code})",
|
||||
redmine_id=redmine_id,
|
||||
)
|
||||
|
||||
if response.status_code == HTTPStatus.NOT_FOUND.value:
|
||||
raise RedmineNotFoundError(
|
||||
f"Not found ({response.status_code})",
|
||||
redmine_id=redmine_id,
|
||||
)
|
||||
|
||||
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS.value:
|
||||
transient_error = RedmineTransientError(
|
||||
"Rate limited (429)", redmine_id=redmine_id
|
||||
)
|
||||
elif response.status_code >= HTTPStatus.INTERNAL_SERVER_ERROR.value:
|
||||
transient_error = RedmineTransientError(
|
||||
f"Server error ({response.status_code})",
|
||||
redmine_id=redmine_id,
|
||||
)
|
||||
else:
|
||||
# 4xx other than auth/404 are permanent.
|
||||
raise RedmineError(
|
||||
f"Client error ({response.status_code}): {response.text}",
|
||||
redmine_id=redmine_id,
|
||||
)
|
||||
|
||||
if transient_error is not None:
|
||||
if attempt == self.retry.max_attempts:
|
||||
raise transient_error
|
||||
logger.warning(
|
||||
"Redmine transient error on attempt %d/%d: %s",
|
||||
attempt,
|
||||
self.retry.max_attempts,
|
||||
transient_error,
|
||||
)
|
||||
|
||||
if attempt < self.retry.max_attempts:
|
||||
delay = min(
|
||||
self.retry.base_delay_seconds * (2 ** (attempt - 1)),
|
||||
self.retry.max_delay_seconds,
|
||||
)
|
||||
logger.info("Retrying Redmine request in %.1f seconds", delay)
|
||||
time.sleep(delay)
|
||||
|
||||
# Should never be reached because the loop raises.
|
||||
raise RedmineTransientError("Unexpected loop exit", redmine_id=redmine_id) from last_error
|
||||
|
||||
def fetch_contacts(
|
||||
self,
|
||||
*,
|
||||
updated_since: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
status_filter: Optional[str] = "active",
|
||||
) -> List[Dict[str, Any]]:
|
||||
params: Dict[str, Any] = {"limit": limit, "offset": offset}
|
||||
if self.project_id is not None:
|
||||
params["project_id"] = self.project_id
|
||||
if updated_since:
|
||||
params["updated_on"] = f">={updated_since}"
|
||||
if status_filter:
|
||||
params["status"] = status_filter
|
||||
|
||||
data = self._request("GET", "/contacts.json", params=params)
|
||||
contacts = data.get("contacts", [])
|
||||
logger.info(
|
||||
"Fetched %d contacts from Redmine (limit=%d, offset=%d)",
|
||||
len(contacts),
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
return contacts
|
||||
|
||||
def fetch_contact(self, contact_id: int) -> Dict[str, Any]:
|
||||
data = self._request(
|
||||
"GET",
|
||||
f"/contacts/{contact_id}.json",
|
||||
redmine_id=contact_id,
|
||||
)
|
||||
return data.get("contact", data)
|
||||
|
||||
def iter_all_contacts(
|
||||
self,
|
||||
*,
|
||||
updated_since: Optional[str] = None,
|
||||
batch_size: int = 100,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Iterate through paginated contacts and collect all results."""
|
||||
all_contacts: List[Dict[str, Any]] = []
|
||||
offset = 0
|
||||
while True:
|
||||
batch = self.fetch_contacts(
|
||||
updated_since=updated_since,
|
||||
limit=batch_size,
|
||||
offset=offset,
|
||||
)
|
||||
if not batch:
|
||||
break
|
||||
all_contacts.extend(batch)
|
||||
if len(batch) < batch_size:
|
||||
break
|
||||
offset += batch_size
|
||||
return all_contacts
|
||||
@@ -0,0 +1,65 @@
|
||||
"""High-level Redmine -> CRM sync orchestrator."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from redmine_crm.crm_client import CrmClient
|
||||
from redmine_crm.mapper import enrich, map_contact
|
||||
from redmine_crm.redmine_client import RedmineClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SyncState:
|
||||
"""Minimal sync state tracker."""
|
||||
|
||||
def __init__(self, storage: Optional[Dict[str, Any]] = None) -> None:
|
||||
self._data = storage if storage is not None else {}
|
||||
|
||||
def get_last_sync(self) -> Optional[str]:
|
||||
return self._data.get("last_sync_timestamp")
|
||||
|
||||
def update(self, timestamp: str, stats: Dict[str, Any]) -> None:
|
||||
self._data["last_sync_timestamp"] = timestamp
|
||||
self._data["last_run_stats"] = stats
|
||||
|
||||
|
||||
def run_sync(
|
||||
redmine: RedmineClient,
|
||||
crm: CrmClient,
|
||||
sync_state: SyncState,
|
||||
*,
|
||||
base_url: str,
|
||||
batch_size: int = 100,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run the full ETL pipeline: Extract, Transform, Load."""
|
||||
last_sync = sync_state.get_last_sync()
|
||||
logger.info("Starting Redmine -> CRM sync; last_sync=%s", last_sync)
|
||||
|
||||
# Extract
|
||||
contacts = redmine.iter_all_contacts(
|
||||
updated_since=last_sync,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
logger.info("Extracted %d Redmine contacts", len(contacts))
|
||||
|
||||
# Transform
|
||||
records: List[Dict[str, Any]] = []
|
||||
for contact in contacts:
|
||||
try:
|
||||
mapped = map_contact(contact, base_url=base_url)
|
||||
enriched, _enrich_meta = enrich(mapped, contact)
|
||||
records.append(enriched)
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to map Redmine contact %s: %s", contact.get("id"), exc)
|
||||
|
||||
# Load
|
||||
stats = crm.sync_batch(records)
|
||||
logger.info("CRM upsert stats: %s", stats)
|
||||
|
||||
import datetime
|
||||
|
||||
sync_state.update(datetime.datetime.now(datetime.timezone.utc).isoformat(), stats)
|
||||
return stats
|
||||
Reference in New Issue
Block a user