Add semantic-index service, deployment assets, and tests

semantic_index/app.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, Optional
+
+from .config import Settings, load_settings
+from .embeddings import OpenAIEmbedder, OpenAIEmbeddingClient
+from .ingest import BackfillService
+from .models import SearchQuery, search_response
+from .qdrant_store import QdrantStore
+from .refresh import FileRefreshState, RedmineRefreshService
+from .redmine import RedmineApiSource, RedmineMapper
+from .search import HybridSearchService
+
+
+def build_services(settings: Optional[Settings] = None) -> Dict[str, Any]:
+    settings = settings or load_settings()
+    embedding_client = OpenAIEmbeddingClient(api_key=settings.openai_api_key)
+    embedder = OpenAIEmbedder(client=embedding_client)
+    store = QdrantStore(
+        url=settings.qdrant_url,
+        api_key=settings.qdrant_api_key,
+        collection=settings.qdrant_collection,
+    )
+    redmine_source = RedmineApiSource(
+        redmine_url=settings.redmine_url,
+        api_key=settings.redmine_api_key or "",
+        project_identifier=settings.redmine_project_identifier,
+    )
+    search_service = HybridSearchService(embedder=embedder, store=store)
+    backfill_service = BackfillService(
+        source=redmine_source,
+        embedder=embedder,
+        store=store,
+        mapper=RedmineMapper(redmine_url=settings.redmine_url, project_identifier=settings.redmine_project_identifier),
+    )
+    refresh_service = RedmineRefreshService(
+        source=redmine_source,
+        embedder=embedder,
+        store=store,
+        mapper=RedmineMapper(redmine_url=settings.redmine_url, project_identifier=settings.redmine_project_identifier),
+        state=FileRefreshState(settings.refresh_state_path),
+    )
+    return {
+        "settings": settings,
+        "search": search_service,
+        "backfill": backfill_service,
+        "refresh": refresh_service,
+        "store": store,
+        "redmine_source": redmine_source,
+    }
+
+
+def create_app(settings: Optional[Settings] = None, service_builder: Optional[Callable[[], Dict[str, Any]]] = None):
+    try:
+        from fastapi import FastAPI, Header, HTTPException
+    except ImportError as exc:
+        raise RuntimeError("Install fastapi and uvicorn to run the HTTP service") from exc
+
+    services: Optional[Dict[str, Any]] = None
+    app = FastAPI(title="Redmine Semantic Index", version="0.1.0")
+
+    def get_services() -> Dict[str, Any]:
+        nonlocal services
+        if services is None:
+            if service_builder is not None:
+                services = service_builder()
+            else:
+                services = build_services(settings)
+        return services
+
+    def authorize(authorization: Optional[str]) -> None:
+        api_key = get_services()["settings"].service_api_key
+        if not api_key:
+            return
+        expected = f"Bearer {api_key}"
+        if authorization != expected:
+            raise HTTPException(status_code=401, detail="unauthorized")
+
+    @app.get("/health")
+    def health() -> Dict[str, str]:
+        return {"status": "ok"}
+
+    @app.post("/sources/redmine/backfill-sample")
+    def backfill(payload: Dict[str, Any] | None = None, authorization: Optional[str] = Header(default=None)) -> Dict[str, Any]:
+        authorize(authorization)
+        active_services = get_services()
+        limit = int((payload or {}).get("limit", active_services["settings"].sample_limit))
+        return active_services["backfill"].backfill_redmine_sample(limit=limit)
+
+    @app.post("/sources/redmine/refresh")
+    def refresh(payload: Dict[str, Any] | None = None, authorization: Optional[str] = Header(default=None)) -> Dict[str, Any]:
+        authorize(authorization)
+        payload = payload or {}
+        project_limits = payload.get("project_limits")
+        if not project_limits:
+            project = payload.get("project_identifier") or get_services()["settings"].redmine_project_identifier
+            if not project:
+                raise HTTPException(status_code=400, detail="project_limits or project_identifier is required")
+            project_limits = {project: int(payload.get("limit", get_services()["settings"].sample_limit))}
+        return get_services()["refresh"].refresh_redmine_project_limits(
+            {str(project): int(limit) for project, limit in project_limits.items()},
+            dry_run=bool(payload.get("dry_run", False)),
+            force_rebuild=bool(payload.get("force_rebuild", False)),
+            overlap_minutes=int(payload.get("overlap_minutes", 15)),
+        )
+
+    @app.post("/search")
+    def search(payload: Dict[str, Any], authorization: Optional[str] = Header(default=None)) -> Dict[str, Any]:
+        authorize(authorization)
+        query = SearchQuery(
+            text=payload.get("query") or payload.get("text") or "",
+            source=payload.get("source"),
+            project_id=payload.get("project_id"),
+            project_identifier=payload.get("project_identifier"),
+            doc_type=payload.get("doc_type"),
+            issue_id=payload.get("issue_id"),
+            contact_id=payload.get("contact_id"),
+            contact_email=payload.get("contact_email"),
+            date_from=payload.get("date_from"),
+            date_to=payload.get("date_to"),
+            limit=int(payload.get("limit", 10)),
+            include_snippets=bool(payload.get("include_snippets", True)),
+        )
+        results = get_services()["search"].search(query)
+        return search_response(query, results)
+
+    @app.get("/projects")
+    def projects(authorization: Optional[str] = Header(default=None)) -> Dict[str, Any]:
+        authorize(authorization)
+        return {"projects": get_services()["store"].list_projects(source="redmine")}
+
+    @app.get("/documents/{document_id}")
+    def document(document_id: str, authorization: Optional[str] = Header(default=None)) -> Dict[str, Any]:
+        authorize(authorization)
+        found = get_services()["search"].get_document(document_id)
+        if found is None:
+            raise HTTPException(status_code=404, detail="not_found")
+        return found
+
+    return app
+
+
+class LazyASGIApp:
+    def __init__(self) -> None:
+        self._app = None
+
+    async def __call__(self, scope, receive, send):
+        if self._app is None:
+            self._app = create_app()
+        await self._app(scope, receive, send)
+
+
+app = LazyASGIApp()