Add semantic-index service, deployment assets, and tests

2026-05-04 09:50:03 -04:00
parent faad70872b
commit b305544f63
42 changed files with 5059 additions and 0 deletions
@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat >&2 <<'EOF'
+Usage:
+  deploy/semantic-index/install.sh [--dry-run] [--apply] [--start] [--no-system] [--skip-deps]
+
+Modes:
+  --dry-run   Print commands that would run. This is the default.
+  --apply     Install files, venv, dependencies, env template, and systemd units.
+  --start     With --apply, reload systemd and start only semantic-index.service.
+  --no-system Skip sudo/systemd operations. Useful for tests and local validation.
+  --skip-deps Skip venv creation and dependency install.
+
+The installer never runs backfill, never enables the refresh timer, and never
+passes --force-rebuild.
+EOF
+}
+
+mode=dry-run
+start_service=0
+system_ops=1
+skip_deps=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --dry-run)
+      mode=dry-run
+      shift
+      ;;
+    --apply)
+      mode=apply
+      shift
+      ;;
+    --start)
+      start_service=1
+      shift
+      ;;
+    --no-system)
+      system_ops=0
+      shift
+      ;;
+    --skip-deps)
+      skip_deps=1
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      usage
+      exit 2
+      ;;
+  esac
+done
+
+if [[ "$start_service" -eq 1 && "$mode" != "apply" ]]; then
+  echo "--start requires --apply" >&2
+  exit 2
+fi
+
+repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+install_dir=${SEMANTIC_INDEX_INSTALL_DIR:-/opt/semantic-index}
+env_file=${SEMANTIC_INDEX_ENV_FILE:-/etc/semantic-index.env}
+state_dir=${SEMANTIC_INDEX_STATE_DIR:-/var/lib/semantic-index}
+log_dir=${SEMANTIC_INDEX_LOG_DIR:-/var/log/semantic-index}
+systemd_dir=${SEMANTIC_INDEX_SYSTEMD_DIR:-/etc/systemd/system}
+python_bin=${PYTHON:-python3}
+
+run() {
+  if [[ "$mode" == "dry-run" ]]; then
+    printf 'would run:'
+    printf ' %q' "$@"
+    printf '\n'
+  else
+    "$@"
+  fi
+}
+
+run_sudo() {
+  if [[ "$system_ops" -eq 0 ]]; then
+    run "$@"
+  else
+    run sudo "$@"
+  fi
+}
+
+install_env_template() {
+  if [[ "$mode" == "dry-run" ]]; then
+    echo "would copy env template only if missing: $env_file"
+    return
+  fi
+  if [[ -e "$env_file" ]]; then
+    echo "keeping existing $env_file"
+    return
+  fi
+  if [[ "$system_ops" -eq 0 ]]; then
+    mkdir -p "$(dirname "$env_file")"
+    cp "$repo_root/deploy/semantic-index/semantic-index.env.example" "$env_file"
+  else
+    sudo install -m 0640 "$repo_root/deploy/semantic-index/semantic-index.env.example" "$env_file"
+  fi
+}
+
+print_next_steps_warning() {
+  cat <<EOF
+
+Semantic Index installed, but deployment is not complete.
+
+Required manual steps:
+1. Edit $env_file and fill real secrets/URLs.
+2. Start or restart the HTTP service:
+   sudo systemctl daemon-reload
+   sudo systemctl start semantic-index.service
+3. Validate:
+   curl -sS http://127.0.0.1:8787/health
+   $install_dir/semantic_index/search.sh "goods return" customer-service 3
+4. Before enabling scheduled refresh, run:
+   SEMANTIC_INDEX_PROJECT_LIMITS='customer-service=5' $install_dir/semantic_index/refresh.sh
+   $install_dir/semantic_index/refresh.sh --apply
+5. Create/confirm a Qdrant snapshot before any production-scale backfill.
+
+The refresh timer was NOT enabled automatically.
+Do not use --force-rebuild unless you intentionally want to pay to re-embed unchanged documents.
+EOF
+}
+
+echo "mode=$mode"
+echo "install_dir=$install_dir"
+echo "env_file=$env_file"
+echo "state_dir=$state_dir"
+echo "log_dir=$log_dir"
+
+run_sudo mkdir -p "$install_dir" "$state_dir" "$log_dir" "$systemd_dir"
+run_sudo rsync -a \
+  --exclude ".env" \
+  --exclude "__pycache__/" \
+  --exclude "*.pyc" \
+  "$repo_root/semantic_index" \
+  "$repo_root/tests" \
+  "$repo_root/docs" \
+  "$repo_root/deploy" \
+  "$repo_root/dist" \
+  "$install_dir/"
+
+if [[ "$skip_deps" -eq 1 ]]; then
+  echo "skipping venv/dependency install because --skip-deps was used"
+elif [[ "$mode" == "apply" && "$system_ops" -eq 0 ]]; then
+  run "$python_bin" -m venv "$install_dir/.venv"
+  run "$install_dir/.venv/bin/pip" install openai qdrant-client fastapi uvicorn
+else
+  run_sudo "$python_bin" -m venv "$install_dir/.venv"
+  run_sudo "$install_dir/.venv/bin/pip" install openai qdrant-client fastapi uvicorn
+fi
+
+install_env_template
+
+run_sudo install -m 0644 "$repo_root/deploy/semantic-index/semantic-index.service" "$systemd_dir/semantic-index.service"
+run_sudo install -m 0644 "$repo_root/deploy/semantic-index/semantic-index-refresh.service" "$systemd_dir/semantic-index-refresh.service"
+run_sudo install -m 0644 "$repo_root/deploy/semantic-index/semantic-index-refresh.timer" "$systemd_dir/semantic-index-refresh.timer"
+
+if [[ "$mode" == "apply" && "$skip_deps" -eq 0 ]]; then
+  "$install_dir/.venv/bin/python" -m py_compile "$install_dir"/semantic_index/*.py
+  "$install_dir/.venv/bin/python" -m unittest discover -s "$install_dir/tests/semantic_index"
+  bash -n "$install_dir/semantic_index/refresh.sh"
+elif [[ "$mode" == "apply" ]]; then
+  echo "skipping installed-code validation because --skip-deps was used"
+fi
+
+if [[ "$mode" == "apply" && "$start_service" -eq 1 ]]; then
+  if [[ "$system_ops" -eq 0 ]]; then
+    echo "skipping systemctl start because --no-system was used"
+  else
+    sudo systemctl daemon-reload
+    sudo systemctl start semantic-index.service
+  fi
+fi
+
+if [[ "$mode" == "apply" ]]; then
+  print_next_steps_warning
+fi