Add semantic-index service, deployment assets, and tests
This commit is contained in:
Jason Thistlethwaite
Executable
+183
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
Usage:
|
||||
deploy/semantic-index/install.sh [--dry-run] [--apply] [--start] [--no-system] [--skip-deps]
|
||||
|
||||
Modes:
|
||||
--dry-run Print commands that would run. This is the default.
|
||||
--apply Install files, venv, dependencies, env template, and systemd units.
|
||||
--start With --apply, reload systemd and start only semantic-index.service.
|
||||
--no-system Skip sudo/systemd operations. Useful for tests and local validation.
|
||||
--skip-deps Skip venv creation and dependency install.
|
||||
|
||||
The installer never runs backfill, never enables the refresh timer, and never
|
||||
passes --force-rebuild.
|
||||
EOF
|
||||
}
|
||||
|
||||
mode=dry-run
|
||||
start_service=0
|
||||
system_ops=1
|
||||
skip_deps=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--dry-run)
|
||||
mode=dry-run
|
||||
shift
|
||||
;;
|
||||
--apply)
|
||||
mode=apply
|
||||
shift
|
||||
;;
|
||||
--start)
|
||||
start_service=1
|
||||
shift
|
||||
;;
|
||||
--no-system)
|
||||
system_ops=0
|
||||
shift
|
||||
;;
|
||||
--skip-deps)
|
||||
skip_deps=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$start_service" -eq 1 && "$mode" != "apply" ]]; then
|
||||
echo "--start requires --apply" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
install_dir=${SEMANTIC_INDEX_INSTALL_DIR:-/opt/semantic-index}
|
||||
env_file=${SEMANTIC_INDEX_ENV_FILE:-/etc/semantic-index.env}
|
||||
state_dir=${SEMANTIC_INDEX_STATE_DIR:-/var/lib/semantic-index}
|
||||
log_dir=${SEMANTIC_INDEX_LOG_DIR:-/var/log/semantic-index}
|
||||
systemd_dir=${SEMANTIC_INDEX_SYSTEMD_DIR:-/etc/systemd/system}
|
||||
python_bin=${PYTHON:-python3}
|
||||
|
||||
run() {
|
||||
if [[ "$mode" == "dry-run" ]]; then
|
||||
printf 'would run:'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
run_sudo() {
|
||||
if [[ "$system_ops" -eq 0 ]]; then
|
||||
run "$@"
|
||||
else
|
||||
run sudo "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
install_env_template() {
|
||||
if [[ "$mode" == "dry-run" ]]; then
|
||||
echo "would copy env template only if missing: $env_file"
|
||||
return
|
||||
fi
|
||||
if [[ -e "$env_file" ]]; then
|
||||
echo "keeping existing $env_file"
|
||||
return
|
||||
fi
|
||||
if [[ "$system_ops" -eq 0 ]]; then
|
||||
mkdir -p "$(dirname "$env_file")"
|
||||
cp "$repo_root/deploy/semantic-index/semantic-index.env.example" "$env_file"
|
||||
else
|
||||
sudo install -m 0640 "$repo_root/deploy/semantic-index/semantic-index.env.example" "$env_file"
|
||||
fi
|
||||
}
|
||||
|
||||
print_next_steps_warning() {
|
||||
cat <<EOF
|
||||
|
||||
Semantic Index installed, but deployment is not complete.
|
||||
|
||||
Required manual steps:
|
||||
1. Edit $env_file and fill real secrets/URLs.
|
||||
2. Start or restart the HTTP service:
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl start semantic-index.service
|
||||
3. Validate:
|
||||
curl -sS http://127.0.0.1:8787/health
|
||||
$install_dir/semantic_index/search.sh "goods return" customer-service 3
|
||||
4. Before enabling scheduled refresh, run:
|
||||
SEMANTIC_INDEX_PROJECT_LIMITS='customer-service=5' $install_dir/semantic_index/refresh.sh
|
||||
$install_dir/semantic_index/refresh.sh --apply
|
||||
5. Create/confirm a Qdrant snapshot before any production-scale backfill.
|
||||
|
||||
The refresh timer was NOT enabled automatically.
|
||||
Do not use --force-rebuild unless you intentionally want to pay to re-embed unchanged documents.
|
||||
EOF
|
||||
}
|
||||
|
||||
echo "mode=$mode"
|
||||
echo "install_dir=$install_dir"
|
||||
echo "env_file=$env_file"
|
||||
echo "state_dir=$state_dir"
|
||||
echo "log_dir=$log_dir"
|
||||
|
||||
run_sudo mkdir -p "$install_dir" "$state_dir" "$log_dir" "$systemd_dir"
|
||||
run_sudo rsync -a \
|
||||
--exclude ".env" \
|
||||
--exclude "__pycache__/" \
|
||||
--exclude "*.pyc" \
|
||||
"$repo_root/semantic_index" \
|
||||
"$repo_root/tests" \
|
||||
"$repo_root/docs" \
|
||||
"$repo_root/deploy" \
|
||||
"$repo_root/dist" \
|
||||
"$install_dir/"
|
||||
|
||||
if [[ "$skip_deps" -eq 1 ]]; then
|
||||
echo "skipping venv/dependency install because --skip-deps was used"
|
||||
elif [[ "$mode" == "apply" && "$system_ops" -eq 0 ]]; then
|
||||
run "$python_bin" -m venv "$install_dir/.venv"
|
||||
run "$install_dir/.venv/bin/pip" install openai qdrant-client fastapi uvicorn
|
||||
else
|
||||
run_sudo "$python_bin" -m venv "$install_dir/.venv"
|
||||
run_sudo "$install_dir/.venv/bin/pip" install openai qdrant-client fastapi uvicorn
|
||||
fi
|
||||
|
||||
install_env_template
|
||||
|
||||
run_sudo install -m 0644 "$repo_root/deploy/semantic-index/semantic-index.service" "$systemd_dir/semantic-index.service"
|
||||
run_sudo install -m 0644 "$repo_root/deploy/semantic-index/semantic-index-refresh.service" "$systemd_dir/semantic-index-refresh.service"
|
||||
run_sudo install -m 0644 "$repo_root/deploy/semantic-index/semantic-index-refresh.timer" "$systemd_dir/semantic-index-refresh.timer"
|
||||
|
||||
if [[ "$mode" == "apply" && "$skip_deps" -eq 0 ]]; then
|
||||
"$install_dir/.venv/bin/python" -m py_compile "$install_dir"/semantic_index/*.py
|
||||
"$install_dir/.venv/bin/python" -m unittest discover -s "$install_dir/tests/semantic_index"
|
||||
bash -n "$install_dir/semantic_index/refresh.sh"
|
||||
elif [[ "$mode" == "apply" ]]; then
|
||||
echo "skipping installed-code validation because --skip-deps was used"
|
||||
fi
|
||||
|
||||
if [[ "$mode" == "apply" && "$start_service" -eq 1 ]]; then
|
||||
if [[ "$system_ops" -eq 0 ]]; then
|
||||
echo "skipping systemctl start because --no-system was used"
|
||||
else
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl start semantic-index.service
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$mode" == "apply" ]]; then
|
||||
print_next_steps_warning
|
||||
fi
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Redmine Semantic Index Rolling Refresh
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/opt/semantic-index
|
||||
EnvironmentFile=/etc/semantic-index.env
|
||||
ExecStart=/bin/bash -lc 'exec /opt/semantic-index/semantic_index/refresh.sh --apply'
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run Redmine Semantic Index Rolling Refresh
|
||||
|
||||
[Timer]
|
||||
OnBootSec=10min
|
||||
OnUnitActiveSec=30min
|
||||
Unit=semantic-index-refresh.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,22 @@
|
||||
# Copy to /etc/semantic-index.env and fill secrets on the target host.
|
||||
# Do not commit real values.
|
||||
|
||||
OPENAI_API_KEY=
|
||||
QDRANT_URL=http://qdrant-host:6333
|
||||
QDRANT_API_KEY=
|
||||
QDRANT_COLLECTION=redmine_semantic_sample
|
||||
|
||||
REDMINE_URL=http://redmine-host
|
||||
REDMINE_API_KEY=
|
||||
REDMINE_PROJECT_IDENTIFIER=
|
||||
REDMINE_SAMPLE_LIMIT=500
|
||||
|
||||
SEMANTIC_INDEX_HOST=127.0.0.1
|
||||
SEMANTIC_INDEX_PORT=8787
|
||||
SEMANTIC_INDEX_API_KEY=
|
||||
SEMANTIC_INDEX_REFRESH_STATE_PATH=/var/lib/semantic-index/refresh_state.json
|
||||
|
||||
SEMANTIC_INDEX_PROJECT_LIMITS=customer-service=500,hiring=200,todo-jason=200,sales-inbox=100,business-development=100,dock-scheduling=100,prep-standardization=100
|
||||
SEMANTIC_INDEX_LOG_DIR=/var/log/semantic-index
|
||||
SEMANTIC_INDEX_STATE_PATH=/var/lib/semantic-index/refresh_state.json
|
||||
SEMANTIC_INDEX_OVERLAP_MINUTES=15
|
||||
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=Redmine Semantic Index HTTP API
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/semantic-index
|
||||
EnvironmentFile=/etc/semantic-index.env
|
||||
ExecStart=/bin/bash -lc 'exec /opt/semantic-index/.venv/bin/uvicorn semantic_index.app:app --host "${SEMANTIC_INDEX_HOST}" --port "${SEMANTIC_INDEX_PORT}"'
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in New Issue
Block a user