diff --git a/.gitignore b/.gitignore index b49eab6..34a71ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /.cache/ +/.venv/ /__pycache__/ /redmine-copy/ /dist/*.tar.gz @@ -7,4 +8,5 @@ redMCP/test.env redMCP/vendor/ redMCP/composer.phar .env +semantic_index/.env *.pyc diff --git a/docs/test_instance_post_import.md b/docs/test_instance_post_import.md index 5cc7797..2ee3930 100644 --- a/docs/test_instance_post_import.md +++ b/docs/test_instance_post_import.md @@ -15,6 +15,92 @@ redMCP testing. - Mailpit ports: HTTP `8025`, SMTP `1025`, POP3 `1110` - POP3 credentials: `test` / `testpass` - SMTP authentication: none +- Shared scratch path: `/opt/lanscratch` +- Post-import payload path: + `/opt/lanscratch/redmine-post-import/repo` +- Post-import status path: + `/opt/lanscratch/redmine-post-import/status` + +## Automated Daily Post-Import + +Stage the post-import payload from this host into the shared LAN scratch folder: + +```sh +./stage_post_import_payload.py +``` + +The default staging mode is a dry run. Review the `rsync` command, then apply: + +```sh +./stage_post_import_payload.py --apply +``` + +After the fresh production database and `/usr/share/redmine` tree have been +copied onto the LAN test host, the test host should run the automation locally: + +```sh +cd /opt/lanscratch/redmine-post-import/repo +./post_import_refresh.py --local --apply +``` + +For manual review on the test host, omit `--apply` first: + +```sh +cd /opt/lanscratch/redmine-post-import/repo +./post_import_refresh.py --local +``` + +This host can check completion by reading: + +```text +/opt/lanscratch/redmine-post-import/status/latest.json +/opt/lanscratch/redmine-post-import/status/latest-success.json +``` + +The automation: + +- verifies the tracked plugin source directories exist locally and that the + remote Redmine path exists; +- overlays remote dev-only files from `/home/reddev/redmine-dev-overrides` when + that directory exists; +- reapplies `redmine_event_outbox`, `redmine_contacts`, and + `redmine_contacts_helpdesk` from this repository into + `/usr/share/redmine/plugins/`; +- runs `RAILS_ENV=production bundle exec rake redmine:plugins:migrate`; +- fixes group-write permissions on attachment, `tmp`, and `log` paths; +- runs `reset_helpdesk_mail_settings.py` unless `--skip-helpdesk-reset` is + passed; +- restarts Passenger with `touch tmp/restart.txt`; +- runs `validate_test_instance.py`; +- checks outbox status and dry-runs a small outbox batch; +- runs a semantic-index dry-run smoke check only. + +Each applied run writes status JSON under +`/opt/lanscratch/redmine-post-import/status/runs/`, updates `latest.json` after +each step, and updates `latest-success.json` only after every step exits +successfully. The JSON includes the run id, host, execution mode, Redmine path, +repo root, failed step when applicable, and per-command return codes. + +Remote write and permission steps use `sudo` by default because a fresh +production file copy may leave `/usr/share/redmine` or attachment paths owned by +another user. This applies in both local and SSH modes. If the dev host already +gives the runner write access to those paths, pass `--no-remote-sudo`. + +The older SSH orchestration path from this host remains available: + +```sh +./post_import_refresh.py +./post_import_refresh.py --apply +``` + +The automation deliberately does **not** run a semantic-index apply refresh, +does **not** use `--force-rebuild`, and does **not** enable the semantic-index +refresh timer. After a fresh database clone, treat semantic-index writes or a +Qdrant rebuild as a separate manual maintenance action with a snapshot or +isolated dev collection first. + +Use the manual sections below for troubleshooting individual steps or for +running the sequence by hand. ## 1. Validate The Fresh Import diff --git a/plugins/redmine_event_outbox/db/migrate/001_create_event_outbox_events.rb b/plugins/redmine_event_outbox/db/migrate/001_create_event_outbox_events.rb index 3645de5..858685e 100644 --- a/plugins/redmine_event_outbox/db/migrate/001_create_event_outbox_events.rb +++ b/plugins/redmine_event_outbox/db/migrate/001_create_event_outbox_events.rb @@ -1,5 +1,7 @@ class CreateEventOutboxEvents < ActiveRecord::Migration def change + return if table_exists?(:event_outbox_events) + create_table :event_outbox_events do |t| t.string :event_type, :null => false t.string :source_type, :null => false diff --git a/post_import_refresh.py b/post_import_refresh.py new file mode 100755 index 0000000..a90a30c --- /dev/null +++ b/post_import_refresh.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +"""Post-import automation for the LAN Redmine development clone. + +Run this after the production database and Redmine tree have been copied onto +the dev/test host. It reapplies this repository's tracked plugin forks, runs +plugin migrations, sanitizes Helpdesk mail settings, restarts Passenger, and +performs validation. The default mode is a dry run. +""" + +import argparse +import datetime as dt +import json +import os +import shlex +import subprocess +import sys +import socket +from pathlib import Path +from typing import List, Optional, Tuple + + +DEFAULT_SSH_HOST = "reddev@192.168.50.170" +DEFAULT_SSH_KEY = Path("/tmp/reddev") +DEFAULT_REMOTE_REDMINE = "/usr/share/redmine" +DEFAULT_REMOTE_OVERRIDES = "/home/reddev/redmine-dev-overrides" +DEFAULT_FILES_ROOT = "/var/lib/redmine/default/files" +DEFAULT_MAILPIT_HOST = "192.168.1.105" +DEFAULT_SEMANTIC_LIMITS = "customer-service=5" +DEFAULT_LANSCRATCH_ROOT = Path("/opt/lanscratch/redmine-post-import") +DEFAULT_STATUS_DIR = DEFAULT_LANSCRATCH_ROOT / "status" +DEFAULT_LOCAL_REPO_ROOT = DEFAULT_LANSCRATCH_ROOT / "repo" + +TRACKED_PLUGINS = ( + "redmine_event_outbox", + "redmine_contacts", + "redmine_contacts_helpdesk", +) + + +class AutomationConfig: + def __init__( + self, + apply=False, + local=False, + repo_root=None, + status_dir=DEFAULT_STATUS_DIR, + ssh_host=DEFAULT_SSH_HOST, + ssh_key=DEFAULT_SSH_KEY, + remote_redmine=DEFAULT_REMOTE_REDMINE, + remote_overrides=DEFAULT_REMOTE_OVERRIDES, + files_root=DEFAULT_FILES_ROOT, + mailpit_host=DEFAULT_MAILPIT_HOST, + semantic_limits=DEFAULT_SEMANTIC_LIMITS, + remote_sudo=True, + skip_semantic_check=False, + skip_helpdesk_reset=False, + ): + self.apply = apply + self.local = local + self.repo_root = repo_root if repo_root is not None else (DEFAULT_LOCAL_REPO_ROOT if local else Path(".")) + self.status_dir = status_dir + self.ssh_host = ssh_host + self.ssh_key = ssh_key + self.remote_redmine = remote_redmine + self.remote_overrides = remote_overrides + self.files_root = files_root + self.mailpit_host = mailpit_host + self.semantic_limits = semantic_limits + self.remote_sudo = remote_sudo + self.skip_semantic_check = skip_semantic_check + self.skip_helpdesk_reset = skip_helpdesk_reset + + +class Step: + def __init__(self, name, commands): + self.name = name + self.commands = commands + + +class StepResult: + def __init__(self, step, command, returncode): + self.step = step + self.command = command + self.returncode = returncode + + +def main() -> int: + args = parse_args() + config = AutomationConfig( + apply=args.apply, + local=args.local, + repo_root=args.repo_root or (DEFAULT_LOCAL_REPO_ROOT if args.local else Path(".")), + status_dir=args.status_dir, + ssh_host=args.ssh_host, + ssh_key=args.ssh_key, + remote_redmine=args.remote_redmine, + remote_overrides=args.remote_overrides, + files_root=args.files_root, + mailpit_host=args.mailpit_host, + semantic_limits=args.semantic_limits, + remote_sudo=not args.no_remote_sudo, + skip_semantic_check=args.skip_semantic_check, + skip_helpdesk_reset=args.skip_helpdesk_reset, + ) + steps = build_steps(config) + run_id = utc_stamp() + results = [] # type: List[StepResult] + + print(f"mode={'apply' if config.apply else 'dry-run'}") + print(f"execution={'local' if config.local else 'remote'}") + if config.apply: + write_status(config, run_id, "running", results) + for step in steps: + print(f"\n== {step.name} ==") + for command in step.commands: + if config.apply: + print(f"running: {command}") + result = subprocess.run(command, shell=True, check=False) + results.append(StepResult(step.name, command, result.returncode)) + write_status(config, run_id, "running", results) + if result.returncode != 0: + print(f"error: command failed with exit {result.returncode}", file=sys.stderr) + write_status(config, run_id, "failed", results, failed_step=step.name) + return result.returncode + else: + print(f"would run: {command}") + + if not config.apply: + print("\nDry run only. Re-run with --apply after reviewing the command list.") + else: + write_status(config, run_id, "success", results) + return 0 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run daily post-import steps for the LAN Redmine dev clone." + ) + parser.add_argument("--apply", action="store_true", help="Run commands. Default is dry-run.") + parser.add_argument( + "--local", + action="store_true", + help="Run directly on the Redmine test host instead of orchestrating over SSH.", + ) + parser.add_argument( + "--repo-root", + type=Path, + default=Path(os.environ["POST_IMPORT_REPO_ROOT"]) if "POST_IMPORT_REPO_ROOT" in os.environ else None, + help="Repository/payload root to copy plugins and run helper scripts from.", + ) + parser.add_argument( + "--status-dir", + type=Path, + default=Path(os.getenv("POST_IMPORT_STATUS_DIR", str(DEFAULT_STATUS_DIR))), + ) + parser.add_argument("--ssh-host", default=os.getenv("REDMINE_SSH_HOST", DEFAULT_SSH_HOST)) + parser.add_argument( + "--ssh-key", + type=Path, + default=Path(os.getenv("REDMINE_SSH_KEY", str(DEFAULT_SSH_KEY))), + ) + parser.add_argument( + "--remote-redmine", + default=os.getenv("REDMINE_REMOTE_PATH", DEFAULT_REMOTE_REDMINE), + ) + parser.add_argument( + "--remote-overrides", + default=os.getenv("REDMINE_REMOTE_OVERRIDES", DEFAULT_REMOTE_OVERRIDES), + help=( + "Remote directory containing dev-only files to overlay after the production copy. " + "If it is absent on the remote host, the step is skipped." + ), + ) + parser.add_argument("--files-root", default=DEFAULT_FILES_ROOT) + parser.add_argument("--mailpit-host", default=DEFAULT_MAILPIT_HOST) + parser.add_argument("--semantic-limits", default=DEFAULT_SEMANTIC_LIMITS) + parser.add_argument( + "--no-remote-sudo", + action="store_true", + help="Do not use sudo for remote write/permission steps.", + ) + parser.add_argument("--skip-semantic-check", action="store_true") + parser.add_argument("--skip-helpdesk-reset", action="store_true") + return parser.parse_args() + + +def build_steps(config): + steps = [ + Step( + "preflight", + preflight_commands(config), + ), + Step( + "restore dev-local overrides", + ( + restore_overrides_command(config), + ), + ), + Step( + "reapply tracked plugins", + ( + copy_plugins_command(config), + ), + ), + Step( + "run plugin migrations", + ( + redmine_command( + config, + "RAILS_ENV=production bundle exec rake redmine:plugins:migrate", + ), + ), + ), + Step( + "fix dev writable paths", + ( + fix_permissions_command(config), + ), + ), + ] + + if not config.skip_helpdesk_reset: + steps.append( + Step( + "reset Helpdesk mail settings", + ( + helpdesk_reset_command(config), + ), + ) + ) + + steps.extend( + [ + Step( + "restart Passenger", + ( + redmine_command(config, "touch tmp/restart.txt"), + ), + ), + Step( + "validate test instance", + ( + validate_command(config), + ), + ), + Step( + "check outbox worker", + outbox_commands(config), + ), + ] + ) + + if not config.skip_semantic_check: + steps.append( + Step( + "validate semantic index dry-run", + ( + semantic_check_command(config), + ), + ) + ) + + return steps + + +def preflight_commands(config): + commands = tuple(local_test(str(config.repo_root / "plugins" / plugin)) for plugin in TRACKED_PLUGINS) + if config.local: + return commands + (f"test -d {q(config.remote_redmine)}",) + return commands + (ssh(config, f"test -d {q(config.remote_redmine)}"),) + + +def semantic_check_command(config): + refresh = config.repo_root / "semantic_index" / "refresh.sh" + python_bin = config.repo_root / ".venv" / "bin" / "python" + command = "SEMANTIC_INDEX_PROJECT_LIMITS={limits} {refresh}".format( + limits=q(config.semantic_limits), + refresh=q(str(refresh)), + ) + if not config.local: + return command + return ( + "if test -x {python}; then PYTHON={python} {command}; " + "else echo 'semantic index runtime missing; skipping dry-run'; fi" + ).format( + python=q(str(python_bin)), + command=command, + ) + + +def local_test(path): + return f"test -d {q(path)}" + + +def restore_overrides_command(config): + command = "if [ -d {overrides} ]; then {sudo}rsync -a {overrides}/ {redmine}/; else echo 'no dev overrides directory: {overrides}'; fi".format( + sudo=remote_sudo_prefix(config), + overrides=q(config.remote_overrides), + redmine=q(config.remote_redmine), + ) + return command if config.local else ssh(config, command) + + +def copy_plugins_command(config): + sources = " ".join(q(str(config.repo_root / "plugins" / plugin)) for plugin in TRACKED_PLUGINS) + if config.local: + return f"rsync -a --delete {sources} {q(config.remote_redmine + '/plugins/')}" + return ( + "rsync -a --delete " + f"{rsync_path_option(config)}" + f"-e {q(ssh_transport(config))} " + f"{sources} " + f"{q(config.ssh_host + ':' + config.remote_redmine + '/plugins/')}" + ) + + +def redmine_command(config, command): + full = f"cd {q(config.remote_redmine)} && {command}" + return full if config.local else ssh(config, full) + + +def fix_permissions_command(config): + command = "{sudo}mkdir -p {files} {redmine}/tmp {redmine}/log && {sudo}chmod -R g+rwX {files} {redmine}/tmp {redmine}/log && {sudo}find {files} -type d -exec chmod g+s {{}} +".format( + sudo=remote_sudo_prefix(config), + files=q(config.files_root), + redmine=q(config.remote_redmine), + ) + return command if config.local else ssh(config, command) + + +def script(config, name): + path = config.repo_root / name + if str(config.repo_root) == ".": + return q(f"./{name}") + return q(str(path)) + + +def helpdesk_reset_command(config): + base = f"{script(config, 'reset_helpdesk_mail_settings.py')} " + if config.local: + return ( + base + + f"--local --remote-redmine {q(config.remote_redmine)} " + f"--mailpit-host {q(config.mailpit_host)}" + ) + return ( + base + + f"--ssh-host {q(config.ssh_host)} " + f"--ssh-key {q(str(config.ssh_key))} " + f"--remote-redmine {q(config.remote_redmine)} " + f"--mailpit-host {q(config.mailpit_host)}" + ) + + +def validate_command(config): + base = f"{script(config, 'validate_test_instance.py')} " + if config.local: + return ( + base + + f"--local --remote-redmine {q(config.remote_redmine)} " + f"--mailpit-host {q(config.mailpit_host)} " + f"--files-root {q(config.files_root)}" + ) + return ( + base + + f"--ssh-host {q(config.ssh_host)} " + f"--ssh-key {q(str(config.ssh_key))} " + f"--remote-redmine {q(config.remote_redmine)} " + f"--mailpit-host {q(config.mailpit_host)} " + f"--files-root {q(config.files_root)}" + ) + + +def outbox_commands(config): + base = script(config, "redmine_outbox_worker.py") + local = " --local" if config.local else "" + return ( + f"{base}{local} --status", + f"{base}{local} --dry-run --batch-size 10", + ) + + +def ssh(config, remote_command): + return ( + "ssh " + f"-i {q(str(config.ssh_key))} " + "-o IdentitiesOnly=yes " + f"{q(config.ssh_host)} " + f"{q(remote_command)}" + ) + + +def ssh_transport(config): + return f"ssh -i {str(config.ssh_key)} -o IdentitiesOnly=yes" + + +def remote_sudo_prefix(config): + return "sudo " if config.remote_sudo else "" + + +def rsync_path_option(config): + return "--rsync-path 'sudo rsync' " if config.remote_sudo else "" + + +def write_status( + config: AutomationConfig, + run_id: str, + status: str, + results, + failed_step=None, +): + now = utc_iso() + document = { + "run_id": run_id, + "started_at": run_id_to_iso(run_id), + "updated_at": now, + "finished_at": now if status in {"success", "failed"} else None, + "mode": "apply" if config.apply else "dry-run", + "execution": "local" if config.local else "remote", + "host": socket.gethostname(), + "remote_redmine": config.remote_redmine, + "repo_root": str(config.repo_root), + "status": status, + "failed_step": failed_step, + "steps": [ + {"step": item.step, "command": item.command, "returncode": item.returncode} + for item in results + ], + } + config.status_dir.mkdir(parents=True, exist_ok=True) + runs_dir = config.status_dir / "runs" + runs_dir.mkdir(parents=True, exist_ok=True) + write_json(runs_dir / f"{run_id}.json", document) + write_json(config.status_dir / "latest.json", document) + if status == "success": + write_json(config.status_dir / "latest-success.json", document) + return document + + +def write_json(path, document): + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8") + tmp.replace(path) + + +def utc_stamp(): + return dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def utc_iso(): + return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def run_id_to_iso(run_id): + try: + parsed = dt.datetime.strptime(run_id, "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc) + return parsed.strftime("%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return run_id + + +def q(value): + return shlex.quote(value) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/redmine_outbox_worker.py b/redmine_outbox_worker.py index 5dc1b0b..c3390bd 100755 --- a/redmine_outbox_worker.py +++ b/redmine_outbox_worker.py @@ -7,8 +7,6 @@ writes deterministic JSONL records, and marks rows processed only after the write succeeds. """ -from __future__ import annotations - import argparse import json import os @@ -17,7 +15,6 @@ import subprocess import sys import time import uuid -from dataclasses import dataclass from pathlib import Path from typing import Any, Iterable @@ -32,15 +29,16 @@ class OutboxWorkerError(RuntimeError): pass -@dataclass(frozen=True) class RemoteRedmine: - ssh_host: str - ssh_key: Path - remote_redmine: str + def __init__(self, ssh_host, ssh_key, remote_redmine, local=False): + self.ssh_host = ssh_host + self.ssh_key = ssh_key + self.remote_redmine = remote_redmine + self.local = local - def mysql_json_lines(self, sql: str) -> list[dict[str, Any]]: + def mysql_json_lines(self, sql): stdout = self.mysql(sql) - rows: list[dict[str, Any]] = [] + rows = [] for line in stdout.splitlines(): if not line.strip(): continue @@ -52,24 +50,29 @@ class RemoteRedmine: raise OutboxWorkerError(f"Remote query returned non-hex row: {line[:200]}") from exc return rows - def mysql(self, sql: str) -> str: - command = [ - "ssh", - "-i", - str(self.ssh_key), - "-o", - "IdentitiesOnly=yes", - self.ssh_host, - self._mysql_runner_command(), - ] + def mysql(self, sql): + command = self._mysql_runner_command() + shell = True + if not self.local: + command = [ + "ssh", + "-i", + str(self.ssh_key), + "-o", + "IdentitiesOnly=yes", + self.ssh_host, + self._mysql_runner_command(), + ] + shell = False try: result = subprocess.run( command, input=sql, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + shell=shell, ) except OSError as exc: raise OutboxWorkerError(f"Could not run ssh: {exc}") from exc @@ -78,7 +81,7 @@ class RemoteRedmine: raise OutboxWorkerError(result.stderr.strip() or "Remote MySQL command failed.") return result.stdout - def _mysql_runner_command(self) -> str: + def _mysql_runner_command(self): ruby = ( "require 'yaml'; " "c = YAML.load_file('config/database.yml')['production']; " @@ -91,10 +94,11 @@ class RemoteRedmine: return f"cd {shell_quote(self.remote_redmine)} && ruby -e {shell_quote(ruby)}" -def main() -> int: +def main(): parser = argparse.ArgumentParser(description="Process Redmine event outbox rows into enriched JSONL documents.") parser.add_argument("--ssh-host", default=os.getenv("REDMINE_SSH_HOST", DEFAULT_SSH_HOST)) parser.add_argument("--ssh-key", type=Path, default=Path(os.getenv("REDMINE_SSH_KEY", str(DEFAULT_SSH_KEY)))) + parser.add_argument("--local", action="store_true", help="Read the Redmine database locally instead of over SSH.") parser.add_argument("--remote-redmine", default=os.getenv("REDMINE_REMOTE_PATH", DEFAULT_REMOTE_REDMINE)) parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) parser.add_argument("--batch-size", type=int, default=20) @@ -111,7 +115,7 @@ def main() -> int: parser.add_argument("--apply-purge", action="store_true", help="Actually delete rows selected by --purge-processed-days.") args = parser.parse_args() - remote = RemoteRedmine(args.ssh_host, args.ssh_key, args.remote_redmine) + remote = RemoteRedmine(args.ssh_host, args.ssh_key, args.remote_redmine, local=args.local) worker_id = make_worker_id() try: @@ -164,7 +168,7 @@ def main() -> int: return 1 -def pending_events(remote: RemoteRedmine, limit: int, max_attempts: int, stale_lock_minutes: int) -> list[dict[str, Any]]: +def pending_events(remote, limit, max_attempts, stale_lock_minutes): return remote.mysql_json_lines( f""" SELECT HEX(CAST(JSON_OBJECT( @@ -190,7 +194,7 @@ LIMIT {sql_int(limit)}; ) -def outbox_status(remote: RemoteRedmine, max_attempts: int, stale_lock_minutes: int) -> dict[str, Any]: +def outbox_status(remote, max_attempts, stale_lock_minutes): rows = remote.mysql_json_lines( f""" SELECT HEX(CAST(JSON_OBJECT( @@ -217,12 +221,12 @@ FROM event_outbox_events; def claim_events( - remote: RemoteRedmine, - worker_id: str, - limit: int, - max_attempts: int, - stale_lock_minutes: int, -) -> list[dict[str, Any]]: + remote, + worker_id, + limit, + max_attempts, + stale_lock_minutes, +): remote.mysql( f""" UPDATE event_outbox_events @@ -258,7 +262,7 @@ LIMIT {sql_int(limit)}; ) -def purge_processed(remote: RemoteRedmine, days: int, apply: bool) -> int: +def purge_processed(remote, days, apply): if days < 0: raise OutboxWorkerError("--purge-processed-days must be zero or greater.") count_sql = f""" @@ -282,9 +286,9 @@ WHERE processed_at IS NOT NULL return count -def enrich_event(remote: RemoteRedmine, event: dict[str, Any]) -> list[dict[str, Any]]: +def enrich_event(remote, event): payload = parse_payload(event.get("payload")) - documents: list[dict[str, Any]] = [event_document(event, payload)] + documents = [event_document(event, payload)] event_type = str(event.get("event_type") or "") if event_type.startswith("helpdesk_ticket."): @@ -301,7 +305,7 @@ def enrich_event(remote: RemoteRedmine, event: dict[str, Any]) -> list[dict[str, return [with_event_context(document, event) for document in documents] -def event_document(event: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]: +def event_document(event, payload): return { "doc_type": "event", "doc_id": f"event:{event.get('id')}", @@ -318,35 +322,35 @@ def event_document(event: dict[str, Any], payload: dict[str, Any]) -> dict[str, } -def fetch_ticket_documents(remote: RemoteRedmine, ids: Iterable[Any]) -> list[dict[str, Any]]: +def fetch_ticket_documents(remote, ids): id_list = sql_id_list(ids) if not id_list: return [] return remote.mysql_json_lines(ticket_sql(f"ht.id IN ({id_list})")) -def fetch_tickets_by_issue(remote: RemoteRedmine, issue_ids: Iterable[Any]) -> list[dict[str, Any]]: +def fetch_tickets_by_issue(remote, issue_ids): id_list = sql_id_list(issue_ids) if not id_list: return [] return remote.mysql_json_lines(ticket_sql(f"ht.issue_id IN ({id_list})")) -def fetch_message_documents(remote: RemoteRedmine, ids: Iterable[Any]) -> list[dict[str, Any]]: +def fetch_message_documents(remote, ids): id_list = sql_id_list(ids) if not id_list: return [] return remote.mysql_json_lines(message_sql(f"jm.id IN ({id_list})")) -def fetch_messages_by_journal(remote: RemoteRedmine, journal_ids: Iterable[Any]) -> list[dict[str, Any]]: +def fetch_messages_by_journal(remote, journal_ids): id_list = sql_id_list(journal_ids) if not id_list: return [] return remote.mysql_json_lines(message_sql(f"jm.journal_id IN ({id_list})")) -def fetch_contact_documents(remote: RemoteRedmine, ids: Iterable[Any]) -> list[dict[str, Any]]: +def fetch_contact_documents(remote, ids): id_list = sql_id_list(ids) if not id_list: return [] @@ -375,7 +379,7 @@ ORDER BY c.id; ) -def ticket_sql(where_clause: str) -> str: +def ticket_sql(where_clause): return f""" SELECT HEX(CAST(JSON_OBJECT( 'doc_type', 'ticket', @@ -423,7 +427,7 @@ ORDER BY ht.id; """ -def message_sql(where_clause: str) -> str: +def message_sql(where_clause): return f""" SELECT HEX(CAST(JSON_OBJECT( 'doc_type', 'message', @@ -474,7 +478,7 @@ ORDER BY jm.id; """ -def with_event_context(document: dict[str, Any], event: dict[str, Any]) -> dict[str, Any]: +def with_event_context(document, event): document["event_id"] = event.get("id") document["event_type"] = event.get("event_type") document["event_occurred_at"] = event.get("occurred_at") @@ -482,7 +486,7 @@ def with_event_context(document: dict[str, Any], event: dict[str, Any]) -> dict[ return document -def append_jsonl(path: Path, documents: Iterable[dict[str, Any]]) -> None: +def append_jsonl(path, documents): path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as handle: for document in documents: @@ -490,7 +494,7 @@ def append_jsonl(path: Path, documents: Iterable[dict[str, Any]]) -> None: handle.write("\n") -def mark_processed(remote: RemoteRedmine, event_id: Any, worker_id: str) -> None: +def mark_processed(remote, event_id, worker_id): remote.mysql( f""" UPDATE event_outbox_events @@ -501,7 +505,7 @@ WHERE id = {sql_int(event_id)} ) -def mark_failed(remote: RemoteRedmine, event_id: Any, worker_id: str, exc: Exception) -> None: +def mark_failed(remote, event_id, worker_id, exc): message = f"{exc.__class__.__name__}: {exc}" remote.mysql( f""" @@ -516,7 +520,7 @@ WHERE id = {sql_int(event_id)} ) -def release_claims(remote: RemoteRedmine, worker_id: str) -> None: +def release_claims(remote, worker_id): remote.mysql( f""" UPDATE event_outbox_events @@ -527,7 +531,7 @@ WHERE processed_at IS NULL ) -def parse_payload(value: Any) -> dict[str, Any]: +def parse_payload(value): if isinstance(value, dict): return value if not value: @@ -539,7 +543,7 @@ def parse_payload(value: Any) -> dict[str, Any]: return parsed if isinstance(parsed, dict) else {} -def sql_id_list(values: Iterable[Any]) -> str: +def sql_id_list(values): ids = [] for value in values: try: @@ -551,22 +555,22 @@ def sql_id_list(values: Iterable[Any]) -> str: return ",".join(sorted(set(ids), key=int)) -def sql_int(value: Any) -> int: +def sql_int(value): try: return max(0, int(value)) except (TypeError, ValueError): return 0 -def sql_string(value: str) -> str: +def sql_string(value): return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'" -def shell_quote(value: str) -> str: +def shell_quote(value): return "'" + value.replace("'", "'\"'\"'") + "'" -def make_worker_id() -> str: +def make_worker_id(): return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:12]}" diff --git a/reset_helpdesk_mail_settings.py b/reset_helpdesk_mail_settings.py index 4252348..53cd8de 100755 --- a/reset_helpdesk_mail_settings.py +++ b/reset_helpdesk_mail_settings.py @@ -7,20 +7,17 @@ settings so test mail flows through Mailpit and imported real credentials cannot be used accidentally. """ -from __future__ import annotations - import argparse import json import os import subprocess import sys -from dataclasses import dataclass from pathlib import Path from typing import Any DEFAULT_SSH_HOST = "reddev@192.168.50.170" -DEFAULT_SSH_KEY = Path("/tmp/reddev") +DEFAULT_SSH_KEY = Path("/home/iadnah/reddev") DEFAULT_REMOTE_REDMINE = "/usr/share/redmine" DEFAULT_MAILPIT_HOST = "192.168.1.105" @@ -55,15 +52,16 @@ class ResetError(RuntimeError): pass -@dataclass(frozen=True) class RemoteRedmine: - ssh_host: str - ssh_key: Path - remote_redmine: str + def __init__(self, ssh_host, ssh_key, remote_redmine, local=False): + self.ssh_host = ssh_host + self.ssh_key = ssh_key + self.remote_redmine = remote_redmine + self.local = local - def mysql_json_lines(self, sql: str) -> list[dict[str, Any]]: + def mysql_json_lines(self, sql): stdout = self.mysql(sql) - rows: list[dict[str, Any]] = [] + rows = [] for line in stdout.splitlines(): if not line.strip(): continue @@ -73,24 +71,29 @@ class RemoteRedmine: raise ResetError(f"Remote query returned an unexpected row: {line[:200]}") from exc return rows - def mysql(self, sql: str) -> str: - command = [ - "ssh", - "-i", - str(self.ssh_key), - "-o", - "IdentitiesOnly=yes", - self.ssh_host, - self._mysql_runner_command(), - ] + def mysql(self, sql): + command = self._mysql_runner_command() + shell = True + if not self.local: + command = [ + "ssh", + "-i", + str(self.ssh_key), + "-o", + "IdentitiesOnly=yes", + self.ssh_host, + self._mysql_runner_command(), + ] + shell = False try: result = subprocess.run( command, input=sql, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + shell=shell, ) except OSError as exc: raise ResetError(f"Could not run ssh: {exc}") from exc @@ -99,7 +102,7 @@ class RemoteRedmine: raise ResetError(result.stderr.strip() or "Remote MySQL command failed.") return result.stdout - def _mysql_runner_command(self) -> str: + def _mysql_runner_command(self): ruby = ( "require 'yaml'; " "c = YAML.load_file('config/database.yml')['production']; " @@ -112,12 +115,13 @@ class RemoteRedmine: return f"cd {shell_quote(self.remote_redmine)} && ruby -e {shell_quote(ruby)}" -def main() -> int: +def main(): parser = argparse.ArgumentParser( description="Reset Helpdesk mail settings for all active projects." ) parser.add_argument("--ssh-host", default=os.getenv("REDMINE_SSH_HOST", DEFAULT_SSH_HOST)) parser.add_argument("--ssh-key", type=Path, default=Path(os.getenv("REDMINE_SSH_KEY", str(DEFAULT_SSH_KEY)))) + parser.add_argument("--local", action="store_true", help="Read the Redmine database locally instead of over SSH.") parser.add_argument("--remote-redmine", default=os.getenv("REDMINE_REMOTE_PATH", DEFAULT_REMOTE_REDMINE)) parser.add_argument("--mailpit-host", default=DEFAULT_MAILPIT_HOST, help="Host Redmine should use to reach Mailpit.") parser.add_argument("--pop3-port", type=int, default=1110) @@ -139,7 +143,7 @@ def main() -> int: parser.add_argument("--dry-run", action="store_true", help="Show affected projects and settings without writing.") args = parser.parse_args() - remote = RemoteRedmine(args.ssh_host, args.ssh_key, args.remote_redmine) + remote = RemoteRedmine(args.ssh_host, args.ssh_key, args.remote_redmine, local=args.local) try: projects = find_active_projects(remote, args.project) @@ -166,7 +170,7 @@ def main() -> int: return 1 -def find_active_projects(remote: RemoteRedmine, filters: list[str]) -> list[dict[str, Any]]: +def find_active_projects(remote, filters): where = ["p.status = 1"] if filters: clauses = [] @@ -190,8 +194,8 @@ ORDER BY p.identifier; ) -def build_values(args: argparse.Namespace, projects: list[dict[str, Any]]) -> list[tuple[int, str, str]]: - rows: list[tuple[int, str, str]] = [] +def build_values(args, projects): + rows = [] for project in projects: project_id = int(project["id"]) answer_from = args.from_pattern.format( @@ -227,7 +231,7 @@ def build_values(args: argparse.Namespace, projects: list[dict[str, Any]]) -> li return rows -def apply_values(remote: RemoteRedmine, rows: list[tuple[int, str, str]]) -> None: +def apply_values(remote, rows): statements = ["START TRANSACTION;"] for project_id, name, value in rows: project_id_sql = sql_int(project_id) @@ -254,8 +258,8 @@ WHERE NOT EXISTS ( remote.mysql("\n".join(statements)) -def print_plan(rows: list[tuple[int, str, str]]) -> None: - current_project_id: int | None = None +def print_plan(rows): + current_project_id = None for project_id, name, value in rows: if project_id != current_project_id: current_project_id = project_id @@ -264,18 +268,18 @@ def print_plan(rows: list[tuple[int, str, str]]) -> None: print(f" {name} = {display_value}") -def sql_int(value: Any) -> int: +def sql_int(value): try: return max(0, int(value)) except (TypeError, ValueError): return 0 -def sql_string(value: Any) -> str: +def sql_string(value): return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'" -def shell_quote(value: str) -> str: +def shell_quote(value): return "'" + value.replace("'", "'\"'\"'") + "'" diff --git a/stage_post_import_payload.py b/stage_post_import_payload.py new file mode 100755 index 0000000..3493258 --- /dev/null +++ b/stage_post_import_payload.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Stage post-import automation files into the shared LAN scratch directory.""" + +import argparse +import shlex +import subprocess +import sys +from pathlib import Path + + +DEFAULT_TARGET = Path("/opt/lanscratch/redmine-post-import/repo") + +PAYLOAD_PATHS = ( + "plugins", + "docs", + "semantic_index", + "deploy", + "dist", + "post_import_refresh.py", + "stage_post_import_payload.py", + "reset_helpdesk_mail_settings.py", + "validate_test_instance.py", + "redmine_outbox_worker.py", + "redMCP", +) + +EXCLUDES = ( + ".env", + ".venv", + ".cache", + "__pycache__/", + "*.pyc", + "*.tar.gz", +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Copy the post-import automation payload into /opt/lanscratch." + ) + parser.add_argument("--target", type=Path, default=DEFAULT_TARGET) + parser.add_argument("--apply", action="store_true", help="Run rsync. Default is dry-run.") + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parent + command = build_rsync_command(repo_root, args.target) + if not args.apply: + print("mode=dry-run") + print(f"would run: mkdir -p {shlex.quote(str(args.target))}") + print(f"would run: {command}") + return 0 + + args.target.mkdir(parents=True, exist_ok=True) + print(f"running: {command}") + result = subprocess.run(command, shell=True, check=False) + return result.returncode + + +def build_rsync_command(repo_root: Path, target: Path) -> str: + exclude_args = " ".join(f"--exclude {shlex.quote(pattern)}" for pattern in EXCLUDES) + sources = " ".join(shlex.quote(str(repo_root / path)) for path in PAYLOAD_PATHS) + return f"rsync -a --delete {exclude_args} {sources} {shlex.quote(str(target))}/" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_event_outbox_migration.py b/tests/test_event_outbox_migration.py new file mode 100644 index 0000000..dd9a673 --- /dev/null +++ b/tests/test_event_outbox_migration.py @@ -0,0 +1,18 @@ +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +MIGRATION = ROOT / "plugins" / "redmine_event_outbox" / "db" / "migrate" / "001_create_event_outbox_events.rb" + + +class EventOutboxMigrationTest(unittest.TestCase): + def test_create_table_migration_is_idempotent_for_imported_dev_clone(self): + source = MIGRATION.read_text() + + self.assertIn("table_exists?(:event_outbox_events)", source) + self.assertIn("return if table_exists?(:event_outbox_events)", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_post_import_refresh.py b/tests/test_post_import_refresh.py new file mode 100644 index 0000000..52f1593 --- /dev/null +++ b/tests/test_post_import_refresh.py @@ -0,0 +1,112 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from post_import_refresh import AutomationConfig, StepResult, build_steps, write_status + + +class PostImportRefreshPlanTest(unittest.TestCase): + def test_dry_run_is_the_default_and_never_enables_index_writes(self): + config = AutomationConfig() + steps = build_steps(config) + commands = "\n".join(command for step in steps for command in step.commands) + + self.assertFalse(config.apply) + self.assertIn("validate semantic index dry-run", [step.name for step in steps]) + self.assertIn("semantic_index/refresh.sh", commands) + self.assertNotIn("semantic_index/refresh.sh --apply", commands) + self.assertNotIn("--force-rebuild", commands) + self.assertNotIn("systemctl enable --now semantic-index-refresh.timer", commands) + + def test_plugin_reapply_happens_before_migrations_and_helpdesk_reset(self): + names = [step.name for step in build_steps(AutomationConfig())] + + self.assertLess(names.index("reapply tracked plugins"), names.index("run plugin migrations")) + self.assertLess(names.index("run plugin migrations"), names.index("reset Helpdesk mail settings")) + + def test_expected_plugins_are_reapplied_to_remote_redmine_tree(self): + steps = build_steps(AutomationConfig()) + plugin_step = next(step for step in steps if step.name == "reapply tracked plugins") + commands = "\n".join(plugin_step.commands) + + self.assertIn("plugins/redmine_event_outbox", commands) + self.assertIn("plugins/redmine_contacts", commands) + self.assertIn("plugins/redmine_contacts_helpdesk", commands) + self.assertNotIn("plugins/redmine_event_outbox/", commands) + self.assertIn("reddev@192.168.50.170:/usr/share/redmine/plugins/", commands) + + def test_apply_mode_runs_mutating_validation_sequence(self): + steps = build_steps(AutomationConfig(apply=True)) + commands = "\n".join(command for step in steps for command in step.commands) + + self.assertIn("bundle exec rake redmine:plugins:migrate", commands) + self.assertIn("./reset_helpdesk_mail_settings.py", commands) + self.assertIn("touch tmp/restart.txt", commands) + self.assertIn("./validate_test_instance.py", commands) + + def test_remote_write_steps_use_sudo_by_default(self): + commands = "\n".join(command for step in build_steps(AutomationConfig()) for command in step.commands) + + self.assertIn("--rsync-path 'sudo rsync'", commands) + self.assertIn("sudo mkdir -p", commands) + self.assertIn("sudo chmod -R g+rwX", commands) + + def test_local_mode_emits_local_commands_without_ssh(self): + config = AutomationConfig(local=True) + commands = "\n".join(command for step in build_steps(config) for command in step.commands) + + self.assertNotIn("ssh -i", commands) + self.assertNotIn("rsync-path", commands) + self.assertIn("reset_helpdesk_mail_settings.py --local", commands) + self.assertIn("validate_test_instance.py --local", commands) + self.assertNotIn("--composer-bin", commands) + self.assertIn("redmine_outbox_worker.py --local --status", commands) + self.assertIn("/opt/lanscratch/redmine-post-import/repo/plugins/redmine_event_outbox", commands) + self.assertIn("/usr/share/redmine/plugins/", commands) + self.assertIn("cd /usr/share/redmine && RAILS_ENV=production bundle exec rake redmine:plugins:migrate", commands) + + def test_local_semantic_check_is_non_blocking_without_staged_venv(self): + config = AutomationConfig(local=True) + semantic_step = next(step for step in build_steps(config) if step.name == "validate semantic index dry-run") + command = semantic_step.commands[0] + + self.assertIn("test -x /opt/lanscratch/redmine-post-import/repo/.venv/bin/python", command) + self.assertIn("semantic index runtime missing; skipping dry-run", command) + self.assertIn("else", command) + + def test_status_paths_default_to_lanscratch(self): + config = AutomationConfig() + + self.assertEqual(Path("/opt/lanscratch/redmine-post-import/status"), config.status_dir) + + def test_write_status_updates_latest_and_success_only_on_success(self): + with tempfile.TemporaryDirectory() as tmp: + config = AutomationConfig(status_dir=Path(tmp)) + failed = write_status( + config, + run_id="20260428T010000Z", + status="failed", + results=[StepResult("preflight", "test -d missing", 1)], + failed_step="preflight", + ) + + self.assertTrue((Path(tmp) / "latest.json").exists()) + self.assertTrue((Path(tmp) / "runs" / "20260428T010000Z.json").exists()) + self.assertFalse((Path(tmp) / "latest-success.json").exists()) + self.assertEqual("failed", failed["status"]) + + successful = write_status( + config, + run_id="20260428T010100Z", + status="success", + results=[StepResult("preflight", "test -d plugins", 0)], + ) + + latest_success = json.loads((Path(tmp) / "latest-success.json").read_text()) + self.assertEqual(successful["run_id"], latest_success["run_id"]) + self.assertEqual("success", latest_success["status"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_python36_compat.py b/tests/test_python36_compat.py new file mode 100644 index 0000000..499ac41 --- /dev/null +++ b/tests/test_python36_compat.py @@ -0,0 +1,23 @@ +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +POST_IMPORT_SCRIPTS = [ + ROOT / "post_import_refresh.py", + ROOT / "stage_post_import_payload.py", + ROOT / "reset_helpdesk_mail_settings.py", + ROOT / "validate_test_instance.py", + ROOT / "redmine_outbox_worker.py", +] + + +class Python36CompatTest(unittest.TestCase): + def test_post_import_scripts_do_not_use_subprocess_text_keyword(self): + for path in POST_IMPORT_SCRIPTS: + with self.subTest(path=path.name): + self.assertNotIn("text=True", path.read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_stage_post_import_payload.py b/tests/test_stage_post_import_payload.py new file mode 100644 index 0000000..b6af343 --- /dev/null +++ b/tests/test_stage_post_import_payload.py @@ -0,0 +1,26 @@ +import unittest +from pathlib import Path + +from stage_post_import_payload import build_rsync_command + + +class StagePostImportPayloadTest(unittest.TestCase): + def test_stage_command_targets_lanscratch_and_excludes_runtime_files(self): + command = build_rsync_command( + repo_root=Path("/repo"), + target=Path("/opt/lanscratch/redmine-post-import/repo"), + ) + + self.assertIn("/opt/lanscratch/redmine-post-import/repo/", command) + self.assertIn("/repo/plugins", command) + self.assertIn("/repo/post_import_refresh.py", command) + self.assertIn("/repo/stage_post_import_payload.py", command) + self.assertIn("--exclude .env", command) + self.assertIn("--exclude .venv", command) + self.assertIn("--exclude .cache", command) + self.assertIn("--exclude __pycache__/", command) + self.assertIn("--exclude '*.tar.gz'", command) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_validate_test_instance.py b/tests/test_validate_test_instance.py new file mode 100644 index 0000000..57e062f --- /dev/null +++ b/tests/test_validate_test_instance.py @@ -0,0 +1,21 @@ +import unittest + +import validate_test_instance + + +class ValidateTestInstanceTest(unittest.TestCase): + def test_missing_controlled_projects_are_warn_for_daily_clone(self): + results = validate_test_instance.controlled_project_check([]) + + self.assertEqual("WARN", results.status) + self.assertIn("optional", results.detail) + + def test_composer_validation_is_skipped_when_disabled(self): + result = validate_test_instance.check_composer(None) + + self.assertEqual("WARN", result.status) + self.assertIn("skipped", result.detail) + + +if __name__ == "__main__": + unittest.main() diff --git a/validate_test_instance.py b/validate_test_instance.py index 559f592..2ec075e 100755 --- a/validate_test_instance.py +++ b/validate_test_instance.py @@ -6,15 +6,12 @@ post-import reset steps. It reports whether the test instance looks ready for Helpdesk and redMCP testing without changing remote state. """ -from __future__ import annotations - import argparse import json import os import shutil import socket import subprocess -from dataclasses import dataclass from pathlib import Path from typing import Any @@ -26,20 +23,30 @@ DEFAULT_MAILPIT_HOST = "192.168.1.105" DEFAULT_FILES_ROOT = "/var/lib/redmine/default/files" -@dataclass(frozen=True) class CheckResult: - status: str - name: str - detail: str + def __init__(self, status, name, detail): + self.status = status + self.name = name + self.detail = detail -@dataclass(frozen=True) class RemoteRedmine: - ssh_host: str - ssh_key: Path - remote_redmine: str + def __init__(self, ssh_host, ssh_key, remote_redmine, local=False): + self.ssh_host = ssh_host + self.ssh_key = ssh_key + self.remote_redmine = remote_redmine + self.local = local - def ssh(self, remote_command: str) -> subprocess.CompletedProcess[str]: + def ssh(self, remote_command): + if self.local: + return subprocess.run( + remote_command, + shell=True, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) return subprocess.run( [ "ssh", @@ -50,43 +57,49 @@ class RemoteRedmine: self.ssh_host, remote_command, ], - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, ) - def mysql_json_lines(self, sql: str) -> list[dict[str, Any]]: + def mysql_json_lines(self, sql): result = self.mysql(sql) - rows: list[dict[str, Any]] = [] + rows = [] for line in result.splitlines(): if not line.strip(): continue rows.append(json.loads(bytes.fromhex(line.strip()).decode("utf-8"))) return rows - def mysql(self, sql: str) -> str: + def mysql(self, sql): + command = self._mysql_runner_command() + shell = True + if not self.local: + command = [ + "ssh", + "-i", + str(self.ssh_key), + "-o", + "IdentitiesOnly=yes", + self.ssh_host, + self._mysql_runner_command(), + ] + shell = False result = subprocess.run( - [ - "ssh", - "-i", - str(self.ssh_key), - "-o", - "IdentitiesOnly=yes", - self.ssh_host, - self._mysql_runner_command(), - ], + command, input=sql, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + shell=shell, ) if result.returncode != 0: raise RuntimeError(result.stderr.strip() or "Remote MySQL command failed.") return result.stdout - def _mysql_runner_command(self) -> str: + def _mysql_runner_command(self): ruby = ( "require 'yaml'; " "c = YAML.load_file('config/database.yml')['production']; " @@ -99,18 +112,23 @@ class RemoteRedmine: return f"cd {shell_quote(self.remote_redmine)} && ruby -e {shell_quote(ruby)}" -def main() -> int: +def main(): parser = argparse.ArgumentParser(description="Read-only checks for the Redmine LAN test instance.") parser.add_argument("--ssh-host", default=os.getenv("REDMINE_SSH_HOST", DEFAULT_SSH_HOST)) parser.add_argument("--ssh-key", type=Path, default=Path(os.getenv("REDMINE_SSH_KEY", str(DEFAULT_SSH_KEY)))) + parser.add_argument("--local", action="store_true", help="Validate local Redmine paths/database instead of using SSH.") parser.add_argument("--remote-redmine", default=os.getenv("REDMINE_REMOTE_PATH", DEFAULT_REMOTE_REDMINE)) parser.add_argument("--mailpit-host", default=DEFAULT_MAILPIT_HOST) parser.add_argument("--files-root", default=DEFAULT_FILES_ROOT) - parser.add_argument("--composer-bin", default=os.getenv("COMPOSER_BIN", "composer")) + parser.add_argument( + "--composer-bin", + default=os.getenv("COMPOSER_BIN"), + help="Optional Composer binary or composer.phar for redMCP validation.", + ) args = parser.parse_args() - remote = RemoteRedmine(args.ssh_host, args.ssh_key, args.remote_redmine) - checks: list[CheckResult] = [] + remote = RemoteRedmine(args.ssh_host, args.ssh_key, args.remote_redmine, local=args.local) + checks = [] checks.extend(check_remote_basics(remote)) checks.extend(check_mailpit_connectivity(remote, args.mailpit_host)) @@ -127,8 +145,8 @@ def main() -> int: return 1 if failures else 0 -def check_remote_basics(remote: RemoteRedmine) -> list[CheckResult]: - results: list[CheckResult] = [] +def check_remote_basics(remote): + results = [] result = remote.ssh("printf ok") if result.returncode == 0 and result.stdout == "ok": results.append(CheckResult("OK", "SSH", f"connected to {remote.ssh_host}")) @@ -155,7 +173,7 @@ def check_remote_basics(remote: RemoteRedmine) -> list[CheckResult]: return results -def check_mailpit_connectivity(remote: RemoteRedmine, host: str) -> list[CheckResult]: +def check_mailpit_connectivity(remote, host): results = [ tcp_check("Mailpit HTTP from local", host, 8025), tcp_check("Mailpit SMTP from local", host, 1025), @@ -181,7 +199,7 @@ def check_mailpit_connectivity(remote: RemoteRedmine, host: str) -> list[CheckRe return results -def check_files_permissions(remote: RemoteRedmine, files_root: str) -> list[CheckResult]: +def check_files_permissions(remote, files_root): command = ( "ruby -e " + shell_quote( @@ -208,8 +226,8 @@ def check_files_permissions(remote: RemoteRedmine, files_root: str) -> list[Chec return [CheckResult("OK", "Attachment directory permissions", detail)] -def check_database_state(remote: RemoteRedmine, mailpit_host: str) -> list[CheckResult]: - results: list[CheckResult] = [] +def check_database_state(remote, mailpit_host): + results = [] try: projects = remote.mysql_json_lines( """ @@ -219,12 +237,7 @@ WHERE identifier IN ('fud-helpdesk', 'fud-nohelpdesk') ORDER BY identifier; """ ) - found = {project["identifier"] for project in projects} - missing = {"fud-helpdesk", "fud-nohelpdesk"} - found - if missing: - results.append(CheckResult("FAIL", "Controlled test projects", "missing " + ", ".join(sorted(missing)))) - else: - results.append(CheckResult("OK", "Controlled test projects", ", ".join(sorted(found)))) + results.append(controlled_project_check(projects)) settings_rows = remote.mysql_json_lines(helpdesk_settings_sql()) failures = helpdesk_setting_failures(settings_rows, mailpit_host) @@ -237,7 +250,7 @@ ORDER BY identifier; return results -def helpdesk_settings_sql() -> str: +def helpdesk_settings_sql(): return """ SELECT HEX(CAST(JSON_OBJECT( 'identifier', p.identifier, @@ -264,7 +277,7 @@ ORDER BY p.identifier; """ -def helpdesk_setting_failures(rows: list[dict[str, Any]], mailpit_host: str) -> list[str]: +def helpdesk_setting_failures(rows, mailpit_host): expected = { "protocol": "pop3", "host": mailpit_host, @@ -281,7 +294,7 @@ def helpdesk_setting_failures(rows: list[dict[str, Any]], mailpit_host: str) -> "smtp_ssl": "0", "smtp_tls": "0", } - failures: list[str] = [] + failures = [] for row in rows: for key, value in expected.items(): if row.get(key) != value: @@ -289,7 +302,22 @@ def helpdesk_setting_failures(rows: list[dict[str, Any]], mailpit_host: str) -> return failures -def check_composer(composer_bin: str) -> CheckResult: +def controlled_project_check(projects): + found = {project["identifier"] for project in projects} + missing = {"fud-helpdesk", "fud-nohelpdesk"} - found + if missing: + return CheckResult( + "WARN", + "Controlled test projects", + "optional smoke-test project(s) missing after production clone: " + + ", ".join(sorted(missing)), + ) + return CheckResult("OK", "Controlled test projects", ", ".join(sorted(found))) + + +def check_composer(composer_bin): + if not composer_bin: + return CheckResult("WARN", "Composer validation", "skipped; pass --composer-bin to enable") composer_path = Path(composer_bin) composer_on_path = shutil.which(composer_bin) if composer_on_path is None and not composer_path.exists(): @@ -302,7 +330,7 @@ def check_composer(composer_bin: str) -> CheckResult: command = [php, composer_bin, "validate", "--working-dir=redMCP"] result = subprocess.run( command, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, @@ -312,7 +340,7 @@ def check_composer(composer_bin: str) -> CheckResult: return CheckResult("FAIL", "Composer validation", (result.stdout + result.stderr).strip()) -def tcp_check(name: str, host: str, port: int) -> CheckResult: +def tcp_check(name, host, port): try: with socket.create_connection((host, port), timeout=5): return CheckResult("OK", name, f"{host}:{port}") @@ -320,7 +348,7 @@ def tcp_check(name: str, host: str, port: int) -> CheckResult: return CheckResult("FAIL", name, f"{host}:{port} {exc.__class__.__name__}: {exc}") -def shell_quote(value: str) -> str: +def shell_quote(value): return "'" + value.replace("'", "'\"'\"'") + "'"