#!/usr/bin/env python3 """Post-import automation for the LAN Redmine development clone. Run this after the production database and Redmine tree have been copied onto the dev/test host. It reapplies this repository's tracked plugin forks, runs plugin migrations, sanitizes Helpdesk mail settings, restarts Passenger, and performs validation. The default mode is a dry run. """ import argparse import datetime as dt import json import os import shlex import subprocess import sys import socket from pathlib import Path from typing import List, Optional, Tuple DEFAULT_SSH_HOST = "reddev@192.168.50.170" DEFAULT_SSH_KEY = Path("/tmp/reddev") DEFAULT_REMOTE_REDMINE = "/usr/share/redmine" DEFAULT_REMOTE_OVERRIDES = "/home/reddev/redmine-dev-overrides" DEFAULT_FILES_ROOT = "/var/lib/redmine/default/files" DEFAULT_MAILPIT_HOST = "192.168.1.105" DEFAULT_SEMANTIC_LIMITS = "customer-service=5" DEFAULT_LANSCRATCH_ROOT = Path("/opt/lanscratch/redmine-post-import") DEFAULT_STATUS_DIR = DEFAULT_LANSCRATCH_ROOT / "status" DEFAULT_LOCAL_REPO_ROOT = DEFAULT_LANSCRATCH_ROOT / "repo" TRACKED_PLUGINS = ( "redmine_event_outbox", "redmine_contacts", "redmine_contacts_helpdesk", ) class AutomationConfig: def __init__( self, apply=False, local=False, repo_root=None, status_dir=DEFAULT_STATUS_DIR, ssh_host=DEFAULT_SSH_HOST, ssh_key=DEFAULT_SSH_KEY, remote_redmine=DEFAULT_REMOTE_REDMINE, remote_overrides=DEFAULT_REMOTE_OVERRIDES, files_root=DEFAULT_FILES_ROOT, mailpit_host=DEFAULT_MAILPIT_HOST, semantic_limits=DEFAULT_SEMANTIC_LIMITS, remote_sudo=True, skip_semantic_check=False, skip_helpdesk_reset=False, ): self.apply = apply self.local = local self.repo_root = repo_root if repo_root is not None else (DEFAULT_LOCAL_REPO_ROOT if local else Path(".")) self.status_dir = status_dir self.ssh_host = ssh_host self.ssh_key = ssh_key self.remote_redmine = remote_redmine self.remote_overrides = remote_overrides self.files_root = files_root self.mailpit_host = mailpit_host self.semantic_limits = semantic_limits self.remote_sudo = remote_sudo self.skip_semantic_check = skip_semantic_check self.skip_helpdesk_reset = skip_helpdesk_reset class Step: def __init__(self, name, commands): self.name = name self.commands = commands class StepResult: def __init__(self, step, command, returncode): self.step = step self.command = command self.returncode = returncode def main() -> int: args = parse_args() config = AutomationConfig( apply=args.apply, local=args.local, repo_root=args.repo_root or (DEFAULT_LOCAL_REPO_ROOT if args.local else Path(".")), status_dir=args.status_dir, ssh_host=args.ssh_host, ssh_key=args.ssh_key, remote_redmine=args.remote_redmine, remote_overrides=args.remote_overrides, files_root=args.files_root, mailpit_host=args.mailpit_host, semantic_limits=args.semantic_limits, remote_sudo=not args.no_remote_sudo, skip_semantic_check=args.skip_semantic_check, skip_helpdesk_reset=args.skip_helpdesk_reset, ) steps = build_steps(config) run_id = utc_stamp() results = [] # type: List[StepResult] print(f"mode={'apply' if config.apply else 'dry-run'}") print(f"execution={'local' if config.local else 'remote'}") if config.apply: write_status(config, run_id, "running", results) for step in steps: print(f"\n== {step.name} ==") for command in step.commands: if config.apply: print(f"running: {command}") result = subprocess.run(command, shell=True, check=False) results.append(StepResult(step.name, command, result.returncode)) write_status(config, run_id, "running", results) if result.returncode != 0: print(f"error: command failed with exit {result.returncode}", file=sys.stderr) write_status(config, run_id, "failed", results, failed_step=step.name) return result.returncode else: print(f"would run: {command}") if not config.apply: print("\nDry run only. Re-run with --apply after reviewing the command list.") else: write_status(config, run_id, "success", results) return 0 def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run daily post-import steps for the LAN Redmine dev clone." ) parser.add_argument("--apply", action="store_true", help="Run commands. Default is dry-run.") parser.add_argument( "--local", action="store_true", help="Run directly on the Redmine test host instead of orchestrating over SSH.", ) parser.add_argument( "--repo-root", type=Path, default=Path(os.environ["POST_IMPORT_REPO_ROOT"]) if "POST_IMPORT_REPO_ROOT" in os.environ else None, help="Repository/payload root to copy plugins and run helper scripts from.", ) parser.add_argument( "--status-dir", type=Path, default=Path(os.getenv("POST_IMPORT_STATUS_DIR", str(DEFAULT_STATUS_DIR))), ) parser.add_argument("--ssh-host", default=os.getenv("REDMINE_SSH_HOST", DEFAULT_SSH_HOST)) parser.add_argument( "--ssh-key", type=Path, default=Path(os.getenv("REDMINE_SSH_KEY", str(DEFAULT_SSH_KEY))), ) parser.add_argument( "--remote-redmine", default=os.getenv("REDMINE_REMOTE_PATH", DEFAULT_REMOTE_REDMINE), ) parser.add_argument( "--remote-overrides", default=os.getenv("REDMINE_REMOTE_OVERRIDES", DEFAULT_REMOTE_OVERRIDES), help=( "Remote directory containing dev-only files to overlay after the production copy. " "If it is absent on the remote host, the step is skipped." ), ) parser.add_argument("--files-root", default=DEFAULT_FILES_ROOT) parser.add_argument("--mailpit-host", default=DEFAULT_MAILPIT_HOST) parser.add_argument("--semantic-limits", default=DEFAULT_SEMANTIC_LIMITS) parser.add_argument( "--no-remote-sudo", action="store_true", help="Do not use sudo for remote write/permission steps.", ) parser.add_argument("--skip-semantic-check", action="store_true") parser.add_argument("--skip-helpdesk-reset", action="store_true") return parser.parse_args() def build_steps(config): steps = [ Step( "preflight", preflight_commands(config), ), Step( "restore dev-local overrides", ( restore_overrides_command(config), ), ), Step( "reapply tracked plugins", ( copy_plugins_command(config), ), ), Step( "run plugin migrations", ( redmine_command( config, "RAILS_ENV=production bundle exec rake redmine:plugins:migrate", ), ), ), Step( "fix dev writable paths", ( fix_permissions_command(config), ), ), ] if not config.skip_helpdesk_reset: steps.append( Step( "reset Helpdesk mail settings", ( helpdesk_reset_command(config), ), ) ) steps.extend( [ Step( "restart Passenger", ( redmine_command(config, "touch tmp/restart.txt"), ), ), Step( "validate test instance", ( validate_command(config), ), ), Step( "check outbox worker", outbox_commands(config), ), ] ) if not config.skip_semantic_check: steps.append( Step( "validate semantic index dry-run", ( semantic_check_command(config), ), ) ) return steps def preflight_commands(config): commands = tuple(local_test(str(config.repo_root / "plugins" / plugin)) for plugin in TRACKED_PLUGINS) if config.local: return commands + (f"test -d {q(config.remote_redmine)}",) return commands + (ssh(config, f"test -d {q(config.remote_redmine)}"),) def semantic_check_command(config): refresh = config.repo_root / "semantic_index" / "refresh.sh" python_bin = config.repo_root / ".venv" / "bin" / "python" command = "SEMANTIC_INDEX_PROJECT_LIMITS={limits} {refresh}".format( limits=q(config.semantic_limits), refresh=q(str(refresh)), ) if not config.local: return command return ( "if test -x {python}; then PYTHON={python} {command}; " "else echo 'semantic index runtime missing; skipping dry-run'; fi" ).format( python=q(str(python_bin)), command=command, ) def local_test(path): return f"test -d {q(path)}" def restore_overrides_command(config): command = "if [ -d {overrides} ]; then {sudo}rsync -a {overrides}/ {redmine}/; else echo 'no dev overrides directory: {overrides}'; fi".format( sudo=remote_sudo_prefix(config), overrides=q(config.remote_overrides), redmine=q(config.remote_redmine), ) return command if config.local else ssh(config, command) def copy_plugins_command(config): sources = " ".join(q(str(config.repo_root / "plugins" / plugin)) for plugin in TRACKED_PLUGINS) if config.local: return f"rsync -a --delete {sources} {q(config.remote_redmine + '/plugins/')}" return ( "rsync -a --delete " f"{rsync_path_option(config)}" f"-e {q(ssh_transport(config))} " f"{sources} " f"{q(config.ssh_host + ':' + config.remote_redmine + '/plugins/')}" ) def redmine_command(config, command): full = f"cd {q(config.remote_redmine)} && {command}" return full if config.local else ssh(config, full) def fix_permissions_command(config): command = "{sudo}mkdir -p {files} {redmine}/tmp {redmine}/log && {sudo}chmod -R g+rwX {files} {redmine}/tmp {redmine}/log && {sudo}find {files} -type d -exec chmod g+s {{}} +".format( sudo=remote_sudo_prefix(config), files=q(config.files_root), redmine=q(config.remote_redmine), ) return command if config.local else ssh(config, command) def script(config, name): path = config.repo_root / name if str(config.repo_root) == ".": return q(f"./{name}") return q(str(path)) def helpdesk_reset_command(config): base = f"{script(config, 'reset_helpdesk_mail_settings.py')} " if config.local: return ( base + f"--local --remote-redmine {q(config.remote_redmine)} " f"--mailpit-host {q(config.mailpit_host)}" ) return ( base + f"--ssh-host {q(config.ssh_host)} " f"--ssh-key {q(str(config.ssh_key))} " f"--remote-redmine {q(config.remote_redmine)} " f"--mailpit-host {q(config.mailpit_host)}" ) def validate_command(config): base = f"{script(config, 'validate_test_instance.py')} " if config.local: return ( base + f"--local --remote-redmine {q(config.remote_redmine)} " f"--mailpit-host {q(config.mailpit_host)} " f"--files-root {q(config.files_root)}" ) return ( base + f"--ssh-host {q(config.ssh_host)} " f"--ssh-key {q(str(config.ssh_key))} " f"--remote-redmine {q(config.remote_redmine)} " f"--mailpit-host {q(config.mailpit_host)} " f"--files-root {q(config.files_root)}" ) def outbox_commands(config): base = script(config, "redmine_outbox_worker.py") local = " --local" if config.local else "" return ( f"{base}{local} --status", f"{base}{local} --dry-run --batch-size 10", ) def ssh(config, remote_command): return ( "ssh " f"-i {q(str(config.ssh_key))} " "-o IdentitiesOnly=yes " f"{q(config.ssh_host)} " f"{q(remote_command)}" ) def ssh_transport(config): return f"ssh -i {str(config.ssh_key)} -o IdentitiesOnly=yes" def remote_sudo_prefix(config): return "sudo " if config.remote_sudo else "" def rsync_path_option(config): return "--rsync-path 'sudo rsync' " if config.remote_sudo else "" def write_status( config: AutomationConfig, run_id: str, status: str, results, failed_step=None, ): now = utc_iso() document = { "run_id": run_id, "started_at": run_id_to_iso(run_id), "updated_at": now, "finished_at": now if status in {"success", "failed"} else None, "mode": "apply" if config.apply else "dry-run", "execution": "local" if config.local else "remote", "host": socket.gethostname(), "remote_redmine": config.remote_redmine, "repo_root": str(config.repo_root), "status": status, "failed_step": failed_step, "steps": [ {"step": item.step, "command": item.command, "returncode": item.returncode} for item in results ], } config.status_dir.mkdir(parents=True, exist_ok=True) runs_dir = config.status_dir / "runs" runs_dir.mkdir(parents=True, exist_ok=True) write_json(runs_dir / f"{run_id}.json", document) write_json(config.status_dir / "latest.json", document) if status == "success": write_json(config.status_dir / "latest-success.json", document) return document def write_json(path, document): tmp = path.with_suffix(path.suffix + ".tmp") tmp.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8") tmp.replace(path) def utc_stamp(): return dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") def utc_iso(): return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def run_id_to_iso(run_id): try: parsed = dt.datetime.strptime(run_id, "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc) return parsed.strftime("%Y-%m-%dT%H:%M:%SZ") except ValueError: return run_id def q(value): return shlex.quote(value) if __name__ == "__main__": raise SystemExit(main())