redmine/post_import_refresh.py

#!/usr/bin/env python3
"""Post-import automation for the LAN Redmine development clone.

Run this after the production database and Redmine tree have been copied onto
the dev/test host. It reapplies this repository's tracked plugin forks, runs
plugin migrations, sanitizes Helpdesk mail settings, restarts Passenger, and
performs validation. The default mode is a dry run.
"""

import argparse
import datetime as dt
import json
import os
import shlex
import subprocess
import sys
import socket
from pathlib import Path
from typing import List, Optional, Tuple


DEFAULT_SSH_HOST = "reddev@192.168.50.170"
DEFAULT_SSH_KEY = Path("/tmp/reddev")
DEFAULT_REMOTE_REDMINE = "/usr/share/redmine"
DEFAULT_REMOTE_OVERRIDES = "/home/reddev/redmine-dev-overrides"
DEFAULT_FILES_ROOT = "/var/lib/redmine/default/files"
DEFAULT_MAILPIT_HOST = "192.168.1.105"
DEFAULT_SEMANTIC_LIMITS = "customer-service=5"
DEFAULT_LANSCRATCH_ROOT = Path("/opt/lanscratch/redmine-post-import")
DEFAULT_STATUS_DIR = DEFAULT_LANSCRATCH_ROOT / "status"
DEFAULT_LOCAL_REPO_ROOT = DEFAULT_LANSCRATCH_ROOT / "repo"

TRACKED_PLUGINS = (
    "redmine_event_outbox",
    "redmine_contacts",
    "redmine_contacts_helpdesk",
)


class AutomationConfig:
    def __init__(
        self,
        apply=False,
        local=False,
        repo_root=None,
        status_dir=DEFAULT_STATUS_DIR,
        ssh_host=DEFAULT_SSH_HOST,
        ssh_key=DEFAULT_SSH_KEY,
        remote_redmine=DEFAULT_REMOTE_REDMINE,
        remote_overrides=DEFAULT_REMOTE_OVERRIDES,
        files_root=DEFAULT_FILES_ROOT,
        mailpit_host=DEFAULT_MAILPIT_HOST,
        semantic_limits=DEFAULT_SEMANTIC_LIMITS,
        remote_sudo=True,
        skip_semantic_check=False,
        skip_helpdesk_reset=False,
    ):
        self.apply = apply
        self.local = local
        self.repo_root = repo_root if repo_root is not None else (DEFAULT_LOCAL_REPO_ROOT if local else Path("."))
        self.status_dir = status_dir
        self.ssh_host = ssh_host
        self.ssh_key = ssh_key
        self.remote_redmine = remote_redmine
        self.remote_overrides = remote_overrides
        self.files_root = files_root
        self.mailpit_host = mailpit_host
        self.semantic_limits = semantic_limits
        self.remote_sudo = remote_sudo
        self.skip_semantic_check = skip_semantic_check
        self.skip_helpdesk_reset = skip_helpdesk_reset


class Step:
    def __init__(self, name, commands):
        self.name = name
        self.commands = commands


class StepResult:
    def __init__(self, step, command, returncode):
        self.step = step
        self.command = command
        self.returncode = returncode


def main() -> int:
    args = parse_args()
    config = AutomationConfig(
        apply=args.apply,
        local=args.local,
        repo_root=args.repo_root or (DEFAULT_LOCAL_REPO_ROOT if args.local else Path(".")),
        status_dir=args.status_dir,
        ssh_host=args.ssh_host,
        ssh_key=args.ssh_key,
        remote_redmine=args.remote_redmine,
        remote_overrides=args.remote_overrides,
        files_root=args.files_root,
        mailpit_host=args.mailpit_host,
        semantic_limits=args.semantic_limits,
        remote_sudo=not args.no_remote_sudo,
        skip_semantic_check=args.skip_semantic_check,
        skip_helpdesk_reset=args.skip_helpdesk_reset,
    )
    steps = build_steps(config)
    run_id = utc_stamp()
    results = []  # type: List[StepResult]

    print(f"mode={'apply' if config.apply else 'dry-run'}")
    print(f"execution={'local' if config.local else 'remote'}")
    if config.apply:
        write_status(config, run_id, "running", results)
    for step in steps:
        print(f"\n== {step.name} ==")
        for command in step.commands:
            if config.apply:
                print(f"running: {command}")
                result = subprocess.run(command, shell=True, check=False)
                results.append(StepResult(step.name, command, result.returncode))
                write_status(config, run_id, "running", results)
                if result.returncode != 0:
                    print(f"error: command failed with exit {result.returncode}", file=sys.stderr)
                    write_status(config, run_id, "failed", results, failed_step=step.name)
                    return result.returncode
            else:
                print(f"would run: {command}")

    if not config.apply:
        print("\nDry run only. Re-run with --apply after reviewing the command list.")
    else:
        write_status(config, run_id, "success", results)
    return 0


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Run daily post-import steps for the LAN Redmine dev clone."
    )
    parser.add_argument("--apply", action="store_true", help="Run commands. Default is dry-run.")
    parser.add_argument(
        "--local",
        action="store_true",
        help="Run directly on the Redmine test host instead of orchestrating over SSH.",
    )
    parser.add_argument(
        "--repo-root",
        type=Path,
        default=Path(os.environ["POST_IMPORT_REPO_ROOT"]) if "POST_IMPORT_REPO_ROOT" in os.environ else None,
        help="Repository/payload root to copy plugins and run helper scripts from.",
    )
    parser.add_argument(
        "--status-dir",
        type=Path,
        default=Path(os.getenv("POST_IMPORT_STATUS_DIR", str(DEFAULT_STATUS_DIR))),
    )
    parser.add_argument("--ssh-host", default=os.getenv("REDMINE_SSH_HOST", DEFAULT_SSH_HOST))
    parser.add_argument(
        "--ssh-key",
        type=Path,
        default=Path(os.getenv("REDMINE_SSH_KEY", str(DEFAULT_SSH_KEY))),
    )
    parser.add_argument(
        "--remote-redmine",
        default=os.getenv("REDMINE_REMOTE_PATH", DEFAULT_REMOTE_REDMINE),
    )
    parser.add_argument(
        "--remote-overrides",
        default=os.getenv("REDMINE_REMOTE_OVERRIDES", DEFAULT_REMOTE_OVERRIDES),
        help=(
            "Remote directory containing dev-only files to overlay after the production copy. "
            "If it is absent on the remote host, the step is skipped."
        ),
    )
    parser.add_argument("--files-root", default=DEFAULT_FILES_ROOT)
    parser.add_argument("--mailpit-host", default=DEFAULT_MAILPIT_HOST)
    parser.add_argument("--semantic-limits", default=DEFAULT_SEMANTIC_LIMITS)
    parser.add_argument(
        "--no-remote-sudo",
        action="store_true",
        help="Do not use sudo for remote write/permission steps.",
    )
    parser.add_argument("--skip-semantic-check", action="store_true")
    parser.add_argument("--skip-helpdesk-reset", action="store_true")
    return parser.parse_args()


def build_steps(config):
    steps = [
        Step(
            "preflight",
            preflight_commands(config),
        ),
        Step(
            "restore dev-local overrides",
            (
                restore_overrides_command(config),
            ),
        ),
        Step(
            "reapply tracked plugins",
            (
                copy_plugins_command(config),
            ),
        ),
        Step(
            "run plugin migrations",
            (
                redmine_command(
                    config,
                    "RAILS_ENV=production bundle exec rake redmine:plugins:migrate",
                ),
            ),
        ),
        Step(
            "fix dev writable paths",
            (
                fix_permissions_command(config),
            ),
        ),
    ]

    if not config.skip_helpdesk_reset:
        steps.append(
            Step(
                "reset Helpdesk mail settings",
                (
                    helpdesk_reset_command(config),
                ),
            )
        )

    steps.extend(
        [
            Step(
                "restart Passenger",
                (
                    redmine_command(config, "touch tmp/restart.txt"),
                ),
            ),
            Step(
                "validate test instance",
                (
                    validate_command(config),
                ),
            ),
            Step(
                "check outbox worker",
                outbox_commands(config),
            ),
        ]
    )

    if not config.skip_semantic_check:
        steps.append(
            Step(
                "validate semantic index dry-run",
                (
                    semantic_check_command(config),
                ),
            )
        )

    return steps


def preflight_commands(config):
    commands = tuple(local_test(str(config.repo_root / "plugins" / plugin)) for plugin in TRACKED_PLUGINS)
    if config.local:
        return commands + (f"test -d {q(config.remote_redmine)}",)
    return commands + (ssh(config, f"test -d {q(config.remote_redmine)}"),)


def semantic_check_command(config):
    refresh = config.repo_root / "semantic_index" / "refresh.sh"
    python_bin = config.repo_root / ".venv" / "bin" / "python"
    command = "SEMANTIC_INDEX_PROJECT_LIMITS={limits} {refresh}".format(
        limits=q(config.semantic_limits),
        refresh=q(str(refresh)),
    )
    if not config.local:
        return command
    return (
        "if test -x {python}; then PYTHON={python} {command}; "
        "else echo 'semantic index runtime missing; skipping dry-run'; fi"
    ).format(
        python=q(str(python_bin)),
        command=command,
    )


def local_test(path):
    return f"test -d {q(path)}"


def restore_overrides_command(config):
    command = "if [ -d {overrides} ]; then {sudo}rsync -a {overrides}/ {redmine}/; else echo 'no dev overrides directory: {overrides}'; fi".format(
        sudo=remote_sudo_prefix(config),
        overrides=q(config.remote_overrides),
        redmine=q(config.remote_redmine),
    )
    return command if config.local else ssh(config, command)


def copy_plugins_command(config):
    sources = " ".join(q(str(config.repo_root / "plugins" / plugin)) for plugin in TRACKED_PLUGINS)
    if config.local:
        return f"rsync -a --delete {sources} {q(config.remote_redmine + '/plugins/')}"
    return (
        "rsync -a --delete "
        f"{rsync_path_option(config)}"
        f"-e {q(ssh_transport(config))} "
        f"{sources} "
        f"{q(config.ssh_host + ':' + config.remote_redmine + '/plugins/')}"
    )


def redmine_command(config, command):
    full = f"cd {q(config.remote_redmine)} && {command}"
    return full if config.local else ssh(config, full)


def fix_permissions_command(config):
    command = "{sudo}mkdir -p {files} {redmine}/tmp {redmine}/log && {sudo}chmod -R g+rwX {files} {redmine}/tmp {redmine}/log && {sudo}find {files} -type d -exec chmod g+s {{}} +".format(
        sudo=remote_sudo_prefix(config),
        files=q(config.files_root),
        redmine=q(config.remote_redmine),
    )
    return command if config.local else ssh(config, command)


def script(config, name):
    path = config.repo_root / name
    if str(config.repo_root) == ".":
        return q(f"./{name}")
    return q(str(path))


def helpdesk_reset_command(config):
    base = f"{script(config, 'reset_helpdesk_mail_settings.py')} "
    if config.local:
        return (
            base +
            f"--local --remote-redmine {q(config.remote_redmine)} "
            f"--mailpit-host {q(config.mailpit_host)}"
        )
    return (
        base +
        f"--ssh-host {q(config.ssh_host)} "
        f"--ssh-key {q(str(config.ssh_key))} "
        f"--remote-redmine {q(config.remote_redmine)} "
        f"--mailpit-host {q(config.mailpit_host)}"
    )


def validate_command(config):
    base = f"{script(config, 'validate_test_instance.py')} "
    if config.local:
        return (
            base +
            f"--local --remote-redmine {q(config.remote_redmine)} "
            f"--mailpit-host {q(config.mailpit_host)} "
            f"--files-root {q(config.files_root)}"
        )
    return (
        base +
        f"--ssh-host {q(config.ssh_host)} "
        f"--ssh-key {q(str(config.ssh_key))} "
        f"--remote-redmine {q(config.remote_redmine)} "
        f"--mailpit-host {q(config.mailpit_host)} "
        f"--files-root {q(config.files_root)}"
    )


def outbox_commands(config):
    base = script(config, "redmine_outbox_worker.py")
    local = " --local" if config.local else ""
    return (
        f"{base}{local} --status",
        f"{base}{local} --dry-run --batch-size 10",
    )


def ssh(config, remote_command):
    return (
        "ssh "
        f"-i {q(str(config.ssh_key))} "
        "-o IdentitiesOnly=yes "
        f"{q(config.ssh_host)} "
        f"{q(remote_command)}"
    )


def ssh_transport(config):
    return f"ssh -i {str(config.ssh_key)} -o IdentitiesOnly=yes"


def remote_sudo_prefix(config):
    return "sudo " if config.remote_sudo else ""


def rsync_path_option(config):
    return "--rsync-path 'sudo rsync' " if config.remote_sudo else ""


def write_status(
    config: AutomationConfig,
    run_id: str,
    status: str,
    results,
    failed_step=None,
):
    now = utc_iso()
    document = {
        "run_id": run_id,
        "started_at": run_id_to_iso(run_id),
        "updated_at": now,
        "finished_at": now if status in {"success", "failed"} else None,
        "mode": "apply" if config.apply else "dry-run",
        "execution": "local" if config.local else "remote",
        "host": socket.gethostname(),
        "remote_redmine": config.remote_redmine,
        "repo_root": str(config.repo_root),
        "status": status,
        "failed_step": failed_step,
        "steps": [
            {"step": item.step, "command": item.command, "returncode": item.returncode}
            for item in results
        ],
    }
    config.status_dir.mkdir(parents=True, exist_ok=True)
    runs_dir = config.status_dir / "runs"
    runs_dir.mkdir(parents=True, exist_ok=True)
    write_json(runs_dir / f"{run_id}.json", document)
    write_json(config.status_dir / "latest.json", document)
    if status == "success":
        write_json(config.status_dir / "latest-success.json", document)
    return document


def write_json(path, document):
    tmp = path.with_suffix(path.suffix + ".tmp")
    tmp.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    tmp.replace(path)


def utc_stamp():
    return dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")


def utc_iso():
    return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


def run_id_to_iso(run_id):
    try:
        parsed = dt.datetime.strptime(run_id, "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc)
        return parsed.strftime("%Y-%m-%dT%H:%M:%SZ")
    except ValueError:
        return run_id


def q(value):
    return shlex.quote(value)


if __name__ == "__main__":
    raise SystemExit(main())