Harden portable Redmine CLI runtime and cache writes

This commit is contained in:
Jason Thistlethwaite
2026-10-18 03:18:36 -04:00
parent f6915face2
commit 2e686215ed
19 changed files with 1353 additions and 224 deletions
+165
View File
@@ -0,0 +1,165 @@
"""Real process tests: no placeholder success assertions or live API access."""
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import pytest
from redmine_cache_io import CacheError, atomic_write_json, cache_lock
ROOT=Path(__file__).resolve().parents[1]
def spawn(code,*args,extra=None):
return subprocess.Popen([sys.executable,'-c',code,*map(str,args)],cwd=ROOT,
env={**os.environ,**(extra or {})},stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
def finish(proc):
out,err=proc.communicate(timeout=15)
assert proc.returncode==0,err
return out,err
def test_distinct_ticket_updates_from_stale_instances_survive(tmp_path):
code='''from redmine_tickets import TicketCache
from pathlib import Path
import sys,time
c=TicketCache(Path(sys.argv[1]),'customer-service')
print('ready',flush=True)
time.sleep(.2)
for n in range(8): c.write_issue({'id':int(sys.argv[2])*100+n,'subject':'fixture','updated_on':str(n)})
'''
children=[spawn(code,tmp_path,n) for n in range(3)]
for child in children:
assert child.stdout.readline().strip()=='ready'
for child in children: finish(child)
data=json.loads((tmp_path/'manifest.json').read_text())
assert len(data['tickets'])==24
assert len(list((tmp_path/'customer-service').glob('*.json')))==24
def test_distinct_contact_updates_survive(tmp_path):
code='''from redmine_ticket_contacts import ContactCache
from pathlib import Path
import sys,time
c=ContactCache(Path(sys.argv[1]))
for n in range(10): c.write_contact({'id':int(sys.argv[2])*100+n,'first_name':'fixture'})
'''
children=[spawn(code,tmp_path,n) for n in range(3)]
for child in children: finish(child)
data=json.loads((tmp_path/'contacts/manifest.json').read_text())
assert len(data['contacts'])==30
def test_readers_see_complete_json_during_writes(tmp_path):
path=tmp_path/'data.json'
atomic_write_json(path,{'n':0,'blob':'x'*10000})
child=spawn('''from redmine_cache_io import atomic_write_json
from pathlib import Path
import sys
for n in range(60): atomic_write_json(Path(sys.argv[1]),{'n':n,'blob':'x'*10000})
''',path)
reads=0
while child.poll() is None:
data=json.loads(path.read_text())
assert len(data['blob'])==10000
reads+=1
finish(child)
assert reads>0
def holder(path):
child=spawn('''from redmine_cache_io import cache_lock
from pathlib import Path
import sys,time
with cache_lock(Path(sys.argv[1])):
print('locked',flush=True)
time.sleep(20)
''',path)
assert child.stdout.readline().strip()=='locked'
return child
def test_contention_timeout_is_visible_and_nonzero(tmp_path):
path=tmp_path/'manifest.json'; child=holder(path)
try:
waiter=spawn('''from redmine_cache_io import atomic_write_json
from pathlib import Path
import sys
atomic_write_json(Path(sys.argv[1]),{'x':1})
''',path,extra={'REDMINE_LOCK_TIMEOUT':'.12'})
out,err=waiter.communicate(timeout=5)
assert waiter.returncode!=0
assert 'Waiting for cache lock' in err
assert 'timeout after 0.12s' in err
finally:
child.kill();child.communicate(timeout=5)
def test_sigkill_releases_existing_lock_file(tmp_path):
path=tmp_path/'manifest.json'; child=holder(path)
child.kill(); child.communicate(timeout=5)
assert path.with_name(path.name+'.lock').exists()
waiter=spawn('''from redmine_cache_io import atomic_write_json
from pathlib import Path
import sys
atomic_write_json(Path(sys.argv[1]),{'recovered':True})
''',path,extra={'REDMINE_LOCK_TIMEOUT':'.2'})
finish(waiter)
assert json.loads(path.read_text())=={'recovered':True}
@pytest.mark.parametrize('value',['bogus','-1','nan','inf'])
def test_invalid_timeout_is_actionable(tmp_path,monkeypatch,value):
monkeypatch.setenv('REDMINE_LOCK_TIMEOUT',value)
with pytest.raises(CacheError,match='REDMINE_LOCK_TIMEOUT'):
atomic_write_json(tmp_path/'x.json',{})
def test_stale_temporary_file_is_removed_on_next_lock(tmp_path):
path=tmp_path/'manifest.json'
stale=tmp_path/'.manifest.json.killed.tmp'
stale.write_text('partial')
atomic_write_json(path,{'recovered':True})
assert not stale.exists()
assert json.loads(path.read_text())=={'recovered':True}
def test_nested_lock_and_atomic_failure_preserves_original(tmp_path):
path=tmp_path/'data.json'
atomic_write_json(path,{'old':True})
with cache_lock(path,timeout=.1):
with pytest.raises(TypeError): atomic_write_json(path,{'bad':object()})
assert json.loads(path.read_text())=={'old':True}
assert not list(tmp_path.glob('*.tmp'))
def test_corrupt_contact_manifest_not_reset(tmp_path):
from redmine_ticket_contacts import ContactCache
c=ContactCache(tmp_path)
c.manifest_path.write_text('{broken')
with pytest.raises(CacheError): c.write_contact({'id':1})
assert c.manifest_path.read_text()=='{broken'
def test_snapshot_fetches_are_serialized_before_replacement(tmp_path):
code='''from redmine_contacts import snapshot_transaction,write_cache
from argparse import Namespace
from pathlib import Path
import sys,time
args=Namespace(cache=Path(sys.argv[1]),base_url='https://test.example',project='p',refresh=True)
@snapshot_transaction
def fetch(args):
print('fetch',flush=True)
time.sleep(float(sys.argv[2]))
write_cache(args.cache,args,[{'id':int(sys.argv[3])}])
fetch(args)
'''
path=tmp_path/'contacts.json'
first=spawn(code,path,.3,1)
assert first.stdout.readline().strip()=='fetch'
second=spawn(code,path,0,2)
finish(first); _,err=finish(second)
assert 'Waiting for cache lock' in err
assert json.loads(path.read_text())['contacts']==[{'id':2}]
+187
View File
@@ -0,0 +1,187 @@
"""Run real wrapper entry points in a relocated checkout with an isolated HOME."""
import json
import os
from pathlib import Path
import shutil
import subprocess
import pytest
ROOT=Path(__file__).resolve().parents[1]
@pytest.fixture
def runtime(tmp_path):
root=tmp_path/'relocated checkout'
(root/'bin').mkdir(parents=True)
shutil.copy(ROOT/'redmine_cli_runtime.py',root)
for name in ('rcontacts-live','rtickets-live','rtl','rtd'):
shutil.copy(ROOT/'bin'/name,root/'bin'/name)
fake='import json,os,sys; print(json.dumps({"argv":sys.argv[1:],"env":{k:v for k,v in os.environ.items() if k.startswith("REDMINE_") or k.startswith("NMI_")}}))\n'
for name in ('redmine_tickets.py','redmine_contacts.py'):
(root/name).write_text(fake)
home=tmp_path/'home'; home.mkdir()
env={'HOME':str(home),'PATH':'/usr/bin:/bin','TMPDIR':os.environ['TMPDIR']}
def run(*args,command='rtl',extra=None):
return subprocess.run([str(root/'bin'/command),*args],cwd=tmp_path,env={**env,**(extra or {})},text=True,capture_output=True,timeout=10)
return root,home,run
def test_config_file_and_export_precedence(runtime):
root,home,run=runtime
(home/'.redmine.live').write_text('export REDMINE_API_KEY="fixture-secret"\nREDMINE_BASE_URL=https://file.example.test\nREDMINE_PROJECT=abc\nREDMINE_BASE_CACHE="data with spaces"\n')
p=run('--show-config',extra={'REDMINE_BASE_URL':'https://env.example.test'})
assert p.returncode==0,p.stderr
d=json.loads(p.stdout)
assert d['base_url']=='https://env.example.test'
assert d['project']=='abc'
assert d['base_cache']==str(home/'data with spaces')
assert d['api_key_present']
assert 'fixture-secret' not in p.stdout+p.stderr
assert 'environment' in d['sources']['base_url']
p=run('--base-url','https://cli.example.test','--show-config',extra={'REDMINE_BASE_URL':'https://env.example.test'})
assert json.loads(p.stdout)['base_url']=='https://cli.example.test'
def test_real_dispatch_preserves_stdin_arguments_and_key(runtime):
root,home,run=runtime
(home/'.redmine.live').write_text('REDMINE_API_KEY=fixture-secret\n')
p=run('--base-cache','my cache','search','--query','two words',command='rcontacts-live')
assert p.returncode==0,p.stderr
d=json.loads(p.stdout)
assert d['env']['REDMINE_API_KEY']=='fixture-secret'
assert d['argv'][-3:]==['search','--query','two words']
cache=d['argv'][d['argv'].index('--cache')+1]
assert cache.endswith('my cache/redmine_contacts/rm.ldrprep.com_customer-service.json')
assert not Path(cache).is_dir()
def test_doctor_missing_key_nonzero_with_search_paths(runtime):
root,home,run=runtime
p=run('--doctor')
assert p.returncode==2
d=json.loads(p.stdout)
assert not d['ready']
assert 'REDMINE_API_KEY' in d['problems'][0]
assert str(home/'.redmine.live') in d['searched_env_paths']
assert not (root/'cache').exists()
p=run('sync')
assert p.returncode==2 and 'REDMINE_API_KEY' in p.stderr
def test_explicit_missing_or_malformed_file_blocks(runtime):
root,home,run=runtime
assert run('--env-file','absent','--doctor').returncode==2
file=home/'bad.env'; file.write_text('not an assignment\n')
p=run('--env-file',str(file),'--show-config')
assert p.returncode==2 and 'Malformed' in p.stderr
def test_help_works_without_config(runtime):
_,_,run=runtime
p=run('--env-file','missing','--help')
assert p.returncode==0
assert 'REDMINE_API_KEY' in p.stdout
def test_aliases_match_and_dev_never_loads_live(runtime):
root,home,run=runtime
(home/'.redmine.live').write_text('REDMINE_API_KEY=fixture-live\n')
a=json.loads(run('--show-config',command='rtl').stdout)
b=json.loads(run('--show-config',command='rtickets-live').stdout)
assert a==b
d=run('--show-config',command='rtd')
data=json.loads(d.stdout)
assert not data['api_key_present']
assert data['nmi_mode']=='dev'
assert 'DEVELOPMENT' in d.stderr
assert data['ticket_cache']==str(root/'cache-dev/redmine_tickets')
def test_cache_overrides_and_scope_isolation(runtime):
root,_,run=runtime
default=json.loads(run('--show-config').stdout)
assert default['ticket_cache']==str(root/'cache/redmine_tickets')
assert default['contacts_cache'].endswith('/cache/redmine_contacts/rm.ldrprep.com_customer-service.json')
assert default['contacts_cache_layout']=='namespaced-file'
other=json.loads(run('--project','meetings','--show-config').stdout)
assert other['ticket_cache']!=default['ticket_cache']
host=json.loads(run('--base-url','https://other.example.test','--show-config').stdout)
assert host['ticket_cache']!=default['ticket_cache']
exact=json.loads(run('--cache-dir','specific path','--show-config').stdout)
assert exact['ticket_cache'].endswith('/specific path')
def test_json_config_and_key_file_propagation(runtime):
root,home,run=runtime
config=home/'live.json'; key=home/'key'; key.write_text('fixture-key-file\n')
config.write_text(json.dumps({'project':'meetings','cache-dir':'mirror','api-key-file':'key'}))
p=run('--config',str(config),'search',extra={'REDMINE_PROJECT':'overridden'})
assert p.returncode==0,p.stderr
d=json.loads(p.stdout)
assert d['env']['REDMINE_API_KEY']=='fixture-key-file'
assert d['env']['REDMINE_PROJECT']=='overridden'
assert '--config' not in d['argv']
assert str(home/'mirror') in d['argv']
def test_subcommand_config_is_not_global(runtime):
_,_,run=runtime
p=run('materialize-queues','--config','queues.json')
assert p.returncode==0,p.stderr
assert json.loads(p.stdout)['argv'][-3:]==['materialize-queues','--config','queues.json']
def test_directory_contacts_cache_rejected(runtime):
root,_,run=runtime
p=run('--cache',str(root),'--show-config',command='rcontacts-live')
assert p.returncode==2 and 'JSON file' in p.stderr
def test_legacy_contacts_snapshot_is_preserved(runtime):
root,_,run=runtime
legacy=root/'cache/redmine_contacts'
legacy.parent.mkdir(parents=True)
legacy.write_text(json.dumps({'base_url':'https://rm.ldrprep.com','project':'customer-service','contacts':[]}))
d=json.loads(run('--show-config').stdout)
assert d['contacts_cache']==str(legacy)
assert d['contacts_cache_layout']=='legacy-single-file'
def test_contacts_handoff_preserves_resolved_context(tmp_path,monkeypatch):
import argparse
import shlex
from redmine_cli_runtime import contacts_handoff
monkeypatch.setenv('REDMINE_ENV_FILE',str(tmp_path/'profile with spaces.env'))
monkeypatch.setenv('REDMINE_CONTACTS_CACHE',str(tmp_path/'cache with spaces/contacts.json'))
args=argparse.Namespace(base_url='https://rm.example.test',project='meetings',cache_dir=tmp_path/'tickets',api_key_file=tmp_path/'key file')
cmd=shlex.split(contacts_handoff(args,42))
assert cmd[cmd.index('--cache')+1]==str(tmp_path/'cache with spaces/contacts.json')
assert cmd[cmd.index('--env-file')+1]==str(tmp_path/'profile with spaces.env')
assert cmd[cmd.index('--api-key-file')+1]==str(tmp_path/'key file')
assert cmd[-3:]==['show','--contact-id','42']
def test_actual_wrapper_invalid_ticket_query_is_clean_error(tmp_path):
env={'HOME':str(tmp_path),'PATH':'/usr/bin:/bin','TMPDIR':os.environ['TMPDIR'],
'REDMINE_API_KEY':'fixture-only','REDMINE_BASE_CACHE':str(tmp_path/'cache')}
p=subprocess.run([str(ROOT/'bin/rtl'),'search','--where','id == -1'],env=env,cwd=tmp_path,
text=True,capture_output=True,timeout=10)
assert p.returncode==2
assert 'Invalid --where expression' in p.stderr
assert 'Traceback' not in p.stderr
def test_actual_wrapper_to_actual_nmi_doctor(tmp_path):
cli=ROOT.parent/'mrscClient-php/bin/nmi'
if not cli.exists(): pytest.skip('Sibling NMI checkout unavailable')
profile=tmp_path/'nmi.env'
profile.write_text('NMI_ENDPOINT=https://fixture.example.test\nNMI_MRSC_ACCESS_CODE=fixture-access\nNMI_SECRETKEY=fixture-secret\n')
redmine=tmp_path/'redmine.env'
redmine.write_text(f'NMI_CLI={cli}\nNMI_ENV_FILE={profile}\nREDMINE_BASE_CACHE={tmp_path}/cache\n')
p=subprocess.run([str(ROOT/'bin/rtickets-live'),'--env-file',str(redmine),'--doctor-nmi'],
env={'HOME':str(tmp_path),'PATH':'/usr/bin:/bin'},cwd=tmp_path,text=True,capture_output=True,timeout=10)
assert p.returncode==0,p.stdout+p.stderr
d=json.loads(p.stdout)
assert d['ready'] and d['mode']=='production'
assert d['env_file']==str(profile)
assert not d['network_checked']
assert 'fixture-secret' not in p.stdout+p.stderr
+160
View File
@@ -0,0 +1,160 @@
"""Portable runtime contract tests; fake child responses never contact NMI."""
import json
import subprocess
from pathlib import Path
import pytest
import redmine_ticket_nmi as bridge
def fake_cli(tmp_path):
p = tmp_path / 'space dir' / 'nmi'
p.parent.mkdir()
p.write_text('<?php // test fixture\n')
return p
def doctor_result(cmd, **kwargs):
return subprocess.CompletedProcess(cmd, 0, json.dumps({
'ready': True, 'mode': 'production', 'env_file': None,
'endpoint': 'https://nmi.example.test/api', 'problem': None,
}), '')
def test_environment_only_runtime_uses_actual_doctor(monkeypatch, tmp_path):
cli = fake_cli(tmp_path)
calls = []
def invoke(cmd, **kwargs):
calls.append((cmd, kwargs))
return doctor_result(cmd, **kwargs)
monkeypatch.setattr(bridge.subprocess, 'run', invoke)
profile = bridge.resolve_runtime_profile(environ={
'NMI_CLI': str(cli), 'NMI_SECRETKEY': 'fixture-secret',
'NMI_MRSC_ACCESS_CODE': 'fixture-access',
'NMI_ENDPOINT': 'https://nmi.example.test/api',
})
assert profile.ready
assert profile.mode == 'production'
assert profile.env_file is None
assert len(calls) == 1
cmd, kwargs = calls[0]
assert '--doctor' in cmd and '--json' in cmd
assert not any(arg.startswith('--env-file') for arg in cmd)
assert 'fixture-secret' not in ' '.join(cmd)
assert kwargs['env']['NMI_SECRETKEY'] == 'fixture-secret'
def test_mode_and_binary_are_resolved_at_call_time(monkeypatch, tmp_path):
cli = fake_cli(tmp_path)
monkeypatch.setenv('NMI_CLI', str(cli))
monkeypatch.setenv('NMI_MODE', 'dev')
monkeypatch.delenv('NMI_ENV_FILE', raising=False)
calls = []
def invoke(cmd, **kwargs):
calls.append(cmd)
return subprocess.CompletedProcess(cmd, 0, '{}', '')
monkeypatch.setattr(bridge.subprocess, 'run', invoke)
bridge.run_nmi(['resolve', '--item-no=200000001'])
assert str(cli) in calls[0]
assert '--mode=dev' in calls[0]
assert not any('None' in value for value in calls[0])
def test_explicit_missing_env_blocks_child(monkeypatch, tmp_path):
monkeypatch.setattr(bridge.subprocess, 'run', lambda *a, **k: pytest.fail('must not spawn'))
p = bridge.resolve_runtime_profile(env_file=str(tmp_path / 'absent'), nmi_bin=str(fake_cli(tmp_path)), environ={})
assert not p.ready
assert 'does not exist' in p.problem
def test_doctor_nonzero_is_not_ready(monkeypatch, tmp_path):
def invoke(cmd, **kwargs):
return subprocess.CompletedProcess(cmd, 2, json.dumps({'ready': False, 'problem': 'NMI_SECRETKEY is missing'}), '')
monkeypatch.setattr(bridge.subprocess, 'run', invoke)
p = bridge.resolve_runtime_profile(nmi_bin=str(fake_cli(tmp_path)), environ={})
assert not p.ready
assert 'NMI_SECRETKEY' in p.problem
def test_doctor_timeout_actionable(monkeypatch, tmp_path):
def invoke(cmd, **kwargs):
raise subprocess.TimeoutExpired(cmd, 15)
monkeypatch.setattr(bridge.subprocess, 'run', invoke)
p = bridge.resolve_runtime_profile(nmi_bin=str(fake_cli(tmp_path)), environ={})
assert not p.ready
assert 'timed out' in p.problem.lower()
def test_brief_executes_resolved_profile_not_import_defaults(monkeypatch, tmp_path):
cli = fake_cli(tmp_path)
env_file = tmp_path / 'selected.env'
env_file.write_text('NMI_MODE=production\n')
profile = bridge.NmiRuntimeProfile('production', env_file, cli, 'https://nmi.example.test/api', True)
monkeypatch.setattr(bridge, 'resolve_runtime_profile', lambda **kwargs: profile)
calls = []
def brief(plan, **kwargs):
calls.append(kwargs)
return {'ok': True, 'text': '=== NMI CS BRIEF ===\nfixture'}
monkeypatch.setattr(bridge, 'try_cs_brief', brief)
result = bridge.brief_for_issue({'id': 1}, [{'type': 'item_no', 'value': '200000001'}], with_history=False)
assert result['ok']
assert calls[0]['nmi_bin'] == str(cli)
assert calls[0]['env_file'] == str(env_file)
assert calls[0]['endpoint'] == profile.endpoint
def test_history_uses_one_item_identifier(monkeypatch):
seen = []
monkeypatch.setattr(bridge, 'try_item_history', lambda item, **kwargs: seen.append(item) or {'ok': True})
bridge.enrich_with_history({'plan': {'item_no': ['200000001', '200000002']}})
assert seen == ['200000001']
def test_real_php_doctor_integration(tmp_path):
"""Executes sibling NMI's real config resolver, not its network API."""
import os
import shutil
cli = Path(bridge.DEFAULT_NMI_BIN)
if not cli.is_file() or not shutil.which('php'):
pytest.skip('Sibling NMI/PHP not available')
env_file = tmp_path / 'test profile.env'
env_file.write_text('NMI_MRSC_ACCESS_CODE=fixture-access\nNMI_SECRETKEY=fixture-secret\nNMI_ENDPOINT=https://fixture.example.test\n')
profile = bridge.resolve_runtime_profile(environ={
'HOME': str(tmp_path), 'PATH': '/usr/bin:/bin', 'NMI_CLI': str(cli),
'NMI_ENV_FILE': str(env_file),
})
assert profile.ready, profile.problem
assert profile.mode == 'production'
assert profile.env_file == env_file
assert profile.endpoint == 'https://fixture.example.test'
def test_real_php_child_receives_same_runtime_for_brief_and_history(tmp_path, monkeypatch):
"""Fake PHP service-facing CLI; exercises real subprocess argv forwarding."""
import shutil
if not shutil.which('php'):
pytest.skip('PHP unavailable')
cli = tmp_path / 'fake nmi.php'
log = tmp_path / 'calls.jsonl'
env_file = tmp_path / 'profile.env'
env_file.write_text('NMI_MODE=dev\n')
cli.write_text('''<?php
file_put_contents(getenv('BRIDGE_TEST_LOG'), json_encode($argv)."\\n", FILE_APPEND);
if (in_array('--doctor', $argv, true)) {
echo json_encode(['ready'=>true,'mode'=>'dev','env_file'=>getenv('NMI_ENV_FILE'),'endpoint'=>'https://fixture.example.test','problem'=>null]);
} elseif (in_array('cs-brief',$argv,true)) {
echo "=== NMI CS BRIEF === fixture only\\n";
} else { echo json_encode(['fixture'=>true]); }
''')
monkeypatch.setenv('NMI_CLI', str(cli))
monkeypatch.setenv('NMI_ENV_FILE', str(env_file))
monkeypatch.setenv('NMI_MODE', 'dev')
monkeypatch.setenv('BRIDGE_TEST_LOG', str(log))
result = bridge.brief_for_issue({'id':1}, [{'type':'item_no','value':'200000001'}])
assert result['ok'] and result['item_history']['ok']
calls = [json.loads(line) for line in log.read_text().splitlines()]
assert len(calls) == 3
for call in calls:
assert '--mode=dev' in call
assert '--env-file='+str(env_file) in call
assert '--item-no=200000001' in calls[-1]
+10 -1
View File
@@ -9,9 +9,18 @@ import pytest
import redmine_ticket_nmi
def test_runtime_profile_honors_explicit_production_mode_and_env_file(tmp_path):
def test_runtime_profile_honors_explicit_production_mode_and_env_file(tmp_path, monkeypatch):
production_env = tmp_path / "production.env"
production_env.write_text("NMI_ENDPOINT_PRODUCTION=https://app.ldrprep.com/api.php\n")
# File existence alone is no longer readiness. Simulate the offline PHP
# diagnostic here; real CLI resolution is exercised by the integration tests.
import json
import subprocess
monkeypatch.setattr(redmine_ticket_nmi.subprocess, "run", lambda cmd, **kwargs:
subprocess.CompletedProcess(cmd, 0, json.dumps({
"ready": True, "mode": "production", "env_file": str(production_env),
"endpoint": "https://app.ldrprep.com/api.php", "problem": None,
}), ""))
profile = redmine_ticket_nmi.resolve_runtime_profile(
environ={
+1 -1
View File
@@ -224,7 +224,7 @@ class RedmineTicketsSyncTest(unittest.TestCase):
self.assertEqual(2, result.seen)
self.assertEqual(1, result.skipped_cached)
self.assertEqual(1, result.fetched)
self.assertEqual([(101, ("journals", "helpdesk"))], client.full_fetches)
self.assertEqual([(101, ("journals", "helpdesk", "attachments"))], client.full_fetches)
self.assertTrue(cache.issue_path(101).is_file())