Jason Thistlethwaite
126 lines
4.2 KiB
Python
126 lines
4.2 KiB
Python
"""POSIX local-filesystem cache transactions; persistent locks, atomic JSON.
|
|
|
|
Locks protect read/modify/write, not only replacement. File existence never
|
|
indicates ownership. Kernel locks are released even when the holder is killed.
|
|
"""
|
|
from __future__ import annotations
|
|
from contextlib import contextmanager
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
|
|
class CacheError(RuntimeError):
|
|
pass
|
|
|
|
_local = threading.local()
|
|
|
|
def lock_timeout():
|
|
raw = os.environ.get('REDMINE_LOCK_TIMEOUT','60')
|
|
try:
|
|
value=float(raw)
|
|
if not math.isfinite(value) or value < 0:
|
|
raise ValueError()
|
|
return value
|
|
except ValueError as exc:
|
|
raise CacheError('REDMINE_LOCK_TIMEOUT must be a finite nonnegative number of seconds.') from exc
|
|
|
|
@contextmanager
|
|
def cache_lock(target, timeout=None):
|
|
target=Path(target).resolve()
|
|
seconds=lock_timeout() if timeout is None else float(timeout)
|
|
if not math.isfinite(seconds) or seconds < 0:
|
|
raise CacheError('Cache lock timeout must be finite and nonnegative.')
|
|
held=getattr(_local,'held',None)
|
|
if held is None:
|
|
held=_local.held={}
|
|
key=(os.getpid(),str(target))
|
|
if key in held:
|
|
yield
|
|
return
|
|
path=target.with_name(target.name+'.lock')
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
fd=os.open(path,os.O_RDWR|os.O_CREAT,0o600)
|
|
acquired=False
|
|
start=time.monotonic()
|
|
warned=False
|
|
try:
|
|
while True:
|
|
try:
|
|
fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
acquired=True
|
|
held[key]=fd
|
|
# A killed writer may leave its same-directory temporary file.
|
|
# Once this lock is ours, no live writer for this target can own
|
|
# one, so removing matching leftovers is safe.
|
|
for stale in target.parent.glob('.' + target.name + '.*.tmp'):
|
|
try:
|
|
stale.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
break
|
|
except BlockingIOError:
|
|
if not warned:
|
|
print(f'Waiting for cache lock: {path} (timeout {seconds:g}s)',file=sys.stderr,flush=True)
|
|
warned=True
|
|
remaining=seconds-(time.monotonic()-start)
|
|
if remaining <= 0:
|
|
raise CacheError(f'Cache lock timeout after {seconds:g}s: {path}. Another writer is active; retry or increase REDMINE_LOCK_TIMEOUT. Do not delete the lock file.')
|
|
time.sleep(min(0.05,remaining))
|
|
yield
|
|
finally:
|
|
if acquired:
|
|
held.pop(key,None)
|
|
fcntl.flock(fd,fcntl.LOCK_UN)
|
|
os.close(fd)
|
|
|
|
|
|
def read_json(path):
|
|
path=Path(path)
|
|
try:
|
|
data=json.loads(path.read_text(encoding='utf-8'))
|
|
except FileNotFoundError:
|
|
return {}
|
|
except (json.JSONDecodeError,UnicodeError) as exc:
|
|
raise CacheError(f'Invalid cache JSON: {path}; refusing to overwrite it. Restore or repair this file.') from exc
|
|
if not isinstance(data,dict):
|
|
raise CacheError(f'Cache JSON must be an object: {path}; refusing to overwrite it.')
|
|
return data
|
|
|
|
|
|
def locked_read_json(path):
|
|
with cache_lock(path):
|
|
return read_json(path)
|
|
|
|
|
|
def atomic_write_json(path,data):
|
|
path=Path(path)
|
|
with cache_lock(path):
|
|
fd,name=tempfile.mkstemp(prefix='.'+path.name+'.',suffix='.tmp',dir=path.parent)
|
|
tmp=Path(name)
|
|
try:
|
|
with os.fdopen(fd,'w',encoding='utf-8') as handle:
|
|
json.dump(data,handle,indent=2,sort_keys=True)
|
|
handle.write('\n')
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
# Keep existing shared-file permissions; new sensitive caches are 0600.
|
|
if path.exists():
|
|
os.chmod(tmp,path.stat().st_mode & 0o777)
|
|
os.replace(tmp,path)
|
|
finally:
|
|
tmp.unlink(missing_ok=True)
|
|
|
|
|
|
def update_manifest_atomic(path,updater):
|
|
with cache_lock(path):
|
|
current=read_json(path)
|
|
updater(current)
|
|
atomic_write_json(path,current)
|
|
return current
|