fix(reset): never take another live reset's fence, and fence absent row registries (#78)
Review round 4 (codex): - R1: fence_registries reads an existing marker under the registry lock and refuses the reset, changing nothing, when another live (or undecidable) reset holds it - registries may be shared between workspaces; only a stale marker is replaced, and the reset's exit removes only its own fences. - R2: the registries to fence come from update.jobs_dirs, the enumeration open_jobs now uses too, whether or not they exist; an absent row registry is created and fenced (a registration would create it), and one that cannot be created refuses the reset. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
568385c77a
commit
b49314c3d0
6 files changed
+213
-74
No files matched your search
@@ -23,8 +23,8 @@ from typing import Dict, Iterator, List, Mapping, Optional, Tuple
|
||||
from lib_py.agents.base import AGENT_SESSION_ENV, POST_DISCOVERED
|
||||
from lib_py.agents.registry import agent_of_row, get_adapter, own_key
|
||||
from lib_py.delegation.mqtt_common import registry_lock
|
||||
from lib_py.delegation.registry import RESET_MARKER
|
||||
from lib_py.update import open_jobs, row_server
|
||||
from lib_py.delegation.registry import RESET_MARKER, reset_marker
|
||||
from lib_py.update import jobs_dirs, open_jobs, row_server
|
||||
from lib_py.verify_session import verify_session_uuid
|
||||
|
||||
SPEC_VERSION = 1
|
||||
@@ -485,35 +485,36 @@ def target_open_jobs(workspace: str, state: dict, env: Mapping[str, str], sessio
|
||||
return out
|
||||
|
||||
|
||||
class FenceBusy(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def fence_registries(workspace: str, state: dict, env: Mapping[str, str]) -> Iterator[str]:
|
||||
"""Write RESET_MARKER into every registry open_jobs reads, each under its
|
||||
"""Write RESET_MARKER into every registry open_jobs reads (jobs_dirs,
|
||||
created when absent - a registration would create it too), each under its
|
||||
registry lock (a registration that read no marker has finished writing),
|
||||
and yield its real path as it is written. This workspace's .mam/jobs and
|
||||
MAM_JOBS_DIR (created when absent) refuse every registration; another
|
||||
control plane's registry named by a row refuses only those addressed to
|
||||
this registry's rows. The marker copies .mam/reset-in-progress."""
|
||||
and yield its real path as it is written. FenceBusy when one is already
|
||||
fenced by another live (or undecidable) reset - registries may be shared
|
||||
between workspaces; its marker is left as it is. This workspace's own
|
||||
registries refuse every registration; another control plane's only those
|
||||
addressed to this registry's rows. The marker copies .mam/reset-in-progress."""
|
||||
with open(os.path.join(workspace, ".mam", "reset-in-progress"), encoding="utf-8") as fh:
|
||||
base = "".join(l + "\n" for l in fh.read().splitlines() if l.split("=", 1)[0] in ("pid", "lstart", "started"))
|
||||
me = dict(l.split("=", 1) for l in base.splitlines()).get("pid", "")
|
||||
base += "workspace=%s\n" % workspace
|
||||
rows = state.get("herdr_sessions") or []
|
||||
sessions = ",".join("herdr:%s" % s["name"] for s in rows if s.get("name"))
|
||||
plan, seen = [], set()
|
||||
for d in (os.path.join(workspace, ".mam", "jobs"), env.get("MAM_JOBS_DIR") or ""):
|
||||
if d:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
plan.append((d, base))
|
||||
for s in rows:
|
||||
d = (s.get("control_plane") or {}).get("mam_jobs_dir") or ""
|
||||
if d and os.path.isdir(d):
|
||||
plan.append((d, base + "sessions=%s\n" % sessions))
|
||||
for d, text in plan:
|
||||
if _real(d) in seen:
|
||||
continue
|
||||
seen.add(_real(d))
|
||||
for d, addressed_only in jobs_dirs(workspace, state, env):
|
||||
os.makedirs(d, exist_ok=True)
|
||||
with registry_lock(d):
|
||||
held, fields = reset_marker(d)
|
||||
if held in ("live", "unknown") and fields.get("pid") != me:
|
||||
raise FenceBusy(
|
||||
"the job registry %s is fenced by a reset of %s (pid %s, holder %s); wait for it to end"
|
||||
% (d, fields.get("workspace", "?"), fields.get("pid", "?"), held))
|
||||
fd, tmp = tempfile.mkstemp(dir=d, prefix=RESET_MARKER + ".")
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(text)
|
||||
fh.write(base + ("sessions=%s\n" % sessions if addressed_only else ""))
|
||||
os.replace(tmp, os.path.join(d, RESET_MARKER))
|
||||
yield _real(d)
|
||||
|
||||
@@ -886,9 +887,12 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
if a.cmd == "fence":
|
||||
try:
|
||||
for d in fence_registries(ws, state, os.environ):
|
||||
print(d, flush=True)
|
||||
print("fenced %s" % d, flush=True)
|
||||
except FenceBusy as exc:
|
||||
print("refused %s" % exc)
|
||||
return 1
|
||||
except OSError as exc:
|
||||
print("ERROR: cannot fence the job registries (%s)" % exc, file=sys.stderr)
|
||||
print("refused the job registries cannot be fenced against new registrations (%s)" % exc)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
Reference in new issue
Block a user