fix(reset): fence every job registry the reset reads, by its physical directory (#78)
Review round 3 R1 (codex): the registration guard chose the workspace marker from the path's spelling, so a symlink alias of .mam/jobs or a MAM_JOBS_DIR registry was not refused. The reset now writes .reset-in-progress into each registry its open-job check reads (.mam/jobs, MAM_JOBS_DIR, a row's other control_plane.mam_jobs_dir - the latter only for this workspace's sessions), each under that registry's lock and before the check, and removes it on exit; register_job reads the marker inside the registry it writes to. The precheck's separate lock barrier is subsumed by the fence. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
548c6854ea
commit
568385c77a
6 files changed
+186
-75
No files matched your search
@@ -68,12 +68,16 @@ def liveness_window(heartbeat_sec: int, heartbeat_misses: int) -> int:
|
||||
|
||||
|
||||
# Issue #78: a reset of a workspace purges its agents' conversations, so no job
|
||||
# may be registered in that workspace's .mam/jobs while one runs. The reset
|
||||
# publishes .mam/reset-in-progress (pid/lstart, loop_lock.sh's format) and then
|
||||
# takes this registry's lock once before it lists the open jobs; the marker is
|
||||
# read under the same lock here, so a registration either is written before
|
||||
# that listing or sees the marker. The reset's own create (the onboarding job
|
||||
# of an agent it recreates) passes the reset's pid in MAM_RESET_OWNER_PID.
|
||||
# may be registered for them while it runs. The reset writes RESET_MARKER into
|
||||
# every registry its open-job check reads (its own .mam/jobs and MAM_JOBS_DIR,
|
||||
# and a row's other control_plane.mam_jobs_dir), each under that registry's
|
||||
# lock and before the check: a registration reads it under the same lock, so it
|
||||
# is either written before the check or refused - by whatever path it names
|
||||
# the directory. The marker carries the reset's pid/lstart (loop_lock.sh's
|
||||
# format), its workspace, and for another control plane's registry the
|
||||
# sessions it covers. The reset's own create (the onboarding job of an agent
|
||||
# it recreates) passes the reset's pid in MAM_RESET_OWNER_PID.
|
||||
RESET_MARKER = ".reset-in-progress"
|
||||
RESET_OWNER_ENV = "MAM_RESET_OWNER_PID"
|
||||
|
||||
|
||||
@@ -90,15 +94,13 @@ def _lstart(pid: str) -> str:
|
||||
return " ".join(out.split())
|
||||
|
||||
|
||||
def reset_refusal(registry_dir: str) -> Optional[str]:
|
||||
"""Why a registration in ``registry_dir`` is refused, or None: a live
|
||||
reset of the workspace that owns it (``<ws>/.mam/jobs``) that is not the
|
||||
caller's own. Holder states as loop_lock.sh decides them: a dead or
|
||||
reused pid is stale; ps failing on this process is unknown (refused)."""
|
||||
jobs = os.path.abspath(registry_dir)
|
||||
if os.path.basename(jobs) != "jobs" or os.path.basename(os.path.dirname(jobs)) != ".mam":
|
||||
return None
|
||||
marker = os.path.join(os.path.dirname(jobs), "reset-in-progress")
|
||||
def reset_refusal(registry_dir: str, agent_session: str) -> Optional[str]:
|
||||
"""Why registering a job for ``agent_session`` in ``registry_dir`` is
|
||||
refused, or None: a live reset fenced this registry (for that session) and
|
||||
the caller is not its own create. Holder states as loop_lock.sh decides
|
||||
them: a dead or reused pid is stale; ps failing on this process is unknown
|
||||
(refused)."""
|
||||
marker = os.path.join(registry_dir, RESET_MARKER)
|
||||
try:
|
||||
with open(marker, encoding="utf-8") as fh:
|
||||
fields = dict(line.split("=", 1) for line in fh.read().splitlines() if "=" in line)
|
||||
@@ -106,6 +108,8 @@ def reset_refusal(registry_dir: str) -> Optional[str]:
|
||||
return None
|
||||
except (OSError, ValueError) as exc:
|
||||
return "the reset marker %s cannot be read (%s)" % (marker, exc)
|
||||
if "sessions" in fields and agent_session not in fields["sessions"].split(","):
|
||||
return None
|
||||
pid = fields.get("pid", "")
|
||||
if not pid.isdigit() or os.environ.get(RESET_OWNER_ENV) == pid:
|
||||
return None
|
||||
@@ -115,7 +119,7 @@ def reset_refusal(registry_dir: str) -> Optional[str]:
|
||||
if not cur or (fields.get("lstart") and cur != fields["lstart"]):
|
||||
return None
|
||||
return ("a reset of %s is in progress (pid %s); its agents are being purged and recreated;"
|
||||
" register the job when it has finished" % (os.path.dirname(os.path.dirname(jobs)), pid))
|
||||
" register the job when it has finished" % (fields.get("workspace", "?"), pid))
|
||||
|
||||
|
||||
def register_job(
|
||||
@@ -213,7 +217,7 @@ def register_job(
|
||||
if dry_run:
|
||||
record["dry_run"] = True
|
||||
with registry_lock(registry_dir):
|
||||
refusal = reset_refusal(registry_dir)
|
||||
refusal = reset_refusal(registry_dir, agent_session)
|
||||
if refusal:
|
||||
raise ResetInProgressError(refusal)
|
||||
if mqtt_common._job_path(job_id, registry_dir).exists():
|
||||
|
||||
Reference in new issue
Block a user