fix(issue-30): review round 2 - the watcher is stopped with KILL, and a strict last gate sits between the notice and the keys

grok N1: the wrapper spent up to 5 s on a TERM budget before it finalized the
job, so when the watcher swallowed the TERM the silent job's close came ~5 s
late and a `completed` written in that gap won (test_issue14 idle-grace).
codex D1: `_notify` blocks on a publish for up to 120 s, and the prompt went
out right after it; a completed, a re-dispatch or an unreadable record in that
window still got the nudge. codex D2: the wrapper pid was checked only at the
loop head, so a watcher whose wrapper vanished during a wait could still
notify and nudge once.

- multi-agent-mux-delegate-job `stop_stall_watch`: KILL + wait, no budget.
  The watcher holds nothing to flush (its TERM handler is a bare os._exit),
  and KILL cannot be swallowed.
- lib_py/stall_watch.py: `_wrapper_gone()` after every wait (before any
  notice) and at the loop head; `_may_prompt()` right before the keys - the
  wrapper is alive and the record, read then, is open and this dispatch;
  `_dispatch_open(strict=True)` treats an unreadable record as "no".
- SKILL.md: the lifetime contract states the one-wait lag and that nothing is
  sent after the wrapper is seen gone.
- tests/test_issue30_stall_watch_stop.py: the stop returns at once against a
  TERM-ignoring stand-in (took 6 s on the round-2 code); completed,
  re-dispatch and a torn record during the notice publish withhold the nudge;
  a wrapper gone during the wait gets no notice and no nudge. All five fail
  on the round-2 code.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
GodopuandClaude Opus 5.5 committed 2026-09-26 15:28:07 +09:00
1 parent af0a2a2557
commit 9200562c91
4 files changed
+144 -31

No files matched your search

+106 -7
View File
@@ -63,10 +63,10 @@ def _until(pred, limit=20.0):
return pred()
def test_stop_stall_watch_is_bounded_when_the_watcher_ignores_term(tmp_path):
def test_stop_stall_watch_returns_at_once_when_the_watcher_ignores_term(tmp_path):
"""The wrapper's stop routine, run as is against a stand-in that ignores
SIGTERM (what a swallowed TERM looks like from the wrapper's side): it
returns within its 5 s budget, says so, and the stand-in is dead."""
returns at once and the stand-in is dead."""
fn = subprocess.run(["sed", "-n", "/^stop_stall_watch()/,/^}/p", str(WRAPPER)],
capture_output=True, text=True, check=True).stdout
assert fn.startswith("stop_stall_watch()") and fn.rstrip().endswith("}")
@@ -79,8 +79,8 @@ def test_stop_stall_watch_is_bounded_when_the_watcher_ignores_term(tmp_path):
took = time.monotonic() - t0
assert res.returncode == 0, res.stdout + res.stderr
assert "stand-in gone" in res.stdout, res.stdout
assert "stall watch for t ignored TERM; killed" in res.stderr, res.stderr
assert 4 < took < 12, took
assert "took=0" in res.stdout or "took=1" in res.stdout, res.stdout
assert took < 5, took
def test_the_wrapper_returns_when_its_watcher_ignores_term(tmp_path, mock_herdr):
@@ -125,11 +125,10 @@ def test_the_wrapper_returns_when_its_watcher_ignores_term(tmp_path, mock_herdr)
if proc.poll() is None:
proc.kill()
assert proc.returncode == 3, f"expected the broker_unavailable path (rc=3), got {proc.returncode}\n{stderr}\n{stdout}"
# ack grace (8 s) + the 5 s TERM budget + slack; an unbounded wait never returns.
# ack grace (8 s) + slack; an unbounded wait never returns.
assert took < 40, took
if pidfile.exists(): # the stand-in got past its shim (a TERM in the shim's own start-up can still end it)
if pidfile.exists(): # the stand-in got past its shim
assert _until(lambda: not _alive(int(pidfile.read_text())), 5), "the watcher outlived the wrapper"
assert re.search(r"stall watch for \S+ ignored TERM", stderr), stderr
def test_the_watcher_stops_once_its_wrapper_is_gone(mam_sandbox, mock_herdr):
@@ -324,3 +323,103 @@ def test_a_refused_idle_notice_withholds_the_nudge(tmp_path, monkeypatch):
return w.WaitOutcome(w.TIMEOUT, error_code="timeout")
rc, prompts, notes, _, _ = _watch_in_process(tmp_path, monkeypatch, [_idle, _idle_then_close], False)
assert rc == 0 and notes == ["stalled:idle"] and prompts == [], (prompts, notes)
# ---------------------------------------------------------------------------
# round-2 review (codex D1/D2) - the last gate sits between the notice and the keys
# ---------------------------------------------------------------------------
def _during_notice(tmp_path, monkeypatch, change, alive=lambda pid: True):
"""Idle past S on the second tick; `change(rec)` runs inside the notice's
publish, as another actor would act while it waits on the wire."""
from lib_py import stall_watch
import argparse
import uuid
jid = uuid.uuid4().hex[:8]
rec = tmp_path / f"{jid}.json"
rec.write_text(json.dumps({"job_id": jid, "status": "running", "dispatch_token": "tok", "attempt_id": 1}))
ticks = []
def fake_wait_once(target, until=(), timeout_ms=None, session=None, run=None):
from lib_py.loop import watcher as w
ticks.append(1)
if len(ticks) == 1:
return w.WaitOutcome(w.MATCHED, agent={"agent_status": "idle", "state_change_seq": 1, "pane_id": "w1:p1"})
if len(ticks) > 2: # a watcher that did not stop: end the run through the record
d = json.loads(rec.read_text())
d["status"] = "completed"
rec.write_text(json.dumps(d))
time.sleep(1.2)
return w.WaitOutcome(w.TIMEOUT, error_code="timeout")
monkeypatch.setattr(stall_watch.watcher, "wait_once", fake_wait_once)
monkeypatch.setattr(stall_watch, "_alive", alive)
monkeypatch.setenv("MAM_STALL_IDLE_S", "1")
args = argparse.Namespace(job=jid, registry_dir=str(tmp_path), target="creator-codex-t", agent="codex",
herdr_session="", expect_dispatch_token="tok", attempt_id=1, watch_pid=4242)
src = _Src()
sw = stall_watch.StallWatch(args, source=src)
notes = []
def notify(detail):
notes.append(detail)
change(rec)
return True # the publisher said rc 0: that is not a statement about the record
monkeypatch.setattr(sw, "_notify", notify)
return sw.run(), src.prompts, notes
def _complete(rec):
d = json.loads(rec.read_text())
d["status"] = "completed"
rec.write_text(json.dumps(d))
def _redispatch(rec):
d = json.loads(rec.read_text())
d["dispatch_token"], d["attempt_id"] = "tok-2", 2
rec.write_text(json.dumps(d))
def test_a_job_completed_while_the_notice_is_published_is_not_nudged(tmp_path, monkeypatch):
rc, prompts, notes = _during_notice(tmp_path, monkeypatch, _complete)
assert rc == 0 and notes == ["stalled:idle"] and prompts == [], (prompts, notes)
def test_a_job_redispatched_while_the_notice_is_published_is_not_nudged(tmp_path, monkeypatch):
rc, prompts, notes = _during_notice(tmp_path, monkeypatch, _redispatch)
assert rc == 0 and notes == ["stalled:idle"] and prompts == [], (prompts, notes)
def test_an_unreadable_record_at_the_last_gate_withholds_the_nudge(tmp_path, monkeypatch):
real = {}
def garble(rec):
real["text"] = rec.read_text()
rec.write_text("{not json") # a torn write, seen once
from lib_py import stall_watch
orig = stall_watch.StallWatch._dispatch_open
def once_then_restore(self, strict=False):
out = orig(self, strict)
if strict and "text" in real:
(Path(self.args.registry_dir) / f"{self.args.job}.json").write_text(real.pop("text"))
return out
monkeypatch.setattr(stall_watch.StallWatch, "_dispatch_open", once_then_restore)
rc, prompts, notes = _during_notice(tmp_path, monkeypatch, garble)
assert rc == 0 and notes == ["stalled:idle"] and prompts == [], (prompts, notes)
def test_a_wrapper_gone_during_the_wait_gets_no_notice_and_no_nudge(tmp_path, monkeypatch):
"""Idle past S and the wrapper vanished during the same wait: the watcher
leaves before judging - no notice, no keys, the record untouched."""
calls = []
def alive(pid):
calls.append(pid)
return len(calls) <= 2 # alive at the first loop head and after the first wait, gone after
rc, prompts, notes = _during_notice(tmp_path, monkeypatch, lambda rec: None, alive=alive)
assert rc == 0 and notes == [] and prompts == [], (prompts, notes)