fix(issue-30): review round 2 - the watcher is stopped with KILL, and a strict last gate sits between the notice and the keys
grok N1: the wrapper spent up to 5 s on a TERM budget before it finalized the job, so when the watcher swallowed the TERM the silent job's close came ~5 s late and a `completed` written in that gap won (test_issue14 idle-grace). codex D1: `_notify` blocks on a publish for up to 120 s, and the prompt went out right after it; a completed, a re-dispatch or an unreadable record in that window still got the nudge. codex D2: the wrapper pid was checked only at the loop head, so a watcher whose wrapper vanished during a wait could still notify and nudge once. - multi-agent-mux-delegate-job `stop_stall_watch`: KILL + wait, no budget. The watcher holds nothing to flush (its TERM handler is a bare os._exit), and KILL cannot be swallowed. - lib_py/stall_watch.py: `_wrapper_gone()` after every wait (before any notice) and at the loop head; `_may_prompt()` right before the keys - the wrapper is alive and the record, read then, is open and this dispatch; `_dispatch_open(strict=True)` treats an unreadable record as "no". - SKILL.md: the lifetime contract states the one-wait lag and that nothing is sent after the wrapper is seen gone. - tests/test_issue30_stall_watch_stop.py: the stop returns at once against a TERM-ignoring stand-in (took 6 s on the round-2 code); completed, re-dispatch and a torn record during the notice publish withhold the nudge; a wrapper gone during the wait gets no notice and no nudge. All five fail on the round-2 code. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
1 parent
af0a2a2557
commit
9200562c91
4 files changed
+144
-31
No files matched your search
@@ -263,18 +263,20 @@ class StallWatch(object):
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def _dispatch_open(self):
|
||||
def _dispatch_open(self, strict=False):
|
||||
"""Whether the job is still open *and* still the dispatch we were started
|
||||
for: a `completed` or a re-dispatch that landed during a blocking herdr
|
||||
wait must not be nudged (review R2). A missing record counts as gone;
|
||||
an unreadable one as a transient."""
|
||||
an unreadable one as a transient - unless `strict` (the check that
|
||||
permits a prompt), where only a readable, open record of this dispatch
|
||||
says yes (round-2 review D1)."""
|
||||
try:
|
||||
with open(os.path.join(self.args.registry_dir, self.args.job + ".json")) as f:
|
||||
rec = json.load(f)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except (OSError, ValueError):
|
||||
return True
|
||||
return not strict
|
||||
if rec.get("status") in TERMINAL_STATUSES:
|
||||
return False
|
||||
tok = self.args.expect_dispatch_token
|
||||
@@ -285,6 +287,19 @@ class StallWatch(object):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _wrapper_gone(self):
|
||||
if self.args.watch_pid and not _alive(self.args.watch_pid):
|
||||
log.info("stall_watch: wrapper pid %d is gone; stall watch stops", self.args.watch_pid)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _may_prompt(self):
|
||||
"""The last gate before keys reach the pane: the wrapper is alive and the
|
||||
record, read now, is this dispatch and open. Anything the notice's
|
||||
publish (up to 120 s) let happen - a completed, a re-dispatch, a dead
|
||||
wrapper, an unreadable record - withholds the nudge."""
|
||||
return not self._wrapper_gone() and self._dispatch_open(strict=True)
|
||||
|
||||
def run(self) -> int:
|
||||
st, sig, agent = self.st, Signals(), self.args.agent
|
||||
if not st.reliable:
|
||||
@@ -295,11 +310,9 @@ class StallWatch(object):
|
||||
sig.job_status = self._job_status()
|
||||
if sig.job_status in TERMINAL_STATUSES or not self._dispatch_open():
|
||||
return RC_TERMINAL
|
||||
if self.args.watch_pid and not _alive(self.args.watch_pid):
|
||||
if self._wrapper_gone():
|
||||
# The wrapper is gone (killed, crashed): nobody will stop us, so
|
||||
# stop ourselves. Checked between herdr calls, so at most one
|
||||
# wait (<= 60 s) late.
|
||||
log.info("stall_watch: wrapper pid %d is gone; stall watch stops", self.args.watch_pid)
|
||||
# stop ourselves - here, after every wait and before a prompt.
|
||||
return RC_TERMINAL
|
||||
if sig.agent_status is None:
|
||||
until, tmo = watcher.VALID_STATES, 1000 # learn the current state
|
||||
@@ -313,6 +326,8 @@ class StallWatch(object):
|
||||
else:
|
||||
until, tmo = ("idle", "done", "blocked"), 60000
|
||||
out = watcher.wait_once(self.args.target, until=until, timeout_ms=tmo, session=self.args.herdr_session)
|
||||
if self._wrapper_gone(): # round-2 review D2: before any notice
|
||||
return RC_TERMINAL
|
||||
if out.kind in (watcher.TRANSPORT_ERROR, watcher.AGENT_NOT_FOUND):
|
||||
code = "agent_not_found" if out.kind == watcher.AGENT_NOT_FOUND else (out.error_code or "transport_error")
|
||||
now = time.monotonic()
|
||||
@@ -390,6 +405,10 @@ class StallWatch(object):
|
||||
if not self._notify("stalled:idle"):
|
||||
log.warning("stall_watch: the stalled:idle notice was refused; nudge withheld")
|
||||
return idle_since
|
||||
if not self._may_prompt(): # the publish can take long; re-read before the keys
|
||||
log.info("stall_watch: job %s closed, re-dispatched or unreadable, or the wrapper left, "
|
||||
"during the notice; nudge withheld", self.args.job)
|
||||
return idle_since
|
||||
self.src.prompt(NUDGE.format(job=self.args.job))
|
||||
return idle_since
|
||||
|
||||
@@ -416,7 +435,7 @@ def main(argv=None) -> int:
|
||||
p.add_argument("--watch-pid", type=int, default=None, help="exit once this process (the wrapper) is gone")
|
||||
args = p.parse_args(argv)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s stall_watch %(levelname)s %(message)s")
|
||||
# The wrapper stops us with SIGTERM once its subscriber ends: leave quietly.
|
||||
# The wrapper stops us with SIGKILL once its subscriber ends; a TERM from anyone else: leave quietly.
|
||||
signal.signal(signal.SIGTERM, lambda *_: os._exit(RC_TERMINAL))
|
||||
return StallWatch(args).run()
|
||||
|
||||
|
||||
Reference in new issue
Block a user