fix(issue-18): admit remote attempts atomically, end timed-out commands with their group, fence heartbeats off the terminal, renew liveness only on news

Review round 2 (planner-reviewer-codex-01, NOT PASS at d09c162):

- R1 remote.mirror_job: admission under the registry lock. The same
  dispatch_token is idempotent (a finished attempt never runs again), a
  lower attempt or the same attempt under another token is refused before
  any write, and only a strictly newer attempt replaces the record
  (register_job(supersede_older_attempt=True); events.log kept, seqs stay
  monotonic).
- R2 remote.handle_dispatch: the --exec command runs in its own session;
  at timeout_sec, or when the runner is interrupted (SIGTERM now unwinds
  it), the whole process group is terminated (TERM, then KILL) before the
  error is published.
- R3 terminal barrier: a terminal publish raises an attempt-scoped barrier
  under the lock that allocates seqs, stops the heartbeat timer, gives it
  MAM_HEARTBEAT_DRAIN_S (default 1 s) to exit and then kills it, and only
  then takes its own seq. Beats are admitted under that lock
  (next_seq(admit=...)); a beat still on the wire is cancelled with the
  timer rather than holding the terminal back; and a late non-terminal
  event never publishes a running state over a terminal record.
- R4 job_subscriber: a retained state renews the idle clock only when its
  seq is newer than everything already accepted; re-delivered or older
  snapshots still recover a terminal status.
- tests: five regressions, all failing on 77f2829; T1-14's wall clock
  15 -> 30 s against public-broker latency.
- docs: MESSAGING 2.5 (barrier, state renewal, detection vs final close),
  heartbeat docstring, loop SKILL.md (remote rows need the Python runner),
  VERSIONS v4.5.0 items 1/4/5 and the test count (25).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HCGpJXhSZasJLU38ksDghe

tests/test_issue18_liveness_contract.py

@@ -30,6 +30,7 @@ import subprocess
 import sys
 import threading
 import time
+import uuid
 from pathlib import Path
 
 import pytest
@@ -505,6 +506,265 @@ def test_remote_runner_ignores_a_resent_dispatch_of_a_finished_attempt(tmp_path,
     assert _job(reg, "feed1234")["dispatch_token"] == "tok-2" and _job(reg, "feed1234")["status"] == "pending"
 
 
+def test_remote_runner_never_rolls_a_newer_attempt_back(tmp_path, monkeypatch):
+    """Review round 2 (codex R1): a validly signed dispatch for attempt 1 that
+    arrives after attempt 2 is stale. It must neither replace the record nor
+    touch its events.log, and it is never run; the same attempt under another
+    token is refused as a conflict."""
+    from lib_py.delegation import remote
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
+    monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
+    reg = tmp_path / "jobs"
+    jid = uuid.uuid4().hex[:8]
+    a1 = {"job_id": jid, "brief": "attempt one", "dispatch_token": "tok-" + uuid.uuid4().hex, "attempt_id": 1,
+          "auth_token": None, "timeout_sec": 30, "heartbeat_sec": 5, "heartbeat_misses": 2, "ack_timeout_sec": 5}
+    b2 = dict(a1, brief="attempt two", dispatch_token="tok-" + uuid.uuid4().hex, attempt_id=2)
+    assert remote.mirror_job(a1, str(reg), "s") == jid
+    assert remote.mirror_job(b2, str(reg), "s") == jid, "a newer attempt is admitted"
+    log = reg / f"{jid}.events.log"
+    log.write_text(json.dumps({"event": "started", "seq": 1, "data": {"dispatch_token": b2["dispatch_token"]}}) + "\n")
+    record_before, log_before = (reg / f"{jid}.json").read_text(), log.read_text()
+
+    assert remote.mirror_job(a1, str(reg), "s") is None, "the replayed attempt-1 dispatch is stale"
+    assert remote.mirror_job(dict(b2, dispatch_token="tok-" + uuid.uuid4().hex), str(reg), "s") is None, \
+        "the same attempt under another token is a conflict"
+    assert (reg / f"{jid}.json").read_text() == record_before, "no record rollback"
+    assert log.read_text() == log_before, "no events.log rollback"
+    ran = tmp_path / "ran"
+    assert remote.handle_dispatch(a1, str(reg), "s", f"touch {ran}") == 0
+    assert not ran.exists(), "a stale dispatch is never executed"
+    rec = _job(reg, jid)
+    assert (rec["attempt_id"], rec["dispatch_token"]) == (2, b2["dispatch_token"])
+    assert (reg / jid / "brief.md").read_text() == "attempt two"
+
+
+def test_remote_command_timeout_ends_the_whole_process_group(tmp_path, monkeypatch):
+    """Review round 2 (codex R2): at timeout_sec the runner stops the whole
+    command - not only its shell - before it reports the error, so no
+    descendant keeps working past the deadline."""
+    from lib_py.delegation import remote
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
+    monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
+    published = []
+    monkeypatch.setattr(remote, "_publish", lambda jid, reg, event, detail, d, heartbeat=None:
+                        published.append((event, detail)) or 0)
+    reg = tmp_path / "jobs"
+    pid_file, marker = tmp_path / "child.pid", tmp_path / "late.marker"
+    child = tmp_path / "child.py"
+    child.write_text("import os, time\n"
+                     f"open({str(pid_file)!r}, 'w').write(str(os.getpid()))\n"
+                     "time.sleep(2)\n"
+                     f"open({str(marker)!r}, 'w').write('too late')\n")
+    d = {"job_id": uuid.uuid4().hex[:8], "brief": "b", "dispatch_token": "tok-" + uuid.uuid4().hex, "attempt_id": 1,
+         "auth_token": None, "timeout_sec": 1, "heartbeat_sec": 5, "heartbeat_misses": 2, "ack_timeout_sec": 5}
+    pid = None
+    try:
+        # `; :` keeps the shell alive as the python child's parent (no exec)
+        rc = remote.handle_dispatch(d, str(reg), "s", f'"{sys.executable}" "{child}"; :')
+        assert rc == 124
+        assert published[-1][0] == "error" and "timeout_sec 1" in published[-1][1], published
+        pid = int(pid_file.read_text())
+        time.sleep(2.5)
+        assert not marker.exists(), "a descendant kept working past the deadline"
+        assert not _alive(pid)
+    finally:
+        if pid and _alive(pid):
+            os.kill(pid, signal.SIGKILL)
+
+
+_PAUSING_TIMER = r'''
+import json, os, sys, time
+from lib_py.delegation import heartbeat as hb, publish_event as pe
+
+capture, paused, go = sys.argv[1], sys.argv[2], sys.argv[3]
+
+
+def fake_publish(config, topic, body, retain, extra=()):
+    with open(capture, "a") as fh:
+        fh.write(json.dumps({"who": "timer", "event": json.loads(body)["event"],
+                             "state": [json.loads(b).get("status") for _t, b, _r in extra]}) + "\n")
+
+
+pe._publish_once = fake_publish
+real_prepare = pe.prepare_event
+
+
+def paused_prepare(*a, **k):
+    open(paused, "w").close()
+    deadline = time.time() + 20
+    while not os.path.exists(go) and time.time() < deadline:
+        time.sleep(0.05)
+    return real_prepare(*a, **k)
+
+
+pe.prepare_event = paused_prepare
+argv = sys.argv[5:]  # argv[4] names the module so stop_timer recognises this process as the job's timer
+job, reg = argv[argv.index("--job") + 1], argv[argv.index("--registry-dir") + 1]
+with open(hb.pidfile_path(job, reg), "w") as fh:
+    fh.write(str(os.getpid()))
+sys.exit(hb.main(argv))
+'''
+
+
+def test_no_heartbeat_follows_the_terminal_event_even_when_one_is_in_flight(tmp_path, monkeypatch):
+    """Review round 2 (codex R3): a beat already running when the agent
+    publishes `completed` must not publish after it. The timer's first beat is
+    held just before it asks for a seq; the terminal publish raises the
+    attempt's barrier, and the beat - released only then - is refused under the
+    seq lock. Nothing follows `completed` on disk or on the (stubbed) wire, and
+    the last retained state stays `completed`."""
+    from lib_py.delegation import publish_event as pe, registry
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
+    monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
+    monkeypatch.setenv("MAM_HEARTBEAT_DRAIN_S", "8")
+    reg = tmp_path / "jobs"
+    jid = registry.register_job("barrier", registry_dir=str(reg), auth_token="tok-" + uuid.uuid4().hex)
+    tok = _job(reg, jid)["dispatch_token"]
+    capture, paused, go = tmp_path / "wire.ndjson", tmp_path / "beat.paused", tmp_path / "beat.go"
+    script = tmp_path / "pausing_timer.py"
+    script.write_text(_PAUSING_TIMER)
+    env = dict(os.environ, PYTHONPATH=str(SKILLS), MAM_ENV_FILE=str(tmp_path / ".mam.env.absent"),
+               DELEGATE_JOB_LOGS_DIR=str(tmp_path / "logs"))
+    timer = subprocess.Popen([sys.executable, str(script), str(capture), str(paused), str(go),
+                              "lib_py.delegation.heartbeat", "--job", jid, "--registry-dir", str(reg),
+                              "--interval", "1", "--expect-dispatch-token", tok],
+                             cwd=str(tmp_path), env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    try:
+        deadline = time.time() + 15
+        while time.time() < deadline and not paused.exists():
+            time.sleep(0.05)
+        assert paused.exists(), "the timer's first beat never reached its publish"
+
+        def fake_publish(config, topic, body, retain, extra=()):
+            with open(capture, "a") as fh:
+                fh.write(json.dumps({"who": "terminal", "event": json.loads(body)["event"],
+                                     "state": [json.loads(b).get("status") for _t, b, _r in extra]}) + "\n")
+
+        monkeypatch.setattr(pe, "_publish_once", fake_publish)
+        result = {}
+        terminal = threading.Thread(target=lambda: result.update(rc=pe.main(
+            ["--registry-dir", str(reg), "--job", jid, "--event", "completed", "--detail", "PASS",
+             "--expect-dispatch-token", tok, "--attempts", "1"])))
+        terminal.start()
+        barrier = reg / f"{jid}.heartbeat.barrier"
+        deadline = time.time() + 10
+        while time.time() < deadline and terminal.is_alive() and not barrier.exists():
+            time.sleep(0.02)
+        go.touch()  # release the in-flight beat
+        terminal.join(timeout=30)
+        assert result.get("rc") == 0
+        assert timer.wait(timeout=30) is not None
+        wire = [json.loads(l) for l in capture.read_text().splitlines() if l.strip()]
+        assert wire and wire[-1]["event"] == "completed", f"something was published after completed: {wire}"
+        assert wire[-1]["state"] == ["completed"], wire
+        local = [e["event"] for e in _events(reg, jid)]
+        assert local and local[-1] == "completed", local
+        assert _job(reg, jid)["status"] == "completed"
+    finally:
+        if timer.poll() is None:
+            timer.kill()
+
+
+def _signed_state(job_id, seq, token, dispatch_token, status="running", attempt_id=1):
+    import hashlib, hmac
+    from lib_py.delegation import publish_event as pe
+    p = pe.build_payload(job_id, seq, "state", "progress", {"dispatch_token": dispatch_token, "attempt_id": attempt_id},
+                         token)
+    p.update(status=status, last_event="progress")
+    p["data"].pop("hmac_sig", None)
+    sp = {k: v for k, v in p.items() if k != "data"}
+    sp["data"] = dict(p["data"])
+    p["data"]["hmac_sig"] = hmac.new(token.encode(), json.dumps(sp, sort_keys=True, separators=(",", ":")).encode(),
+                                     hashlib.sha256).hexdigest()
+    return p
+
+
+def test_subscriber_state_renews_liveness_only_when_it_advances(tmp_path, monkeypatch):
+    """Review round 2 (codex R4): the same retained snapshot re-delivered (as on
+    every reconnect) or an older one is still queued - a terminal status must
+    be recoverable - but does not renew the idle clock; only a state newer than
+    every event and state already accepted does."""
+    from lib_py.delegation import job_subscriber as js
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
+    token = "t" * 24
+    w = js._Watcher({"j1"}, {"j1": token}, {"j1": 0}, {"j1": "d1"}, {"j1": 1})
+    w.on_message(None, None, _Msg("x/events", _signed("j1", 1, "started", token)))
+    w.on_message(None, None, _Msg("x/events", _signed("j1", 10, "progress", token)))
+    for _ in range(3):  # duplicate delivery of an old snapshot
+        w.on_message(None, None, _Msg("x/state", _signed("j1", 1, "state", token, status="running", last_event="started")))
+    w.on_message(None, None, _Msg("x/state", _signed("j1", 10, "state", token, status="running", last_event="progress")))
+    w.on_message(None, None, _Msg("x/state", _signed("j1", 11, "state", token, status="running", last_event="progress")))
+    states = [p for _t, p in list(w.events.queue) if p["event"] == "state"]
+    assert [p["_renews_liveness"] for p in states] == [False, False, False, False, True]
+
+
+def test_a_replayed_retained_state_does_not_keep_a_silent_job_alive(tmp_path, monkeypatch):
+    """Review round 2 (codex R4), end to end: the agent went silent after seq 10
+    and the broker keeps re-delivering an old retained snapshot, as it does on
+    every reconnect. The subscriber must still reach its idle deadline rather
+    than run into the wall clock."""
+    from lib_py.delegation import job_subscriber as js, registry
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
+    monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
+    reg = tmp_path / "jobs"
+    token = "tok-" + uuid.uuid4().hex
+    jid = registry.register_job("replay", registry_dir=str(reg), auth_token=token)
+    rec = _job(reg, jid)
+    rec["last_seq"] = 10
+    (reg / f"{jid}.json").write_text(json.dumps(rec))
+    state = _signed_state(jid, 1, token, rec["dispatch_token"])
+    topic = rec["topic_prefix"] + "/state"
+
+    class FakeClient:
+        def __init__(self):
+            self.on_message = self.on_connect = self.on_subscribe = self.on_disconnect = None
+            self.stopped = threading.Event()
+
+        def reconnect_delay_set(self, **_kw):
+            pass
+
+        def connect(self, *_a, **_kw):
+            return 0
+
+        def subscribe(self, *_a, **_kw):
+            return (0, 1)
+
+        def disconnect(self):
+            pass
+
+        def loop_stop(self):
+            self.stopped.set()
+
+        def loop_start(self):
+            def run():
+                self.on_connect(self, None, None, 0, None)
+                self.on_subscribe(self, None, 1, [1], None)
+                while not self.stopped.is_set():
+                    self.on_message(self, None, _Msg(topic, state))
+                    time.sleep(0.3)
+            threading.Thread(target=run, daemon=True).start()
+
+    fake = FakeClient()
+    monkeypatch.setattr(js, "make_client", lambda *_a, **_kw: fake)
+    started = time.monotonic()
+    rc = js.main(["--job", jid, "--registry-dir", str(reg), "--idle-timeout", "2", "--ack-timeout", "0",
+                  "--timeout", "20"])
+    elapsed = time.monotonic() - started
+    fake.stopped.set()
+    assert rc == 2
+    assert elapsed < 8, f"the replayed state kept the silent job alive for {elapsed:.1f}s"
+
+
 # ---------------------------------------------------------------------------
 # 5. wrapper acceptance criteria (broker)
 # ---------------------------------------------------------------------------

tests/test_issue6_fail_closed.py

@@ -615,7 +615,7 @@ def test_t1_14_working_agent_reobserve_completed(tmp_path, mock_herdr):
             "--prompt", "test T1-14",
             "--registry-dir", str(reg_dir),
             "--workdir", str(tmp_path),
-            "--timeout", "15",
+            "--timeout", "30",
             "--idle-timeout", "3",
         ],
         cwd=tmp_path,