fix(issue-18): bind the heartbeat timer to the agent process; scope started to the attempt; ignore re-sent finished dispatches

Review round 1 (claude NOT PASS) corrections:
- B2: the timer now watches a process - `--watch-pid`, or by default the
  nearest agent-CLI ancestor of the publishing command - and stops within
  one period once it is gone, so a crashed agent stops heartbeating and is
  closed after K x N; a remote runner watches itself. Documented as process
  liveness (an alive-but-stuck agent keeps heartbeating).
- R1: `_wait_started` accepts an events.log `started` only when it carries
  the current dispatch token (loop jobs share one events.log).
- R2: the remote runner ignores a re-sent dispatch of an attempt that
  already reached its terminal.
- R3: registry.md EOF blank line (git diff --check clean).
- R4: the ack error states the effective budget when the wall clock cut it.
- P3: stop_timer verifies the pidfile pid is this job's timer before
  signalling; a retained state message keeps its own seq watermark so it
  can never make the monotonic guard drop its own event.
- MULTI_AGENT_RULES.md timeout section updated for the contract.
- tests: agent-process death stops the timer, recycled-pid safety,
  attempt-scoped started, re-sent dispatch; acceptance 3 now kills the
  agent process instead of the timer. Help captures refreshed
  (--watch-pid). The wrapper can be sourced with
  MAM_DELEGATE_JOB_NO_MAIN=1 for helper-level tests.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff

tests/fixtures/issue_5/phase1/help-publish_event.txt

@@ -7,7 +7,8 @@ usage: publish_event.py [-h] (--job JOB | --pick-pending)
                         [--side-effect KIND=REFERENCE]
                         [--attempt-id ATTEMPT_ID]
                         [--expect-dispatch-token EXPECT_DISPATCH_TOKEN]
-                        [--heartbeat N] [--no-state] [-v]
+                        [--heartbeat N] [--no-state] [--watch-pid WATCH_PID]
+                        [-v]
 
 Publish a Job event to MQTT
 
@@ -39,4 +40,7 @@ optional arguments:
                         the job ends (Issue #18; not with a terminal event)
   --no-state            do not publish the retained <topic_prefix>/state
                         message
+  --watch-pid WATCH_PID
+                        bind the heartbeat timer to this process (default: the
+                        nearest agent-CLI ancestor; 0 = no watch)
   -v, --verbose

tests/test_issue18_liveness_contract.py

@@ -220,6 +220,61 @@ def test_heartbeat_timer_stops_on_its_own_when_the_job_is_redispatched(tmp_path)
         _kill_timer(reg, jid)
 
 
+def test_heartbeat_stops_when_the_watched_agent_process_dies(tmp_path):
+    """Review B2: the timer follows the agent's life. The 'agent' here is a
+    shell that runs the start command with --watch-pid $$ and then idles;
+    killing that shell must stop the heartbeats within about one period."""
+    reg = tmp_path / "jobs"
+    env = _env(tmp_path, UNREACHABLE)
+    jid = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "register",
+                          "--prompt", "hb"], capture_output=True, text=True, env=env, cwd=str(tmp_path)).stdout.strip()
+    agent = subprocess.Popen(
+        ["bash", "-c", f'"{sys.executable}" "{PUBLISH_PY}" --registry-dir "{reg}" --job {jid} --event started '
+                       f'--heartbeat 1 --attempts 1 --watch-pid $$ >/dev/null 2>&1; sleep 120'],
+        cwd=str(tmp_path), env=env)
+    try:
+        deadline = time.time() + 15
+        while time.time() < deadline and not _timer_pid(reg, jid):
+            time.sleep(0.1)
+        pid = _timer_pid(reg, jid)
+        assert pid and _alive(pid)
+        time.sleep(2.5)
+        before = len([e for e in _events(reg, jid) if e["event"] == "progress"])
+        assert before >= 2
+        agent.kill()  # the agent crashes
+        agent.wait(timeout=10)
+        deadline = time.time() + 6
+        while time.time() < deadline and _alive(pid):
+            time.sleep(0.1)
+        assert not _alive(pid), "timer must stop once the watched agent is gone"
+        time.sleep(1.5)
+        after = len([e for e in _events(reg, jid) if e["event"] == "progress"])
+        assert after <= before + 2, f"heartbeats kept coming after the agent died: {before} -> {after}"
+        assert _job(reg, jid)["status"] == "running", "the timer stops; the orchestrator's K x N closes the job"
+    finally:
+        if agent.poll() is None:
+            agent.kill()
+        _kill_timer(reg, jid)
+
+
+def test_stop_timer_never_signals_a_recycled_pid(tmp_path, monkeypatch):
+    """A stale pidfile (timer crashed) may name a pid that now belongs to
+    something else; stop_timer must only signal a heartbeat timer of this job."""
+    from lib_py.delegation import heartbeat
+
+    reg = tmp_path / "jobs"
+    reg.mkdir()
+    bystander = subprocess.Popen(["sleep", "30"])
+    try:
+        (reg / "job1.heartbeat.pid").write_text(str(bystander.pid))
+        assert heartbeat.running_timer("job1", str(reg)) == bystander.pid
+        assert heartbeat.stop_timer("job1", str(reg)) is False
+        assert bystander.poll() is None, "the bystander must not be killed"
+        assert not (reg / "job1.heartbeat.pid").exists(), "the stale pidfile is removed"
+    finally:
+        bystander.kill()
+
+
 def test_heartbeat_is_refused_on_a_terminal_event(tmp_path, monkeypatch):
     from lib_py.delegation import publish_event as pe, registry
 
@@ -402,6 +457,54 @@ def test_send_keys_safe_ignores_pane_change_and_busy_token(tmp_path):
         assert "PASTE_PATH_USED" not in res.stderr
 
 
+def test_wait_started_only_accepts_this_attempts_started(tmp_path):
+    """Review R1: a loop job keeps one events.log across iterations. A
+    `started` left by the previous attempt (other dispatch token) must not
+    acknowledge the new injection; one with the current token must."""
+    reg = tmp_path / "jobs"
+    reg.mkdir()
+    jid = "abcd1234"
+    (reg / f"{jid}.json").write_text(json.dumps({"job_id": jid, "status": "pending", "dispatch_token": "new-token"}))
+    log = reg / f"{jid}.events.log"
+    log.write_text(json.dumps({"event": "started", "seq": 1, "data": {"dispatch_token": "old-token"}}) + "\n")
+    env = _env(tmp_path, {"MAM_DELEGATE_JOB_NO_MAIN": "1"})
+
+    def wait_started(extra: str) -> subprocess.CompletedProcess:
+        # the wrapper sets -e: capture the helper's rc instead of dying on it
+        script = (f'source "{WRAPPER}"; set +e; REGISTRY_DIR="{reg}"; PY="{sys.executable}"; TIMEOUT=60; START_TIME=$(date +%s); '
+                  f'_disp_tok="new-token"; {extra} _wait_started {jid} 3; echo "rc=$?"')
+        return subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env, cwd=str(tmp_path), timeout=60)
+
+    res = wait_started("")
+    assert "rc=1" in res.stdout, f"old attempt's started must not count\n{res.stdout}\n{res.stderr}"
+    with open(log, "a") as fh:
+        fh.write(json.dumps({"event": "started", "seq": 2, "data": {"dispatch_token": "new-token"}}) + "\n")
+    res = wait_started("")
+    assert "rc=0" in res.stdout and "recorded 'started' in events.log" in res.stderr, res.stderr
+
+
+def test_remote_runner_ignores_a_resent_dispatch_of_a_finished_attempt(tmp_path, monkeypatch):
+    """Review R2: a retained dispatch delivered again (runner restart, broker
+    replay) must not re-run an attempt that already reached its terminal."""
+    from lib_py.delegation import remote, registry
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
+    monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
+    reg = tmp_path / "jobs"
+    d = {"job_id": "feed1234", "brief": "b", "dispatch_token": "tok-1", "attempt_id": 1, "auth_token": None,
+         "timeout_sec": 30, "heartbeat_sec": 5, "heartbeat_misses": 2, "ack_timeout_sec": 5}
+    assert remote.mirror_job(d, str(reg), "s") == "feed1234"
+    assert remote.mirror_job(d, str(reg), "s") == "feed1234", "same attempt, still open: idempotent"
+    from lib_py.delegation import mqtt_common
+    mqtt_common.update_job_status("feed1234", str(reg), _source="registry_cli", status="completed")
+    assert remote.mirror_job(d, str(reg), "s") is None, "finished attempt: the re-sent dispatch is ignored"
+    assert remote.handle_dispatch(d, str(reg), "s", "echo should-not-run") == 0
+    d2 = dict(d, dispatch_token="tok-2", attempt_id=2)
+    assert remote.mirror_job(d2, str(reg), "s") == "feed1234", "a new attempt is mirrored afresh"
+    assert _job(reg, "feed1234")["dispatch_token"] == "tok-2" and _job(reg, "feed1234")["status"] == "pending"
+
+
 # ---------------------------------------------------------------------------
 # 5. wrapper acceptance criteria (broker)
 # ---------------------------------------------------------------------------
@@ -490,14 +593,28 @@ def test_acceptance3_lost_heartbeats_close_the_job_and_a_late_completed_is_refus
                         ["--heartbeat", "1", "--heartbeat-misses", "3", "--ack-timeout", "40"], timeout_s=60)
     jid = _wait_job_id(reg)
     t0 = None
+    agent = None
     try:
         time.sleep(1)
-        assert _publish(reg, jid, "started", env, "--heartbeat", "1").returncode == 0
+        tok = _job(reg, jid)["dispatch_token"]
+        # the agent: a shell that runs the injected start command (timer bound
+        # to it via --watch-pid $$) and then keeps "working" silently
+        agent = subprocess.Popen(
+            ["bash", "-c", f'"{sys.executable}" "{PUBLISH_PY}" --registry-dir "{reg}" --job {jid} --event started '
+                           f'--expect-dispatch-token {tok} --heartbeat 1 --watch-pid $$; sleep 300'],
+            cwd=str(tmp_path), env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        deadline = time.time() + 30
+        while time.time() < deadline and not _timer_pid(reg, jid):
+            time.sleep(0.2)
+        assert _timer_pid(reg, jid), "the start command must have armed the timer"
         time.sleep(4)
-        _kill_timer(reg, jid)  # the agent dies: no more heartbeats
+        agent.kill()  # the agent dies: its timer stops, no more heartbeats
+        agent.wait(timeout=10)
         t0 = time.time()
         stdout, stderr = proc.communicate(timeout=120)
     finally:
+        if agent is not None and agent.poll() is None:
+            agent.kill()
         _kill_timer(reg, jid)
         if proc.poll() is None:
             proc.kill()