fix(issue-18): serialize the heartbeat timer lifecycle per job

Review round 4 (codex R3): two concurrent `start_timer` calls for the same
attempt both passed the "no timer" check, both spawned, and the pidfile
tracked only one - the untracked timer could still publish a retained
`running` after `completed`. A per-job flock (`<job>.heartbeat.lock`,
separate from the registry/seq lock so in-flight beats can still take
their seq) now makes observe/drain/spawn/publish-pid one ownership
operation; the terminal stop takes the same lock; a start that finds the
attempt ended (terminal record or raised barrier) does not arm; the timer
removes its own pidfile only under the lock so a replacement's pid is never
deleted. Regressions: concurrent starts leave exactly one tracked owner that
the terminal stop drains; a start after the barrier arms nothing.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff

tests/test_issue18_liveness_contract.py

@@ -258,6 +258,72 @@ def test_heartbeat_stops_when_the_watched_agent_process_dies(tmp_path):
         _kill_timer(reg, jid)
 
 
+def test_concurrent_start_timer_calls_leave_exactly_one_tracked_owner(tmp_path, monkeypatch):
+    """Review round 4 (codex R3): two `start_timer` calls for the same
+    job/attempt at once - both see no timer, both spawn, one pidfile - left an
+    untracked timer that could publish after the terminal event. The lifecycle
+    lock serialises observe/drain/spawn/publish-pid, so exactly one timer
+    survives, the pidfile names it, and the terminal stop drains it."""
+    from lib_py.delegation import heartbeat, registry
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
+    monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
+    monkeypatch.setenv("MQTT_BROKER", "127.0.0.1")
+    monkeypatch.setenv("MQTT_PORT", "9")
+    reg = tmp_path / "jobs"
+    jid = registry.register_job("race", registry_dir=str(reg))
+    tok = _job(reg, jid)["dispatch_token"]
+    pids = []
+    errors = []
+
+    def start():
+        try:
+            pids.append(heartbeat.start_timer(jid, str(reg), 1, dispatch_token=tok, attempt_id=1, watch_pid=0))
+        except Exception as exc:  # pragma: no cover - surfaced below
+            errors.append(exc)
+
+    threads = [threading.Thread(target=start) for _ in range(2)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join(timeout=60)
+    try:
+        assert not errors, errors
+        assert len(pids) == 2 and all(pids)
+        time.sleep(1.0)
+        alive = [p for p in pids if _alive(p)]
+        assert len(alive) == 1, f"exactly one timer may survive two concurrent starts, alive={alive} pids={pids}"
+        assert _timer_pid(reg, jid) == alive[0], "the pidfile names the surviving owner"
+        # the terminal path drains the tracked owner and nothing else is left
+        assert heartbeat.stop_timer(jid, str(reg), drain=5.0) is True
+        assert not any(_alive(p) for p in pids)
+        assert _timer_pid(reg, jid) is None
+    finally:
+        for p in pids:
+            try:
+                os.kill(p, signal.SIGKILL)
+            except OSError:
+                pass
+
+
+def test_start_timer_after_the_terminal_barrier_does_not_arm(tmp_path, monkeypatch):
+    """A `started --heartbeat` that loses the race against the terminal
+    publish must not leave a timer behind."""
+    from lib_py.delegation import heartbeat, registry
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
+    monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
+    reg = tmp_path / "jobs"
+    jid = registry.register_job("late-start", registry_dir=str(reg))
+    tok = _job(reg, jid)["dispatch_token"]
+    heartbeat.raise_barrier(jid, str(reg), dispatch_token=tok, attempt_id=1)
+    assert heartbeat.start_timer(jid, str(reg), 1, dispatch_token=tok, attempt_id=1, watch_pid=0) == 0
+    assert _timer_pid(reg, jid) is None
+    assert not (reg / f"{jid}.heartbeat.pid").exists()
+
+
 def test_stop_timer_never_signals_a_recycled_pid(tmp_path, monkeypatch):
     """A stale pidfile (timer crashed) may name a pid that now belongs to
     something else; stop_timer must only signal a heartbeat timer of this job."""