fix(issue-18): let an exiting heartbeat timer skip the busy lifecycle lock; kill a timer whose pid cannot be recorded

Review round 5 (planner-reviewer-codex-01, NOT PASS at 2b1d590):

- F1 bounded lock cycle: the stopper holds the per-job lifecycle lock while
  it waits for the timer to exit, and the timer's exit cleanup waited for
  that same lock, so every stop ran out the drain and ended in SIGKILL with
  a false "did not drain" warning. The exiting timer now tries the lock
  without blocking (lifecycle_lock(blocking=False)); when it is busy the
  ownership operation holding it removes the pidfile, and when it is free
  the own-pid comparison is kept.
- F2 untracked owner: when writing the new timer's pid fails after the
  spawn, _spawn_timer_locked kills and reaps that exact process while the
  lock is still held and drops only a partial pidfile of that attempt,
  then re-raises.
- tests: an idle timer stops well before a 10 s drain without SIGKILL;
  a pidfile failure after spawn leaves no survivor, no pidfile and a free
  lock (both fail on 2b1d590); a spawn failure leaves nothing behind and
  the next start works.
- docs: MESSAGING 2.5 and VERSIONS item 1 describe the lifecycle lock;
  VERSIONS test count 31.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HCGpJXhSZasJLU38ksDghe

.agents/skills/lib_py/delegation/heartbeat.py

@@ -184,7 +184,7 @@ def lifecycle_lock_path(job_id: str, registry_dir: str) -> str:
 
 
 @contextlib.contextmanager
-def lifecycle_lock(job_id: str, registry_dir: str):
+def lifecycle_lock(job_id: str, registry_dir: str, blocking: bool = True):
     """The per-job timer ownership lock (review round 4, codex R3).
 
     Observing/draining the previous owner, spawning the replacement and
@@ -195,12 +195,17 @@ def lifecycle_lock(job_id: str, registry_dir: str):
     not arm at all. This is a separate flock from the registry/seq lock on
     purpose: an in-flight beat needs that lock to take its seq, and the drain
     waits for exactly those beats, so holding the seq lock across the drain
-    would deadlock."""
+    would deadlock. With ``blocking=False`` it yields False instead of waiting
+    when the lock is busy - an exiting timer uses that (review round 5, codex F1)."""
     os.makedirs(registry_dir, exist_ok=True)
     with open(lifecycle_lock_path(job_id, registry_dir), "a+") as fh:
-        fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
         try:
-            yield
+            fcntl.flock(fh.fileno(), fcntl.LOCK_EX if blocking else fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            yield False  # busy and non-blocking: the caller must not touch owned state
+            return
+        try:
+            yield True
         finally:
             fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
 
@@ -335,8 +340,28 @@ def _spawn_timer_locked(job_id, registry_dir, interval, dispatch_token, attempt_
     with open(log_path, "ab") as log:
         proc = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT,
                                 start_new_session=True, env=env, cwd=os.getcwd())
-    with open(pidfile_path(job_id, registry_dir), "w") as fh:
-        fh.write(str(proc.pid))
+    pidfile = pidfile_path(job_id, registry_dir)
+    try:
+        with open(pidfile, "w") as fh:
+            fh.write(str(proc.pid))
+    except BaseException:
+        # Review round 5 (codex F2): a timer whose pid could not be published
+        # would be an owner nobody can find or stop. End and reap it here, while
+        # the lifecycle lock is still held, and drop only a partial pidfile of
+        # this attempted owner (the previous owner's was removed by the stop).
+        try:
+            proc.kill()
+            proc.wait(timeout=5)
+        except Exception:
+            pass
+        try:
+            with open(pidfile) as fh:
+                partial = fh.read().strip()
+            if not partial or str(proc.pid).startswith(partial):
+                os.unlink(pidfile)
+        except (OSError, ValueError):
+            pass
+        raise
     logger.info("heartbeat timer pid %d for job %s every %ss (watching pid %s)",
                 proc.pid, job_id, interval, watch_pid or "none")
     return proc.pid
@@ -439,13 +464,18 @@ def run(job_id: str, registry_dir: str, interval: int, dispatch_token=None,
     for t in in_flight:
         t.join(timeout=15)
     # Under the lifecycle lock: a replacement may be writing its own pid to
-    # this file right now, and that owner's pidfile must not be deleted.
+    # this file right now, and that owner's pidfile must not be deleted. Never
+    # *wait* for the lock here (review round 5, codex F1): a stopper holds it
+    # while it waits for this very exit, so waiting would only end in SIGKILL at
+    # its drain deadline. When the lock is busy its holder is an ownership
+    # operation, and that operation removes the pidfile itself.
     try:
-        with lifecycle_lock(job_id, registry_dir):
-            with open(own_pidfile) as fh:
-                mine = fh.read().strip() == str(os.getpid())
-            if mine:
-                os.unlink(own_pidfile)
+        with lifecycle_lock(job_id, registry_dir, blocking=False) as owned:
+            if owned:
+                with open(own_pidfile) as fh:
+                    mine = fh.read().strip() == str(os.getpid())
+                if mine:
+                    os.unlink(own_pidfile)
     except (OSError, ValueError):
         pass
     return 0