feat(issue-18): decide liveness and delivery by the event protocol, not the screen

Contract (no new event types): heartbeat every N s (automatic timer),
unresponsive after K x N s without an event, `started` within T s of the
injection, retained per-job state, and a remote transport.

- registry: heartbeat_sec / heartbeat_misses / ack_timeout_sec / transport
  fields; idle_timeout_sec derives K x N unless explicit (defaults 30/4/60
  keep the historical 120 s idle window).
- heartbeat.py + publish_event --heartbeat N: detached fixed-cadence timer
  publishing signed `progress` heartbeats (threaded wire publish, local
  record first); stopped by the terminal event via pidfile, by re-dispatch
  and by lifetime. Terminal events refuse --heartbeat.
- publish_event: retained `<prefix>/state` (status, last_event, seq, HMAC)
  sent on the same connection after every event (--no-state to skip);
  publisher.publish_raw takes extra messages.
- job_subscriber: subscribes the state topic, recovers terminal status from
  retained state, resets idle on any accepted message, records seq_gap and
  reconnected in the audit log, idle budget before the first event is
  T + K x N (--ack-timeout), --remote mirrors started/terminal events into
  the registry through the transition boundary.
- lib.sh send_keys_safe: an accepted-but-unconfirmed submission is confirmed
  by agent_status working or the job's own `started` (_sks_job_started);
  pane change and busy token are no longer delivery signals.
- wrapper: --heartbeat/--heartbeat-misses/--ack-timeout (MAM_* env),
  injected start command arms the timer, brief states the contract,
  _wait_started (registry / events.log / subscriber log, bounded by T and
  the wall clock) replaces the 10 s inject grace, R1 grace uses only the
  on-disk event age (herdr/pane logged once as a diagnostic),
  remote:<name> sessions skip preflight/idle/pane and dispatch over the
  broker (remote_dispatch).
- remote.py (+ scripts shim): signed retained dispatch (MAM_DISPATCH_SECRET)
  and a runner that mirrors the job into its own registry with the
  orchestrator's identity, publishes started --heartbeat, runs --exec and
  publishes the terminal event.
- loop runner: passes MAM_HEARTBEAT_SEC / MAM_HEARTBEAT_MISSES /
  MAM_ACK_TIMEOUT_S through; YAML rows with transport: remote delegate as
  remote:<name>.
- docs: MESSAGING.md 2.5 contract + remote transport, delegate-job SKILL.md,
  registry.md, loop SKILL.md.
- tests: tests/test_issue18_liveness_contract.py (16); issue 14 pane-signal
  tests flipped to started-based; issue 6 working-agent tests acknowledge
  with started; CLI snapshot/help captures refreshed.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff

tests/test_issue14_delivery_ack.py

@@ -85,24 +85,63 @@ def _assert_single_injection(res: subprocess.CompletedProcess):
 
 
 # ---------------------------------------------------------------------------
-# lib.sh: accepted submission confirmed through the pane
+# lib.sh: accepted submission confirmed by the job's own `started` (Issue #18)
 # ---------------------------------------------------------------------------
-def test_accepted_timeout_confirmed_by_busy_token(tmp_path):
+def _job_that_starts(tmp_path, delay_s: float):
+    """A registered job whose `started` lands `delay_s` after the call - the
+    protocol signal that replaced the pane (Issue #18). Returns (env, job_id)."""
+    import threading
+    from lib_py.delegation import registry
+
+    reg = tmp_path / "jobs"
+    env = {"MAM_ENV_FILE": str(tmp_path / ".mam.env.absent"), "DELEGATE_JOB_LOGS_DIR": str(tmp_path / "logs"),
+           "MQTT_BROKER": "127.0.0.1", "MQTT_PORT": "9", "SKS_REGISTRY_DIR": str(reg)}
+    saved = {k: os.environ.get(k) for k in env}
+    os.environ.update(env)
+    try:
+        jid = registry.register_job("issue 14 ack", registry_dir=str(reg))
+    finally:
+        for k, v in saved.items():
+            if v is None:
+                os.environ.pop(k, None)
+            else:
+                os.environ[k] = v
+
+    def start():
+        time.sleep(delay_s)
+        subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg), "--job", jid,
+                        "--event", "started", "--attempts", "1"], capture_output=True, env=_base_env(tmp_path, env))
+
+    threading.Thread(target=start, daemon=True).start()
+    return env, jid
+
+
+def test_accepted_timeout_confirmed_by_started_event_busy_token_alone_is_not_enough(tmp_path):
+    """The pane shows the busy token, but that is not delivery any more: the
+    job's `started` (published 2 s in) is what confirms it, and the busy token
+    on its own (second run) still fails closed."""
+    env, jid = _job_that_starts(tmp_path, 2.0)
     script = _codex_like_mock(CODEX_IDLE, CODEX_WORKING) + \
-        'send_keys_safe "reviewer-codex-01" "review the triage report" "job_14a"'
-    res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "5"})
+        f'send_keys_safe "reviewer-codex-01" "review the triage report" "{jid}"'
+    res = _run_lib_sh(script, tmp_path, env={**env, "SKS_DELIVERY_GRACE_S": "8"})
     assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
     _assert_single_injection(res)
     assert "delivery unknown" not in res.stderr
 
+    script = _codex_like_mock(CODEX_IDLE, CODEX_WORKING) + \
+        'send_keys_safe "reviewer-codex-01" "review the triage report" "job_14a"'
+    res = _run_lib_sh(script, tmp_path, env={**env, "SKS_DELIVERY_GRACE_S": "2"})
+    assert res.returncode == 4, f"busy token must not confirm delivery: rc={res.returncode}\n{res.stderr}"
+    _assert_single_injection(res)
 
-def test_accepted_timeout_confirmed_by_pane_change_after_fast_turn(tmp_path):
-    """A turn that already finished before the first poll shows no busy token,
-    only the answer; the pane moving on from its pre-submit snapshot is enough."""
+
+def test_accepted_timeout_pane_change_after_fast_turn_is_not_delivery(tmp_path):
+    """A turn that already finished shows only the answer. The pane moving on
+    used to count; under the protocol contract only `started` does."""
     script = _codex_like_mock(CODEX_IDLE, CODEX_ANSWERED) + \
         'send_keys_safe "reviewer-codex-01" "Reply with OK" "job_14b"'
-    res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "5"})
-    assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
+    res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "2"})
+    assert res.returncode == 4, f"rc={res.returncode}\n{res.stderr}"
     _assert_single_injection(res)
 
 
@@ -147,7 +186,8 @@ def test_transport_error_is_not_treated_as_accepted(tmp_path):
 def test_mock_herdr_timeout_with_idle_status_is_confirmed(tmp_path, mock_herdr):
     """Same scenario through the shared mock herdr: `timeout` from agent prompt,
     agent_status pinned to idle, the pane gaining the busy token after
-    submission. Exactly one prompt call, no paste-buffer fallback."""
+    submission - and the job's `started` landing 2 s in, which is what
+    confirms it (Issue #18). Exactly one prompt call, no paste-buffer fallback."""
     state_file = Path(mock_herdr)
     state = json.loads(state_file.read_text())
     state["agents"]["agt_codex"] = {
@@ -158,9 +198,10 @@ def test_mock_herdr_timeout_with_idle_status_is_confirmed(tmp_path, mock_herdr):
     }
     state_file.write_text(json.dumps(state, indent=2))
 
+    env, jid = _job_that_starts(tmp_path, 2.0)
     res = _run_lib_sh(
-        'send_keys_safe "agt_codex" "review please" "job_14f"',
-        tmp_path, env={"SKS_IDLE_HANDSHAKE": "0", "SKS_DELIVERY_GRACE_S": "5"},
+        f'send_keys_safe "agt_codex" "review please" "{jid}"',
+        tmp_path, env={**env, "SKS_IDLE_HANDSHAKE": "0", "SKS_DELIVERY_GRACE_S": "8"},
     )
     assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
 
@@ -243,13 +284,14 @@ def test_wrapper_honours_registry_ack_over_unconfirmed_inject(tmp_path, mock_her
 
 
 # ---------------------------------------------------------------------------
-# wrapper: idle-timeout grace (R1) must see a working codex through the pane
+# wrapper: idle-timeout grace (R1) is decided by the contract, not the pane
 # ---------------------------------------------------------------------------
-def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr):
-    """herdr keeps reporting `idle` for a codex-like agent that is visibly
-    working (busy token in the pane). When the subscriber idles out, the
-    wrapper must keep waiting for the on-disk result instead of closing the
-    job as deadline_timeout mid-turn; the agent then completes out of band."""
+def test_wrapper_idle_timeout_grace_ignores_pane_busy_token(tmp_path, mock_herdr):
+    """Issue #18 flips the Issue #14 expectation: a codex-like agent that is
+    visibly working (busy token in the pane) but publishes nothing after its
+    `started` is unresponsive once K x N (here --idle-timeout 3 s) has passed.
+    The wrapper closes it as deadline_timeout and a `completed` written 14 s
+    in is refused by the CAS table - the pane never extended the window."""
     reg_dir = tmp_path / "jobs"
     reg_dir.mkdir(parents=True, exist_ok=True)
 
@@ -265,7 +307,7 @@ def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr
     }
     state_file.write_text(json.dumps(state, indent=2))
 
-    env = _base_env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_DELIVERY_GRACE_S": "5"})
+    env = _base_env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_DELIVERY_GRACE_S": "2", "MAM_INJECT_ACK_GRACE_S": "20"})
     proc = subprocess.Popen(
         [
             "bash", str(WRAPPER), "submit",
@@ -289,19 +331,29 @@ def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr
         time.sleep(0.1)
     assert jid, "job was not registered"
 
-    # Well past the 3 s idle timeout, the agent finishes.
+    # The agent acknowledges (started) without arming a heartbeat timer ...
+    time.sleep(1)
+    tok = json.loads((reg_dir / f"{jid}.json").read_text())["dispatch_token"]
+    ack = subprocess.run(
+        [sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg_dir), "--job", jid, "--event", "started",
+         "--expect-dispatch-token", tok],
+        capture_output=True, text=True, cwd=str(tmp_path), env=env,
+    )
+    assert ack.returncode == 0, ack.stderr
+    # ... and goes silent while its pane keeps showing the busy token.
     time.sleep(14)
     done = subprocess.run(
         [sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg_dir),
          "update", "--job", jid, "--status", "completed"],
         capture_output=True, text=True, cwd=str(tmp_path), env=env,
     )
-    assert done.returncode == 0, done.stderr
 
     stdout, stderr = proc.communicate(timeout=90)
     job_data = json.loads((reg_dir / f"{jid}.json").read_text())
-    assert job_data.get("status") == "completed", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
-    assert proc.returncode == 0, f"expected rc=0 after the working grace, got {proc.returncode}\n{stdout}\n{stderr}"
+    assert proc.returncode == 2, f"expected deadline_timeout (rc=2): the pane is not liveness, got {proc.returncode}\n{stdout}\n{stderr}"
+    assert job_data.get("status") == "error", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
+    assert done.returncode != 0 or "refused" in done.stderr, "the late completed must be refused"
+    assert "diagnostic only" in stderr
 
 
 # ---------------------------------------------------------------------------
@@ -451,8 +503,9 @@ def test_wrapper_idle_grace_never_outlives_the_wall_clock(tmp_path):
     # The heartbeats were honoured: the job was not closed after the first idle
     # window, it stayed open up to the wall clock ...
     assert elapsed >= timeout_s - 1, f"closed before the wall clock: {elapsed:.1f}s\n{stderr}"
-    # ... and not a moment past it beyond closing and publishing the error.
-    assert elapsed <= timeout_s + 12, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}"
+    # ... and not a moment past it beyond closing and publishing the error
+    # (the error publish alone can take ~10 s against the public broker).
+    assert elapsed <= timeout_s + 15, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}"
 
 
 # ---------------------------------------------------------------------------
@@ -489,7 +542,7 @@ def test_publish_event_records_the_heartbeat_before_the_wire_publish(tmp_path, m
     events_log = reg_dir / f"{jid}.events.log"
     on_disk_at_publish = []
 
-    def slow_broker_publish(config, topic, body, retain):
+    def slow_broker_publish(config, topic, body, retain, extra=()):
         on_disk_at_publish.append([e.get("event") for e in _logged_events(events_log)])
 
     monkeypatch.setattr(pe, "_publish_once", slow_broker_publish)
@@ -509,7 +562,7 @@ def test_publish_event_failed_publish_keeps_one_local_record(tmp_path, monkeypat
     events_log = reg_dir / f"{jid}.events.log"
     lines_seen = []
 
-    def unreachable_broker(config, topic, body, retain):
+    def unreachable_broker(config, topic, body, retain, extra=()):
         lines_seen.append(len(_logged_events(events_log)))
         raise ConnectionError("broker unreachable")