feat(issue-18): decide liveness and delivery by the event protocol, not the screen

Contract (no new event types): heartbeat every N s (automatic timer),
unresponsive after K x N s without an event, `started` within T s of the
injection, retained per-job state, and a remote transport.

- registry: heartbeat_sec / heartbeat_misses / ack_timeout_sec / transport
  fields; idle_timeout_sec derives K x N unless explicit (defaults 30/4/60
  keep the historical 120 s idle window).
- heartbeat.py + publish_event --heartbeat N: detached fixed-cadence timer
  publishing signed `progress` heartbeats (threaded wire publish, local
  record first); stopped by the terminal event via pidfile, by re-dispatch
  and by lifetime. Terminal events refuse --heartbeat.
- publish_event: retained `<prefix>/state` (status, last_event, seq, HMAC)
  sent on the same connection after every event (--no-state to skip);
  publisher.publish_raw takes extra messages.
- job_subscriber: subscribes the state topic, recovers terminal status from
  retained state, resets idle on any accepted message, records seq_gap and
  reconnected in the audit log, idle budget before the first event is
  T + K x N (--ack-timeout), --remote mirrors started/terminal events into
  the registry through the transition boundary.
- lib.sh send_keys_safe: an accepted-but-unconfirmed submission is confirmed
  by agent_status working or the job's own `started` (_sks_job_started);
  pane change and busy token are no longer delivery signals.
- wrapper: --heartbeat/--heartbeat-misses/--ack-timeout (MAM_* env),
  injected start command arms the timer, brief states the contract,
  _wait_started (registry / events.log / subscriber log, bounded by T and
  the wall clock) replaces the 10 s inject grace, R1 grace uses only the
  on-disk event age (herdr/pane logged once as a diagnostic),
  remote:<name> sessions skip preflight/idle/pane and dispatch over the
  broker (remote_dispatch).
- remote.py (+ scripts shim): signed retained dispatch (MAM_DISPATCH_SECRET)
  and a runner that mirrors the job into its own registry with the
  orchestrator's identity, publishes started --heartbeat, runs --exec and
  publishes the terminal event.
- loop runner: passes MAM_HEARTBEAT_SEC / MAM_HEARTBEAT_MISSES /
  MAM_ACK_TIMEOUT_S through; YAML rows with transport: remote delegate as
  remote:<name>.
- docs: MESSAGING.md 2.5 contract + remote transport, delegate-job SKILL.md,
  registry.md, loop SKILL.md.
- tests: tests/test_issue18_liveness_contract.py (16); issue 14 pane-signal
  tests flipped to started-based; issue 6 working-agent tests acknowledge
  with started; CLI snapshot/help captures refreshed.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
GodopuandClaude Fable 5.1 committed 2026-09-13 19:06:12 +09:00
1 parent 139bdd4b4c
commit 897b505266
21 files changed
+1989 -129

No files matched your search

+36 -6
View File
@@ -603,8 +603,31 @@ def test_t1_14_working_agent_reobserve_completed(tmp_path, mock_herdr):
assert len(agent_gets) >= 1, "R1 loop must query agent get"
def _publish_started_for_registered_job(reg_dir, cwd, env=None):
"""Issue #18: a working agent that never publishes `started` is a delivery
failure (rc 1) under the liveness contract. The R1 re-observation cases
below are about an agent that *did* start and then went silent, so the
watcher threads acknowledge the job the way the injected command does."""
deadline = time.time() + 20
while time.time() < deadline:
files = list(Path(reg_dir).glob("*.json"))
if files:
jid = files[0].stem
tok = json.loads(files[0].read_text()).get("dispatch_token") or ""
args = [sys.executable, str(SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"),
"--registry-dir", str(reg_dir), "--job", jid, "--event", "started", "--attempts", "1"]
if tok:
args += ["--expect-dispatch-token", tok]
run_env = dict(os.environ)
run_env.update(env or {})
subprocess.run(args, capture_output=True, text=True, cwd=str(cwd), env=run_env, timeout=60)
return jid
time.sleep(0.1)
return None
# ---------------------------------------------------------------------------
# T1-15: R1 working 고정 + 미발행 -> 벽시계 --timeout 만료 후 rc=2 error
# T1-15: R1 working 고정 + 미발행 -> Issue #18: K×N 후 rc=2 error (working 은 진단일 뿐)
# ---------------------------------------------------------------------------
def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
reg_dir = tmp_path / "jobs"
@@ -628,6 +651,7 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
st["agents"]["agt_w2"]["agent_status"] = "working"
state_file.write_text(json.dumps(st, indent=2))
_publish_started_for_registered_job(reg_dir, tmp_path)
break
except Exception:
pass
@@ -637,6 +661,10 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
t.daemon = True
t.start()
# Issue #18: `working` in herdr is a diagnostic, not liveness. An agent
# that acknowledged (started) and then publishes nothing is closed after
# K x N (--idle-timeout 2) plus one grace window - well before the wall
# clock - even though herdr keeps saying `working` the whole time.
start_t = time.time()
res = _run_wrapper(
[
@@ -646,16 +674,16 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
"--prompt", "test T1-15",
"--registry-dir", str(reg_dir),
"--workdir", str(tmp_path),
"--timeout", "5",
"--timeout", "40",
"--idle-timeout", "2",
],
cwd=tmp_path,
timeout=60,
timeout=90,
)
stop_ev.set()
elapsed = time.time() - start_t
assert res.returncode == 2, f"Expected rc=2 on wallclock timeout, got {res.returncode}"
assert elapsed >= 4.0, f"Must wait up to wallclock timeout, elapsed={elapsed}"
assert res.returncode == 2, f"Expected rc=2 (deadline_timeout), got {res.returncode}\n{res.stderr}"
assert elapsed < 39.0, f"a silent `working` agent must not be kept open to the wall clock, elapsed={elapsed}"
job_files = list(reg_dir.glob("*.json"))
assert len(job_files) == 1
@@ -663,7 +691,8 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
assert job_data.get("status") == "error"
final_st = json.loads(state_file.read_text())
agent_gets = [c for c in final_st.get("calls", []) if "agent" in c and "get" in c]
assert len(agent_gets) >= 2, "R1 loop must repeatedly query agent get during wallclock wait"
assert len(agent_gets) >= 2, "the idle handshake and the one diagnostic probe query agent get"
assert "diagnostic only" in res.stderr, "Issue #18: herdr status is logged, not used as liveness"
# ---------------------------------------------------------------------------
@@ -691,6 +720,7 @@ def test_t1_16_working_agent_done_without_event(tmp_path, mock_herdr):
if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
st["agents"]["agt_w3"]["agent_status"] = "working"
state_file.write_text(json.dumps(st, indent=2))
_publish_started_for_registered_job(reg_dir, tmp_path)
break
except Exception:
pass