feat(issue-18): decide liveness and delivery by the event protocol, not the screen
Contract (no new event types): heartbeat every N s (automatic timer), unresponsive after K x N s without an event, `started` within T s of the injection, retained per-job state, and a remote transport. - registry: heartbeat_sec / heartbeat_misses / ack_timeout_sec / transport fields; idle_timeout_sec derives K x N unless explicit (defaults 30/4/60 keep the historical 120 s idle window). - heartbeat.py + publish_event --heartbeat N: detached fixed-cadence timer publishing signed `progress` heartbeats (threaded wire publish, local record first); stopped by the terminal event via pidfile, by re-dispatch and by lifetime. Terminal events refuse --heartbeat. - publish_event: retained `<prefix>/state` (status, last_event, seq, HMAC) sent on the same connection after every event (--no-state to skip); publisher.publish_raw takes extra messages. - job_subscriber: subscribes the state topic, recovers terminal status from retained state, resets idle on any accepted message, records seq_gap and reconnected in the audit log, idle budget before the first event is T + K x N (--ack-timeout), --remote mirrors started/terminal events into the registry through the transition boundary. - lib.sh send_keys_safe: an accepted-but-unconfirmed submission is confirmed by agent_status working or the job's own `started` (_sks_job_started); pane change and busy token are no longer delivery signals. - wrapper: --heartbeat/--heartbeat-misses/--ack-timeout (MAM_* env), injected start command arms the timer, brief states the contract, _wait_started (registry / events.log / subscriber log, bounded by T and the wall clock) replaces the 10 s inject grace, R1 grace uses only the on-disk event age (herdr/pane logged once as a diagnostic), remote:<name> sessions skip preflight/idle/pane and dispatch over the broker (remote_dispatch). - remote.py (+ scripts shim): signed retained dispatch (MAM_DISPATCH_SECRET) and a runner that mirrors the job into its own registry with the orchestrator's identity, publishes started --heartbeat, runs --exec and publishes the terminal event. - loop runner: passes MAM_HEARTBEAT_SEC / MAM_HEARTBEAT_MISSES / MAM_ACK_TIMEOUT_S through; YAML rows with transport: remote delegate as remote:<name>. - docs: MESSAGING.md 2.5 contract + remote transport, delegate-job SKILL.md, registry.md, loop SKILL.md. - tests: tests/test_issue18_liveness_contract.py (16); issue 14 pane-signal tests flipped to started-based; issue 6 working-agent tests acknowledge with started; CLI snapshot/help captures refreshed. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
1 parent
139bdd4b4c
commit
897b505266
21 files changed
+1989
-129
No files matched your search
@@ -603,8 +603,31 @@ def test_t1_14_working_agent_reobserve_completed(tmp_path, mock_herdr):
|
||||
assert len(agent_gets) >= 1, "R1 loop must query agent get"
|
||||
|
||||
|
||||
def _publish_started_for_registered_job(reg_dir, cwd, env=None):
|
||||
"""Issue #18: a working agent that never publishes `started` is a delivery
|
||||
failure (rc 1) under the liveness contract. The R1 re-observation cases
|
||||
below are about an agent that *did* start and then went silent, so the
|
||||
watcher threads acknowledge the job the way the injected command does."""
|
||||
deadline = time.time() + 20
|
||||
while time.time() < deadline:
|
||||
files = list(Path(reg_dir).glob("*.json"))
|
||||
if files:
|
||||
jid = files[0].stem
|
||||
tok = json.loads(files[0].read_text()).get("dispatch_token") or ""
|
||||
args = [sys.executable, str(SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"),
|
||||
"--registry-dir", str(reg_dir), "--job", jid, "--event", "started", "--attempts", "1"]
|
||||
if tok:
|
||||
args += ["--expect-dispatch-token", tok]
|
||||
run_env = dict(os.environ)
|
||||
run_env.update(env or {})
|
||||
subprocess.run(args, capture_output=True, text=True, cwd=str(cwd), env=run_env, timeout=60)
|
||||
return jid
|
||||
time.sleep(0.1)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# T1-15: R1 working 고정 + 미발행 -> 벽시계 --timeout 만료 후 rc=2 error
|
||||
# T1-15: R1 working 고정 + 미발행 -> Issue #18: K×N 후 rc=2 error (working 은 진단일 뿐)
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
reg_dir = tmp_path / "jobs"
|
||||
@@ -628,6 +651,7 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
|
||||
st["agents"]["agt_w2"]["agent_status"] = "working"
|
||||
state_file.write_text(json.dumps(st, indent=2))
|
||||
_publish_started_for_registered_job(reg_dir, tmp_path)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
@@ -637,6 +661,10 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
# Issue #18: `working` in herdr is a diagnostic, not liveness. An agent
|
||||
# that acknowledged (started) and then publishes nothing is closed after
|
||||
# K x N (--idle-timeout 2) plus one grace window - well before the wall
|
||||
# clock - even though herdr keeps saying `working` the whole time.
|
||||
start_t = time.time()
|
||||
res = _run_wrapper(
|
||||
[
|
||||
@@ -646,16 +674,16 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
"--prompt", "test T1-15",
|
||||
"--registry-dir", str(reg_dir),
|
||||
"--workdir", str(tmp_path),
|
||||
"--timeout", "5",
|
||||
"--timeout", "40",
|
||||
"--idle-timeout", "2",
|
||||
],
|
||||
cwd=tmp_path,
|
||||
timeout=60,
|
||||
timeout=90,
|
||||
)
|
||||
stop_ev.set()
|
||||
elapsed = time.time() - start_t
|
||||
assert res.returncode == 2, f"Expected rc=2 on wallclock timeout, got {res.returncode}"
|
||||
assert elapsed >= 4.0, f"Must wait up to wallclock timeout, elapsed={elapsed}"
|
||||
assert res.returncode == 2, f"Expected rc=2 (deadline_timeout), got {res.returncode}\n{res.stderr}"
|
||||
assert elapsed < 39.0, f"a silent `working` agent must not be kept open to the wall clock, elapsed={elapsed}"
|
||||
|
||||
job_files = list(reg_dir.glob("*.json"))
|
||||
assert len(job_files) == 1
|
||||
@@ -663,7 +691,8 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
assert job_data.get("status") == "error"
|
||||
final_st = json.loads(state_file.read_text())
|
||||
agent_gets = [c for c in final_st.get("calls", []) if "agent" in c and "get" in c]
|
||||
assert len(agent_gets) >= 2, "R1 loop must repeatedly query agent get during wallclock wait"
|
||||
assert len(agent_gets) >= 2, "the idle handshake and the one diagnostic probe query agent get"
|
||||
assert "diagnostic only" in res.stderr, "Issue #18: herdr status is logged, not used as liveness"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -691,6 +720,7 @@ def test_t1_16_working_agent_done_without_event(tmp_path, mock_herdr):
|
||||
if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
|
||||
st["agents"]["agt_w3"]["agent_status"] = "working"
|
||||
state_file.write_text(json.dumps(st, indent=2))
|
||||
_publish_started_for_registered_job(reg_dir, tmp_path)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
Reference in new issue
Block a user