feat(issue-18): decide liveness and delivery by the event protocol, not the screen

Contract (no new event types): heartbeat every N s (automatic timer),
unresponsive after K x N s without an event, `started` within T s of the
injection, retained per-job state, and a remote transport.

- registry: heartbeat_sec / heartbeat_misses / ack_timeout_sec / transport
  fields; idle_timeout_sec derives K x N unless explicit (defaults 30/4/60
  keep the historical 120 s idle window).
- heartbeat.py + publish_event --heartbeat N: detached fixed-cadence timer
  publishing signed `progress` heartbeats (threaded wire publish, local
  record first); stopped by the terminal event via pidfile, by re-dispatch
  and by lifetime. Terminal events refuse --heartbeat.
- publish_event: retained `<prefix>/state` (status, last_event, seq, HMAC)
  sent on the same connection after every event (--no-state to skip);
  publisher.publish_raw takes extra messages.
- job_subscriber: subscribes the state topic, recovers terminal status from
  retained state, resets idle on any accepted message, records seq_gap and
  reconnected in the audit log, idle budget before the first event is
  T + K x N (--ack-timeout), --remote mirrors started/terminal events into
  the registry through the transition boundary.
- lib.sh send_keys_safe: an accepted-but-unconfirmed submission is confirmed
  by agent_status working or the job's own `started` (_sks_job_started);
  pane change and busy token are no longer delivery signals.
- wrapper: --heartbeat/--heartbeat-misses/--ack-timeout (MAM_* env),
  injected start command arms the timer, brief states the contract,
  _wait_started (registry / events.log / subscriber log, bounded by T and
  the wall clock) replaces the 10 s inject grace, R1 grace uses only the
  on-disk event age (herdr/pane logged once as a diagnostic),
  remote:<name> sessions skip preflight/idle/pane and dispatch over the
  broker (remote_dispatch).
- remote.py (+ scripts shim): signed retained dispatch (MAM_DISPATCH_SECRET)
  and a runner that mirrors the job into its own registry with the
  orchestrator's identity, publishes started --heartbeat, runs --exec and
  publishes the terminal event.
- loop runner: passes MAM_HEARTBEAT_SEC / MAM_HEARTBEAT_MISSES /
  MAM_ACK_TIMEOUT_S through; YAML rows with transport: remote delegate as
  remote:<name>.
- docs: MESSAGING.md 2.5 contract + remote transport, delegate-job SKILL.md,
  registry.md, loop SKILL.md.
- tests: tests/test_issue18_liveness_contract.py (16); issue 14 pane-signal
  tests flipped to started-based; issue 6 working-agent tests acknowledge
  with started; CLI snapshot/help captures refreshed.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
GodopuandClaude Fable 5.1 committed 2026-09-13 19:06:12 +09:00
1 parent 139bdd4b4c
commit 897b505266
21 files changed
+1989 -129

No files matched your search

+5 -1
View File
@@ -2,7 +2,7 @@ usage: job_subscriber.py [-h] (--job JOB | --wait-any) [--timeout TIMEOUT]
[--idle-timeout IDLE_TIMEOUT] [--expect-retention]
[--expect-dispatch-token EXPECT_DISPATCH_TOKEN]
[--attempt-id ATTEMPT_ID]
[--registry-dir REGISTRY_DIR] [-v]
[--registry-dir REGISTRY_DIR] [--remote] [-v]
Subscribe to Job events on MQTT
@@ -21,4 +21,8 @@ optional arguments:
--attempt-id ATTEMPT_ID
minimum attempt id to accept events for
--registry-dir REGISTRY_DIR
--remote the agent shares no disk with this registry: mirror
started/terminal events into the job record (Issue
#18; automatic for jobs registered with --transport
remote)
-v, --verbose
+7 -1
View File
@@ -6,7 +6,8 @@ usage: publish_event.py [-h] (--job JOB | --pick-pending)
[--finalize-report PATH]
[--side-effect KIND=REFERENCE]
[--attempt-id ATTEMPT_ID]
[--expect-dispatch-token EXPECT_DISPATCH_TOKEN] [-v]
[--expect-dispatch-token EXPECT_DISPATCH_TOKEN]
[--heartbeat N] [--no-state] [-v]
Publish a Job event to MQTT
@@ -33,4 +34,9 @@ optional arguments:
job's current iteration)
--expect-dispatch-token EXPECT_DISPATCH_TOKEN
expected dispatch token for CAS verification
--heartbeat N after publishing, start a detached timer that
publishes a `progress` heartbeat every N seconds until
the job ends (Issue #18; not with a terminal event)
--no-state do not publish the retained <topic_prefix>/state
message
-v, --verbose
+80 -27
View File
@@ -85,24 +85,63 @@ def _assert_single_injection(res: subprocess.CompletedProcess):
# ---------------------------------------------------------------------------
# lib.sh: accepted submission confirmed through the pane
# lib.sh: accepted submission confirmed by the job's own `started` (Issue #18)
# ---------------------------------------------------------------------------
def test_accepted_timeout_confirmed_by_busy_token(tmp_path):
def _job_that_starts(tmp_path, delay_s: float):
"""A registered job whose `started` lands `delay_s` after the call - the
protocol signal that replaced the pane (Issue #18). Returns (env, job_id)."""
import threading
from lib_py.delegation import registry
reg = tmp_path / "jobs"
env = {"MAM_ENV_FILE": str(tmp_path / ".mam.env.absent"), "DELEGATE_JOB_LOGS_DIR": str(tmp_path / "logs"),
"MQTT_BROKER": "127.0.0.1", "MQTT_PORT": "9", "SKS_REGISTRY_DIR": str(reg)}
saved = {k: os.environ.get(k) for k in env}
os.environ.update(env)
try:
jid = registry.register_job("issue 14 ack", registry_dir=str(reg))
finally:
for k, v in saved.items():
if v is None:
os.environ.pop(k, None)
else:
os.environ[k] = v
def start():
time.sleep(delay_s)
subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg), "--job", jid,
"--event", "started", "--attempts", "1"], capture_output=True, env=_base_env(tmp_path, env))
threading.Thread(target=start, daemon=True).start()
return env, jid
def test_accepted_timeout_confirmed_by_started_event_busy_token_alone_is_not_enough(tmp_path):
"""The pane shows the busy token, but that is not delivery any more: the
job's `started` (published 2 s in) is what confirms it, and the busy token
on its own (second run) still fails closed."""
env, jid = _job_that_starts(tmp_path, 2.0)
script = _codex_like_mock(CODEX_IDLE, CODEX_WORKING) + \
'send_keys_safe "reviewer-codex-01" "review the triage report" "job_14a"'
res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "5"})
f'send_keys_safe "reviewer-codex-01" "review the triage report" "{jid}"'
res = _run_lib_sh(script, tmp_path, env={**env, "SKS_DELIVERY_GRACE_S": "8"})
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
_assert_single_injection(res)
assert "delivery unknown" not in res.stderr
script = _codex_like_mock(CODEX_IDLE, CODEX_WORKING) + \
'send_keys_safe "reviewer-codex-01" "review the triage report" "job_14a"'
res = _run_lib_sh(script, tmp_path, env={**env, "SKS_DELIVERY_GRACE_S": "2"})
assert res.returncode == 4, f"busy token must not confirm delivery: rc={res.returncode}\n{res.stderr}"
_assert_single_injection(res)
def test_accepted_timeout_confirmed_by_pane_change_after_fast_turn(tmp_path):
"""A turn that already finished before the first poll shows no busy token,
only the answer; the pane moving on from its pre-submit snapshot is enough."""
def test_accepted_timeout_pane_change_after_fast_turn_is_not_delivery(tmp_path):
"""A turn that already finished shows only the answer. The pane moving on
used to count; under the protocol contract only `started` does."""
script = _codex_like_mock(CODEX_IDLE, CODEX_ANSWERED) + \
'send_keys_safe "reviewer-codex-01" "Reply with OK" "job_14b"'
res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "5"})
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "2"})
assert res.returncode == 4, f"rc={res.returncode}\n{res.stderr}"
_assert_single_injection(res)
@@ -147,7 +186,8 @@ def test_transport_error_is_not_treated_as_accepted(tmp_path):
def test_mock_herdr_timeout_with_idle_status_is_confirmed(tmp_path, mock_herdr):
"""Same scenario through the shared mock herdr: `timeout` from agent prompt,
agent_status pinned to idle, the pane gaining the busy token after
submission. Exactly one prompt call, no paste-buffer fallback."""
submission - and the job's `started` landing 2 s in, which is what
confirms it (Issue #18). Exactly one prompt call, no paste-buffer fallback."""
state_file = Path(mock_herdr)
state = json.loads(state_file.read_text())
state["agents"]["agt_codex"] = {
@@ -158,9 +198,10 @@ def test_mock_herdr_timeout_with_idle_status_is_confirmed(tmp_path, mock_herdr):
}
state_file.write_text(json.dumps(state, indent=2))
env, jid = _job_that_starts(tmp_path, 2.0)
res = _run_lib_sh(
'send_keys_safe "agt_codex" "review please" "job_14f"',
tmp_path, env={"SKS_IDLE_HANDSHAKE": "0", "SKS_DELIVERY_GRACE_S": "5"},
f'send_keys_safe "agt_codex" "review please" "{jid}"',
tmp_path, env={**env, "SKS_IDLE_HANDSHAKE": "0", "SKS_DELIVERY_GRACE_S": "8"},
)
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
@@ -243,13 +284,14 @@ def test_wrapper_honours_registry_ack_over_unconfirmed_inject(tmp_path, mock_her
# ---------------------------------------------------------------------------
# wrapper: idle-timeout grace (R1) must see a working codex through the pane
# wrapper: idle-timeout grace (R1) is decided by the contract, not the pane
# ---------------------------------------------------------------------------
def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr):
"""herdr keeps reporting `idle` for a codex-like agent that is visibly
working (busy token in the pane). When the subscriber idles out, the
wrapper must keep waiting for the on-disk result instead of closing the
job as deadline_timeout mid-turn; the agent then completes out of band."""
def test_wrapper_idle_timeout_grace_ignores_pane_busy_token(tmp_path, mock_herdr):
"""Issue #18 flips the Issue #14 expectation: a codex-like agent that is
visibly working (busy token in the pane) but publishes nothing after its
`started` is unresponsive once K x N (here --idle-timeout 3 s) has passed.
The wrapper closes it as deadline_timeout and a `completed` written 14 s
in is refused by the CAS table - the pane never extended the window."""
reg_dir = tmp_path / "jobs"
reg_dir.mkdir(parents=True, exist_ok=True)
@@ -265,7 +307,7 @@ def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr
}
state_file.write_text(json.dumps(state, indent=2))
env = _base_env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_DELIVERY_GRACE_S": "5"})
env = _base_env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_DELIVERY_GRACE_S": "2", "MAM_INJECT_ACK_GRACE_S": "20"})
proc = subprocess.Popen(
[
"bash", str(WRAPPER), "submit",
@@ -289,19 +331,29 @@ def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr
time.sleep(0.1)
assert jid, "job was not registered"
# Well past the 3 s idle timeout, the agent finishes.
# The agent acknowledges (started) without arming a heartbeat timer ...
time.sleep(1)
tok = json.loads((reg_dir / f"{jid}.json").read_text())["dispatch_token"]
ack = subprocess.run(
[sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg_dir), "--job", jid, "--event", "started",
"--expect-dispatch-token", tok],
capture_output=True, text=True, cwd=str(tmp_path), env=env,
)
assert ack.returncode == 0, ack.stderr
# ... and goes silent while its pane keeps showing the busy token.
time.sleep(14)
done = subprocess.run(
[sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg_dir),
"update", "--job", jid, "--status", "completed"],
capture_output=True, text=True, cwd=str(tmp_path), env=env,
)
assert done.returncode == 0, done.stderr
stdout, stderr = proc.communicate(timeout=90)
job_data = json.loads((reg_dir / f"{jid}.json").read_text())
assert job_data.get("status") == "completed", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
assert proc.returncode == 0, f"expected rc=0 after the working grace, got {proc.returncode}\n{stdout}\n{stderr}"
assert proc.returncode == 2, f"expected deadline_timeout (rc=2): the pane is not liveness, got {proc.returncode}\n{stdout}\n{stderr}"
assert job_data.get("status") == "error", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
assert done.returncode != 0 or "refused" in done.stderr, "the late completed must be refused"
assert "diagnostic only" in stderr
# ---------------------------------------------------------------------------
@@ -451,8 +503,9 @@ def test_wrapper_idle_grace_never_outlives_the_wall_clock(tmp_path):
# The heartbeats were honoured: the job was not closed after the first idle
# window, it stayed open up to the wall clock ...
assert elapsed >= timeout_s - 1, f"closed before the wall clock: {elapsed:.1f}s\n{stderr}"
# ... and not a moment past it beyond closing and publishing the error.
assert elapsed <= timeout_s + 12, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}"
# ... and not a moment past it beyond closing and publishing the error
# (the error publish alone can take ~10 s against the public broker).
assert elapsed <= timeout_s + 15, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}"
# ---------------------------------------------------------------------------
@@ -489,7 +542,7 @@ def test_publish_event_records_the_heartbeat_before_the_wire_publish(tmp_path, m
events_log = reg_dir / f"{jid}.events.log"
on_disk_at_publish = []
def slow_broker_publish(config, topic, body, retain):
def slow_broker_publish(config, topic, body, retain, extra=()):
on_disk_at_publish.append([e.get("event") for e in _logged_events(events_log)])
monkeypatch.setattr(pe, "_publish_once", slow_broker_publish)
@@ -509,7 +562,7 @@ def test_publish_event_failed_publish_keeps_one_local_record(tmp_path, monkeypat
events_log = reg_dir / f"{jid}.events.log"
lines_seen = []
def unreachable_broker(config, topic, body, retain):
def unreachable_broker(config, topic, body, retain, extra=()):
lines_seen.append(len(_logged_events(events_log)))
raise ConnectionError("broker unreachable")
+617
View File
@@ -0,0 +1,617 @@
#!/usr/bin/env python3
"""Issue #18 — liveness and delivery are decided by the event protocol, not by
screen scraping.
Contract (registry fields, wrapper options):
* heartbeat obligation - the agent publishes `progress` at least every N s;
`publish_event --event started --heartbeat N` starts an automatic timer
that does it for the agent and stops itself at the terminal event.
* unresponsive - no event (disk or wire) for K x N s closes the job
as deadline_timeout; herdr status and pane text are diagnostics only.
* delivery - `started` must arrive within T s of the injection,
else the inject fails closed (rc 1, registry error, single injection).
* wire durability - every event is followed by a retained
`<prefix>/state` message on the same connection; a subscriber that
(re)connects recovers the status at once and records seq gaps.
* remote transport - `--agent-session remote:<name>` delivers the brief
over a signed retained `<prefix>/dispatch`; a runner on another host with
its own registry and broker credentials runs it, and the orchestrator's
subscriber mirrors its events into the orchestrator's registry.
Tests marked "broker" talk to the configured broker (the public default in the
sandbox); each stays under a minute.
"""
from __future__ import annotations
import json
import os
import signal
import subprocess
import sys
import threading
import time
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parent.parent
SKILLS = REPO_ROOT / ".agents" / "skills"
LIB_SH = SKILLS / "lib.sh"
WRAPPER = SKILLS / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
SCRIPTS = SKILLS / "multi-agent-mux-delegate-job" / "scripts"
PUBLISH_PY = SCRIPTS / "publish_event.py"
REGISTRY_PY = SCRIPTS / "registry.py"
REMOTE_PY = SCRIPTS / "remote.py"
UNREACHABLE = {"MQTT_BROKER": "127.0.0.1", "MQTT_PORT": "9", "MQTT_TLS": "0"}
def _env(cwd: Path, extra=None) -> dict:
env = dict(os.environ)
env["DELEGATE_JOB_PYTHON"] = sys.executable
env["AGENT_PYTHON_BIN"] = sys.executable
env["WORKSPACE_ROOT"] = str(cwd)
env["HOME"] = str(cwd)
env["MAM_ENV_FILE"] = str(cwd / ".mam.env.absent")
env["DELEGATE_JOB_LOGS_DIR"] = str(cwd / "logs")
env["PYTHONPATH"] = str(SKILLS)
for k in ("MQTT_BROKER", "MQTT_HOST", "MQTT_PORT", "MQTT_TLS", "MQTT_USERNAME", "MQTT_PASSWORD"):
env.pop(k, None)
if extra:
env.update({k: str(v) for k, v in extra.items()})
return env
def _events(reg_dir: Path, jid: str) -> list:
p = reg_dir / f"{jid}.events.log"
if not p.exists():
return []
return [json.loads(l) for l in p.read_text().splitlines() if l.strip()]
def _job(reg_dir: Path, jid: str) -> dict:
return json.loads((reg_dir / f"{jid}.json").read_text())
def _wait_job_id(reg_dir: Path, timeout=20) -> str:
deadline = time.time() + timeout
while time.time() < deadline:
files = list(reg_dir.glob("*.json"))
if files:
return files[0].stem
time.sleep(0.1)
raise AssertionError("job was not registered")
def _publish(reg_dir: Path, jid: str, event: str, env: dict, *extra) -> subprocess.CompletedProcess:
"""Publish as the agent would: with the dispatch identity the wrapper's
injected command carries (`--expect-dispatch-token`)."""
tok = _job(reg_dir, jid).get("dispatch_token")
ident = ["--expect-dispatch-token", tok] if tok else []
return subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg_dir), "--job", jid,
"--event", event, "--attempts", "2", *ident, *extra],
capture_output=True, text=True, env=env, cwd=str(reg_dir.parent), timeout=120)
def _timer_pid(reg_dir: Path, jid: str):
p = reg_dir / f"{jid}.heartbeat.pid"
try:
return int(p.read_text().strip())
except (OSError, ValueError):
return None
def _alive(pid) -> bool:
if not pid:
return False
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
return subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True).stdout.strip() not in ("", "Z", "Z+")
def _kill_timer(reg_dir: Path, jid: str):
pid = _timer_pid(reg_dir, jid)
if pid:
try:
os.kill(pid, signal.SIGKILL)
except OSError:
pass
# ---------------------------------------------------------------------------
# 1. registry: the contract is part of the record
# ---------------------------------------------------------------------------
def test_register_derives_the_idle_window_from_the_contract(tmp_path, monkeypatch):
from lib_py.delegation import registry
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
reg = tmp_path / "jobs"
jid = registry.register_job("contract", registry_dir=str(reg))
rec = _job(reg, jid)
assert (rec["heartbeat_sec"], rec["heartbeat_misses"], rec["ack_timeout_sec"]) == (30, 4, 60)
assert rec["idle_timeout_sec"] == 120 == registry.liveness_window(30, 4)
assert rec["transport"] == "herdr"
jid2 = registry.register_job("contract", registry_dir=str(reg), heartbeat_sec=5, heartbeat_misses=3)
assert _job(reg, jid2)["idle_timeout_sec"] == 15
jid3 = registry.register_job("contract", registry_dir=str(reg), heartbeat_sec=5, heartbeat_misses=3,
idle_timeout_sec=40)
assert _job(reg, jid3)["idle_timeout_sec"] == 40, "an explicit idle timeout is kept"
def test_wrapper_registers_the_contract_and_derives_idle_timeout(tmp_path):
"""--dry-run registers the job without any herdr; K x N shows up as idle."""
reg = tmp_path / "jobs"
reg.mkdir()
res = subprocess.run(
["bash", str(WRAPPER), "submit", "--agent", "human", "--prompt", "contract", "--dry-run",
"--registry-dir", str(reg), "--workdir", str(tmp_path),
"--heartbeat", "7", "--heartbeat-misses", "3", "--ack-timeout", "11"],
capture_output=True, text=True, env=_env(tmp_path, UNREACHABLE), cwd=str(tmp_path), timeout=60)
assert res.returncode == 0, res.stderr
jid = _wait_job_id(reg)
rec = _job(reg, jid)
assert (rec["heartbeat_sec"], rec["heartbeat_misses"], rec["ack_timeout_sec"], rec["idle_timeout_sec"]) == (7, 3, 11, 21)
assert "--event started --heartbeat 7" in res.stdout, "the injected start command must arm the timer"
assert "within 11 s" in res.stdout
# ---------------------------------------------------------------------------
# 2. the automatic heartbeat (no broker needed: events land on disk first)
# ---------------------------------------------------------------------------
def test_started_with_heartbeat_beats_on_disk_until_the_terminal_event(tmp_path):
reg = tmp_path / "jobs"
env = _env(tmp_path, UNREACHABLE)
jid = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "register",
"--prompt", "hb", "--heartbeat-sec", "1", "--heartbeat-misses", "3"],
capture_output=True, text=True, env=env, cwd=str(tmp_path)).stdout.strip()
assert jid
try:
res = _publish(reg, jid, "started", env, "--heartbeat", "1", "--attempts", "1")
assert res.returncode == 2, "the broker is unreachable, the event is still recorded locally"
pid = _timer_pid(reg, jid)
assert pid and _alive(pid), "a detached timer must be running"
time.sleep(4.5)
beats = [e for e in _events(reg, jid) if e["event"] == "progress"]
assert len(beats) >= 3, [e["event"] for e in _events(reg, jid)]
assert all(e["detail"] == "heartbeat" for e in beats)
assert all(e["data"].get("dispatch_token") == _job(reg, jid)["dispatch_token"] for e in beats), \
"heartbeats carry the dispatch identity"
seqs = [e["seq"] for e in _events(reg, jid)]
assert seqs == sorted(seqs) and len(set(seqs)) == len(seqs)
res = _publish(reg, jid, "completed", env, "--attempts", "1")
assert res.returncode == 2
deadline = time.time() + 5
while time.time() < deadline and _alive(pid):
time.sleep(0.1)
assert not _alive(pid), "the terminal event must stop the timer"
assert _timer_pid(reg, jid) is None
assert _events(reg, jid)[-1]["event"] == "completed", "no heartbeat may trail the terminal event"
assert _job(reg, jid)["status"] == "completed"
finally:
_kill_timer(reg, jid)
def test_heartbeat_timer_stops_on_its_own_when_the_job_is_redispatched(tmp_path):
reg = tmp_path / "jobs"
env = _env(tmp_path, UNREACHABLE)
jid = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "register",
"--prompt", "hb"], capture_output=True, text=True, env=env, cwd=str(tmp_path)).stdout.strip()
try:
_publish(reg, jid, "started", env, "--heartbeat", "1", "--attempts", "1")
pid = _timer_pid(reg, jid)
assert _alive(pid)
# An operator moves the job to the next attempt: the old attempt's timer must not keep beating.
_publish(reg, jid, "completed", env, "--attempts", "1")
subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "update", "--job", jid,
"--status", "pending", "--attempt-id", "2"], capture_output=True, text=True, env=env, cwd=str(tmp_path))
deadline = time.time() + 5
while time.time() < deadline and _alive(pid):
time.sleep(0.1)
assert not _alive(pid)
finally:
_kill_timer(reg, jid)
def test_heartbeat_is_refused_on_a_terminal_event(tmp_path, monkeypatch):
from lib_py.delegation import publish_event as pe, registry
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
reg = tmp_path / "jobs"
jid = registry.register_job("hb", registry_dir=str(reg))
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "completed", "--heartbeat", "5"]) == 1
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "progress", "--heartbeat", "0"]) == 1
assert _events(reg, jid) == [], "a refused call consumes no seq and records nothing"
# ---------------------------------------------------------------------------
# 3. wire durability: retained state next to every event, seq gaps detected
# ---------------------------------------------------------------------------
def test_every_event_carries_a_signed_retained_state_message(tmp_path, monkeypatch):
from lib_py.delegation import mqtt_common, publish_event as pe, registry
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
reg = tmp_path / "jobs"
jid = registry.register_job("state", registry_dir=str(reg), auth_token="tok-" + "a" * 20)
sent = []
def fake_publish(config, topic, body, retain, extra=()):
sent.append((topic, json.loads(body), retain, [(t, json.loads(b), r) for t, b, r in extra]))
monkeypatch.setattr(pe, "_publish_once", fake_publish)
tok = _job(reg, jid)["dispatch_token"]
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "started", "--expect-dispatch-token", tok]) == 0
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "completed", "--detail", "PASS",
"--expect-dispatch-token", tok]) == 0
prefix = _job(reg, jid)["topic_prefix"]
assert [s[0] for s in sent] == [f"{prefix}/events"] * 2
for (topic, ev, retain, extra), status in zip(sent, ("running", "completed")):
assert len(extra) == 1
st_topic, st, st_retain = extra[0]
assert st_topic == f"{prefix}/state" and st_retain is True
assert st["event"] == "state" and st["status"] == status and st["last_event"] == ev["event"]
assert st["seq"] == ev["seq"] and st["job_id"] == jid
assert mqtt_common.verify_hmac(st, "tok-" + "a" * 20), "state must be signed with the job token"
assert st["data"]["dispatch_token"] == ev["data"]["dispatch_token"]
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "progress", "--no-state",
"--expect-dispatch-token", tok]) == 0
assert len(sent) == 3 and sent[-1][3] == [], "--no-state publishes the event alone"
class _Msg:
def __init__(self, topic, payload):
self.topic = topic
self.payload = json.dumps(payload).encode()
def _signed(job_id, seq, event, token, **fields):
from lib_py.delegation import publish_event as pe
p = pe.build_payload(job_id, seq, event, "", {"dispatch_token": "d1", "attempt_id": 1}, token)
if fields:
# re-sign after adding state fields, as build_state_payload does
import hashlib, hmac
p.update(fields)
p["data"].pop("hmac_sig", None)
sp = {k: v for k, v in p.items() if k != "data"}
sp["data"] = dict(p["data"])
p["data"]["hmac_sig"] = hmac.new(token.encode(), json.dumps(sp, sort_keys=True, separators=(",", ":")).encode(),
hashlib.sha256).hexdigest()
return p
def test_subscriber_records_seq_gaps_and_recovers_terminal_from_state(tmp_path, monkeypatch):
from lib_py.delegation import job_subscriber as js, mqtt_common
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
token = "t" * 24
w = js._Watcher({"j1"}, {"j1": token}, {"j1": 0}, {"j1": "d1"}, {"j1": 1})
w.on_message(None, None, _Msg("x/events", _signed("j1", 1, "started", token)))
w.on_message(None, None, _Msg("x/events", _signed("j1", 2, "progress", token)))
w.on_message(None, None, _Msg("x/events", _signed("j1", 5, "progress", token))) # 3 and 4 were lost
audit = [e for e in mqtt_common.iter_logged_events("j1", mqtt_common.get_logs_dir()) if e["event"] == "seq_gap"]
assert len(audit) == 1 and (audit[0]["expected_seq"], audit[0]["seq"], audit[0]["via"]) == (3, 5, "events")
assert w.events.qsize() == 3
# a retained state after a reconnect: newer seq than anything seen, terminal status
st = _signed("j1", 8, "state", token, status="completed", last_event="completed")
w.on_message(None, None, _Msg("x/state", st))
assert w.events.qsize() == 4
topic, payload = list(w.events.queue)[-1]
assert topic == "x/state" and payload["event"] == "state" and payload["status"] == "completed"
gaps = [e for e in mqtt_common.iter_logged_events("j1", mqtt_common.get_logs_dir()) if e["event"] == "seq_gap"]
assert len(gaps) == 2 and gaps[-1]["via"] == "state"
# a forged state (wrong token) is dropped like a forged event
bad = _signed("j1", 9, "state", "wrong", status="error", last_event="error")
w.on_message(None, None, _Msg("x/state", bad))
assert w.events.qsize() == 4
# a state for a stale attempt (other dispatch token) is dropped
stale = _signed("j1", 10, "state", token, status="error", last_event="error")
stale["data"]["dispatch_token"] = "d0"
w.on_message(None, None, _Msg("x/state", stale))
assert w.events.qsize() == 4
def test_subscriber_subscribes_to_the_state_topic(tmp_path):
src = (SKILLS / "lib_py" / "delegation" / "job_subscriber.py").read_text()
assert 'state_topics.append(f"{prefix}/state")' in src
assert "for topic in subscribed_topics + state_topics:" in src
# ---------------------------------------------------------------------------
# 4. delivery: `started` decides, screen signals do not
# ---------------------------------------------------------------------------
def _codex_like_mock(pane_before: str, pane_after: str, code: str = "timeout") -> str:
return f'''
_pane_quiescent() {{ return 0; }}
_pane_dialog_open() {{ return 1; }}
FLAG="$WORKSPACE_ROOT/.prompt-submitted"
rm -f "$FLAG"
_sks_herdr() {{
case "${{1:-}}" in
agent)
case "${{2:-}}" in
prompt) touch "$FLAG"; echo '{{"error":{{"code":"{code}","message":"x"}}}}' >&2; return 1 ;;
get) echo '{{"result":{{"agent":{{"agent_status":"idle"}}}}}}'; return 0 ;;
esac ;;
capture-pane)
if [ -e "$FLAG" ]; then printf '%s\\n' "{pane_after}"; else printf '%s\\n' "{pane_before}"; fi
return 0 ;;
paste-buffer|set-buffer|delete-buffer|send-keys)
echo "PASTE_PATH_USED: $*" >&2; return 0 ;;
esac
return 0
}}
'''
def _run_lib_sh(snippet: str, cwd: Path, env=None, timeout=60):
return subprocess.run(["bash", "-c", f'source "{LIB_SH}" >/dev/null 2>&1; {snippet}'],
capture_output=True, text=True, cwd=str(cwd), env=_env(cwd, env), timeout=timeout)
def test_send_keys_safe_is_confirmed_by_the_job_started_event(tmp_path, monkeypatch):
"""herdr says `timeout`, agent_status stays idle, the pane never changes:
the only thing that confirms delivery is the job's own `started`."""
from lib_py.delegation import registry
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
reg = tmp_path / "jobs"
jid = registry.register_job("ack", registry_dir=str(reg))
env = _env(tmp_path, {**UNREACHABLE, "SKS_DELIVERY_GRACE_S": "14", "SKS_REGISTRY_DIR": str(reg)})
def agent_starts_later():
time.sleep(2)
subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg), "--job", jid,
"--event", "started", "--attempts", "1"], capture_output=True, env=env, cwd=str(tmp_path))
t = threading.Thread(target=agent_starts_later, daemon=True)
t.start()
started = time.time()
res = subprocess.run(["bash", "-c", f'source "{LIB_SH}" >/dev/null 2>&1; '
+ _codex_like_mock("idle", "idle") + f'send_keys_safe "reviewer-codex-01" "prompt" "{jid}"'],
capture_output=True, text=True, cwd=str(tmp_path), env=env, timeout=60)
t.join()
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
assert time.time() - started < 11, "confirmed once started landed, not at the end of the 14 s grace"
assert "PASTE_PATH_USED" not in res.stderr
def test_send_keys_safe_ignores_pane_change_and_busy_token(tmp_path):
"""The screen moving on, or showing a busy token, is not delivery any more:
with no `started` the accepted submission still fails closed (rc 4)."""
for after in ("Working (0s • esc to interrupt)\\n› Ask Codex", "• OK\\n› Ask Codex"):
res = _run_lib_sh(_codex_like_mock("idle", after) + 'send_keys_safe "reviewer-codex-01" "prompt" "adhoc"',
tmp_path, env={**UNREACHABLE, "SKS_DELIVERY_GRACE_S": "2"})
assert res.returncode == 4, f"rc={res.returncode}\n{res.stderr}"
assert "delivery unknown/failed" in res.stderr
assert "PASTE_PATH_USED" not in res.stderr
# ---------------------------------------------------------------------------
# 5. wrapper acceptance criteria (broker)
# ---------------------------------------------------------------------------
def _headless_agent(state_file: Path, name: str, agent: str = "codex") -> None:
state = json.loads(state_file.read_text())
# agent_status pinned idle, prompt `timeout`, empty pane: nothing to scrape
state["agents"][name] = {"agent": agent, "agent_status": "idle", "prompt_wait_result": "timeout", "buffer": ""}
state_file.write_text(json.dumps(state, indent=2))
def _submit(tmp_path: Path, reg: Path, session: str, agent: str, extra_args, env_extra=None, timeout_s=60):
env = _env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_EMPTY_GIVEUP": "1", **(env_extra or {})})
proc = subprocess.Popen(
["bash", str(WRAPPER), "submit", "--agent", agent, "--agent-session", session, "--prompt", "issue 18",
"--registry-dir", str(reg), "--workdir", str(tmp_path), "--timeout", str(timeout_s), *extra_args],
cwd=str(tmp_path), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
return proc, env
def _prompt_calls(state_file: Path) -> int:
"""`agent prompt` invocations; the isolated wrapper prefixes `--session <name>`."""
n = 0
for c in json.loads(state_file.read_text()).get("calls", []):
c = list(c)
if len(c) >= 2 and c[0] == "--session":
c = c[2:]
if len(c) >= 2 and c[0] == "agent" and c[1] == "prompt":
n += 1
return n
def test_acceptance1_headless_agent_survives_on_heartbeats_alone(tmp_path, mock_herdr):
"""herdr idle throughout, empty pane: `started --heartbeat 2` keeps the job
alive past several K x N windows (2 x 2 = 4 s) and `completed` yields rc 0."""
reg = tmp_path / "jobs"
reg.mkdir()
_headless_agent(Path(mock_herdr), "headless-codex")
proc, env = _submit(tmp_path, reg, "herdr:headless-codex", "codex",
["--heartbeat", "2", "--heartbeat-misses", "2", "--ack-timeout", "40"], timeout_s=90)
jid = _wait_job_id(reg)
try:
time.sleep(1)
assert _publish(reg, jid, "started", env, "--heartbeat", "2").returncode == 0
time.sleep(16) # four idle windows without any agent-published event
assert _job(reg, jid)["status"] == "running", "heartbeats alone must keep the job open"
assert _publish(reg, jid, "completed", env, "--detail", "PASS").returncode == 0
stdout, stderr = proc.communicate(timeout=120)
finally:
_kill_timer(reg, jid)
if proc.poll() is None:
proc.kill()
assert proc.returncode == 0, f"rc={proc.returncode}\n{stdout}\n{stderr}"
assert _job(reg, jid)["status"] == "completed"
assert _prompt_calls(Path(mock_herdr)) == 1
beats = [e for e in _events(reg, jid) if e["event"] == "progress"]
assert len(beats) >= 5, len(beats)
brief = (reg / jid / "brief.md").read_text()
assert "heartbeat every 2 s" in brief and "after 2 missed heartbeats" in brief and "within 40 s" in brief
def test_acceptance2_no_started_within_T_fails_closed_without_reinjection(tmp_path, mock_herdr):
reg = tmp_path / "jobs"
reg.mkdir()
_headless_agent(Path(mock_herdr), "mute-codex")
started = time.time()
proc, env = _submit(tmp_path, reg, "herdr:mute-codex", "codex",
["--ack-timeout", "4", "--heartbeat", "2", "--heartbeat-misses", "2"],
env_extra={"SKS_DELIVERY_GRACE_S": "1"}, timeout_s=60)
stdout, stderr = proc.communicate(timeout=120)
jid = _wait_job_id(reg)
assert proc.returncode == 1, f"rc={proc.returncode}\n{stdout}\n{stderr}"
assert "no 'started' within 4s" in stderr
assert _job(reg, jid)["status"] == "error"
assert _prompt_calls(Path(mock_herdr)) == 1, "single injection: never re-injected"
assert time.time() - started < 45, "the ack wait is bounded by T, not the wall clock"
ev = [e for e in _events(reg, jid) if e["event"] == "error"]
assert ev and ev[-1]["detail"] == "agent_blocked"
def test_acceptance3_lost_heartbeats_close_the_job_and_a_late_completed_is_refused(tmp_path, mock_herdr):
reg = tmp_path / "jobs"
reg.mkdir()
_headless_agent(Path(mock_herdr), "dying-codex")
proc, env = _submit(tmp_path, reg, "herdr:dying-codex", "codex",
["--heartbeat", "1", "--heartbeat-misses", "3", "--ack-timeout", "40"], timeout_s=60)
jid = _wait_job_id(reg)
t0 = None
try:
time.sleep(1)
assert _publish(reg, jid, "started", env, "--heartbeat", "1").returncode == 0
time.sleep(4)
_kill_timer(reg, jid) # the agent dies: no more heartbeats
t0 = time.time()
stdout, stderr = proc.communicate(timeout=120)
finally:
_kill_timer(reg, jid)
if proc.poll() is None:
proc.kill()
assert proc.returncode == 2, f"rc={proc.returncode}\n{stdout}\n{stderr}"
assert _job(reg, jid)["status"] == "error"
assert "herdr/pane says busy=" in stderr, "screen state is logged as a diagnostic only"
# K x N = 3 s idle + one grace window: closed long before the 60 s wall clock
assert time.time() - t0 < 30, f"took {time.time() - t0:.0f}s"
late = _publish(reg, jid, "completed", env, "--detail", "too late")
assert late.returncode in (0, 2, 4)
assert _job(reg, jid)["status"] == "error", "terminal immutability (CAS) refuses the late completed"
# ---------------------------------------------------------------------------
# 6. remote transport (broker): no shared disk, no pane, only the wire
# ---------------------------------------------------------------------------
def test_acceptance5_remote_runner_completes_a_job_over_the_broker_only(tmp_path):
orch = tmp_path / "orchestrator"
remote = tmp_path / "remote-host"
for d in (orch, remote):
(d / "jobs").mkdir(parents=True)
secret = "issue18-shared-secret"
runner_env = _env(remote, {"MAM_DISPATCH_SECRET": secret, "MAM_JOBS_DIR": str(remote / "jobs")})
runner_log = remote / "runner.log"
with open(runner_log, "wb") as log:
runner = subprocess.Popen(
[sys.executable, str(REMOTE_PY), "run", "--session", "sim-worker", "--registry-dir", str(remote / "jobs"),
"--exec", 'echo "remote work done for $JOB_ID"', "--once", "--timeout", "150"],
cwd=str(remote), env=runner_env, stdout=log, stderr=subprocess.STDOUT)
try:
deadline = time.time() + 45
while time.time() < deadline and "SUBSCRIBED" not in runner_log.read_text(errors="replace"):
time.sleep(0.2)
if "SUBSCRIBED" not in runner_log.read_text(errors="replace"):
pytest.skip("broker unreachable: " + runner_log.read_text(errors="replace")[-300:])
reg = orch / "jobs"
proc, env = _submit(orch, reg, "remote:sim-worker", "remote",
["--heartbeat", "2", "--heartbeat-misses", "3", "--ack-timeout", "60"],
env_extra={"MAM_DISPATCH_SECRET": secret}, timeout_s=120)
stdout, stderr = proc.communicate(timeout=180)
jid = _wait_job_id(reg)
assert proc.returncode == 0, f"rc={proc.returncode}\n{stdout}\n{stderr}\nrunner:\n{runner_log.read_text()}"
assert "subscriber received started" in stderr or "advanced to 'running'" in stderr
assert _job(reg, jid)["status"] == "completed", \
f"the subscriber mirrors the remote terminal into the registry\n{stdout}\n{stderr}\nrunner:\n{runner_log.read_text()}"
assert _job(reg, jid)["transport"] == "remote"
# The runner's own status sync follows its wire publish, so the
# orchestrator can observe `completed` a moment before the remote
# record says so: wait for the runner before reading its registry.
runner.wait(timeout=60)
# the remote side kept its own record with the orchestrator's identity
mirror = _job(remote / "jobs", jid)
assert mirror["status"] == "completed" and mirror["transport"] == "remote"
assert mirror["dispatch_token"] == _job(reg, jid)["dispatch_token"]
assert mirror["auth_token"] == _job(reg, jid)["auth_token"]
assert (remote / "jobs" / jid / "brief.md").read_text().startswith("issue 18")
# the agent command's last stdout line became the completed detail
assert "remote work done for " + jid in stdout, stdout
assert runner.returncode == 0
assert "살아있는 에이전트 세션" not in stdout, "no pane injection path for a remote session"
finally:
if runner.poll() is None:
runner.kill()
def test_remote_runner_refuses_unsigned_dispatches(tmp_path, monkeypatch):
from lib_py.delegation import remote
d = {"kind": "dispatch", "job_id": "j", "session": "s", "schema_version": 1}
assert not remote.verify_dispatch(d, "secret")
signed = remote.sign_dispatch(d, "secret")
assert remote.verify_dispatch(signed, "secret")
assert not remote.verify_dispatch(signed, "other")
assert not remote.verify_dispatch(dict(signed, session="hijacked"), "secret")
assert not remote.verify_dispatch(signed, None)
monkeypatch.delenv("MAM_DISPATCH_SECRET", raising=False)
assert remote.run_runner("s", str(tmp_path), allow_unsigned=False) == 1, "a runner without a secret refuses to run"
# ---------------------------------------------------------------------------
# 7. loop runner: contract knobs pass through; remote rows get remote:<name>
# ---------------------------------------------------------------------------
def test_loop_runner_passes_the_contract_and_remote_label(tmp_path, monkeypatch):
from lib_py.loop import runner as loop_runner
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("WORKSPACE_ROOT", str(tmp_path))
monkeypatch.setenv("MAM_STATE_JSON", json.dumps({"herdr_sessions": [
{"name": "creator-claude-01", "agent": "claude", "role": "creator", "status": "running"},
{"name": "reviewer-far-01", "agent": "claude", "role": "reviewer", "status": "running", "transport": "remote"},
]}))
monkeypatch.setenv("MAM_HEARTBEAT_SEC", "20")
monkeypatch.setenv("MAM_HEARTBEAT_MISSES", "3")
monkeypatch.setenv("MAM_ACK_TIMEOUT_S", "45")
r = loop_runner.LoopRunner(["--creator", "creator-claude-01", "--task", "t"])
r.parse_args()
assert r.session_label("creator-claude-01") == "herdr:creator-claude-01"
assert r.session_label("reviewer-far-01") == "remote:reviewer-far-01"
assert r.contract_args() == ["--heartbeat", "20", "--heartbeat-misses", "3", "--ack-timeout", "45"]
seen = []
def fake_run(cmd, **_kw):
seen.append(cmd)
class P: # noqa: D401 - tiny stand-in
returncode = 0
stdout = "registered job: deadbeef\n"
stderr = ""
return P()
monkeypatch.setattr(loop_runner.subprocess, "run", fake_run)
assert r.delegate_job("reviewer-far-01", "claude", "Reviewer", "review") == "deadbeef"
cmd = seen[0]
assert cmd[cmd.index("--agent-session") + 1] == "remote:reviewer-far-01"
assert cmd[-6:] == ["--heartbeat", "20", "--heartbeat-misses", "3", "--ack-timeout", "45"]
+8 -1
View File
@@ -338,7 +338,14 @@ def test_registry_cli_contract_snapshot(mam_sandbox):
assert reg["--agent-session"][2] == "herdr:claude"
assert reg["--role"][2] == "Worker"
assert reg["--timeout"][2] == 3600
assert reg["--idle-timeout"][2] == 120
# Issue #18: the idle window derives from the liveness contract (K x N =
# 4 x 30 = 120 s, the historical default) unless given explicitly.
assert reg["--idle-timeout"][2] is None
assert reg["--heartbeat-sec"][2] == 30
assert reg["--heartbeat-misses"][2] == 4
assert reg["--ack-timeout"][2] == 60
assert reg["--transport"][1] == ("herdr", "remote")
assert registry.liveness_window(reg["--heartbeat-sec"][2], reg["--heartbeat-misses"][2]) == 120
assert reg["--bits"][2] == 32
assert reg["--job-type"][1] == ("direct", "loop", "discuss")
assert reg["--max-iterations"][2] == 5
+36 -6
View File
@@ -603,8 +603,31 @@ def test_t1_14_working_agent_reobserve_completed(tmp_path, mock_herdr):
assert len(agent_gets) >= 1, "R1 loop must query agent get"
def _publish_started_for_registered_job(reg_dir, cwd, env=None):
"""Issue #18: a working agent that never publishes `started` is a delivery
failure (rc 1) under the liveness contract. The R1 re-observation cases
below are about an agent that *did* start and then went silent, so the
watcher threads acknowledge the job the way the injected command does."""
deadline = time.time() + 20
while time.time() < deadline:
files = list(Path(reg_dir).glob("*.json"))
if files:
jid = files[0].stem
tok = json.loads(files[0].read_text()).get("dispatch_token") or ""
args = [sys.executable, str(SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"),
"--registry-dir", str(reg_dir), "--job", jid, "--event", "started", "--attempts", "1"]
if tok:
args += ["--expect-dispatch-token", tok]
run_env = dict(os.environ)
run_env.update(env or {})
subprocess.run(args, capture_output=True, text=True, cwd=str(cwd), env=run_env, timeout=60)
return jid
time.sleep(0.1)
return None
# ---------------------------------------------------------------------------
# T1-15: R1 working 고정 + 미발행 -> 벽시계 --timeout 만료 후 rc=2 error
# T1-15: R1 working 고정 + 미발행 -> Issue #18: K×N 후 rc=2 error (working 은 진단일 뿐)
# ---------------------------------------------------------------------------
def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
reg_dir = tmp_path / "jobs"
@@ -628,6 +651,7 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
st["agents"]["agt_w2"]["agent_status"] = "working"
state_file.write_text(json.dumps(st, indent=2))
_publish_started_for_registered_job(reg_dir, tmp_path)
break
except Exception:
pass
@@ -637,6 +661,10 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
t.daemon = True
t.start()
# Issue #18: `working` in herdr is a diagnostic, not liveness. An agent
# that acknowledged (started) and then publishes nothing is closed after
# K x N (--idle-timeout 2) plus one grace window - well before the wall
# clock - even though herdr keeps saying `working` the whole time.
start_t = time.time()
res = _run_wrapper(
[
@@ -646,16 +674,16 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
"--prompt", "test T1-15",
"--registry-dir", str(reg_dir),
"--workdir", str(tmp_path),
"--timeout", "5",
"--timeout", "40",
"--idle-timeout", "2",
],
cwd=tmp_path,
timeout=60,
timeout=90,
)
stop_ev.set()
elapsed = time.time() - start_t
assert res.returncode == 2, f"Expected rc=2 on wallclock timeout, got {res.returncode}"
assert elapsed >= 4.0, f"Must wait up to wallclock timeout, elapsed={elapsed}"
assert res.returncode == 2, f"Expected rc=2 (deadline_timeout), got {res.returncode}\n{res.stderr}"
assert elapsed < 39.0, f"a silent `working` agent must not be kept open to the wall clock, elapsed={elapsed}"
job_files = list(reg_dir.glob("*.json"))
assert len(job_files) == 1
@@ -663,7 +691,8 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
assert job_data.get("status") == "error"
final_st = json.loads(state_file.read_text())
agent_gets = [c for c in final_st.get("calls", []) if "agent" in c and "get" in c]
assert len(agent_gets) >= 2, "R1 loop must repeatedly query agent get during wallclock wait"
assert len(agent_gets) >= 2, "the idle handshake and the one diagnostic probe query agent get"
assert "diagnostic only" in res.stderr, "Issue #18: herdr status is logged, not used as liveness"
# ---------------------------------------------------------------------------
@@ -691,6 +720,7 @@ def test_t1_16_working_agent_done_without_event(tmp_path, mock_herdr):
if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
st["agents"]["agt_w3"]["agent_status"] = "working"
state_file.write_text(json.dumps(st, indent=2))
_publish_started_for_registered_job(reg_dir, tmp_path)
break
except Exception:
pass