feat(issue-18): decide liveness and delivery by the event protocol, not the screen

Contract (no new event types): heartbeat every N s (automatic timer),
unresponsive after K x N s without an event, `started` within T s of the
injection, retained per-job state, and a remote transport.

- registry: heartbeat_sec / heartbeat_misses / ack_timeout_sec / transport
  fields; idle_timeout_sec derives K x N unless explicit (defaults 30/4/60
  keep the historical 120 s idle window).
- heartbeat.py + publish_event --heartbeat N: detached fixed-cadence timer
  publishing signed `progress` heartbeats (threaded wire publish, local
  record first); stopped by the terminal event via pidfile, by re-dispatch
  and by lifetime. Terminal events refuse --heartbeat.
- publish_event: retained `<prefix>/state` (status, last_event, seq, HMAC)
  sent on the same connection after every event (--no-state to skip);
  publisher.publish_raw takes extra messages.
- job_subscriber: subscribes the state topic, recovers terminal status from
  retained state, resets idle on any accepted message, records seq_gap and
  reconnected in the audit log, idle budget before the first event is
  T + K x N (--ack-timeout), --remote mirrors started/terminal events into
  the registry through the transition boundary.
- lib.sh send_keys_safe: an accepted-but-unconfirmed submission is confirmed
  by agent_status working or the job's own `started` (_sks_job_started);
  pane change and busy token are no longer delivery signals.
- wrapper: --heartbeat/--heartbeat-misses/--ack-timeout (MAM_* env),
  injected start command arms the timer, brief states the contract,
  _wait_started (registry / events.log / subscriber log, bounded by T and
  the wall clock) replaces the 10 s inject grace, R1 grace uses only the
  on-disk event age (herdr/pane logged once as a diagnostic),
  remote:<name> sessions skip preflight/idle/pane and dispatch over the
  broker (remote_dispatch).
- remote.py (+ scripts shim): signed retained dispatch (MAM_DISPATCH_SECRET)
  and a runner that mirrors the job into its own registry with the
  orchestrator's identity, publishes started --heartbeat, runs --exec and
  publishes the terminal event.
- loop runner: passes MAM_HEARTBEAT_SEC / MAM_HEARTBEAT_MISSES /
  MAM_ACK_TIMEOUT_S through; YAML rows with transport: remote delegate as
  remote:<name>.
- docs: MESSAGING.md 2.5 contract + remote transport, delegate-job SKILL.md,
  registry.md, loop SKILL.md.
- tests: tests/test_issue18_liveness_contract.py (16); issue 14 pane-signal
  tests flipped to started-based; issue 6 working-agent tests acknowledge
  with started; CLI snapshot/help captures refreshed.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
GodopuandClaude Fable 5.1 committed 2026-09-13 19:06:12 +09:00
1 parent 139bdd4b4c
commit 897b505266
21 files changed
+1989 -129

No files matched your search

+63 -4
View File
@@ -47,6 +47,22 @@ def generate_job_id(bits: int = 32) -> str:
return uuid.uuid4().hex[:nibbles]
# Issue #18: the liveness contract's defaults. An agent heartbeats (publishes
# `progress`) at least every N seconds; the orchestrator treats K missed
# heartbeats (K x N seconds since the last event) as unresponsive; `started`
# must arrive within T seconds of the prompt injection. K x N with these
# defaults equals the historical 120 s idle timeout, so a caller that never
# asked for the contract sees the same idle window as before.
DEFAULT_HEARTBEAT_SEC = 30
DEFAULT_HEARTBEAT_MISSES = 4
DEFAULT_ACK_TIMEOUT_SEC = 60
def liveness_window(heartbeat_sec: int, heartbeat_misses: int) -> int:
"""K x N: seconds without any event after which a job is unresponsive."""
return max(1, int(heartbeat_sec)) * max(1, int(heartbeat_misses))
def register_job(
prompt: str,
agent: str = "claude-code",
@@ -54,7 +70,7 @@ def register_job(
role: str = "Worker",
broker: Optional[Dict[str, Any]] = None,
timeout_sec: int = 3600,
idle_timeout_sec: int = 120,
idle_timeout_sec: Optional[int] = None,
registry_dir: str = DEFAULT_REGISTRY_DIR,
job_id: Optional[str] = None,
expected_artifacts: Optional[List[str]] = None,
@@ -64,13 +80,31 @@ def register_job(
reviewer: Optional[str] = None,
reviewer_session: Optional[str] = None,
max_iterations: int = 5,
heartbeat_sec: int = DEFAULT_HEARTBEAT_SEC,
heartbeat_misses: int = DEFAULT_HEARTBEAT_MISSES,
ack_timeout_sec: int = DEFAULT_ACK_TIMEOUT_SEC,
dispatch_token: Optional[str] = None,
attempt_id: int = 1,
transport: str = "herdr",
) -> str:
"""Create a new ``pending`` job record and return its id.
``broker`` defaults to the current environment's resolved broker block, so
the registry alone is enough for ``publish_event.py`` to connect later.
Issue #18: ``idle_timeout_sec`` left as None derives from the liveness
contract (``heartbeat_misses x heartbeat_sec``); an explicit value is kept
as given. ``dispatch_token`` / ``attempt_id`` are minted here unless a
remote runner mirrors a job the orchestrator already registered, in which
case it passes the orchestrator's so the events it publishes carry the
identity the orchestrator's subscriber expects. ``transport`` records how
the prompt reaches the agent (``herdr`` pane or ``remote`` dispatch topic).
"""
job_id = job_id or generate_job_id(bits)
heartbeat_sec = max(1, int(heartbeat_sec))
heartbeat_misses = max(1, int(heartbeat_misses))
if idle_timeout_sec is None:
idle_timeout_sec = liveness_window(heartbeat_sec, heartbeat_misses)
if broker is None:
broker = broker_config_from_env().to_registry_block()
if auth_token is None:
@@ -101,8 +135,13 @@ def register_job(
"reviewer_session": reviewer_session,
"max_iterations": int(max_iterations),
"iteration": 1,
"dispatch_token": uuid.uuid4().hex,
"attempt_id": 1,
"dispatch_token": dispatch_token or uuid.uuid4().hex,
"attempt_id": int(attempt_id),
# Issue #18 liveness contract
"heartbeat_sec": heartbeat_sec,
"heartbeat_misses": heartbeat_misses,
"ack_timeout_sec": max(1, int(ack_timeout_sec)),
"transport": transport,
}
with registry_lock(registry_dir):
if mqtt_common._job_path(job_id, registry_dir).exists():
@@ -276,7 +315,20 @@ def _build_parser() -> argparse.ArgumentParser:
p_reg.add_argument("--agent-session", default="herdr:claude")
p_reg.add_argument("--role", default="Worker", help="logical role for the delegated agent (e.g. Worker, Planner, Reviewer)")
p_reg.add_argument("--timeout", type=int, default=3600)
p_reg.add_argument("--idle-timeout", type=int, default=120)
p_reg.add_argument("--idle-timeout", type=int, default=None,
help="seconds without an event before the job is unresponsive "
"(default: --heartbeat-misses x --heartbeat-sec)")
p_reg.add_argument("--heartbeat-sec", type=int, default=DEFAULT_HEARTBEAT_SEC,
help="agent heartbeat period N in seconds (Issue #18)")
p_reg.add_argument("--heartbeat-misses", type=int, default=DEFAULT_HEARTBEAT_MISSES,
help="missed heartbeats K before the job counts as unresponsive")
p_reg.add_argument("--ack-timeout", type=int, default=DEFAULT_ACK_TIMEOUT_SEC,
help="seconds T after injection within which `started` must arrive")
p_reg.add_argument("--transport", default="herdr", choices=["herdr", "remote"],
help="how the prompt reaches the agent")
p_reg.add_argument("--dispatch-token", default=None, help=argparse.SUPPRESS)
p_reg.add_argument("--attempt-id", type=int, default=1, help=argparse.SUPPRESS)
p_reg.add_argument("--job-id", default=None, help=argparse.SUPPRESS)
p_reg.add_argument("--bits", type=int, default=32, help="32 (PoC) or 128 (prod)")
p_reg.add_argument("--artifact", action="append", default=[], dest="artifacts")
p_reg.add_argument("--auth-token", default=None, help="HMAC auth token for the job (auto-generated if secure broker is detected)")
@@ -430,6 +482,13 @@ def main(argv: Optional[List[str]] = None) -> int:
reviewer=args.reviewer,
reviewer_session=args.reviewer_session,
max_iterations=args.max_iterations,
heartbeat_sec=args.heartbeat_sec,
heartbeat_misses=args.heartbeat_misses,
ack_timeout_sec=args.ack_timeout,
transport=args.transport,
dispatch_token=args.dispatch_token,
attempt_id=args.attempt_id,
job_id=args.job_id,
)
print(job_id)
return 0