feat(issue-18): decide liveness and delivery by the event protocol, not the screen

Contract (no new event types): heartbeat every N s (automatic timer),
unresponsive after K x N s without an event, `started` within T s of the
injection, retained per-job state, and a remote transport.

- registry: heartbeat_sec / heartbeat_misses / ack_timeout_sec / transport
  fields; idle_timeout_sec derives K x N unless explicit (defaults 30/4/60
  keep the historical 120 s idle window).
- heartbeat.py + publish_event --heartbeat N: detached fixed-cadence timer
  publishing signed `progress` heartbeats (threaded wire publish, local
  record first); stopped by the terminal event via pidfile, by re-dispatch
  and by lifetime. Terminal events refuse --heartbeat.
- publish_event: retained `<prefix>/state` (status, last_event, seq, HMAC)
  sent on the same connection after every event (--no-state to skip);
  publisher.publish_raw takes extra messages.
- job_subscriber: subscribes the state topic, recovers terminal status from
  retained state, resets idle on any accepted message, records seq_gap and
  reconnected in the audit log, idle budget before the first event is
  T + K x N (--ack-timeout), --remote mirrors started/terminal events into
  the registry through the transition boundary.
- lib.sh send_keys_safe: an accepted-but-unconfirmed submission is confirmed
  by agent_status working or the job's own `started` (_sks_job_started);
  pane change and busy token are no longer delivery signals.
- wrapper: --heartbeat/--heartbeat-misses/--ack-timeout (MAM_* env),
  injected start command arms the timer, brief states the contract,
  _wait_started (registry / events.log / subscriber log, bounded by T and
  the wall clock) replaces the 10 s inject grace, R1 grace uses only the
  on-disk event age (herdr/pane logged once as a diagnostic),
  remote:<name> sessions skip preflight/idle/pane and dispatch over the
  broker (remote_dispatch).
- remote.py (+ scripts shim): signed retained dispatch (MAM_DISPATCH_SECRET)
  and a runner that mirrors the job into its own registry with the
  orchestrator's identity, publishes started --heartbeat, runs --exec and
  publishes the terminal event.
- loop runner: passes MAM_HEARTBEAT_SEC / MAM_HEARTBEAT_MISSES /
  MAM_ACK_TIMEOUT_S through; YAML rows with transport: remote delegate as
  remote:<name>.
- docs: MESSAGING.md 2.5 contract + remote transport, delegate-job SKILL.md,
  registry.md, loop SKILL.md.
- tests: tests/test_issue18_liveness_contract.py (16); issue 14 pane-signal
  tests flipped to started-based; issue 6 working-agent tests acknowledge
  with started; CLI snapshot/help captures refreshed.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
GodopuandClaude Fable 5.1 committed 2026-09-13 19:06:12 +09:00
1 parent 139bdd4b4c
commit 897b505266
21 files changed
+1989 -129

No files matched your search

+45 -22
View File
@@ -2280,6 +2280,39 @@ except Exception:
' <<< "$raw" 2>/dev/null || true ' <<< "$raw" 2>/dev/null || true
} }
# _sks_job_started <job_id> — Issue #18 delivery signal: the job's own
# `started` (registry status left `pending`, or a started line in the job's
# events.log). Reads the registry the wrapper exports (SKS_REGISTRY_DIR), else
# MAM_JOBS_DIR, else $WORKSPACE_ROOT/.mam/jobs. "adhoc" (no job) is never started.
_sks_job_started() {
local job_id="${1:-adhoc}" rd
[ -n "$job_id" ] && [ "$job_id" != "adhoc" ] || return 1
rd="${SKS_REGISTRY_DIR:-${MAM_JOBS_DIR:-${WORKSPACE_ROOT:-$PWD}/.mam/jobs}}"
"$(_delegate_py_bin)" - "$rd" "$job_id" <<'PYEOF'
import json, os, sys
rd, jid = sys.argv[1], sys.argv[2]
rec = os.path.join(rd, jid + ".json")
try:
st = json.load(open(rec)).get("status")
if st and st != "pending":
sys.exit(0)
except Exception:
pass
log = os.path.join(rd, jid + ".events.log")
try:
with open(log) as fh:
for line in fh:
try:
if json.loads(line).get("event") == "started":
sys.exit(0)
except Exception:
continue
except OSError:
pass
sys.exit(1)
PYEOF
}
# _wait_session_gone <sess> [max_sec=5] # _wait_session_gone <sess> [max_sec=5]
# Reactive loop to wait until a herdr session goes away (happy path returns early). # Reactive loop to wait until a herdr session goes away (happy path returns early).
_wait_session_gone() { _wait_session_gone() {
@@ -2494,9 +2527,6 @@ send_keys_safe() {
# Native herdr 0.8+ fast path: agent prompt handles atomic submission # Native herdr 0.8+ fast path: agent prompt handles atomic submission
if [ "${SKS_PROMPT_WAIT:-1}" = "1" ]; then if [ "${SKS_PROMPT_WAIT:-1}" = "1" ]; then
local p_out="" p_rc=0 local p_out="" p_rc=0
# ISSUE-14: pre-submit snapshot for the delivery-confirmation poll below.
local rpc_pre_pane=""
[ "$_q_rc" != "2" ] && rpc_pre_pane=$(_pane_capture "$sess")
local _p_tmp local _p_tmp
_p_tmp=$(mktemp 2>/dev/null || echo "/tmp/sks_p_${sess}_$$") _p_tmp=$(mktemp 2>/dev/null || echo "/tmp/sks_p_${sess}_$$")
if _sks_herdr agent prompt "$agent_target" "$text" --wait --until working --timeout 5000 >"$_p_tmp" 2>&1; then if _sks_herdr agent prompt "$agent_target" "$text" --wait --until working --timeout 5000 >"$_p_tmp" 2>&1; then
@@ -2534,29 +2564,22 @@ send_keys_safe() {
if [ "$(_sks_agent_status "$sess" "$agent_target")" = "working" ]; then if [ "$(_sks_agent_status "$sess" "$agent_target")" = "working" ]; then
return 0 return 0
fi fi
# ISSUE-14: `timeout` / `agent_prompt_stalled` mean herdr ACCEPTED the # ISSUE-14 / ISSUE-18: `timeout` / `agent_prompt_stalled` mean herdr
# submission and sent the input - only its post-submit status watch came # ACCEPTED the submission and sent the input - only its post-submit
# up empty. An agent whose agent_status never leaves `idle` mid-turn # status watch came up empty (codex under herdr 0.9.0 never reports
# (codex under herdr 0.9.0) therefore failed every delivered prompt here: # `working`). For an accepted submission keep observing for
# the job was closed as agent_blocked and the agent's own `started` / # SKS_DELIVERY_GRACE_S (default 15) and take a *protocol* signal only:
# `completed`, published seconds later, were refused as late. For an # agent_status working, or the job's own `started` in the registry
# accepted submission keep observing for SKS_DELIVERY_GRACE_S (default # (_sks_job_started). Pane changes and busy tokens are no longer
# 15) and take any positive signal: agent_status working, a busy token # delivery evidence (Issue #18: they are CLI-version-specific screen
# the pane did not show before the prompt, or the pane moving on from # scraping that a headless or remote agent never produces). No signal
# its pre-submit snapshot. No signal at all still fails closed (rc 4). # still fails closed (rc 4); the wrapper then waits for `started` up to
# the job's ack timeout T before finalising the inject as failed.
if printf '%s\n' "$p_out" | grep -Eq '"code": *"(timeout|agent_prompt_stalled)"'; then if printf '%s\n' "$p_out" | grep -Eq '"code": *"(timeout|agent_prompt_stalled)"'; then
local g_deadline=$(( $(date +%s) + ${SKS_DELIVERY_GRACE_S:-15} )) local g_deadline=$(( $(date +%s) + ${SKS_DELIVERY_GRACE_S:-15} ))
local pre_busy_rpc=0 cur_pane
printf '%s\n' "$rpc_pre_pane" | grep -Fq "esc to interrupt" && pre_busy_rpc=1
while [ "$(date +%s)" -lt "$g_deadline" ]; do while [ "$(date +%s)" -lt "$g_deadline" ]; do
[ "$(_sks_agent_status "$sess" "$agent_target")" = "working" ] && return 0 [ "$(_sks_agent_status "$sess" "$agent_target")" = "working" ] && return 0
if [ "$_q_rc" != "2" ]; then _sks_job_started "$job_id" && return 0
cur_pane=$(_pane_capture "$sess")
if [ "$pre_busy_rpc" -eq 0 ] && printf '%s\n' "$cur_pane" | grep -Fq "esc to interrupt"; then
return 0
fi
[ -n "$cur_pane" ] && [ "$cur_pane" != "$rpc_pre_pane" ] && return 0
fi
sleep 1 sleep 1
done done
fi fi
+4
View File
@@ -35,6 +35,8 @@ def _build_parser() -> argparse.ArgumentParser:
p_pub.add_argument("--job", required=True) p_pub.add_argument("--job", required=True)
p_pub.add_argument("--event", default="progress", choices=publish_event.VALID_EVENTS) p_pub.add_argument("--event", default="progress", choices=publish_event.VALID_EVENTS)
p_pub.add_argument("--detail", default="") p_pub.add_argument("--detail", default="")
p_pub.add_argument("--heartbeat", type=int, default=None, metavar="N",
help="start the automatic heartbeat timer (Issue #18)")
p_pub.add_argument("--data", default=None) p_pub.add_argument("--data", default=None)
p_pub.add_argument("--retained", action="store_true") p_pub.add_argument("--retained", action="store_true")
p_pub.add_argument("--registry-dir", default=None) p_pub.add_argument("--registry-dir", default=None)
@@ -74,6 +76,8 @@ def main(argv: Optional[List[str]] = None) -> int:
forward += ["--expect-dispatch-token", args.expect_dispatch_token] forward += ["--expect-dispatch-token", args.expect_dispatch_token]
if args.attempt_id is not None: if args.attempt_id is not None:
forward += ["--attempt-id", str(args.attempt_id)] forward += ["--attempt-id", str(args.attempt_id)]
if getattr(args, "heartbeat", None) is not None:
forward += ["--heartbeat", str(args.heartbeat)]
if args.verbose: if args.verbose:
forward += ["--verbose"] forward += ["--verbose"]
return publish_event.main(forward) return publish_event.main(forward)
@@ -0,0 +1,239 @@
"""heartbeat.py — the agent-side automatic heartbeat timer (Issue #18).
The liveness contract obliges an agent to publish `progress` at least every N
seconds while it works. Relying on the brief to say so meant an agent that
forgot, or one whose turn was one long tool call, was closed as unresponsive
while it was fine. This timer publishes the heartbeat for it.
`publish_event.py --event started --heartbeat N` starts one detached timer per
job; it runs in its own session so the agent's shell may return at once. The
timer stops on its own when
* the job record reaches a terminal status (completed / error / cancelled),
* the record's dispatch_token or attempt_id changes (the job was
re-dispatched: this attempt's heartbeats would be refused anyway),
* the job record disappears, or
* its lifetime exceeds the job's wall-clock timeout plus one period.
A terminal publish also stops it directly through the pidfile
(`<registry>/<job_id>.heartbeat.pid`) so no heartbeat can trail a `completed`.
Each beat is an ordinary `publish_event` call in this process: it takes the
next seq, is signed with the job's auth_token, carries the dispatch token and
attempt id the timer was started with, and is recorded in events.log before
the wire publish - exactly what the orchestrator's disk and wire liveness
checks read.
"""
from __future__ import annotations
import argparse
import logging
import os
import signal
import subprocess
import sys
import threading
import time
from typing import List, Optional
from lib_py.delegation import mqtt_common
from lib_py.delegation.mqtt_common import default_registry_dir, load_job
logger = logging.getLogger("delegate_job.heartbeat")
TERMINAL_STATUSES = ("completed", "error", "cancelled")
HEARTBEAT_DETAIL = "heartbeat"
MAX_IN_FLIGHT = 8
def pidfile_path(job_id: str, registry_dir: str) -> str:
return os.path.join(registry_dir, "{}.heartbeat.pid".format(job_id))
def _pid_alive(pid: int) -> bool:
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def running_timer(job_id: str, registry_dir: str) -> Optional[int]:
"""PID of a live timer for this job, or None."""
try:
with open(pidfile_path(job_id, registry_dir)) as fh:
pid = int(fh.read().strip() or 0)
except (OSError, ValueError):
return None
if pid > 0 and _pid_alive(pid):
return pid
return None
def stop_timer(job_id: str, registry_dir: str) -> bool:
"""Terminate the job's timer if one runs. Returns True when one was signalled."""
pid = running_timer(job_id, registry_dir)
path = pidfile_path(job_id, registry_dir)
if pid is None:
try:
os.unlink(path)
except OSError:
pass
return False
try:
os.kill(pid, signal.SIGTERM)
except OSError:
pass
try:
os.unlink(path)
except OSError:
pass
return True
def start_timer(job_id: str, registry_dir: str, interval: int,
dispatch_token: Optional[str] = None, attempt_id: Optional[int] = None,
python: Optional[str] = None) -> int:
"""Spawn the detached timer process and return its pid.
A timer that already runs for the job is stopped first: a second `started`
for the same job (a retry of the same command) must not double the rate.
"""
stop_timer(job_id, registry_dir)
cmd: List[str] = [python or sys.executable, "-m", "lib_py.delegation.heartbeat",
"--job", job_id, "--registry-dir", registry_dir,
"--interval", str(int(interval))]
if dispatch_token:
cmd += ["--expect-dispatch-token", dispatch_token]
if attempt_id is not None:
cmd += ["--attempt-id", str(int(attempt_id))]
env = dict(os.environ)
# The timer imports lib_py from the same tree as this process.
pkg_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
env["PYTHONPATH"] = pkg_root + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
log_path = os.path.join(registry_dir, "{}.heartbeat.log".format(job_id))
with open(log_path, "ab") as log:
proc = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT,
start_new_session=True, env=env, cwd=os.getcwd())
with open(pidfile_path(job_id, registry_dir), "w") as fh:
fh.write(str(proc.pid))
logger.info("heartbeat timer pid %d for job %s every %ss", proc.pid, job_id, interval)
return proc.pid
def _should_stop(job_id: str, registry_dir: str, dispatch_token, attempt_id) -> Optional[str]:
try:
rec = load_job(job_id, registry_dir)
except FileNotFoundError:
return "job record gone"
if rec.get("status") in TERMINAL_STATUSES:
return "job is {}".format(rec.get("status"))
if dispatch_token and rec.get("dispatch_token") != dispatch_token:
return "dispatch token changed (re-dispatched)"
if attempt_id is not None and rec.get("attempt_id", rec.get("iteration", 1)) != attempt_id:
return "attempt changed (re-dispatched)"
return None
def run(job_id: str, registry_dir: str, interval: int, dispatch_token=None,
attempt_id=None, max_lifetime: Optional[float] = None) -> int:
"""The timer loop. Runs until a stop condition holds."""
from lib_py.delegation import publish_event # late import: the CLI imports us
interval = max(1, int(interval))
if max_lifetime is None:
try:
max_lifetime = float(load_job(job_id, registry_dir).get("timeout_sec", 3600)) + interval
except FileNotFoundError:
return 1
stop = {"flag": False}
def _on_term(_signum, _frame):
stop["flag"] = True
signal.signal(signal.SIGTERM, _on_term)
signal.signal(signal.SIGINT, _on_term)
started = time.monotonic()
own_pidfile = pidfile_path(job_id, registry_dir)
# Fixed cadence: the next beat is due `interval` after the previous one
# was *due*, not after its publish returned, so a slow broker (seconds per
# ACK) does not stretch the period the contract promises. Each beat is
# recorded on disk before its wire publish, so the on-disk cadence holds
# even when the wire lags. Sleep first: `started` was just published.
next_due = time.monotonic() + interval
in_flight: List[threading.Thread] = []
while not stop["flag"]:
while time.monotonic() < next_due and not stop["flag"]:
time.sleep(min(0.5, max(0.0, next_due - time.monotonic())))
if stop["flag"]:
break
next_due += interval
if next_due < time.monotonic():
next_due = time.monotonic() + interval # fell far behind: resync
reason = _should_stop(job_id, registry_dir, dispatch_token, attempt_id)
if reason:
logger.info("heartbeat timer for %s stopping: %s", job_id, reason)
break
if time.monotonic() - started > max_lifetime:
logger.info("heartbeat timer for %s stopping: lifetime %.0fs exceeded", job_id, max_lifetime)
break
argv = ["--registry-dir", registry_dir, "--job", job_id, "--event", "progress",
"--detail", HEARTBEAT_DETAIL, "--attempts", "2"]
if dispatch_token:
argv += ["--expect-dispatch-token", dispatch_token]
if attempt_id is not None:
argv += ["--attempt-id", str(attempt_id)]
# Each beat publishes in its own thread: the local record (seq, HMAC,
# events.log) is taken at once, and the wire publish - seconds per ACK
# on a slow broker - must not delay the next beat. In-flight beats are
# capped so a dead broker cannot pile up threads without bound.
in_flight[:] = [t for t in in_flight if t.is_alive()]
if len(in_flight) >= MAX_IN_FLIGHT:
logger.warning("heartbeat for %s skipped: %d publishes still in flight", job_id, len(in_flight))
continue
def _beat(argv=argv):
try:
rc = publish_event.main(argv)
except Exception as exc: # pragma: no cover - defensive
logger.warning("heartbeat publish raised: %s", exc)
rc = 2
if rc == 4:
logger.info("heartbeat publish refused (stale attempt); timer for %s will stop", job_id)
stop["flag"] = True
t = threading.Thread(target=_beat, daemon=True)
t.start()
in_flight.append(t)
for t in in_flight:
t.join(timeout=15)
try:
with open(own_pidfile) as fh:
if fh.read().strip() == str(os.getpid()):
os.unlink(own_pidfile)
except (OSError, ValueError):
pass
return 0
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description="automatic heartbeat timer for a delegated job")
parser.add_argument("--job", required=True)
parser.add_argument("--registry-dir", default=default_registry_dir())
parser.add_argument("--interval", type=int, required=True, help="heartbeat period N in seconds")
parser.add_argument("--expect-dispatch-token", default=None)
parser.add_argument("--attempt-id", type=int, default=None)
parser.add_argument("--max-lifetime", type=float, default=None)
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args(argv)
mqtt_common.ensure_env_loaded()
mqtt_common.setup_logging(logging.DEBUG if args.verbose else logging.INFO)
return run(args.job, args.registry_dir, args.interval, args.expect_dispatch_token,
args.attempt_id, args.max_lifetime)
if __name__ == "__main__":
sys.exit(main())
@@ -44,6 +44,8 @@ from lib_py.delegation.mqtt_common import (
logger = logging.getLogger("delegate_job.job_subscriber") logger = logging.getLogger("delegate_job.job_subscriber")
TERMINAL_EVENTS = ("completed", "error") TERMINAL_EVENTS = ("completed", "error")
TERMINAL_STATUSES = ("completed", "error", "cancelled")
STATE_EVENT = "state"
def _format_line(topic: str, payload: Dict[str, Any]) -> str: def _format_line(topic: str, payload: Dict[str, Any]) -> str:
@@ -133,6 +135,46 @@ class _Watcher:
# by the transition boundary as an idempotent ack, so a replay after # by the transition boundary as an idempotent ack, so a replay after
# restart cannot double-apply either. # restart cannot double-apply either.
self.seen_intents = {} self.seen_intents = {}
# Issue #18: jobs for which at least one wire event was accepted. A seq
# gap is only meaningful after that - the initial last_seq comes from
# the registry and may be behind the wire (or ahead of a fresh remote).
self.accepted_any: Set[str] = set()
def _note_seq_gap(self, jid: str, expected: int, got: int, via: str, topic: str) -> None:
"""Issue #18: a seq the subscriber never saw is a detected loss."""
logger.warning("seq gap for job %s: expected %d, got %d (via %s)", jid, expected, got, via)
mqtt_common.append_event(jid, {
"event": "seq_gap",
"expected_seq": expected,
"seq": got,
"via": via,
"topic": topic,
})
def _on_state(self, msg, payload: Dict[str, Any], jid: str) -> None:
"""A retained `<prefix>/state` message (Issue #18): learn the job's
status and last seq at (re)connect without waiting for the next
heartbeat. Signed and token-checked like an event. Delivered to the
main loop as a state item so the idle window resets and a terminal
status ends the wait."""
seq = payload.get("seq")
if not isinstance(seq, int):
logger.warning("drop state for job %s: missing or invalid seq", jid)
return
last = self.last_seq.get(jid, 0)
if jid in self.accepted_any and seq > last + 1:
self._note_seq_gap(jid, last + 1, seq, "state", msg.topic)
if seq > last:
self.last_seq[jid] = seq
mqtt_common.append_event(jid, {
"event": "received",
"source_event": STATE_EVENT,
"seq": seq,
"topic": msg.topic,
"timestamp": payload.get("timestamp"),
"detail": "status={} last_event={}".format(payload.get("status"), payload.get("last_event")),
})
self.events.put((msg.topic, payload))
def on_message(self, _client, _userdata, msg) -> None: def on_message(self, _client, _userdata, msg) -> None:
# --- defensive parsing ------------------------------------------- # --- defensive parsing -------------------------------------------
@@ -175,6 +217,9 @@ class _Watcher:
logger.warning("drop event for job %s: attempt_id %r is older than expected %r", logger.warning("drop event for job %s: attempt_id %r is older than expected %r",
jid, ev_att, expected_att) jid, ev_att, expected_att)
return return
if payload.get("event") == STATE_EVENT:
self._on_state(msg, payload, jid)
return
# --- replay attack defense: check monotonic sequence --- # --- replay attack defense: check monotonic sequence ---
seq = payload.get("seq") seq = payload.get("seq")
if seq is None or not isinstance(seq, int): if seq is None or not isinstance(seq, int):
@@ -210,7 +255,10 @@ class _Watcher:
logger.warning("drop event for job %s: seq %d is not monotonically increasing (last %d)", logger.warning("drop event for job %s: seq %d is not monotonically increasing (last %d)",
jid, seq, self.last_seq.get(jid, 0)) jid, seq, self.last_seq.get(jid, 0))
return return
if jid in self.accepted_any and seq > self.last_seq.get(jid, 0) + 1:
self._note_seq_gap(jid, self.last_seq.get(jid, 0) + 1, seq, "events", msg.topic)
self.last_seq[jid] = seq self.last_seq[jid] = seq
self.accepted_any.add(jid)
# Recorded only once the event is actually accepted. A late packet that the # Recorded only once the event is actually accepted. A late packet that the
# sequence guard refuses must not mark its intent as seen, or the genuine # sequence guard refuses must not mark its intent as seen, or the genuine
# retry that follows would be mistaken for the replay of an event this # retry that follows would be mistaken for the replay of an event this
@@ -232,6 +280,22 @@ class _Watcher:
self.events.put((msg.topic, payload)) self.events.put((msg.topic, payload))
def _sync_remote_status(jid: str, status: str, registry_dir: str,
dispatch_token: Optional[str]) -> None:
"""Issue #18: mirror a wire event into the orchestrator's registry for a
job whose agent shares no disk with it. Goes through the same transition
boundary as the publisher's own status sync; a refused (late/duplicate)
transition is logged, never raised."""
try:
res = mqtt_common.transition_job_status(
jid, registry_dir, _source="job_subscriber", status=status,
expected_dispatch_token=dispatch_token)
if not res.committed and not res.idempotent:
logger.warning("remote status sync refused for %s -> %s: %s", jid, status, res.reason)
except Exception as exc: # pragma: no cover - best effort
logger.warning("remote status sync failed for %s: %s", jid, exc)
def _collect_jobs(args) -> List[Dict[str, Any]]: def _collect_jobs(args) -> List[Dict[str, Any]]:
"""Resolve the list of job records this invocation should watch.""" """Resolve the list of job records this invocation should watch."""
if args.wait_any: if args.wait_any:
@@ -276,6 +340,13 @@ def _run_subscriber(args) -> int:
base_job = jobs[0] base_job = jobs[0]
wall_timeout = args.timeout if args.timeout is not None else float(base_job.get("timeout_sec", 3600)) wall_timeout = args.timeout if args.timeout is not None else float(base_job.get("timeout_sec", 3600))
idle_timeout = args.idle_timeout if args.idle_timeout is not None else float(base_job.get("idle_timeout_sec", 120)) idle_timeout = args.idle_timeout if args.idle_timeout is not None else float(base_job.get("idle_timeout_sec", 120))
# Issue #18: until the first accepted event the "last event" is the
# injection itself, which happens after this subscriber started and may
# take up to the ack timeout T to produce `started` (a remote runner has
# to receive the dispatch first). The idle budget before the first event
# is therefore T + K x N; from the first event on it is K x N.
ack_timeout = args.ack_timeout if args.ack_timeout is not None else float(base_job.get("ack_timeout_sec", 0) or 0)
first_event_seen = False
# All watched jobs share a broker in practice; connect using the first # All watched jobs share a broker in practice; connect using the first
# job's broker and subscribe to each job's events topic. # job's broker and subscribe to each job's events topic.
@@ -283,16 +354,31 @@ def _run_subscriber(args) -> int:
client = make_client("subscriber", config) client = make_client("subscriber", config)
client.on_message = watcher.on_message client.on_message = watcher.on_message
# Issue #18: a remote job (no shared disk with the agent) has nobody else
# to move its registry record - the subscriber does it from the wire.
remote_sync = bool(args.remote) or any(j.get("transport") == "remote" for j in jobs)
subscribed_topics = [] subscribed_topics = []
state_topics = []
for job in jobs: for job in jobs:
prefix = job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"]) prefix = job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"])
subscribed_topics.append(f"{prefix}/events") subscribed_topics.append(f"{prefix}/events")
state_topics.append(f"{prefix}/state")
connects = {"count": 0}
def on_connect(_c, _u, _flags, reason_code, _props): def on_connect(_c, _u, _flags, reason_code, _props):
if mqtt_common.reason_code_value(reason_code) != 0: if mqtt_common.reason_code_value(reason_code) != 0:
logger.error("broker connection failed: rc=%s", reason_code) logger.error("broker connection failed: rc=%s", reason_code)
return return
for topic in subscribed_topics: connects["count"] += 1
if connects["count"] > 1:
# Issue #18: paho reconnected on its own; re-subscribing here is
# what makes the retained state message arrive again at once.
logger.warning("broker reconnected (connect #%d); re-subscribing", connects["count"])
for jid in expected_ids:
mqtt_common.append_event(jid, {"event": "reconnected", "connect_count": connects["count"]})
for topic in subscribed_topics + state_topics:
_c.subscribe(topic, qos=1) _c.subscribe(topic, qos=1)
logger.info("subscribed to %s", topic) logger.info("subscribed to %s", topic)
@@ -361,7 +447,7 @@ def _run_subscriber(args) -> int:
return 2 return 2
if not subscribed: if not subscribed:
last_event = now last_event = now
idle_left = idle_timeout - (now - last_event) idle_left = (idle_timeout if first_event_seen else idle_timeout + ack_timeout) - (now - last_event)
if idle_left <= 0: if idle_left <= 0:
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts) _check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
if not pending: if not pending:
@@ -386,10 +472,30 @@ def _run_subscriber(args) -> int:
last_event = time.monotonic() last_event = time.monotonic()
retention_checked = True retention_checked = True
print(_format_line(topic, payload), flush=True) first_event_seen = True
jid = payload["job_id"] jid = payload["job_id"]
event = payload.get("event") event = payload.get("event")
if event == STATE_EVENT:
# Issue #18: retained job state. Any state resets the idle
# window (the agent is alive enough to have published); a
# terminal status ends the wait exactly like the terminal event.
status = payload.get("status")
if status in TERMINAL_STATUSES and jid not in terminal:
synth = "completed" if status == "completed" else "error"
payload = dict(payload, event=synth, source="state",
detail="terminal status '{}' recovered from retained state".format(status))
print(_format_line(topic, payload), flush=True)
terminal[jid] = synth
pending.discard(jid)
if remote_sync:
_sync_remote_status(jid, status, args.registry_dir, dispatch_tokens.get(jid))
continue
print(_format_line(topic, payload), flush=True)
if remote_sync and event in ("started", "completed", "error", "cancelled"):
_sync_remote_status(jid, {"started": "running"}.get(event, event),
args.registry_dir, dispatch_tokens.get(jid))
if event in TERMINAL_EVENTS: if event in TERMINAL_EVENTS:
if jid in terminal: if jid in terminal:
# Already finalised: ignore duplicates / late reorders. # Already finalised: ignore duplicates / late reorders.
@@ -421,6 +527,9 @@ def main(argv=None) -> int:
help="wall-clock budget in seconds (default: job.timeout_sec or 3600)") help="wall-clock budget in seconds (default: job.timeout_sec or 3600)")
parser.add_argument("--idle-timeout", type=float, default=None, parser.add_argument("--idle-timeout", type=float, default=None,
help="max seconds with no new event (default: job.idle_timeout_sec or 120)") help="max seconds with no new event (default: job.idle_timeout_sec or 120)")
parser.add_argument("--ack-timeout", type=float, default=None,
help="extra idle budget before the first event, for the injection to "
"produce `started` (default: job.ack_timeout_sec; Issue #18)")
parser.add_argument("--expect-retention", action="store_true", parser.add_argument("--expect-retention", action="store_true",
help="warn if no retained terminal event arrives promptly") help="warn if no retained terminal event arrives promptly")
parser.add_argument("--expect-dispatch-token", default=None, parser.add_argument("--expect-dispatch-token", default=None,
@@ -428,6 +537,10 @@ def main(argv=None) -> int:
parser.add_argument("--attempt-id", type=int, default=None, parser.add_argument("--attempt-id", type=int, default=None,
help="minimum attempt id to accept events for") help="minimum attempt id to accept events for")
parser.add_argument("--registry-dir", default=default_registry_dir()) parser.add_argument("--registry-dir", default=default_registry_dir())
parser.add_argument("--remote", action="store_true",
help="the agent shares no disk with this registry: mirror started/"
"terminal events into the job record (Issue #18; automatic "
"for jobs registered with --transport remote)")
parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args(argv) args = parser.parse_args(argv)
@@ -29,6 +29,7 @@ import logging
import sys import sys
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from lib_py.delegation import heartbeat as heartbeat_timer
from lib_py.delegation import mqtt_common from lib_py.delegation import mqtt_common
from lib_py.delegation import registry from lib_py.delegation import registry
from lib_py.delegation.mqtt_common import ( from lib_py.delegation.mqtt_common import (
@@ -92,15 +93,49 @@ def build_payload(
return payload return payload
def _publish_once(config, topic: str, body: bytes, retain: bool) -> None: def _publish_once(config, topic: str, body: bytes, retain: bool, extra=()) -> None:
"""Connect, publish one QoS-1 message, wait for the broker ACK, disconnect. """Connect, publish one QoS-1 message, wait for the broker ACK, disconnect.
Delegates to the transport module so the CLI path and the service path share Delegates to the transport module so the CLI path and the service path share
one implementation. Keeping a second copy here is what let the persistent one implementation. Keeping a second copy here is what let the persistent
transport's handshake and not-connected bugs exist in only one of them. transport's handshake and not-connected bugs exist in only one of them.
Raises on any failure so ``with_retry`` can re-run the whole sequence. Raises on any failure so ``with_retry`` can re-run the whole sequence.
``extra`` (Issue #18): further ``(topic, body, retain)`` messages sent over
the same connection after the event - the retained job-state message, so
it costs no second broker handshake.
""" """
OneShotPublisher().publish_raw(config, topic, body, retain) OneShotPublisher().publish_raw(config, topic, body, retain, extra=extra)
def state_topic_for(job: Dict[str, Any]) -> str:
prefix = job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"])
return f"{prefix}/state"
def build_state_payload(job: Dict[str, Any], prepared: "PreparedEvent") -> Dict[str, Any]:
"""The retained job-state message published next to every event (Issue #18).
A subscriber that (re)connects receives it at once and learns the job's
status and the last seq without waiting for the next heartbeat; a seq it
never saw in between is a detected loss, not a silent one. Signed like an
event so a forged state cannot close a job.
"""
status = EVENT_TO_STATUS.get(prepared.event) or "running"
data = {k: v for k, v in (prepared.payload.get("data") or {}).items()
if k in ("dispatch_token", "attempt_id")}
payload = build_payload(job_id=prepared.job_id, seq=prepared.seq, event="state",
detail=prepared.event, data=data, auth_token=job.get("auth_token"))
payload["status"] = status
payload["last_event"] = prepared.event
if job.get("auth_token"):
# re-sign: `status`/`last_event` were added after build_payload signed
payload["data"].pop("hmac_sig", None)
sign_payload = {k: v for k, v in payload.items() if k != "data"}
sign_payload["data"] = dict(payload["data"])
msg = json.dumps(sign_payload, sort_keys=True, separators=(",", ":")).encode()
payload["data"]["hmac_sig"] = hmac.new(job["auth_token"].encode(), msg, hashlib.sha256).hexdigest()
return payload
class PreparedEvent(object): class PreparedEvent(object):
@@ -256,9 +291,19 @@ def main(argv=None) -> int:
"job's current iteration)") "job's current iteration)")
parser.add_argument("--expect-dispatch-token", default=None, parser.add_argument("--expect-dispatch-token", default=None,
help="expected dispatch token for CAS verification") help="expected dispatch token for CAS verification")
parser.add_argument("--heartbeat", type=int, default=None, metavar="N",
help="after publishing, start a detached timer that publishes a "
"`progress` heartbeat every N seconds until the job ends "
"(Issue #18; not with a terminal event)")
parser.add_argument("--no-state", action="store_true",
help="do not publish the retained <topic_prefix>/state message")
parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args(argv) args = parser.parse_args(argv)
if args.heartbeat is not None and (args.event in TERMINAL_EVENTS or args.heartbeat < 1):
logger.error("--heartbeat needs a positive period and a non-terminal event")
return 1
mqtt_common.ensure_env_loaded() mqtt_common.ensure_env_loaded()
mqtt_common.setup_logging(logging.DEBUG if args.verbose else logging.WARNING) mqtt_common.setup_logging(logging.DEBUG if args.verbose else logging.WARNING)
@@ -341,6 +386,12 @@ def main(argv=None) -> int:
check_token, actual_tok) check_token, actual_tok)
return 4 return 4
# Issue #18: a terminal event ends the heartbeat obligation. Stop the timer
# before taking this event's seq so no heartbeat can follow `completed`.
if args.event in TERMINAL_EVENTS:
if heartbeat_timer.stop_timer(job_id, args.registry_dir):
logger.info("stopped the heartbeat timer for job %s", job_id)
try: try:
prepared = prepare_event( prepared = prepare_event(
job_id=job_id, job_id=job_id,
@@ -356,6 +407,12 @@ def main(argv=None) -> int:
logger.error("%s", exc) logger.error("%s", exc)
return 1 return 1
extra_messages = []
if not args.no_state and rec is not None:
state_payload = build_state_payload(rec, prepared)
extra_messages.append((state_topic_for(rec),
json.dumps(state_payload, ensure_ascii=False).encode("utf-8"), True))
# Local first: the event reaches events.log before the publish can block on # Local first: the event reaches events.log before the publish can block on
# the broker ACK and its retries (see record_local_event). # the broker ACK and its retries (see record_local_event).
record_local_event(prepared, args.registry_dir) record_local_event(prepared, args.registry_dir)
@@ -370,7 +427,7 @@ def main(argv=None) -> int:
publish_ok = True publish_ok = True
publish_error: Optional[str] = None publish_error: Optional[str] = None
try: try:
publish(prepared.config, prepared.topic, prepared.body, prepared.retain) publish(prepared.config, prepared.topic, prepared.body, prepared.retain, extra_messages)
except Exception as exc: except Exception as exc:
publish_ok = False publish_ok = False
publish_error = str(exc) publish_error = str(exc)
@@ -388,6 +445,22 @@ def main(argv=None) -> int:
expected_attempt=sync_expected_attempt, expected_attempt=sync_expected_attempt,
local_event_recorded=True) local_event_recorded=True)
# Issue #18: the automatic heartbeat. Started whether or not the wire
# publish succeeded - the agent is working either way, and each beat lands
# in events.log before its own publish, which is what local liveness reads.
if args.heartbeat is not None:
hb_token = sync_expected_token
hb_attempt = sync_expected_attempt
if rec is not None:
hb_token = hb_token or rec.get("dispatch_token")
hb_attempt = hb_attempt if hb_attempt is not None else rec.get("attempt_id", rec.get("iteration", 1))
try:
pid = heartbeat_timer.start_timer(job_id, args.registry_dir, args.heartbeat,
dispatch_token=hb_token, attempt_id=hb_attempt)
logger.info("heartbeat timer started (pid %d, every %ss)", pid, args.heartbeat)
except OSError as exc:
logger.error("could not start the heartbeat timer: %s", exc)
if not publish_ok: if not publish_ok:
return 2 return 2
@@ -79,11 +79,13 @@ class OneShotPublisher(_BasePublisher):
def publish(self, prepared): def publish(self, prepared):
self.publish_raw(prepared.config, prepared.topic, prepared.body, prepared.retain) self.publish_raw(prepared.config, prepared.topic, prepared.body, prepared.retain)
def publish_raw(self, config, topic: str, body: bytes, retain: bool) -> None: def publish_raw(self, config, topic: str, body: bytes, retain: bool, extra=()) -> None:
"""Connect, publish, wait for the ACK, disconnect. """Connect, publish, wait for the ACK, disconnect.
The single implementation of one-shot publishing: ``publish_event``'s The single implementation of one-shot publishing: ``publish_event``'s
CLI path calls straight into this, so the two entry points cannot drift. CLI path calls straight into this, so the two entry points cannot drift.
``extra`` messages (``(topic, body, retain)`` tuples, Issue #18's retained
job state) go out over the same connection, each awaited like the first.
""" """
client = self.client_factory("publisher", config) client = self.client_factory("publisher", config)
self.connect_count += 1 self.connect_count += 1
@@ -104,6 +106,8 @@ class OneShotPublisher(_BasePublisher):
if mqtt_common.reason_code_value(connected["rc"]) != 0: if mqtt_common.reason_code_value(connected["rc"]) != 0:
raise ConnectionError("broker refused connection: rc={}".format(connected["rc"])) raise ConnectionError("broker refused connection: rc={}".format(connected["rc"]))
_send(client, topic, body, retain) _send(client, topic, body, retain)
for x_topic, x_body, x_retain in (extra or ()):
_send(client, x_topic, x_body, x_retain)
finally: finally:
client.loop_stop() client.loop_stop()
try: try:
+63 -4
View File
@@ -47,6 +47,22 @@ def generate_job_id(bits: int = 32) -> str:
return uuid.uuid4().hex[:nibbles] return uuid.uuid4().hex[:nibbles]
# Issue #18: the liveness contract's defaults. An agent heartbeats (publishes
# `progress`) at least every N seconds; the orchestrator treats K missed
# heartbeats (K x N seconds since the last event) as unresponsive; `started`
# must arrive within T seconds of the prompt injection. K x N with these
# defaults equals the historical 120 s idle timeout, so a caller that never
# asked for the contract sees the same idle window as before.
DEFAULT_HEARTBEAT_SEC = 30
DEFAULT_HEARTBEAT_MISSES = 4
DEFAULT_ACK_TIMEOUT_SEC = 60
def liveness_window(heartbeat_sec: int, heartbeat_misses: int) -> int:
"""K x N: seconds without any event after which a job is unresponsive."""
return max(1, int(heartbeat_sec)) * max(1, int(heartbeat_misses))
def register_job( def register_job(
prompt: str, prompt: str,
agent: str = "claude-code", agent: str = "claude-code",
@@ -54,7 +70,7 @@ def register_job(
role: str = "Worker", role: str = "Worker",
broker: Optional[Dict[str, Any]] = None, broker: Optional[Dict[str, Any]] = None,
timeout_sec: int = 3600, timeout_sec: int = 3600,
idle_timeout_sec: int = 120, idle_timeout_sec: Optional[int] = None,
registry_dir: str = DEFAULT_REGISTRY_DIR, registry_dir: str = DEFAULT_REGISTRY_DIR,
job_id: Optional[str] = None, job_id: Optional[str] = None,
expected_artifacts: Optional[List[str]] = None, expected_artifacts: Optional[List[str]] = None,
@@ -64,13 +80,31 @@ def register_job(
reviewer: Optional[str] = None, reviewer: Optional[str] = None,
reviewer_session: Optional[str] = None, reviewer_session: Optional[str] = None,
max_iterations: int = 5, max_iterations: int = 5,
heartbeat_sec: int = DEFAULT_HEARTBEAT_SEC,
heartbeat_misses: int = DEFAULT_HEARTBEAT_MISSES,
ack_timeout_sec: int = DEFAULT_ACK_TIMEOUT_SEC,
dispatch_token: Optional[str] = None,
attempt_id: int = 1,
transport: str = "herdr",
) -> str: ) -> str:
"""Create a new ``pending`` job record and return its id. """Create a new ``pending`` job record and return its id.
``broker`` defaults to the current environment's resolved broker block, so ``broker`` defaults to the current environment's resolved broker block, so
the registry alone is enough for ``publish_event.py`` to connect later. the registry alone is enough for ``publish_event.py`` to connect later.
Issue #18: ``idle_timeout_sec`` left as None derives from the liveness
contract (``heartbeat_misses x heartbeat_sec``); an explicit value is kept
as given. ``dispatch_token`` / ``attempt_id`` are minted here unless a
remote runner mirrors a job the orchestrator already registered, in which
case it passes the orchestrator's so the events it publishes carry the
identity the orchestrator's subscriber expects. ``transport`` records how
the prompt reaches the agent (``herdr`` pane or ``remote`` dispatch topic).
""" """
job_id = job_id or generate_job_id(bits) job_id = job_id or generate_job_id(bits)
heartbeat_sec = max(1, int(heartbeat_sec))
heartbeat_misses = max(1, int(heartbeat_misses))
if idle_timeout_sec is None:
idle_timeout_sec = liveness_window(heartbeat_sec, heartbeat_misses)
if broker is None: if broker is None:
broker = broker_config_from_env().to_registry_block() broker = broker_config_from_env().to_registry_block()
if auth_token is None: if auth_token is None:
@@ -101,8 +135,13 @@ def register_job(
"reviewer_session": reviewer_session, "reviewer_session": reviewer_session,
"max_iterations": int(max_iterations), "max_iterations": int(max_iterations),
"iteration": 1, "iteration": 1,
"dispatch_token": uuid.uuid4().hex, "dispatch_token": dispatch_token or uuid.uuid4().hex,
"attempt_id": 1, "attempt_id": int(attempt_id),
# Issue #18 liveness contract
"heartbeat_sec": heartbeat_sec,
"heartbeat_misses": heartbeat_misses,
"ack_timeout_sec": max(1, int(ack_timeout_sec)),
"transport": transport,
} }
with registry_lock(registry_dir): with registry_lock(registry_dir):
if mqtt_common._job_path(job_id, registry_dir).exists(): if mqtt_common._job_path(job_id, registry_dir).exists():
@@ -276,7 +315,20 @@ def _build_parser() -> argparse.ArgumentParser:
p_reg.add_argument("--agent-session", default="herdr:claude") p_reg.add_argument("--agent-session", default="herdr:claude")
p_reg.add_argument("--role", default="Worker", help="logical role for the delegated agent (e.g. Worker, Planner, Reviewer)") p_reg.add_argument("--role", default="Worker", help="logical role for the delegated agent (e.g. Worker, Planner, Reviewer)")
p_reg.add_argument("--timeout", type=int, default=3600) p_reg.add_argument("--timeout", type=int, default=3600)
p_reg.add_argument("--idle-timeout", type=int, default=120) p_reg.add_argument("--idle-timeout", type=int, default=None,
help="seconds without an event before the job is unresponsive "
"(default: --heartbeat-misses x --heartbeat-sec)")
p_reg.add_argument("--heartbeat-sec", type=int, default=DEFAULT_HEARTBEAT_SEC,
help="agent heartbeat period N in seconds (Issue #18)")
p_reg.add_argument("--heartbeat-misses", type=int, default=DEFAULT_HEARTBEAT_MISSES,
help="missed heartbeats K before the job counts as unresponsive")
p_reg.add_argument("--ack-timeout", type=int, default=DEFAULT_ACK_TIMEOUT_SEC,
help="seconds T after injection within which `started` must arrive")
p_reg.add_argument("--transport", default="herdr", choices=["herdr", "remote"],
help="how the prompt reaches the agent")
p_reg.add_argument("--dispatch-token", default=None, help=argparse.SUPPRESS)
p_reg.add_argument("--attempt-id", type=int, default=1, help=argparse.SUPPRESS)
p_reg.add_argument("--job-id", default=None, help=argparse.SUPPRESS)
p_reg.add_argument("--bits", type=int, default=32, help="32 (PoC) or 128 (prod)") p_reg.add_argument("--bits", type=int, default=32, help="32 (PoC) or 128 (prod)")
p_reg.add_argument("--artifact", action="append", default=[], dest="artifacts") p_reg.add_argument("--artifact", action="append", default=[], dest="artifacts")
p_reg.add_argument("--auth-token", default=None, help="HMAC auth token for the job (auto-generated if secure broker is detected)") p_reg.add_argument("--auth-token", default=None, help="HMAC auth token for the job (auto-generated if secure broker is detected)")
@@ -430,6 +482,13 @@ def main(argv: Optional[List[str]] = None) -> int:
reviewer=args.reviewer, reviewer=args.reviewer,
reviewer_session=args.reviewer_session, reviewer_session=args.reviewer_session,
max_iterations=args.max_iterations, max_iterations=args.max_iterations,
heartbeat_sec=args.heartbeat_sec,
heartbeat_misses=args.heartbeat_misses,
ack_timeout_sec=args.ack_timeout,
transport=args.transport,
dispatch_token=args.dispatch_token,
attempt_id=args.attempt_id,
job_id=args.job_id,
) )
print(job_id) print(job_id)
return 0 return 0
+381
View File
@@ -0,0 +1,381 @@
"""remote.py — deliver a job to an agent on another host over the broker (Issue #18).
The herdr transport injects the prompt into a pane on this machine and reads
the agent's `started`/`progress`/`completed` events back over the broker. A
remote agent shares neither the pane nor the orchestrator's disk, so the
prompt itself has to travel over the wire too:
orchestrator --retained `<topic_prefix>/dispatch`--> remote runner
remote runner --`<topic_prefix>/events` + `/state`--> orchestrator's subscriber
`publish_dispatch` (wrapper side) publishes one retained dispatch message
carrying the brief, the job's auth_token / dispatch_token / attempt_id and the
liveness contract (heartbeat N, misses K, ack T). `run_runner` (remote side)
subscribes for its session name, mirrors the job into its own registry,
publishes `started` with the automatic heartbeat, runs the agent command, and
publishes the terminal event. Every event it publishes is signed with the
job's auth_token and carries the orchestrator's dispatch identity, so the
orchestrator's unchanged subscriber accepts it; the subscriber's `--remote`
mode mirrors those events into the orchestrator's registry.
Trust: a dispatch makes a runner execute a command, so it is signed with a
pre-shared secret (`MAM_DISPATCH_SECRET`, from each host's own .mam.env) and a
runner ignores anything unsigned or mis-signed. The auth_token travels inside
that signed message; use this only on a private broker (see MESSAGING.md).
"""
from __future__ import annotations
import argparse
import hashlib
import hmac
import json
import logging
import os
import queue
import shlex
import socket
import subprocess
import sys
import time
from typing import Any, Dict, Optional
from lib_py.delegation import mqtt_common, registry
from lib_py.delegation.mqtt_common import (
DEFAULT_TOPIC_ROOT,
SCHEMA_VERSION,
broker_config_from_env,
broker_config_from_job,
default_registry_dir,
load_job,
make_client,
)
from lib_py.delegation.publisher import OneShotPublisher
logger = logging.getLogger("delegate_job.remote")
DISPATCH_KIND = "dispatch"
SECRET_ENV = "MAM_DISPATCH_SECRET"
SESSION_PREFIX = "remote:"
def is_remote_session(agent_session: Optional[str]) -> bool:
return bool(agent_session) and str(agent_session).startswith(SESSION_PREFIX)
def session_name(agent_session: str) -> str:
return agent_session[len(SESSION_PREFIX):] if is_remote_session(agent_session) else agent_session
def dispatch_topic_for(job: Dict[str, Any]) -> str:
prefix = job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"])
return f"{prefix}/dispatch"
def _canonical(payload: Dict[str, Any]) -> bytes:
body = {k: v for k, v in payload.items() if k != "sig"}
return json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
def sign_dispatch(payload: Dict[str, Any], secret: str) -> Dict[str, Any]:
payload = dict(payload)
payload["sig"] = hmac.new(secret.encode("utf-8"), _canonical(payload), hashlib.sha256).hexdigest()
return payload
def verify_dispatch(payload: Dict[str, Any], secret: Optional[str]) -> bool:
"""True when the dispatch carries a valid signature for ``secret``. With
no secret configured on the runner side nothing is accepted."""
if not secret:
return False
sig = payload.get("sig")
if not isinstance(sig, str):
return False
expected = hmac.new(secret.encode("utf-8"), _canonical(payload), hashlib.sha256).hexdigest()
return hmac.compare_digest(sig, expected)
def build_dispatch(job: Dict[str, Any], session: str, instructions: str) -> Dict[str, Any]:
return {
"schema_version": SCHEMA_VERSION,
"kind": DISPATCH_KIND,
"job_id": job["job_id"],
"session": session,
"instructions": instructions,
"brief": job.get("prompt", ""),
"role": job.get("role"),
"agent": job.get("agent"),
"auth_token": job.get("auth_token"),
"dispatch_token": job.get("dispatch_token"),
"attempt_id": job.get("attempt_id", job.get("iteration", 1)),
"timeout_sec": job.get("timeout_sec", 3600),
"idle_timeout_sec": job.get("idle_timeout_sec"),
"heartbeat_sec": job.get("heartbeat_sec", registry.DEFAULT_HEARTBEAT_SEC),
"heartbeat_misses": job.get("heartbeat_misses", registry.DEFAULT_HEARTBEAT_MISSES),
"ack_timeout_sec": job.get("ack_timeout_sec", registry.DEFAULT_ACK_TIMEOUT_SEC),
"topic_prefix": job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"]),
"origin": socket.gethostname(),
"timestamp": mqtt_common._utcnow(),
}
def publish_dispatch(job_id: str, registry_dir: str, session: str, instructions: str,
secret: Optional[str] = None, allow_unsigned: bool = False) -> str:
"""Publish the retained dispatch for ``job_id``; returns the topic."""
job = load_job(job_id, registry_dir)
secret = secret or os.environ.get(SECRET_ENV)
payload = build_dispatch(job, session, instructions)
if secret:
payload = sign_dispatch(payload, secret)
elif not allow_unsigned:
raise RuntimeError("{} is not set: refusing to publish an unsigned dispatch "
"(set it in .mam.env on both hosts, or pass --allow-unsigned)".format(SECRET_ENV))
topic = dispatch_topic_for(job)
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
mqtt_common.with_retry(OneShotPublisher().publish_raw, attempts=3,
exceptions=(OSError, TimeoutError, ConnectionError, ValueError))(
broker_config_from_job(job), topic, body, True)
mqtt_common.append_event(job_id, {"event": "dispatched", "topic": topic, "session": session,
"attempt_id": payload["attempt_id"]})
logger.info("dispatched job %s to remote session %s on %s", job_id, session, topic)
return topic
def clear_dispatch(config, topic: str) -> None:
"""Retire a retained dispatch (empty retained payload) so a runner that
restarts does not execute it again."""
try:
OneShotPublisher().publish_raw(config, topic, b"", True)
except Exception as exc: # pragma: no cover - best effort
logger.warning("could not clear dispatch %s: %s", topic, exc)
# ---------------------------------------------------------------------------
# remote side
# ---------------------------------------------------------------------------
def mirror_job(d: Dict[str, Any], registry_dir: str, session: str) -> str:
"""Create the local job record a remote agent publishes against.
The record carries the orchestrator's job_id, auth_token, dispatch_token
and attempt_id, so `publish_event` on this host produces events the
orchestrator's subscriber accepts. A previous attempt's record (a
re-dispatch) is retired first: creation is the only write allowed here.
"""
jid = d["job_id"]
path = mqtt_common._job_path(jid, registry_dir)
if path.exists():
old = load_job(jid, registry_dir)
if (old.get("dispatch_token") == d.get("dispatch_token")
and old.get("status") not in ("completed", "error", "cancelled")):
return jid # same attempt, already mirrored (duplicate delivery)
with mqtt_common.registry_lock(registry_dir):
path.unlink()
log = mqtt_common._job_path(jid, registry_dir).with_suffix(".events.log")
if log.exists():
log.unlink()
registry.register_job(
prompt=d.get("brief", ""),
agent=d.get("agent") or "remote",
agent_session=SESSION_PREFIX + session,
role=d.get("role") or "Worker",
timeout_sec=int(d.get("timeout_sec") or 3600),
idle_timeout_sec=d.get("idle_timeout_sec"),
registry_dir=registry_dir,
job_id=jid,
auth_token=d.get("auth_token"),
heartbeat_sec=int(d.get("heartbeat_sec") or registry.DEFAULT_HEARTBEAT_SEC),
heartbeat_misses=int(d.get("heartbeat_misses") or registry.DEFAULT_HEARTBEAT_MISSES),
ack_timeout_sec=int(d.get("ack_timeout_sec") or registry.DEFAULT_ACK_TIMEOUT_SEC),
dispatch_token=d.get("dispatch_token"),
attempt_id=int(d.get("attempt_id") or 1),
transport="remote",
)
brief_dir = os.path.join(registry_dir, jid)
os.makedirs(brief_dir, exist_ok=True)
with open(os.path.join(brief_dir, "brief.md"), "w", encoding="utf-8") as fh:
fh.write(d.get("brief", ""))
return jid
def _publish(jid: str, registry_dir: str, event: str, detail: str, d: Dict[str, Any],
heartbeat: Optional[int] = None) -> int:
from lib_py.delegation import publish_event
argv = ["--registry-dir", registry_dir, "--job", jid, "--event", event, "--detail", detail]
if d.get("dispatch_token"):
argv += ["--expect-dispatch-token", d["dispatch_token"]]
if d.get("attempt_id") is not None:
argv += ["--attempt-id", str(d["attempt_id"])]
if heartbeat:
argv += ["--heartbeat", str(heartbeat)]
return publish_event.main(argv)
def handle_dispatch(d: Dict[str, Any], registry_dir: str, session: str,
exec_cmd: Optional[str], python: Optional[str] = None) -> int:
"""Run one dispatched job to its terminal event. Returns the agent rc."""
jid = mirror_job(d, registry_dir, session)
heartbeat = int(d.get("heartbeat_sec") or registry.DEFAULT_HEARTBEAT_SEC)
rc_started = _publish(jid, registry_dir, "started", "remote runner {} claimed the job".format(session),
d, heartbeat=heartbeat)
if rc_started not in (0, 2):
logger.error("could not publish started for %s (rc %s)", jid, rc_started)
return 1
job = load_job(jid, registry_dir)
rc = 0
if exec_cmd:
py = python or sys.executable
skills_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
env = dict(os.environ)
env["PYTHONPATH"] = skills_root + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
env.update({
"JOB_ID": jid,
"MAM_JOB_ID": jid,
"MAM_JOB_REGISTRY_DIR": registry_dir,
"MAM_JOB_BRIEF": os.path.join(registry_dir, jid, "brief.md"),
"MAM_JOB_INSTRUCTIONS": d.get("instructions", ""),
"MAM_PUBLISH_CMD": " ".join(shlex.quote(x) for x in [
py, "-m", "lib_py.delegation.publish_event", "--registry-dir", registry_dir, "--job", jid]
+ (["--expect-dispatch-token", d["dispatch_token"]] if d.get("dispatch_token") else [])
+ (["--attempt-id", str(d["attempt_id"])] if d.get("attempt_id") is not None else [])),
})
timeout = float(job.get("timeout_sec", 3600))
try:
res = subprocess.run(exec_cmd, shell=True, env=env, capture_output=True, text=True,
timeout=timeout, cwd=os.getcwd())
rc = res.returncode
tail = (res.stdout or "").strip().splitlines()
detail = tail[-1][:200] if tail else "exit {}".format(rc)
if rc != 0:
err = (res.stderr or "").strip().splitlines()
detail = "exit {}: {}".format(rc, (err[-1] if err else detail)[:200])
except subprocess.TimeoutExpired:
rc, detail = 124, "agent command exceeded timeout_sec {}".format(int(timeout))
cur = load_job(jid, registry_dir)
if cur.get("status") not in ("completed", "error", "cancelled"):
_publish(jid, registry_dir, "completed" if rc == 0 else "error", detail, d)
return rc
# No command: an agent on this host publishes the terminal event itself.
deadline = time.monotonic() + float(job.get("timeout_sec", 3600))
while time.monotonic() < deadline:
cur = load_job(jid, registry_dir)
if cur.get("status") in ("completed", "error", "cancelled"):
return 0 if cur.get("status") == "completed" else 1
time.sleep(1.0)
_publish(jid, registry_dir, "error", "deadline_timeout (remote runner)", d)
return 2
def run_runner(session: str, registry_dir: str, exec_cmd: Optional[str] = None, once: bool = False,
secret: Optional[str] = None, allow_unsigned: bool = False,
timeout: Optional[float] = None, python: Optional[str] = None) -> int:
secret = secret or os.environ.get(SECRET_ENV)
if not secret and not allow_unsigned:
logger.error("%s is not set: a runner must not execute unsigned dispatches "
"(pass --allow-unsigned only on a private broker)", SECRET_ENV)
return 1
cfg = broker_config_from_env()
client = make_client("remote-runner", cfg)
inbox: "queue.Queue[Dict[str, Any]]" = queue.Queue()
topic = f"{DEFAULT_TOPIC_ROOT}/+/dispatch"
def on_connect(_c, _u, _flags, reason_code, _props):
if mqtt_common.reason_code_value(reason_code) != 0:
logger.error("broker connection failed: rc=%s", reason_code)
return
_c.subscribe(topic, qos=1)
print(f"SUBSCRIBED {topic}", flush=True)
def on_message(_c, _u, msg):
if not msg.payload:
return # a cleared (retired) dispatch
try:
d = json.loads(msg.payload.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
logger.warning("drop unparseable dispatch on %s", msg.topic)
return
if not isinstance(d, dict) or d.get("kind") != DISPATCH_KIND or d.get("schema_version") != SCHEMA_VERSION:
return
if d.get("session") != session:
return
if secret and not verify_dispatch(d, secret):
logger.warning("drop dispatch for %s: bad or missing signature", d.get("job_id"))
return
if not secret and not allow_unsigned:
return
d["_topic"] = msg.topic
inbox.put(d)
client.on_connect = on_connect
client.on_message = on_message
client.reconnect_delay_set(min_delay=1, max_delay=16)
mqtt_common.with_retry(lambda: client.connect(cfg.host, cfg.port, cfg.keepalive),
attempts=3, base_delay=0.2, max_delay=1.0)()
client.loop_start()
deadline = time.monotonic() + timeout if timeout else None
handled = 0
try:
while True:
wait = 1.0 if deadline is None else max(0.0, min(1.0, deadline - time.monotonic()))
try:
d = inbox.get(timeout=wait)
except queue.Empty:
if deadline is not None and time.monotonic() >= deadline:
logger.warning("runner timeout after %d job(s)", handled)
return 0 if handled else 2
continue
print("DISPATCH job={} attempt={} from={}".format(d.get("job_id"), d.get("attempt_id"), d.get("origin")),
flush=True)
# Retire the retained dispatch before running so a restart cannot replay it.
clear_dispatch(cfg, d["_topic"])
rc = handle_dispatch(d, registry_dir, session, exec_cmd, python=python)
handled += 1
print("DONE job={} rc={}".format(d.get("job_id"), rc), flush=True)
if once:
return 0
finally:
client.loop_stop()
try:
client.disconnect()
except Exception:
pass
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description="remote transport for delegated jobs (Issue #18)")
sub = parser.add_subparsers(dest="command", required=True)
p_d = sub.add_parser("dispatch", help="publish the retained dispatch for a registered job")
p_d.add_argument("--job", required=True)
p_d.add_argument("--registry-dir", default=default_registry_dir())
p_d.add_argument("--session", required=True, help="remote session name (without the remote: prefix)")
p_d.add_argument("--instructions", default="")
p_d.add_argument("--allow-unsigned", action="store_true")
p_r = sub.add_parser("run", help="run a remote session: execute dispatches addressed to it")
p_r.add_argument("--session", required=True)
p_r.add_argument("--registry-dir", default=default_registry_dir())
p_r.add_argument("--exec", dest="exec_cmd", default=None,
help="shell command run per job (env: JOB_ID, MAM_JOB_BRIEF, MAM_PUBLISH_CMD, ...)")
p_r.add_argument("--once", action="store_true", help="exit after one job")
p_r.add_argument("--timeout", type=float, default=None, help="exit after this many seconds")
p_r.add_argument("--allow-unsigned", action="store_true")
p_r.add_argument("--python", default=None)
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args(argv)
mqtt_common.ensure_env_loaded()
mqtt_common.setup_logging(logging.DEBUG if args.verbose else logging.INFO)
if args.command == "dispatch":
try:
print(publish_dispatch(args.job, args.registry_dir, args.session, args.instructions,
allow_unsigned=args.allow_unsigned))
except (FileNotFoundError, RuntimeError) as exc:
logger.error("%s", exc)
return 1
except Exception as exc:
logger.error("dispatch publish failed: %s", exc)
return 2
return 0
return run_runner(args.session, args.registry_dir, args.exec_cmd, args.once,
allow_unsigned=args.allow_unsigned, timeout=args.timeout, python=args.python)
if __name__ == "__main__":
sys.exit(main())
+28 -2
View File
@@ -241,6 +241,32 @@ class LoopRunner:
return "codex" return "codex"
return "claude" return "claude"
def session_label(self, name: str) -> str:
"""`herdr:<name>` for a pane-backed session, `remote:<name>` for a row
registered with `transport: remote` (Issue #18: the prompt then goes
over the broker and no herdr/pane/lsof/ps path is taken)."""
state = getattr(self, "_sessions_state", None)
if state is None:
state = self.load_sessions_state()
self._sessions_state = state
for s in state.get("herdr_sessions", []):
if s.get("name") == name and str(s.get("transport") or "").lower() == "remote":
return f"remote:{name}"
return f"herdr:{name}"
def contract_args(self) -> List[str]:
"""Liveness-contract knobs passed through to every delegate job
(Issue #18): MAM_HEARTBEAT_SEC (N), MAM_HEARTBEAT_MISSES (K),
MAM_ACK_TIMEOUT_S (T). The wrapper derives --idle-timeout as K x N."""
args: List[str] = []
for env_name, flag in (("MAM_HEARTBEAT_SEC", "--heartbeat"),
("MAM_HEARTBEAT_MISSES", "--heartbeat-misses"),
("MAM_ACK_TIMEOUT_S", "--ack-timeout")):
val = os.environ.get(env_name)
if val and val.isdigit() and int(val) > 0:
args += [flag, val]
return args
def resolve_all_reviewers(self, state: Dict[str, Any]) -> List[str]: def resolve_all_reviewers(self, state: Dict[str, Any]) -> List[str]:
"""Resolve running reviewer sessions excluding creator.""" """Resolve running reviewer sessions excluding creator."""
reviewers = [] reviewers = []
@@ -266,12 +292,12 @@ class LoopRunner:
def _run_cmd(p_text: str): def _run_cmd(p_text: str):
cmd = [ cmd = [
"bash", str(wrapper_script), "submit", "bash", str(wrapper_script), "submit",
"--agent-session", f"herdr:{agent_session}", "--agent-session", self.session_label(agent_session),
"--agent", agent_type, "--agent", agent_type,
"--type", "direct", "--type", "direct",
"--role", role, "--role", role,
"--prompt", p_text, "--prompt", p_text,
] ] + self.contract_args()
env = dict(os.environ) env = dict(os.environ)
if "DELEGATE_JOB_PYTHON" not in env: if "DELEGATE_JOB_PYTHON" not in env:
env["DELEGATE_JOB_PYTHON"] = sys.executable env["DELEGATE_JOB_PYTHON"] = sys.executable
@@ -40,7 +40,16 @@ multi-agent-mux-delegate-job submit \
--agent-session herdr:<session_name> \ --agent-session herdr:<session_name> \
--prompt "Task description or instructions here" \ --prompt "Task description or instructions here" \
--role <Worker|Planner|Reviewer> \ --role <Worker|Planner|Reviewer> \
--timeout 3600 --idle-timeout 120 --timeout 3600 [--heartbeat 30 --heartbeat-misses 4 --ack-timeout 60] [--idle-timeout <sec>]
# Liveness contract (Issue #18): the agent heartbeats every N s (automatic),
# the job is unresponsive after K x N s (--idle-timeout overrides), `started`
# must arrive within T s of the injection or the inject fails closed.
# 1b) Same, for an agent on another host (no herdr pane, no shared disk):
# the brief travels over the broker as a signed retained dispatch and a
# `remote.py run --session <name>` runner on that host executes it.
multi-agent-mux-delegate-job submit --agent remote --agent-session remote:<runner_session> \
--prompt "Task description" --role Worker # needs MAM_DISPATCH_SECRET on both hosts
# 2) Submit a job with a feedback loop (Worker-Reviewer Loop) # 2) Submit a job with a feedback loop (Worker-Reviewer Loop)
multi-agent-mux-delegate-job submit \ multi-agent-mux-delegate-job submit \
@@ -77,8 +86,8 @@ Under fail-closed operation (`MAM_DELEGATE_FAIL_CLOSED=1`, default), the CLI ret
| Exit Code | Meaning | Final Registry Status | Failure Behavior & Resilience | | Exit Code | Meaning | Final Registry Status | Failure Behavior & Resilience |
|:---:|:---:|:---:|---| |:---:|:---:|:---:|---|
| `0` | Success | `completed` | Job succeeded and completed event received or verified on disk with approval (`PASS`/`AGREE`). | | `0` | Success | `completed` | Job succeeded and completed event received or verified on disk with approval (`PASS`/`AGREE`). |
| `1` | Failure | `error` | Execution failure / Agent injection error (`inject_agent`). | | `1` | Failure | `error` | Execution failure / delivery failure: no `started` within `--ack-timeout` T s after the injection (`agent_blocked`; the prompt is never re-injected), or the input never reached the pane. |
| `2` | Timeout | `error` (`deadline_timeout`) | Subscriber timeout (total budget or idle timeout reached without terminal event). **R1 Working-Agent Resilience**: If the agent in Herdr is still actively `working`, the wrapper does not immediately fail-close; it re-observes disk status at intervals up to the wall-clock `--timeout` deadline before fail-closing. | | `2` | Timeout | `error` (`deadline_timeout`) | Subscriber timeout (total budget or idle timeout reached without terminal event). **R1 liveness grace (Issue #18)**: after the subscriber idles out the wrapper keeps re-observing the disk while the job's events.log shows an event at most K x N (`--idle-timeout`) old - the automatic heartbeat counts - up to the wall-clock `--timeout`. herdr status and the pane are logged as a diagnostic only. |
| `3` | Broker Error | `error` (`deadline_timeout`) | Broker unavailable (`MQTT_BROKER_UNAVAILABLE`) and terminal status not found on disk during deadline. | | `3` | Broker Error | `error` (`deadline_timeout`) | Broker unavailable (`MQTT_BROKER_UNAVAILABLE`) and terminal status not found on disk during deadline. |
| `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed (prompt never delivered, retryable). | | `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed (prompt never delivered, retryable). |
@@ -131,11 +140,12 @@ a fresh interpreter — but a test harness could.
Every agent participating in the delegation contract must follow the same lifecycle publishing protocol using `publish_event.py`: Every agent participating in the delegation contract must follow the same lifecycle publishing protocol using `publish_event.py`:
1. **On Start**: Publish `started` event. 1. **On Start** (first thing, within the ack timeout T): Publish `started` and arm the heartbeat timer.
`python3 .agents/skills/multi-agent-mux-delegate-job/scripts/publish_event.py --job "$JOB_ID" --event started` `python3 .agents/skills/multi-agent-mux-delegate-job/scripts/publish_event.py --job "$JOB_ID" --event started --heartbeat <N>`
The timer publishes `progress` every N s until your terminal event (Issue #18); the brief and the injected instructions carry the exact command.
2. **On Tool/Permission Prompt**: Publish `permission_required` event. 2. **On Tool/Permission Prompt**: Publish `permission_required` event.
`python3 ... --job "$JOB_ID" --event permission_required --detail "<tool>:<reason>"` `python3 ... --job "$JOB_ID" --event permission_required --detail "<tool>:<reason>"`
3. **On Progress Update (Optional)**: Publish `progress` event. 3. **On Progress Update**: Publish `progress` at check-points (the heartbeat timer already satisfies the every-N-seconds obligation).
`python3 ... --job "$JOB_ID" --event progress --detail "<status_update>"` `python3 ... --job "$JOB_ID" --event progress --detail "<status_update>"`
4. **On Success**: Publish `completed` event. 4. **On Success**: Publish `completed` event.
`python3 ... --job "$JOB_ID" --event completed --detail "<summary>"` (Reviewer should include `"PASS"` in the detail to approve). `python3 ... --job "$JOB_ID" --event completed --detail "<summary>"` (Reviewer should include `"PASS"` in the detail to approve).
@@ -76,11 +76,17 @@ multi-agent-mux-delegate-job <command> [options]
submit --agent <name> --prompt <text> [--workdir <dir>] [--agent-session <label>] submit --agent <name> --prompt <text> [--workdir <dir>] [--agent-session <label>]
[--timeout <sec>] [--idle-timeout <sec>] [--validate <script>] [--timeout <sec>] [--idle-timeout <sec>] [--validate <script>]
[--heartbeat <sec>] [--heartbeat-misses <count>] [--ack-timeout <sec>]
[--registry-dir <dir>] [--dry-run] [--role <role_name>] [--registry-dir <dir>] [--dry-run] [--role <role_name>]
[--type <direct|loop|discuss>] [--reviewer <reviewer_agent>] [--type <direct|loop|discuss>] [--reviewer <reviewer_agent>]
[--reviewer-session <reviewer_session>] [--max-iterations <count>] [--reviewer-session <reviewer_session>] [--max-iterations <count>]
[--counterpart-role <role_name>] [--strict-role-check] [--counterpart-role <role_name>] [--strict-role-check]
# The skill is herdr-interactive only; --mode print was removed. # --agent-session herdr:<name> injects into a local herdr pane;
# --agent-session remote:<name> dispatches over the broker (Issue #18).
# Liveness contract: the agent heartbeats every --heartbeat N s
# (default 30, automatic), the job is unresponsive after
# --heartbeat-misses K x N s (default 4 x 30 = 120; --idle-timeout
# overrides), `started` must arrive within --ack-timeout T s (60).
status --job <id> [--registry-dir <dir>] status --job <id> [--registry-dir <dir>]
list [--registry-dir <dir>] list [--registry-dir <dir>]
verify --job <id> --validate <script> [--registry-dir <dir>] verify --job <id> --validate <script> [--registry-dir <dir>]
@@ -91,7 +97,9 @@ EOF
# ---- arg parsing helpers -------------------------------------------------- # ---- arg parsing helpers --------------------------------------------------
AGENT="claude-code"; PROMPT=""; WORKDIR="$(pwd)"; AGENT_SESSION="herdr:claude" AGENT="claude-code"; PROMPT=""; WORKDIR="$(pwd)"; AGENT_SESSION="herdr:claude"
TIMEOUT=3600; IDLE_TIMEOUT=120; VALIDATE=""; DRY_RUN=0 TIMEOUT=3600; IDLE_TIMEOUT=""; VALIDATE=""; DRY_RUN=0
# Issue #18 liveness contract (N / K / T). An unset --idle-timeout derives K x N.
HEARTBEAT_SEC="${MAM_HEARTBEAT_SEC:-30}"; HEARTBEAT_MISSES="${MAM_HEARTBEAT_MISSES:-4}"; ACK_TIMEOUT="${MAM_ACK_TIMEOUT_S:-60}"
JOB_ID=""; REGISTRY_DIR="$REGISTRY_DIR_DEFAULT"; DELEGATE_ROLE="Worker" JOB_ID=""; REGISTRY_DIR="$REGISTRY_DIR_DEFAULT"; DELEGATE_ROLE="Worker"
TYPE="direct"; REVIEWER="hermes"; REVIEWER_SESSION="herdr:hermes"; MAX_ITERATIONS=5 TYPE="direct"; REVIEWER="hermes"; REVIEWER_SESSION="herdr:hermes"; MAX_ITERATIONS=5
DEFAULT_COUNTERPART_ROLE="Reviewer" DEFAULT_COUNTERPART_ROLE="Reviewer"
@@ -109,6 +117,9 @@ parse_opts() {
--agent-session) AGENT_SESSION="$2"; shift 2;; --agent-session) AGENT_SESSION="$2"; shift 2;;
--timeout) TIMEOUT="$2"; shift 2;; --timeout) TIMEOUT="$2"; shift 2;;
--idle-timeout) IDLE_TIMEOUT="$2"; shift 2;; --idle-timeout) IDLE_TIMEOUT="$2"; shift 2;;
--heartbeat) HEARTBEAT_SEC="$2"; shift 2;;
--heartbeat-misses) HEARTBEAT_MISSES="$2"; shift 2;;
--ack-timeout) ACK_TIMEOUT="$2"; shift 2;;
--validate) VALIDATE="$2"; shift 2;; --validate) VALIDATE="$2"; shift 2;;
--job) JOB_ID="$2"; shift 2;; --job) JOB_ID="$2"; shift 2;;
--registry-dir) REGISTRY_DIR="$2"; shift 2;; --registry-dir) REGISTRY_DIR="$2"; shift 2;;
@@ -123,8 +134,14 @@ parse_opts() {
*) echo "unknown option: $1" >&2; usage; exit 1;; *) echo "unknown option: $1" >&2; usage; exit 1;;
esac esac
done done
[ -n "$IDLE_TIMEOUT" ] || IDLE_TIMEOUT=$(( HEARTBEAT_SEC * HEARTBEAT_MISSES ))
# lib.sh's send_keys_safe reads the job's own `started` from this registry.
export SKS_REGISTRY_DIR="$REGISTRY_DIR"
} }
# Issue #18: `remote:<name>` sessions get the prompt over the broker, not a pane.
_is_remote() { case "${1:-}" in remote:*) return 0;; *) return 1;; esac; }
_to_lower() { _to_lower() {
printf '%s' "$1" | tr '[:upper:]' '[:lower:]' printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
} }
@@ -140,6 +157,8 @@ cmd_submit() {
JOB_ID="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" register \ JOB_ID="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" register \
--prompt "$PROMPT" --agent "$AGENT" --agent-session "$AGENT_SESSION" --role "$DELEGATE_ROLE" \ --prompt "$PROMPT" --agent "$AGENT" --agent-session "$AGENT_SESSION" --role "$DELEGATE_ROLE" \
--timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \ --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
--heartbeat-sec "$HEARTBEAT_SEC" --heartbeat-misses "$HEARTBEAT_MISSES" --ack-timeout "$ACK_TIMEOUT" \
--transport "$(_is_remote "$AGENT_SESSION" && echo remote || echo herdr)" \
--job-type "$TYPE" --reviewer "$REVIEWER" --reviewer-session "$REVIEWER_SESSION" \ --job-type "$TYPE" --reviewer "$REVIEWER" --reviewer-session "$REVIEWER_SESSION" \
--max-iterations "$MAX_ITERATIONS")" --max-iterations "$MAX_ITERATIONS")"
echo "registered job: $JOB_ID" echo "registered job: $JOB_ID"
@@ -155,6 +174,7 @@ cmd_submit() {
- **Target Agent**: $AGENT (session: $AGENT_SESSION) - **Target Agent**: $AGENT (session: $AGENT_SESSION)
- **Role**: $DELEGATE_ROLE - **Role**: $DELEGATE_ROLE
- **Timeout**: $TIMEOUT s (Idle: $IDLE_TIMEOUT s) - **Timeout**: $TIMEOUT s (Idle: $IDLE_TIMEOUT s)
- **Liveness contract**: heartbeat every $HEARTBEAT_SEC s (automatic once you publish \`started --heartbeat $HEARTBEAT_SEC\`), unresponsive after $HEARTBEAT_MISSES missed heartbeats, \`started\` expected within $ACK_TIMEOUT s
- **Output Report Path**: .mam/jobs/$JOB_ID/$AGENT-reports/report-final.md - **Output Report Path**: .mam/jobs/$JOB_ID/$AGENT-reports/report-final.md
## 🔎 Task Description ## 🔎 Task Description
@@ -176,7 +196,7 @@ EOF
[ -n "$_disp_tok" ] && pub_extra+=(--expect-dispatch-token "$_disp_tok") [ -n "$_disp_tok" ] && pub_extra+=(--expect-dispatch-token "$_disp_tok")
[ -n "$_attempt_id" ] && pub_extra+=(--attempt-id "$_attempt_id") [ -n "$_attempt_id" ] && pub_extra+=(--attempt-id "$_attempt_id")
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${pub_extra[*]}" local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${pub_extra[*]}"
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'." local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started --heartbeat $HEARTBEAT_SEC' (run it FIRST, within $ACK_TIMEOUT s; it also starts an automatic heartbeat timer), success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
if [[ "$DRY_RUN" == "1" ]]; then if [[ "$DRY_RUN" == "1" ]]; then
run_agent "$JOB_ID" "$instructions" run_agent "$JOB_ID" "$instructions"
@@ -185,8 +205,8 @@ EOF
return 0 return 0
fi fi
# 1) Preflight # 1) Preflight (a remote session has no local pane to check: Issue #18)
if ! preflight_agent "$JOB_ID" "$AGENT_SESSION" 0; then if ! _is_remote "$AGENT_SESSION" && ! preflight_agent "$JOB_ID" "$AGENT_SESSION" 0; then
finalize_unfinished "$JOB_ID" "error" 1 "preflight" finalize_unfinished "$JOB_ID" "error" 1 "preflight"
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}" local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
echo "$logs_root/$JOB_ID" echo "$logs_root/$JOB_ID"
@@ -194,7 +214,7 @@ EOF
fi fi
# 2) TUI idle handshake # 2) TUI idle handshake
if [ "$AGENT" != "human" ]; then if [ "$AGENT" != "human" ] && ! _is_remote "$AGENT_SESSION"; then
local idle_rc=0 local idle_rc=0
wait_for_agent_idle "$AGENT_SESSION" || idle_rc=$? wait_for_agent_idle "$AGENT_SESSION" || idle_rc=$?
export _SKS_IDLE_RC="$idle_rc" export _SKS_IDLE_RC="$idle_rc"
@@ -210,6 +230,7 @@ EOF
local logf="$REGISTRY_DIR/$JOB_ID.subscriber.out" local logf="$REGISTRY_DIR/$JOB_ID.subscriber.out"
local sub_extra_args=() local sub_extra_args=()
[ -n "$_disp_tok" ] && sub_extra_args+=(--expect-dispatch-token "$_disp_tok") [ -n "$_disp_tok" ] && sub_extra_args+=(--expect-dispatch-token "$_disp_tok")
_is_remote "$AGENT_SESSION" && sub_extra_args+=(--remote)
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \ "$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \ --job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
${sub_extra_args[@]+"${sub_extra_args[@]}"} \ ${sub_extra_args[@]+"${sub_extra_args[@]}"} \
@@ -250,8 +271,8 @@ EOF
echo "WARNING: subscriber subscribe handshake timed out — falling back to proceed" >&2 echo "WARNING: subscriber subscribe handshake timed out — falling back to proceed" >&2
fi fi
# 5) Inject agent # 5) Deliver the prompt (herdr pane or remote dispatch) and wait for `started`
if ! inject_agent "$JOB_ID" "$instructions" "$AGENT_SESSION"; then if ! deliver_prompt "$JOB_ID" "$instructions" "$AGENT_SESSION" "$logf"; then
reap_subscriber "$sub_pid" reap_subscriber "$sub_pid"
finalize_unfinished "$JOB_ID" "error" 1 "inject" finalize_unfinished "$JOB_ID" "error" 1 "inject"
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}" local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
@@ -323,6 +344,7 @@ EOF
- **Target Agent/Session**: $current_session - **Target Agent/Session**: $current_session
- **Role**: $display_role - **Role**: $display_role
- **Iteration**: $iteration - **Iteration**: $iteration
- **Liveness contract**: heartbeat every $HEARTBEAT_SEC s (automatic once you publish \`started --heartbeat $HEARTBEAT_SEC\`), unresponsive after $HEARTBEAT_MISSES missed heartbeats, \`started\` expected within $ACK_TIMEOUT s
- **Output Report Path**: .mam/jobs/$JOB_ID/${clean_session}-reports/report-final.md - **Output Report Path**: .mam/jobs/$JOB_ID/${clean_session}-reports/report-final.md
## 🔎 Task Description ## 🔎 Task Description
@@ -341,8 +363,8 @@ EOF
force_warn_only=1 force_warn_only=1
fi fi
# Preflight # Preflight (skipped for a remote session: Issue #18)
if ! preflight_agent "$JOB_ID" "$current_session" "$force_warn_only"; then if ! _is_remote "$current_session" && ! preflight_agent "$JOB_ID" "$current_session" "$force_warn_only"; then
finalize_unfinished "$JOB_ID" "error" 1 "preflight" finalize_unfinished "$JOB_ID" "error" 1 "preflight"
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}" local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
echo "$logs_root/$JOB_ID" echo "$logs_root/$JOB_ID"
@@ -352,7 +374,7 @@ EOF
# Idle wait # Idle wait
local cur_agent="$AGENT" local cur_agent="$AGENT"
[ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER" [ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER"
if [ "$cur_agent" != "human" ]; then if [ "$cur_agent" != "human" ] && ! _is_remote "$current_session"; then
local idle_rc=0 local idle_rc=0
wait_for_agent_idle "$current_session" || idle_rc=$? wait_for_agent_idle "$current_session" || idle_rc=$?
export _SKS_IDLE_RC="$idle_rc" export _SKS_IDLE_RC="$idle_rc"
@@ -381,6 +403,7 @@ EOF
local logf="$REGISTRY_DIR/${JOB_ID}.iter_${iteration}_${display_role}.subscriber.out" local logf="$REGISTRY_DIR/${JOB_ID}.iter_${iteration}_${display_role}.subscriber.out"
local loop_sub_args=(--attempt-id "$attempt") local loop_sub_args=(--attempt-id "$attempt")
[ -n "$_disp_tok" ] && loop_sub_args+=(--expect-dispatch-token "$_disp_tok") [ -n "$_disp_tok" ] && loop_sub_args+=(--expect-dispatch-token "$_disp_tok")
_is_remote "$current_session" && loop_sub_args+=(--remote)
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \ "$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \ --job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
"${loop_sub_args[@]}" \ "${loop_sub_args[@]}" \
@@ -425,10 +448,10 @@ EOF
local loop_pub_extra=(--attempt-id "$attempt") local loop_pub_extra=(--attempt-id "$attempt")
[ -n "$_disp_tok" ] && loop_pub_extra+=(--expect-dispatch-token "$_disp_tok") [ -n "$_disp_tok" ] && loop_pub_extra+=(--expect-dispatch-token "$_disp_tok")
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${loop_pub_extra[*]}" local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${loop_pub_extra[*]}"
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths for iteration $iteration are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'." local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths for iteration $iteration are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started --heartbeat $HEARTBEAT_SEC' (run it FIRST, within $ACK_TIMEOUT s; it also starts an automatic heartbeat timer), success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
# Trigger agent # Trigger agent
if ! inject_agent "$JOB_ID" "$instructions" "$current_session"; then if ! deliver_prompt "$JOB_ID" "$instructions" "$current_session" "$logf"; then
reap_subscriber "$sub_pid" reap_subscriber "$sub_pid"
finalize_unfinished "$JOB_ID" "error" 1 "inject" finalize_unfinished "$JOB_ID" "error" 1 "inject"
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}" local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
@@ -564,23 +587,37 @@ _disk_status() {
"$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$jid.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true "$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$jid.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true
} }
# ISSUE-14: send_keys_safe can only report what herdr lets it observe. When it # ISSUE-18: delivery is confirmed by the protocol, not the screen. After the
# gives up, the prompt may still have landed - codex under herdr 0.9.0 never # prompt was handed over (herdr accepted the submission, or the dispatch was
# reports `working`, yet publishes `started` within seconds. Closing such a job # published for a remote session) the job's own `started` must arrive within
# as agent_blocked strands a live agent whose later `completed` the CAS table # the ack timeout T: the registry leaves `pending` (the agent's publish_event
# then refuses. Before finalising, give the registry MAM_INJECT_ACK_GRACE_S # syncs it on this host), a started line lands in events.log, or the
# (default 10 s) to advance past pending. Any other status means the job moved # subscriber prints a started line (a remote agent - no shared disk). No
# on - normally the agent's own `started` - so the prompt is treated as # `started` within T is a delivery failure: fail closed, never re-inject
# delivered. The status is not pinned to this dispatch and may come from an # (single-injection contract). The status is not pinned to this dispatch and
# external writer; the terminal handling after this still keeps such an # may come from an external writer; the terminal handling after this still
# error/cancelled as it is rather than turning it into success. # keeps such an error/cancelled as it is rather than turning it into success.
_inject_acknowledged() { _wait_started() {
local job_id="$1" deadline st local job_id="$1" ack_s="${2:-$ACK_TIMEOUT}" sub_log="${3:-}" deadline wall st
deadline=$(( $(date +%s) + ${MAM_INJECT_ACK_GRACE_S:-10} )) deadline=$(( $(date +%s) + ack_s ))
# The ack wait never outlives this dispatch's wall clock (START_TIME + TIMEOUT).
wall=$(( ${START_TIME:-$(date +%s)} + ${TIMEOUT:-3600} ))
[ "$deadline" -gt "$wall" ] && deadline=$wall
local ev_log="$REGISTRY_DIR/$job_id.events.log"
while :; do while :; do
st="$(_disk_status "$job_id")" st="$(_disk_status "$job_id")"
if [ -n "$st" ] && [ "$st" != "pending" ]; then if [ -n "$st" ] && [ "$st" != "pending" ]; then
echo "inject: herdr could not confirm delivery, but job $job_id already advanced to '$st' in the registry; treating the prompt as delivered and continuing." >&2 echo "inject: job $job_id advanced to '$st' in the registry; treating the prompt as delivered and continuing." >&2
return 0
fi
# publish_event records `started` here before its wire publish and before
# the status sync, so a slow broker cannot delay the acknowledgement.
if [ -f "$ev_log" ] && grep -Eq '"event": *"started"' "$ev_log" 2>/dev/null; then
echo "inject: job $job_id recorded 'started' in events.log; treating the prompt as delivered and continuing." >&2
return 0
fi
if [ -n "$sub_log" ] && [ -f "$sub_log" ] && grep -Eq '^[^ ]+ job=[^ ]+ seq=[^ ]+ started( |$)' "$sub_log" 2>/dev/null; then
echo "inject: subscriber received started for job $job_id; treating the prompt as delivered and continuing." >&2
return 0 return 0
fi fi
[ "$(date +%s)" -ge "$deadline" ] && return 1 [ "$(date +%s)" -ge "$deadline" ] && return 1
@@ -588,11 +625,15 @@ _inject_acknowledged() {
done done
} }
# ISSUE-14: herdr's agent_status is blind for some agents (codex stays `idle` # Legacy name retained for callers/tests of the ISSUE-14 guard.
# for the whole turn), so "still working" must also accept the pane's own busy _inject_acknowledged() { _wait_started "$1" "${MAM_INJECT_ACK_GRACE_S:-$ACK_TIMEOUT}" "${2:-}"; }
# token - the same evidence send_keys_safe uses to confirm delivery. Without it
# a codex review longer than the idle timeout was closed as deadline_timeout # ISSUE-14 / ISSUE-18: herdr's agent_status and the pane busy token describe
# mid-turn and its eventual `completed` refused. # the screen, not the protocol. They are no longer liveness *evidence* (a
# headless or remote agent shows neither, and a CLI update can change both
# silently); finalize_unfinished logs them as a diagnostic so an operator can
# tell "dead" from "silent" on the same host. Liveness itself is the contract:
# an event on disk or on the wire within K x N seconds.
_agent_still_working() { _agent_still_working() {
local sess="$1" target="$2" local sess="$1" target="$2"
[ "$(_sks_agent_status "$sess" "$target")" = "working" ] && return 0 [ "$(_sks_agent_status "$sess" "$target")" = "working" ] && return 0
@@ -724,23 +765,27 @@ finalize_unfinished() {
local target_agent local target_agent
target_agent=$(_sanitize_herdr_agent_name "$sess") target_agent=$(_sanitize_herdr_agent_name "$sess")
# ISSUE-14: the subscriber's idle clock only counts events it received over # ISSUE-14 / ISSUE-18: the subscriber's idle clock only counts events it
# the wire. Keep re-observing the disk while liveness evidence keeps # received over the wire. Keep re-observing the disk while the *contract*
# turning up - herdr `working`, the pane busy token, or a heartbeat in # says the agent is alive: an event in events.log at most IDLE_TIMEOUT
# events.log at most IDLE_TIMEOUT old (see _recent_disk_heartbeat). # (K x N) old (_recent_disk_heartbeat - publish_event records every event
# Window policy: entering this grace opens one IDLE_TIMEOUT window and # there before its wire publish, the automatic heartbeat timer included).
# every sighting re-opens it from the time of that poll. An agent that # herdr status and the pane are NOT evidence any more; they are logged
# leaves no evidence at all is therefore closed about 2 x IDLE_TIMEOUT # once here as a diagnostic, so an operator can tell a dead agent from a
# after its last wire event (the subscriber's idle wait plus this window; # silent one on the same host. Window policy: entering this grace opens
# it was IDLE_TIMEOUT before disk heartbeats counted), and heartbeats may # one IDLE_TIMEOUT window and every disk event re-opens it from the time
# be up to about 2 x IDLE_TIMEOUT apart, since one still counts until it # of that poll, so an agent that leaves no event at all is closed about
# is IDLE_TIMEOUT old. Evidence only ever extends the idle window, never # 2 x IDLE_TIMEOUT after its last wire event. Evidence only ever extends
# this dispatch's wall clock: the loop ends at START_TIME + TIMEOUT read # the idle window, never this dispatch's wall clock: the loop ends at
# off the clock on every pass, and the herdr probe is cut off at that # START_TIME + TIMEOUT read off the clock on every pass.
# deadline too (_within_deadline).
local deadline evidence_until left disk_st local deadline evidence_until left disk_st
deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT )) deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT )) evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
if [ "$AGENT" != "human" ] && [ "$sess" != "human" ] && [ -n "$sess" ] && ! _is_remote "$target_sess"; then
local diag="no"
_within_deadline "$deadline" _agent_still_working "$sess" "$target_agent" && diag="yes"
echo "liveness: no wire event for ${IDLE_TIMEOUT}s on job $job_id; herdr/pane says busy=$diag (diagnostic only, not evidence)" >&2
fi
while [ "$(date +%s)" -lt "$deadline" ]; do while [ "$(date +%s)" -lt "$deadline" ]; do
disk_st="$(_disk_status "$job_id")" disk_st="$(_disk_status "$job_id")"
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
@@ -749,8 +794,7 @@ finalize_unfinished() {
[ "$disk_st" = "completed" ] && return 0 [ "$disk_st" = "completed" ] && return 0
break break
fi fi
if { [ "$sess" != "human" ] && [ -n "$sess" ] && _within_deadline "$deadline" _agent_still_working "$sess" "$target_agent"; } \ if _recent_disk_heartbeat "$job_id" "$IDLE_TIMEOUT"; then
|| _recent_disk_heartbeat "$job_id" "$IDLE_TIMEOUT"; then
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT )) evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
elif [ "$(date +%s)" -ge "$evidence_until" ]; then elif [ "$(date +%s)" -ge "$evidence_until" ]; then
break break
@@ -876,6 +920,7 @@ else:
inject_agent() { inject_agent() {
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}" local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}"
# $4: subscriber log (a started line there also confirms delivery)
if [ "$AGENT" = "human" ]; then if [ "$AGENT" = "human" ]; then
echo "[human agent] complete the task, then run publish_event.py --event completed" echo "[human agent] complete the task, then run publish_event.py --event completed"
return 0 return 0
@@ -889,9 +934,24 @@ inject_agent() {
fi fi
echo "살아있는 에이전트 세션 '$sess'에 작업을 위임합니다..." echo "살아있는 에이전트 세션 '$sess'에 작업을 위임합니다..."
if ! send_keys_safe "$sess" "$instructions" "$job_id" && ! _inject_acknowledged "$job_id"; then # ISSUE-18: send_keys_safe is the injection *means*. rc 0 (accepted, herdr
# saw it working) and rc 4 (accepted, no screen signal) both hand over to
# the protocol: the job's own `started` within ACK_TIMEOUT decides. Any
# other rc means the input never reached the agent (pane never quiesced,
# dialog, paste failure) - fail at once, never re-inject.
local sks_rc=0
send_keys_safe "$sess" "$instructions" "$job_id" || sks_rc=$?
case "$sks_rc" in
0|4) ;;
*)
trap - EXIT
echo "ERROR: 프롬프트 주입 실패 (send_keys_safe rc=$sks_rc) — 세션 '$sess'" >&2
return 1 ;;
esac
local ack_s="${MAM_INJECT_ACK_GRACE_S:-$ACK_TIMEOUT}"
if ! _wait_started "$job_id" "$ack_s" "${4:-}"; then
trap - EXIT trap - EXIT
echo "ERROR: 프롬프트 주입 실패 — 세션 '$sess' (프롬프트 잠금 의심)" >&2 echo "ERROR: 프롬프트 주입 실패 — 세션 '$sess': no 'started' within ${ack_s}s (ack timeout); not re-injecting" >&2
return 1 return 1
fi fi
@@ -900,6 +960,38 @@ inject_agent() {
return 0 return 0
} }
# remote_dispatch <job_id> <instructions> <remote:session> <subscriber_log>
# ISSUE-18: no pane, no shared disk. Publish the retained dispatch on the job's
# topic and wait for the agent's own `started` (seen by the subscriber) within
# ACK_TIMEOUT. The dispatch is signed with MAM_DISPATCH_SECRET (.mam.env).
remote_dispatch() {
local job_id="$1" instructions="$2" target_session="$3" sub_log="${4:-}"
local sess="${target_session#remote:}"
echo "원격 세션 '$sess'에 작업을 dispatch 합니다 (broker)..."
local d_args=(--job "$job_id" --registry-dir "$REGISTRY_DIR" --session "$sess" --instructions "$instructions")
[ "${MAM_ALLOW_UNSIGNED_DISPATCH:-0}" = "1" ] && d_args+=(--allow-unsigned)
if ! "$PY" "$SCRIPT_DIR/scripts/remote.py" dispatch "${d_args[@]}" >/dev/null; then
echo "ERROR: dispatch publish failed for job $job_id (remote:$sess)" >&2
return 1
fi
local ack_s="${MAM_INJECT_ACK_GRACE_S:-$ACK_TIMEOUT}"
if ! _wait_started "$job_id" "$ack_s" "$sub_log"; then
echo "ERROR: remote session '$sess' published no 'started' within ${ack_s}s (ack timeout); not re-dispatching" >&2
return 1
fi
echo "작업이 원격 세션 '$sess'에 전달되었습니다."
return 0
}
# deliver_prompt <job_id> <instructions> <session> <subscriber_log>
deliver_prompt() {
if _is_remote "$3"; then
remote_dispatch "$1" "$2" "$3" "${4:-}"
else
inject_agent "$1" "$2" "$3" "${4:-}"
fi
}
run_agent() { run_agent() {
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}" force_warn_only="${4:-0}" local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}" force_warn_only="${4:-0}"
local sess="${target_session#herdr:}" local sess="${target_session#herdr:}"
@@ -193,3 +193,16 @@ $PY scripts/registry.py logs --list # every job, live status
--- ---
> **구현 위치 각주 (v4.3.0)**: 본 문서가 인용하는 `scripts/*.py` 경로는 유효하며 계약도 불변이다. 실제 구현은 issue #5 phase 1에서 `.agents/skills/lib_py/delegation/`으로 이동했고, `scripts/*.py`는 그 패키지를 자기 옆 트리에 바인딩하는 호환 shim이다. > **구현 위치 각주 (v4.3.0)**: 본 문서가 인용하는 `scripts/*.py` 경로는 유효하며 계약도 불변이다. 실제 구현은 issue #5 phase 1에서 `.agents/skills/lib_py/delegation/`으로 이동했고, `scripts/*.py`는 그 패키지를 자기 옆 트리에 바인딩하는 호환 shim이다.
## Liveness contract fields (Issue #18)
| Field | Meaning | Default |
|---|---|---|
| `heartbeat_sec` | N: the agent publishes `progress` at least every N s (the `--heartbeat N` timer does it) | 30 |
| `heartbeat_misses` | K: missed heartbeats before the job is unresponsive | 4 |
| `idle_timeout_sec` | K x N unless `--idle-timeout` was given | 120 |
| `ack_timeout_sec` | T: `started` must arrive within T s of the injection | 60 |
| `transport` | `herdr` (pane injection) or `remote` (signed retained dispatch over the broker) | `herdr` |
`<registry>/<job>.heartbeat.pid` / `.heartbeat.log` belong to the running heartbeat timer; a terminal publish removes the pidfile.
@@ -0,0 +1,37 @@
#!/usr/bin/env python3
"""Compatibility shim -- the implementation lives in lib_py.delegation.remote.
This file's path is a wire contract (agent instructions, the bash wrapper,
SKILL.md, BOOTSTRAP.md, deploy/install.sh, tests). It binds the
``lib_py.delegation`` *package* to the tree next to this file, once per process,
without touching ``sys.path``; ``import remote`` then yields the very same module
object as ``lib_py.delegation.remote``.
Why the package and not this one module: a per-module bind leaves a non-leaf
module (one that imports its siblings) importing dependencies from whichever tree
happens to be on the path, mixing a frozen copy with a live one. Why bind-once and
never rebind: replacing a bound package would split existing references and
break monkeypatching. See docs/contracts/issue-5/ (D2) for the context matrix.
"""
import importlib, importlib.util, os, sys
_SKILLS = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
_PKG = "lib_py.delegation"
if _PKG not in sys.modules: # bind once per process; never rebind
try:
import lib_py # parent resolved normally; shims never rebind it
except ModuleNotFoundError:
sys.path.append(_SKILLS) # standalone only: no lib_py exists anywhere to shadow
import lib_py
_d = os.path.join(_SKILLS, "lib_py", "delegation")
_spec = importlib.util.spec_from_file_location(
_PKG, os.path.join(_d, "__init__.py"), submodule_search_locations=[_d])
_pkg = importlib.util.module_from_spec(_spec)
sys.modules[_PKG] = _pkg # register BEFORE exec so __init__'s own imports resolve here
_spec.loader.exec_module(_pkg)
lib_py.delegation = _pkg
_impl = importlib.import_module(_PKG + ".remote") # resolved through the bound package's __path__
if __name__ != "__main__":
sys.modules[__name__] = _impl # `import remote` is lib_py.delegation.remote
else:
_main = getattr(_impl, "main", None) # a module without a CLI keeps exit status 0
sys.exit(_main() if _main is not None else 0)
@@ -36,6 +36,27 @@ Run an autonomous planning-execution-review loop using multiple agents (Planner,
--- ---
## Liveness contract and remote sessions (Issue #18)
Every delegate job the loop submits carries the liveness contract of
`multi-agent-mux-delegate-job`: agents heartbeat automatically once they
publish `started --heartbeat N`, a job with no event for K x N seconds is
closed as `deadline_timeout`, and `started` must arrive within T seconds of
the injection. Tune it for a whole loop with environment variables, which the
runner passes through as `--heartbeat` / `--heartbeat-misses` / `--ack-timeout`:
```bash
MAM_HEARTBEAT_SEC=30 MAM_HEARTBEAT_MISSES=4 MAM_ACK_TIMEOUT_S=60 \
bash .agents/skills/multi-agent-mux-loop/scripts/run_loop.sh --creator creator-claude-01 --all-reviewer --task "..."
```
A session row in `.mam/agent-sessions.yaml` with `transport: remote` is
delegated as `remote:<name>` (the brief goes over the broker to a
`remote.py run --session <name>` runner on another host; no herdr, pane or
process probing happens for it). The row still needs `role` and
`status: running` to be picked up by `--all-reviewer`.
## Roles & Responsibilities ## Roles & Responsibilities
협업 시스템은 각 에이전트의 책임 영역을 명확히 격리하여 상호 교차 검증을 강제합니다. 협업 시스템은 각 에이전트의 책임 영역을 명확히 격리하여 상호 교차 검증을 강제합니다.
+50 -2
View File
@@ -154,7 +154,7 @@ Every event payload must adhere to the following schema structure:
* **Example Detail**: `"Job 918b0612 started"` * **Example Detail**: `"Job 918b0612 started"`
#### 2. `progress` #### 2. `progress`
* **Emit Trigger**: Optional. Emitted at major check-points, long loops, or sub-task boundaries. * **Emit Trigger**: At least every `heartbeat_sec` while working (Issue #18 heartbeat obligation; `publish_event.py --event started --heartbeat N` automates it with `detail: heartbeat`), plus any check-point the agent wants to report.
* **Payload Constraints**: None. * **Payload Constraints**: None.
* **Example Detail**: `"Section 1: MQTT Broker Architecture completed"` * **Example Detail**: `"Section 1: MQTT Broker Architecture completed"`
@@ -185,6 +185,53 @@ To prevent unauthorized users from hijacking or spoofing events on public broker
--- ---
### 2.5 Liveness and Delivery Contract (Issue #18)
No new event types: the existing events are promoted to a contract with
obligations, periods and timeouts, so that an agent's life and the delivery of
its prompt are decided by what the agent **publishes**, never by scraping a
terminal (`herdr agent_status`, pane text, screen changes). This is what lets
an agent on another host, which shares neither the pane nor the
orchestrator's disk, take part on equal terms.
| Rule | Contract | Where |
|---|---|---|
| **Heartbeat obligation** | While working, the agent publishes `progress` at least every **N** s (`heartbeat_sec`, default 30). `publish_event.py --event started --heartbeat N` starts a detached timer that publishes `progress` (`detail: heartbeat`) every N s with the same `dispatch_token`/`attempt_id`, so the agent does not have to remember. The timer stops itself at the terminal event (which also kills it through `<registry>/<job>.heartbeat.pid`), on re-dispatch, or after `timeout_sec`. | `lib_py/delegation/heartbeat.py`, `publish_event.py --heartbeat` |
| **Unresponsive** | The orchestrator treats a job as unresponsive after **K x N** s without any event (`heartbeat_misses`, default 4 → 120 s; `--idle-timeout` overrides). The subscriber's idle clock counts every accepted message (`progress` and retained state included); the wrapper's post-idle grace re-reads only the job's events.log (`_recent_disk_heartbeat`, written before every wire publish). herdr status and the pane are logged once as a *diagnostic* (`liveness: ... herdr/pane says busy=yes/no (diagnostic only)`) and never decide. | `multi-agent-mux-delegate-job` (`finalize_unfinished`), `job_subscriber.py` |
| **Delivery** | After the prompt is handed over (herdr accepted the submission, or the remote dispatch was published), the agent's `started` must arrive within **T** s (`ack_timeout_sec`, default 60): the registry leaves `pending`, a `started` line lands in events.log, or the subscriber prints one. No `started` within T → the inject fails closed: rc 1, registry `error` (`agent_blocked`), and the prompt is **never re-injected**. `send_keys_safe` confirms an accepted-but-unconfirmed submission only by `agent_status == working` or the job's `started`; pane changes and busy tokens are no longer signals. | `_wait_started`, `lib.sh: _sks_job_started` |
| **Termination** | Only `completed` / `error` (and an operator `cancelled`) end a job. Terminal immutability (Issue #6 CAS) refuses anything that arrives after `deadline_timeout` closed the job. | `engine.py` |
| **Wire durability** | Every event (QoS 1) is followed, on the same connection, by a **retained** `<topic_prefix>/state` message (`event: state`, `status`, `last_event`, `seq`, signed like an event). A subscriber that connects or reconnects receives it at once: a terminal status ends the wait, any state resets the idle window, and a `seq` beyond the last one seen is recorded as `seq_gap` in the audit log (also for gaps between events). paho reconnects on its own; `on_connect` re-subscribes both topics and logs `reconnected`. | `publish_event.py: build_state_payload`, `job_subscriber.py: _on_state` |
| **Credentials** | Each host connects with its own `.mam.env` (Issue #12: the password is never in a record). Every event and state message carries the job's HMAC (`auth_token`). A remote dispatch is signed with the pre-shared `MAM_DISPATCH_SECRET`. | `mqtt_common.py`, `remote.py` |
| **What screen scraping is still for** | (a) operations that need the screen: prompt injection into a pane, trust/permission dialogs, the pre-injection idle handshake; (b) diagnostics on the same host when events stop. Neither is a liveness or delivery decision. | `lib.sh` |
Registry record fields: `heartbeat_sec`, `heartbeat_misses`, `ack_timeout_sec`,
`idle_timeout_sec` (derived K x N unless explicit), `transport`
(`herdr` / `remote`). Wrapper options: `--heartbeat`, `--heartbeat-misses`,
`--ack-timeout` (also `MAM_HEARTBEAT_SEC`, `MAM_HEARTBEAT_MISSES`,
`MAM_ACK_TIMEOUT_S` for the loop runner).
#### Remote transport (`--agent-session remote:<name>`)
```
orchestrator ── retained <prefix>/dispatch (signed) ──▶ remote runner (other host)
orchestrator ◀── <prefix>/events + retained <prefix>/state ── remote runner / agent
```
`remote.py dispatch` publishes the brief, the job's `auth_token`,
`dispatch_token`, `attempt_id` and the contract as one retained, signed
message. `remote.py run --session <name> [--exec CMD]` on the other host
mirrors the job into its own registry (same job id and identity), retires the
retained dispatch, publishes `started --heartbeat N`, runs the command (env:
`JOB_ID`, `MAM_JOB_BRIEF`, `MAM_PUBLISH_CMD`, ...) and publishes the terminal
event unless the agent already did. The orchestrator's subscriber runs with
`--remote` (automatic for `transport: remote` jobs) and mirrors `started` and
terminal events into the orchestrator's registry through the transition
boundary, because nothing else on that host will. The wrapper takes no
herdr/pane/lsof/ps path for a remote session. Because the dispatch carries the
job's `auth_token`, use the remote transport only on a private broker.
---
## 3. Job Lifecycle & State Transitions ## 3. Job Lifecycle & State Transitions
The lifecycle of a delegated job progresses through a highly coordinated state machine, involving file-based registry claiming, asynchronous message subscription, and multi-faceted event publishing. The lifecycle of a delegated job progresses through a highly coordinated state machine, involving file-based registry claiming, asynchronous message subscription, and multi-faceted event publishing.
@@ -291,7 +338,8 @@ The subscriber acts as the central execution watchdog:
``` ```
* **Dual Timeout Semantics**: * **Dual Timeout Semantics**:
1. **Wall-Clock Timeout**: Calculated relative to absolute startup time (`wall_deadline = start + wall_timeout`). It acts as a hard budget limit, guarding against an agent hanging indefinitely. 1. **Wall-Clock Timeout**: Calculated relative to absolute startup time (`wall_deadline = start + wall_timeout`). It acts as a hard budget limit, guarding against an agent hanging indefinitely.
2. **Activity Idle Timeout**: Measured as the difference between the current monotonic time and the last packet arrival time (`idle_left = idle_timeout - (now - last_event)`). If the agent fails to print logs or publish progress updates for the duration of the idle window, the subscriber aborts and exits with code 2. 2. **Activity Idle Timeout**: Measured as the difference between the current monotonic time and the last packet arrival time (`idle_left = idle_timeout - (now - last_event)`). If the agent fails to publish any event (heartbeats and retained state included) for the duration of the idle window - K x N under the Issue #18 contract - the subscriber aborts and exits with code 2.
* **Retained state and seq gaps (Issue #18)**: the subscriber also subscribes to `<topic_prefix>/state`; a retained terminal state resolves the job at (re)connect, any state resets the idle clock, and a seq jump on either topic is written to the audit log as `seq_gap`. With `--remote` (or `transport: remote`) accepted `started`/terminal events are mirrored into the local job record.
--- ---
+5 -1
View File
@@ -2,7 +2,7 @@ usage: job_subscriber.py [-h] (--job JOB | --wait-any) [--timeout TIMEOUT]
[--idle-timeout IDLE_TIMEOUT] [--expect-retention] [--idle-timeout IDLE_TIMEOUT] [--expect-retention]
[--expect-dispatch-token EXPECT_DISPATCH_TOKEN] [--expect-dispatch-token EXPECT_DISPATCH_TOKEN]
[--attempt-id ATTEMPT_ID] [--attempt-id ATTEMPT_ID]
[--registry-dir REGISTRY_DIR] [-v] [--registry-dir REGISTRY_DIR] [--remote] [-v]
Subscribe to Job events on MQTT Subscribe to Job events on MQTT
@@ -21,4 +21,8 @@ optional arguments:
--attempt-id ATTEMPT_ID --attempt-id ATTEMPT_ID
minimum attempt id to accept events for minimum attempt id to accept events for
--registry-dir REGISTRY_DIR --registry-dir REGISTRY_DIR
--remote the agent shares no disk with this registry: mirror
started/terminal events into the job record (Issue
#18; automatic for jobs registered with --transport
remote)
-v, --verbose -v, --verbose
+7 -1
View File
@@ -6,7 +6,8 @@ usage: publish_event.py [-h] (--job JOB | --pick-pending)
[--finalize-report PATH] [--finalize-report PATH]
[--side-effect KIND=REFERENCE] [--side-effect KIND=REFERENCE]
[--attempt-id ATTEMPT_ID] [--attempt-id ATTEMPT_ID]
[--expect-dispatch-token EXPECT_DISPATCH_TOKEN] [-v] [--expect-dispatch-token EXPECT_DISPATCH_TOKEN]
[--heartbeat N] [--no-state] [-v]
Publish a Job event to MQTT Publish a Job event to MQTT
@@ -33,4 +34,9 @@ optional arguments:
job's current iteration) job's current iteration)
--expect-dispatch-token EXPECT_DISPATCH_TOKEN --expect-dispatch-token EXPECT_DISPATCH_TOKEN
expected dispatch token for CAS verification expected dispatch token for CAS verification
--heartbeat N after publishing, start a detached timer that
publishes a `progress` heartbeat every N seconds until
the job ends (Issue #18; not with a terminal event)
--no-state do not publish the retained <topic_prefix>/state
message
-v, --verbose -v, --verbose
+80 -27
View File
@@ -85,24 +85,63 @@ def _assert_single_injection(res: subprocess.CompletedProcess):
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# lib.sh: accepted submission confirmed through the pane # lib.sh: accepted submission confirmed by the job's own `started` (Issue #18)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def test_accepted_timeout_confirmed_by_busy_token(tmp_path): def _job_that_starts(tmp_path, delay_s: float):
"""A registered job whose `started` lands `delay_s` after the call - the
protocol signal that replaced the pane (Issue #18). Returns (env, job_id)."""
import threading
from lib_py.delegation import registry
reg = tmp_path / "jobs"
env = {"MAM_ENV_FILE": str(tmp_path / ".mam.env.absent"), "DELEGATE_JOB_LOGS_DIR": str(tmp_path / "logs"),
"MQTT_BROKER": "127.0.0.1", "MQTT_PORT": "9", "SKS_REGISTRY_DIR": str(reg)}
saved = {k: os.environ.get(k) for k in env}
os.environ.update(env)
try:
jid = registry.register_job("issue 14 ack", registry_dir=str(reg))
finally:
for k, v in saved.items():
if v is None:
os.environ.pop(k, None)
else:
os.environ[k] = v
def start():
time.sleep(delay_s)
subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg), "--job", jid,
"--event", "started", "--attempts", "1"], capture_output=True, env=_base_env(tmp_path, env))
threading.Thread(target=start, daemon=True).start()
return env, jid
def test_accepted_timeout_confirmed_by_started_event_busy_token_alone_is_not_enough(tmp_path):
"""The pane shows the busy token, but that is not delivery any more: the
job's `started` (published 2 s in) is what confirms it, and the busy token
on its own (second run) still fails closed."""
env, jid = _job_that_starts(tmp_path, 2.0)
script = _codex_like_mock(CODEX_IDLE, CODEX_WORKING) + \ script = _codex_like_mock(CODEX_IDLE, CODEX_WORKING) + \
'send_keys_safe "reviewer-codex-01" "review the triage report" "job_14a"' f'send_keys_safe "reviewer-codex-01" "review the triage report" "{jid}"'
res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "5"}) res = _run_lib_sh(script, tmp_path, env={**env, "SKS_DELIVERY_GRACE_S": "8"})
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}" assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
_assert_single_injection(res) _assert_single_injection(res)
assert "delivery unknown" not in res.stderr assert "delivery unknown" not in res.stderr
script = _codex_like_mock(CODEX_IDLE, CODEX_WORKING) + \
'send_keys_safe "reviewer-codex-01" "review the triage report" "job_14a"'
res = _run_lib_sh(script, tmp_path, env={**env, "SKS_DELIVERY_GRACE_S": "2"})
assert res.returncode == 4, f"busy token must not confirm delivery: rc={res.returncode}\n{res.stderr}"
_assert_single_injection(res)
def test_accepted_timeout_confirmed_by_pane_change_after_fast_turn(tmp_path):
"""A turn that already finished before the first poll shows no busy token, def test_accepted_timeout_pane_change_after_fast_turn_is_not_delivery(tmp_path):
only the answer; the pane moving on from its pre-submit snapshot is enough.""" """A turn that already finished shows only the answer. The pane moving on
used to count; under the protocol contract only `started` does."""
script = _codex_like_mock(CODEX_IDLE, CODEX_ANSWERED) + \ script = _codex_like_mock(CODEX_IDLE, CODEX_ANSWERED) + \
'send_keys_safe "reviewer-codex-01" "Reply with OK" "job_14b"' 'send_keys_safe "reviewer-codex-01" "Reply with OK" "job_14b"'
res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "5"}) res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "2"})
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}" assert res.returncode == 4, f"rc={res.returncode}\n{res.stderr}"
_assert_single_injection(res) _assert_single_injection(res)
@@ -147,7 +186,8 @@ def test_transport_error_is_not_treated_as_accepted(tmp_path):
def test_mock_herdr_timeout_with_idle_status_is_confirmed(tmp_path, mock_herdr): def test_mock_herdr_timeout_with_idle_status_is_confirmed(tmp_path, mock_herdr):
"""Same scenario through the shared mock herdr: `timeout` from agent prompt, """Same scenario through the shared mock herdr: `timeout` from agent prompt,
agent_status pinned to idle, the pane gaining the busy token after agent_status pinned to idle, the pane gaining the busy token after
submission. Exactly one prompt call, no paste-buffer fallback.""" submission - and the job's `started` landing 2 s in, which is what
confirms it (Issue #18). Exactly one prompt call, no paste-buffer fallback."""
state_file = Path(mock_herdr) state_file = Path(mock_herdr)
state = json.loads(state_file.read_text()) state = json.loads(state_file.read_text())
state["agents"]["agt_codex"] = { state["agents"]["agt_codex"] = {
@@ -158,9 +198,10 @@ def test_mock_herdr_timeout_with_idle_status_is_confirmed(tmp_path, mock_herdr):
} }
state_file.write_text(json.dumps(state, indent=2)) state_file.write_text(json.dumps(state, indent=2))
env, jid = _job_that_starts(tmp_path, 2.0)
res = _run_lib_sh( res = _run_lib_sh(
'send_keys_safe "agt_codex" "review please" "job_14f"', f'send_keys_safe "agt_codex" "review please" "{jid}"',
tmp_path, env={"SKS_IDLE_HANDSHAKE": "0", "SKS_DELIVERY_GRACE_S": "5"}, tmp_path, env={**env, "SKS_IDLE_HANDSHAKE": "0", "SKS_DELIVERY_GRACE_S": "8"},
) )
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}" assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
@@ -243,13 +284,14 @@ def test_wrapper_honours_registry_ack_over_unconfirmed_inject(tmp_path, mock_her
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# wrapper: idle-timeout grace (R1) must see a working codex through the pane # wrapper: idle-timeout grace (R1) is decided by the contract, not the pane
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr): def test_wrapper_idle_timeout_grace_ignores_pane_busy_token(tmp_path, mock_herdr):
"""herdr keeps reporting `idle` for a codex-like agent that is visibly """Issue #18 flips the Issue #14 expectation: a codex-like agent that is
working (busy token in the pane). When the subscriber idles out, the visibly working (busy token in the pane) but publishes nothing after its
wrapper must keep waiting for the on-disk result instead of closing the `started` is unresponsive once K x N (here --idle-timeout 3 s) has passed.
job as deadline_timeout mid-turn; the agent then completes out of band.""" The wrapper closes it as deadline_timeout and a `completed` written 14 s
in is refused by the CAS table - the pane never extended the window."""
reg_dir = tmp_path / "jobs" reg_dir = tmp_path / "jobs"
reg_dir.mkdir(parents=True, exist_ok=True) reg_dir.mkdir(parents=True, exist_ok=True)
@@ -265,7 +307,7 @@ def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr
} }
state_file.write_text(json.dumps(state, indent=2)) state_file.write_text(json.dumps(state, indent=2))
env = _base_env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_DELIVERY_GRACE_S": "5"}) env = _base_env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_DELIVERY_GRACE_S": "2", "MAM_INJECT_ACK_GRACE_S": "20"})
proc = subprocess.Popen( proc = subprocess.Popen(
[ [
"bash", str(WRAPPER), "submit", "bash", str(WRAPPER), "submit",
@@ -289,19 +331,29 @@ def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr
time.sleep(0.1) time.sleep(0.1)
assert jid, "job was not registered" assert jid, "job was not registered"
# Well past the 3 s idle timeout, the agent finishes. # The agent acknowledges (started) without arming a heartbeat timer ...
time.sleep(1)
tok = json.loads((reg_dir / f"{jid}.json").read_text())["dispatch_token"]
ack = subprocess.run(
[sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg_dir), "--job", jid, "--event", "started",
"--expect-dispatch-token", tok],
capture_output=True, text=True, cwd=str(tmp_path), env=env,
)
assert ack.returncode == 0, ack.stderr
# ... and goes silent while its pane keeps showing the busy token.
time.sleep(14) time.sleep(14)
done = subprocess.run( done = subprocess.run(
[sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg_dir), [sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg_dir),
"update", "--job", jid, "--status", "completed"], "update", "--job", jid, "--status", "completed"],
capture_output=True, text=True, cwd=str(tmp_path), env=env, capture_output=True, text=True, cwd=str(tmp_path), env=env,
) )
assert done.returncode == 0, done.stderr
stdout, stderr = proc.communicate(timeout=90) stdout, stderr = proc.communicate(timeout=90)
job_data = json.loads((reg_dir / f"{jid}.json").read_text()) job_data = json.loads((reg_dir / f"{jid}.json").read_text())
assert job_data.get("status") == "completed", f"status={job_data.get('status')}\n{stdout}\n{stderr}" assert proc.returncode == 2, f"expected deadline_timeout (rc=2): the pane is not liveness, got {proc.returncode}\n{stdout}\n{stderr}"
assert proc.returncode == 0, f"expected rc=0 after the working grace, got {proc.returncode}\n{stdout}\n{stderr}" assert job_data.get("status") == "error", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
assert done.returncode != 0 or "refused" in done.stderr, "the late completed must be refused"
assert "diagnostic only" in stderr
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -451,8 +503,9 @@ def test_wrapper_idle_grace_never_outlives_the_wall_clock(tmp_path):
# The heartbeats were honoured: the job was not closed after the first idle # The heartbeats were honoured: the job was not closed after the first idle
# window, it stayed open up to the wall clock ... # window, it stayed open up to the wall clock ...
assert elapsed >= timeout_s - 1, f"closed before the wall clock: {elapsed:.1f}s\n{stderr}" assert elapsed >= timeout_s - 1, f"closed before the wall clock: {elapsed:.1f}s\n{stderr}"
# ... and not a moment past it beyond closing and publishing the error. # ... and not a moment past it beyond closing and publishing the error
assert elapsed <= timeout_s + 12, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}" # (the error publish alone can take ~10 s against the public broker).
assert elapsed <= timeout_s + 15, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -489,7 +542,7 @@ def test_publish_event_records_the_heartbeat_before_the_wire_publish(tmp_path, m
events_log = reg_dir / f"{jid}.events.log" events_log = reg_dir / f"{jid}.events.log"
on_disk_at_publish = [] on_disk_at_publish = []
def slow_broker_publish(config, topic, body, retain): def slow_broker_publish(config, topic, body, retain, extra=()):
on_disk_at_publish.append([e.get("event") for e in _logged_events(events_log)]) on_disk_at_publish.append([e.get("event") for e in _logged_events(events_log)])
monkeypatch.setattr(pe, "_publish_once", slow_broker_publish) monkeypatch.setattr(pe, "_publish_once", slow_broker_publish)
@@ -509,7 +562,7 @@ def test_publish_event_failed_publish_keeps_one_local_record(tmp_path, monkeypat
events_log = reg_dir / f"{jid}.events.log" events_log = reg_dir / f"{jid}.events.log"
lines_seen = [] lines_seen = []
def unreachable_broker(config, topic, body, retain): def unreachable_broker(config, topic, body, retain, extra=()):
lines_seen.append(len(_logged_events(events_log))) lines_seen.append(len(_logged_events(events_log)))
raise ConnectionError("broker unreachable") raise ConnectionError("broker unreachable")
+617
View File
@@ -0,0 +1,617 @@
#!/usr/bin/env python3
"""Issue #18 — liveness and delivery are decided by the event protocol, not by
screen scraping.
Contract (registry fields, wrapper options):
* heartbeat obligation - the agent publishes `progress` at least every N s;
`publish_event --event started --heartbeat N` starts an automatic timer
that does it for the agent and stops itself at the terminal event.
* unresponsive - no event (disk or wire) for K x N s closes the job
as deadline_timeout; herdr status and pane text are diagnostics only.
* delivery - `started` must arrive within T s of the injection,
else the inject fails closed (rc 1, registry error, single injection).
* wire durability - every event is followed by a retained
`<prefix>/state` message on the same connection; a subscriber that
(re)connects recovers the status at once and records seq gaps.
* remote transport - `--agent-session remote:<name>` delivers the brief
over a signed retained `<prefix>/dispatch`; a runner on another host with
its own registry and broker credentials runs it, and the orchestrator's
subscriber mirrors its events into the orchestrator's registry.
Tests marked "broker" talk to the configured broker (the public default in the
sandbox); each stays under a minute.
"""
from __future__ import annotations
import json
import os
import signal
import subprocess
import sys
import threading
import time
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parent.parent
SKILLS = REPO_ROOT / ".agents" / "skills"
LIB_SH = SKILLS / "lib.sh"
WRAPPER = SKILLS / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
SCRIPTS = SKILLS / "multi-agent-mux-delegate-job" / "scripts"
PUBLISH_PY = SCRIPTS / "publish_event.py"
REGISTRY_PY = SCRIPTS / "registry.py"
REMOTE_PY = SCRIPTS / "remote.py"
UNREACHABLE = {"MQTT_BROKER": "127.0.0.1", "MQTT_PORT": "9", "MQTT_TLS": "0"}
def _env(cwd: Path, extra=None) -> dict:
env = dict(os.environ)
env["DELEGATE_JOB_PYTHON"] = sys.executable
env["AGENT_PYTHON_BIN"] = sys.executable
env["WORKSPACE_ROOT"] = str(cwd)
env["HOME"] = str(cwd)
env["MAM_ENV_FILE"] = str(cwd / ".mam.env.absent")
env["DELEGATE_JOB_LOGS_DIR"] = str(cwd / "logs")
env["PYTHONPATH"] = str(SKILLS)
for k in ("MQTT_BROKER", "MQTT_HOST", "MQTT_PORT", "MQTT_TLS", "MQTT_USERNAME", "MQTT_PASSWORD"):
env.pop(k, None)
if extra:
env.update({k: str(v) for k, v in extra.items()})
return env
def _events(reg_dir: Path, jid: str) -> list:
p = reg_dir / f"{jid}.events.log"
if not p.exists():
return []
return [json.loads(l) for l in p.read_text().splitlines() if l.strip()]
def _job(reg_dir: Path, jid: str) -> dict:
return json.loads((reg_dir / f"{jid}.json").read_text())
def _wait_job_id(reg_dir: Path, timeout=20) -> str:
deadline = time.time() + timeout
while time.time() < deadline:
files = list(reg_dir.glob("*.json"))
if files:
return files[0].stem
time.sleep(0.1)
raise AssertionError("job was not registered")
def _publish(reg_dir: Path, jid: str, event: str, env: dict, *extra) -> subprocess.CompletedProcess:
"""Publish as the agent would: with the dispatch identity the wrapper's
injected command carries (`--expect-dispatch-token`)."""
tok = _job(reg_dir, jid).get("dispatch_token")
ident = ["--expect-dispatch-token", tok] if tok else []
return subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg_dir), "--job", jid,
"--event", event, "--attempts", "2", *ident, *extra],
capture_output=True, text=True, env=env, cwd=str(reg_dir.parent), timeout=120)
def _timer_pid(reg_dir: Path, jid: str):
p = reg_dir / f"{jid}.heartbeat.pid"
try:
return int(p.read_text().strip())
except (OSError, ValueError):
return None
def _alive(pid) -> bool:
if not pid:
return False
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
return subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True).stdout.strip() not in ("", "Z", "Z+")
def _kill_timer(reg_dir: Path, jid: str):
pid = _timer_pid(reg_dir, jid)
if pid:
try:
os.kill(pid, signal.SIGKILL)
except OSError:
pass
# ---------------------------------------------------------------------------
# 1. registry: the contract is part of the record
# ---------------------------------------------------------------------------
def test_register_derives_the_idle_window_from_the_contract(tmp_path, monkeypatch):
from lib_py.delegation import registry
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
reg = tmp_path / "jobs"
jid = registry.register_job("contract", registry_dir=str(reg))
rec = _job(reg, jid)
assert (rec["heartbeat_sec"], rec["heartbeat_misses"], rec["ack_timeout_sec"]) == (30, 4, 60)
assert rec["idle_timeout_sec"] == 120 == registry.liveness_window(30, 4)
assert rec["transport"] == "herdr"
jid2 = registry.register_job("contract", registry_dir=str(reg), heartbeat_sec=5, heartbeat_misses=3)
assert _job(reg, jid2)["idle_timeout_sec"] == 15
jid3 = registry.register_job("contract", registry_dir=str(reg), heartbeat_sec=5, heartbeat_misses=3,
idle_timeout_sec=40)
assert _job(reg, jid3)["idle_timeout_sec"] == 40, "an explicit idle timeout is kept"
def test_wrapper_registers_the_contract_and_derives_idle_timeout(tmp_path):
"""--dry-run registers the job without any herdr; K x N shows up as idle."""
reg = tmp_path / "jobs"
reg.mkdir()
res = subprocess.run(
["bash", str(WRAPPER), "submit", "--agent", "human", "--prompt", "contract", "--dry-run",
"--registry-dir", str(reg), "--workdir", str(tmp_path),
"--heartbeat", "7", "--heartbeat-misses", "3", "--ack-timeout", "11"],
capture_output=True, text=True, env=_env(tmp_path, UNREACHABLE), cwd=str(tmp_path), timeout=60)
assert res.returncode == 0, res.stderr
jid = _wait_job_id(reg)
rec = _job(reg, jid)
assert (rec["heartbeat_sec"], rec["heartbeat_misses"], rec["ack_timeout_sec"], rec["idle_timeout_sec"]) == (7, 3, 11, 21)
assert "--event started --heartbeat 7" in res.stdout, "the injected start command must arm the timer"
assert "within 11 s" in res.stdout
# ---------------------------------------------------------------------------
# 2. the automatic heartbeat (no broker needed: events land on disk first)
# ---------------------------------------------------------------------------
def test_started_with_heartbeat_beats_on_disk_until_the_terminal_event(tmp_path):
reg = tmp_path / "jobs"
env = _env(tmp_path, UNREACHABLE)
jid = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "register",
"--prompt", "hb", "--heartbeat-sec", "1", "--heartbeat-misses", "3"],
capture_output=True, text=True, env=env, cwd=str(tmp_path)).stdout.strip()
assert jid
try:
res = _publish(reg, jid, "started", env, "--heartbeat", "1", "--attempts", "1")
assert res.returncode == 2, "the broker is unreachable, the event is still recorded locally"
pid = _timer_pid(reg, jid)
assert pid and _alive(pid), "a detached timer must be running"
time.sleep(4.5)
beats = [e for e in _events(reg, jid) if e["event"] == "progress"]
assert len(beats) >= 3, [e["event"] for e in _events(reg, jid)]
assert all(e["detail"] == "heartbeat" for e in beats)
assert all(e["data"].get("dispatch_token") == _job(reg, jid)["dispatch_token"] for e in beats), \
"heartbeats carry the dispatch identity"
seqs = [e["seq"] for e in _events(reg, jid)]
assert seqs == sorted(seqs) and len(set(seqs)) == len(seqs)
res = _publish(reg, jid, "completed", env, "--attempts", "1")
assert res.returncode == 2
deadline = time.time() + 5
while time.time() < deadline and _alive(pid):
time.sleep(0.1)
assert not _alive(pid), "the terminal event must stop the timer"
assert _timer_pid(reg, jid) is None
assert _events(reg, jid)[-1]["event"] == "completed", "no heartbeat may trail the terminal event"
assert _job(reg, jid)["status"] == "completed"
finally:
_kill_timer(reg, jid)
def test_heartbeat_timer_stops_on_its_own_when_the_job_is_redispatched(tmp_path):
reg = tmp_path / "jobs"
env = _env(tmp_path, UNREACHABLE)
jid = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "register",
"--prompt", "hb"], capture_output=True, text=True, env=env, cwd=str(tmp_path)).stdout.strip()
try:
_publish(reg, jid, "started", env, "--heartbeat", "1", "--attempts", "1")
pid = _timer_pid(reg, jid)
assert _alive(pid)
# An operator moves the job to the next attempt: the old attempt's timer must not keep beating.
_publish(reg, jid, "completed", env, "--attempts", "1")
subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "update", "--job", jid,
"--status", "pending", "--attempt-id", "2"], capture_output=True, text=True, env=env, cwd=str(tmp_path))
deadline = time.time() + 5
while time.time() < deadline and _alive(pid):
time.sleep(0.1)
assert not _alive(pid)
finally:
_kill_timer(reg, jid)
def test_heartbeat_is_refused_on_a_terminal_event(tmp_path, monkeypatch):
from lib_py.delegation import publish_event as pe, registry
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
reg = tmp_path / "jobs"
jid = registry.register_job("hb", registry_dir=str(reg))
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "completed", "--heartbeat", "5"]) == 1
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "progress", "--heartbeat", "0"]) == 1
assert _events(reg, jid) == [], "a refused call consumes no seq and records nothing"
# ---------------------------------------------------------------------------
# 3. wire durability: retained state next to every event, seq gaps detected
# ---------------------------------------------------------------------------
def test_every_event_carries_a_signed_retained_state_message(tmp_path, monkeypatch):
from lib_py.delegation import mqtt_common, publish_event as pe, registry
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
reg = tmp_path / "jobs"
jid = registry.register_job("state", registry_dir=str(reg), auth_token="tok-" + "a" * 20)
sent = []
def fake_publish(config, topic, body, retain, extra=()):
sent.append((topic, json.loads(body), retain, [(t, json.loads(b), r) for t, b, r in extra]))
monkeypatch.setattr(pe, "_publish_once", fake_publish)
tok = _job(reg, jid)["dispatch_token"]
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "started", "--expect-dispatch-token", tok]) == 0
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "completed", "--detail", "PASS",
"--expect-dispatch-token", tok]) == 0
prefix = _job(reg, jid)["topic_prefix"]
assert [s[0] for s in sent] == [f"{prefix}/events"] * 2
for (topic, ev, retain, extra), status in zip(sent, ("running", "completed")):
assert len(extra) == 1
st_topic, st, st_retain = extra[0]
assert st_topic == f"{prefix}/state" and st_retain is True
assert st["event"] == "state" and st["status"] == status and st["last_event"] == ev["event"]
assert st["seq"] == ev["seq"] and st["job_id"] == jid
assert mqtt_common.verify_hmac(st, "tok-" + "a" * 20), "state must be signed with the job token"
assert st["data"]["dispatch_token"] == ev["data"]["dispatch_token"]
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "progress", "--no-state",
"--expect-dispatch-token", tok]) == 0
assert len(sent) == 3 and sent[-1][3] == [], "--no-state publishes the event alone"
class _Msg:
def __init__(self, topic, payload):
self.topic = topic
self.payload = json.dumps(payload).encode()
def _signed(job_id, seq, event, token, **fields):
from lib_py.delegation import publish_event as pe
p = pe.build_payload(job_id, seq, event, "", {"dispatch_token": "d1", "attempt_id": 1}, token)
if fields:
# re-sign after adding state fields, as build_state_payload does
import hashlib, hmac
p.update(fields)
p["data"].pop("hmac_sig", None)
sp = {k: v for k, v in p.items() if k != "data"}
sp["data"] = dict(p["data"])
p["data"]["hmac_sig"] = hmac.new(token.encode(), json.dumps(sp, sort_keys=True, separators=(",", ":")).encode(),
hashlib.sha256).hexdigest()
return p
def test_subscriber_records_seq_gaps_and_recovers_terminal_from_state(tmp_path, monkeypatch):
from lib_py.delegation import job_subscriber as js, mqtt_common
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
token = "t" * 24
w = js._Watcher({"j1"}, {"j1": token}, {"j1": 0}, {"j1": "d1"}, {"j1": 1})
w.on_message(None, None, _Msg("x/events", _signed("j1", 1, "started", token)))
w.on_message(None, None, _Msg("x/events", _signed("j1", 2, "progress", token)))
w.on_message(None, None, _Msg("x/events", _signed("j1", 5, "progress", token))) # 3 and 4 were lost
audit = [e for e in mqtt_common.iter_logged_events("j1", mqtt_common.get_logs_dir()) if e["event"] == "seq_gap"]
assert len(audit) == 1 and (audit[0]["expected_seq"], audit[0]["seq"], audit[0]["via"]) == (3, 5, "events")
assert w.events.qsize() == 3
# a retained state after a reconnect: newer seq than anything seen, terminal status
st = _signed("j1", 8, "state", token, status="completed", last_event="completed")
w.on_message(None, None, _Msg("x/state", st))
assert w.events.qsize() == 4
topic, payload = list(w.events.queue)[-1]
assert topic == "x/state" and payload["event"] == "state" and payload["status"] == "completed"
gaps = [e for e in mqtt_common.iter_logged_events("j1", mqtt_common.get_logs_dir()) if e["event"] == "seq_gap"]
assert len(gaps) == 2 and gaps[-1]["via"] == "state"
# a forged state (wrong token) is dropped like a forged event
bad = _signed("j1", 9, "state", "wrong", status="error", last_event="error")
w.on_message(None, None, _Msg("x/state", bad))
assert w.events.qsize() == 4
# a state for a stale attempt (other dispatch token) is dropped
stale = _signed("j1", 10, "state", token, status="error", last_event="error")
stale["data"]["dispatch_token"] = "d0"
w.on_message(None, None, _Msg("x/state", stale))
assert w.events.qsize() == 4
def test_subscriber_subscribes_to_the_state_topic(tmp_path):
src = (SKILLS / "lib_py" / "delegation" / "job_subscriber.py").read_text()
assert 'state_topics.append(f"{prefix}/state")' in src
assert "for topic in subscribed_topics + state_topics:" in src
# ---------------------------------------------------------------------------
# 4. delivery: `started` decides, screen signals do not
# ---------------------------------------------------------------------------
def _codex_like_mock(pane_before: str, pane_after: str, code: str = "timeout") -> str:
return f'''
_pane_quiescent() {{ return 0; }}
_pane_dialog_open() {{ return 1; }}
FLAG="$WORKSPACE_ROOT/.prompt-submitted"
rm -f "$FLAG"
_sks_herdr() {{
case "${{1:-}}" in
agent)
case "${{2:-}}" in
prompt) touch "$FLAG"; echo '{{"error":{{"code":"{code}","message":"x"}}}}' >&2; return 1 ;;
get) echo '{{"result":{{"agent":{{"agent_status":"idle"}}}}}}'; return 0 ;;
esac ;;
capture-pane)
if [ -e "$FLAG" ]; then printf '%s\\n' "{pane_after}"; else printf '%s\\n' "{pane_before}"; fi
return 0 ;;
paste-buffer|set-buffer|delete-buffer|send-keys)
echo "PASTE_PATH_USED: $*" >&2; return 0 ;;
esac
return 0
}}
'''
def _run_lib_sh(snippet: str, cwd: Path, env=None, timeout=60):
return subprocess.run(["bash", "-c", f'source "{LIB_SH}" >/dev/null 2>&1; {snippet}'],
capture_output=True, text=True, cwd=str(cwd), env=_env(cwd, env), timeout=timeout)
def test_send_keys_safe_is_confirmed_by_the_job_started_event(tmp_path, monkeypatch):
"""herdr says `timeout`, agent_status stays idle, the pane never changes:
the only thing that confirms delivery is the job's own `started`."""
from lib_py.delegation import registry
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
reg = tmp_path / "jobs"
jid = registry.register_job("ack", registry_dir=str(reg))
env = _env(tmp_path, {**UNREACHABLE, "SKS_DELIVERY_GRACE_S": "14", "SKS_REGISTRY_DIR": str(reg)})
def agent_starts_later():
time.sleep(2)
subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg), "--job", jid,
"--event", "started", "--attempts", "1"], capture_output=True, env=env, cwd=str(tmp_path))
t = threading.Thread(target=agent_starts_later, daemon=True)
t.start()
started = time.time()
res = subprocess.run(["bash", "-c", f'source "{LIB_SH}" >/dev/null 2>&1; '
+ _codex_like_mock("idle", "idle") + f'send_keys_safe "reviewer-codex-01" "prompt" "{jid}"'],
capture_output=True, text=True, cwd=str(tmp_path), env=env, timeout=60)
t.join()
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
assert time.time() - started < 11, "confirmed once started landed, not at the end of the 14 s grace"
assert "PASTE_PATH_USED" not in res.stderr
def test_send_keys_safe_ignores_pane_change_and_busy_token(tmp_path):
"""The screen moving on, or showing a busy token, is not delivery any more:
with no `started` the accepted submission still fails closed (rc 4)."""
for after in ("Working (0s • esc to interrupt)\\n› Ask Codex", "• OK\\n› Ask Codex"):
res = _run_lib_sh(_codex_like_mock("idle", after) + 'send_keys_safe "reviewer-codex-01" "prompt" "adhoc"',
tmp_path, env={**UNREACHABLE, "SKS_DELIVERY_GRACE_S": "2"})
assert res.returncode == 4, f"rc={res.returncode}\n{res.stderr}"
assert "delivery unknown/failed" in res.stderr
assert "PASTE_PATH_USED" not in res.stderr
# ---------------------------------------------------------------------------
# 5. wrapper acceptance criteria (broker)
# ---------------------------------------------------------------------------
def _headless_agent(state_file: Path, name: str, agent: str = "codex") -> None:
state = json.loads(state_file.read_text())
# agent_status pinned idle, prompt `timeout`, empty pane: nothing to scrape
state["agents"][name] = {"agent": agent, "agent_status": "idle", "prompt_wait_result": "timeout", "buffer": ""}
state_file.write_text(json.dumps(state, indent=2))
def _submit(tmp_path: Path, reg: Path, session: str, agent: str, extra_args, env_extra=None, timeout_s=60):
env = _env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_EMPTY_GIVEUP": "1", **(env_extra or {})})
proc = subprocess.Popen(
["bash", str(WRAPPER), "submit", "--agent", agent, "--agent-session", session, "--prompt", "issue 18",
"--registry-dir", str(reg), "--workdir", str(tmp_path), "--timeout", str(timeout_s), *extra_args],
cwd=str(tmp_path), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
return proc, env
def _prompt_calls(state_file: Path) -> int:
"""`agent prompt` invocations; the isolated wrapper prefixes `--session <name>`."""
n = 0
for c in json.loads(state_file.read_text()).get("calls", []):
c = list(c)
if len(c) >= 2 and c[0] == "--session":
c = c[2:]
if len(c) >= 2 and c[0] == "agent" and c[1] == "prompt":
n += 1
return n
def test_acceptance1_headless_agent_survives_on_heartbeats_alone(tmp_path, mock_herdr):
"""herdr idle throughout, empty pane: `started --heartbeat 2` keeps the job
alive past several K x N windows (2 x 2 = 4 s) and `completed` yields rc 0."""
reg = tmp_path / "jobs"
reg.mkdir()
_headless_agent(Path(mock_herdr), "headless-codex")
proc, env = _submit(tmp_path, reg, "herdr:headless-codex", "codex",
["--heartbeat", "2", "--heartbeat-misses", "2", "--ack-timeout", "40"], timeout_s=90)
jid = _wait_job_id(reg)
try:
time.sleep(1)
assert _publish(reg, jid, "started", env, "--heartbeat", "2").returncode == 0
time.sleep(16) # four idle windows without any agent-published event
assert _job(reg, jid)["status"] == "running", "heartbeats alone must keep the job open"
assert _publish(reg, jid, "completed", env, "--detail", "PASS").returncode == 0
stdout, stderr = proc.communicate(timeout=120)
finally:
_kill_timer(reg, jid)
if proc.poll() is None:
proc.kill()
assert proc.returncode == 0, f"rc={proc.returncode}\n{stdout}\n{stderr}"
assert _job(reg, jid)["status"] == "completed"
assert _prompt_calls(Path(mock_herdr)) == 1
beats = [e for e in _events(reg, jid) if e["event"] == "progress"]
assert len(beats) >= 5, len(beats)
brief = (reg / jid / "brief.md").read_text()
assert "heartbeat every 2 s" in brief and "after 2 missed heartbeats" in brief and "within 40 s" in brief
def test_acceptance2_no_started_within_T_fails_closed_without_reinjection(tmp_path, mock_herdr):
reg = tmp_path / "jobs"
reg.mkdir()
_headless_agent(Path(mock_herdr), "mute-codex")
started = time.time()
proc, env = _submit(tmp_path, reg, "herdr:mute-codex", "codex",
["--ack-timeout", "4", "--heartbeat", "2", "--heartbeat-misses", "2"],
env_extra={"SKS_DELIVERY_GRACE_S": "1"}, timeout_s=60)
stdout, stderr = proc.communicate(timeout=120)
jid = _wait_job_id(reg)
assert proc.returncode == 1, f"rc={proc.returncode}\n{stdout}\n{stderr}"
assert "no 'started' within 4s" in stderr
assert _job(reg, jid)["status"] == "error"
assert _prompt_calls(Path(mock_herdr)) == 1, "single injection: never re-injected"
assert time.time() - started < 45, "the ack wait is bounded by T, not the wall clock"
ev = [e for e in _events(reg, jid) if e["event"] == "error"]
assert ev and ev[-1]["detail"] == "agent_blocked"
def test_acceptance3_lost_heartbeats_close_the_job_and_a_late_completed_is_refused(tmp_path, mock_herdr):
reg = tmp_path / "jobs"
reg.mkdir()
_headless_agent(Path(mock_herdr), "dying-codex")
proc, env = _submit(tmp_path, reg, "herdr:dying-codex", "codex",
["--heartbeat", "1", "--heartbeat-misses", "3", "--ack-timeout", "40"], timeout_s=60)
jid = _wait_job_id(reg)
t0 = None
try:
time.sleep(1)
assert _publish(reg, jid, "started", env, "--heartbeat", "1").returncode == 0
time.sleep(4)
_kill_timer(reg, jid) # the agent dies: no more heartbeats
t0 = time.time()
stdout, stderr = proc.communicate(timeout=120)
finally:
_kill_timer(reg, jid)
if proc.poll() is None:
proc.kill()
assert proc.returncode == 2, f"rc={proc.returncode}\n{stdout}\n{stderr}"
assert _job(reg, jid)["status"] == "error"
assert "herdr/pane says busy=" in stderr, "screen state is logged as a diagnostic only"
# K x N = 3 s idle + one grace window: closed long before the 60 s wall clock
assert time.time() - t0 < 30, f"took {time.time() - t0:.0f}s"
late = _publish(reg, jid, "completed", env, "--detail", "too late")
assert late.returncode in (0, 2, 4)
assert _job(reg, jid)["status"] == "error", "terminal immutability (CAS) refuses the late completed"
# ---------------------------------------------------------------------------
# 6. remote transport (broker): no shared disk, no pane, only the wire
# ---------------------------------------------------------------------------
def test_acceptance5_remote_runner_completes_a_job_over_the_broker_only(tmp_path):
orch = tmp_path / "orchestrator"
remote = tmp_path / "remote-host"
for d in (orch, remote):
(d / "jobs").mkdir(parents=True)
secret = "issue18-shared-secret"
runner_env = _env(remote, {"MAM_DISPATCH_SECRET": secret, "MAM_JOBS_DIR": str(remote / "jobs")})
runner_log = remote / "runner.log"
with open(runner_log, "wb") as log:
runner = subprocess.Popen(
[sys.executable, str(REMOTE_PY), "run", "--session", "sim-worker", "--registry-dir", str(remote / "jobs"),
"--exec", 'echo "remote work done for $JOB_ID"', "--once", "--timeout", "150"],
cwd=str(remote), env=runner_env, stdout=log, stderr=subprocess.STDOUT)
try:
deadline = time.time() + 45
while time.time() < deadline and "SUBSCRIBED" not in runner_log.read_text(errors="replace"):
time.sleep(0.2)
if "SUBSCRIBED" not in runner_log.read_text(errors="replace"):
pytest.skip("broker unreachable: " + runner_log.read_text(errors="replace")[-300:])
reg = orch / "jobs"
proc, env = _submit(orch, reg, "remote:sim-worker", "remote",
["--heartbeat", "2", "--heartbeat-misses", "3", "--ack-timeout", "60"],
env_extra={"MAM_DISPATCH_SECRET": secret}, timeout_s=120)
stdout, stderr = proc.communicate(timeout=180)
jid = _wait_job_id(reg)
assert proc.returncode == 0, f"rc={proc.returncode}\n{stdout}\n{stderr}\nrunner:\n{runner_log.read_text()}"
assert "subscriber received started" in stderr or "advanced to 'running'" in stderr
assert _job(reg, jid)["status"] == "completed", \
f"the subscriber mirrors the remote terminal into the registry\n{stdout}\n{stderr}\nrunner:\n{runner_log.read_text()}"
assert _job(reg, jid)["transport"] == "remote"
# The runner's own status sync follows its wire publish, so the
# orchestrator can observe `completed` a moment before the remote
# record says so: wait for the runner before reading its registry.
runner.wait(timeout=60)
# the remote side kept its own record with the orchestrator's identity
mirror = _job(remote / "jobs", jid)
assert mirror["status"] == "completed" and mirror["transport"] == "remote"
assert mirror["dispatch_token"] == _job(reg, jid)["dispatch_token"]
assert mirror["auth_token"] == _job(reg, jid)["auth_token"]
assert (remote / "jobs" / jid / "brief.md").read_text().startswith("issue 18")
# the agent command's last stdout line became the completed detail
assert "remote work done for " + jid in stdout, stdout
assert runner.returncode == 0
assert "살아있는 에이전트 세션" not in stdout, "no pane injection path for a remote session"
finally:
if runner.poll() is None:
runner.kill()
def test_remote_runner_refuses_unsigned_dispatches(tmp_path, monkeypatch):
from lib_py.delegation import remote
d = {"kind": "dispatch", "job_id": "j", "session": "s", "schema_version": 1}
assert not remote.verify_dispatch(d, "secret")
signed = remote.sign_dispatch(d, "secret")
assert remote.verify_dispatch(signed, "secret")
assert not remote.verify_dispatch(signed, "other")
assert not remote.verify_dispatch(dict(signed, session="hijacked"), "secret")
assert not remote.verify_dispatch(signed, None)
monkeypatch.delenv("MAM_DISPATCH_SECRET", raising=False)
assert remote.run_runner("s", str(tmp_path), allow_unsigned=False) == 1, "a runner without a secret refuses to run"
# ---------------------------------------------------------------------------
# 7. loop runner: contract knobs pass through; remote rows get remote:<name>
# ---------------------------------------------------------------------------
def test_loop_runner_passes_the_contract_and_remote_label(tmp_path, monkeypatch):
from lib_py.loop import runner as loop_runner
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("WORKSPACE_ROOT", str(tmp_path))
monkeypatch.setenv("MAM_STATE_JSON", json.dumps({"herdr_sessions": [
{"name": "creator-claude-01", "agent": "claude", "role": "creator", "status": "running"},
{"name": "reviewer-far-01", "agent": "claude", "role": "reviewer", "status": "running", "transport": "remote"},
]}))
monkeypatch.setenv("MAM_HEARTBEAT_SEC", "20")
monkeypatch.setenv("MAM_HEARTBEAT_MISSES", "3")
monkeypatch.setenv("MAM_ACK_TIMEOUT_S", "45")
r = loop_runner.LoopRunner(["--creator", "creator-claude-01", "--task", "t"])
r.parse_args()
assert r.session_label("creator-claude-01") == "herdr:creator-claude-01"
assert r.session_label("reviewer-far-01") == "remote:reviewer-far-01"
assert r.contract_args() == ["--heartbeat", "20", "--heartbeat-misses", "3", "--ack-timeout", "45"]
seen = []
def fake_run(cmd, **_kw):
seen.append(cmd)
class P: # noqa: D401 - tiny stand-in
returncode = 0
stdout = "registered job: deadbeef\n"
stderr = ""
return P()
monkeypatch.setattr(loop_runner.subprocess, "run", fake_run)
assert r.delegate_job("reviewer-far-01", "claude", "Reviewer", "review") == "deadbeef"
cmd = seen[0]
assert cmd[cmd.index("--agent-session") + 1] == "remote:reviewer-far-01"
assert cmd[-6:] == ["--heartbeat", "20", "--heartbeat-misses", "3", "--ack-timeout", "45"]
+8 -1
View File
@@ -338,7 +338,14 @@ def test_registry_cli_contract_snapshot(mam_sandbox):
assert reg["--agent-session"][2] == "herdr:claude" assert reg["--agent-session"][2] == "herdr:claude"
assert reg["--role"][2] == "Worker" assert reg["--role"][2] == "Worker"
assert reg["--timeout"][2] == 3600 assert reg["--timeout"][2] == 3600
assert reg["--idle-timeout"][2] == 120 # Issue #18: the idle window derives from the liveness contract (K x N =
# 4 x 30 = 120 s, the historical default) unless given explicitly.
assert reg["--idle-timeout"][2] is None
assert reg["--heartbeat-sec"][2] == 30
assert reg["--heartbeat-misses"][2] == 4
assert reg["--ack-timeout"][2] == 60
assert reg["--transport"][1] == ("herdr", "remote")
assert registry.liveness_window(reg["--heartbeat-sec"][2], reg["--heartbeat-misses"][2]) == 120
assert reg["--bits"][2] == 32 assert reg["--bits"][2] == 32
assert reg["--job-type"][1] == ("direct", "loop", "discuss") assert reg["--job-type"][1] == ("direct", "loop", "discuss")
assert reg["--max-iterations"][2] == 5 assert reg["--max-iterations"][2] == 5
+36 -6
View File
@@ -603,8 +603,31 @@ def test_t1_14_working_agent_reobserve_completed(tmp_path, mock_herdr):
assert len(agent_gets) >= 1, "R1 loop must query agent get" assert len(agent_gets) >= 1, "R1 loop must query agent get"
def _publish_started_for_registered_job(reg_dir, cwd, env=None):
"""Issue #18: a working agent that never publishes `started` is a delivery
failure (rc 1) under the liveness contract. The R1 re-observation cases
below are about an agent that *did* start and then went silent, so the
watcher threads acknowledge the job the way the injected command does."""
deadline = time.time() + 20
while time.time() < deadline:
files = list(Path(reg_dir).glob("*.json"))
if files:
jid = files[0].stem
tok = json.loads(files[0].read_text()).get("dispatch_token") or ""
args = [sys.executable, str(SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"),
"--registry-dir", str(reg_dir), "--job", jid, "--event", "started", "--attempts", "1"]
if tok:
args += ["--expect-dispatch-token", tok]
run_env = dict(os.environ)
run_env.update(env or {})
subprocess.run(args, capture_output=True, text=True, cwd=str(cwd), env=run_env, timeout=60)
return jid
time.sleep(0.1)
return None
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# T1-15: R1 working 고정 + 미발행 -> 벽시계 --timeout 만료 후 rc=2 error # T1-15: R1 working 고정 + 미발행 -> Issue #18: K×N 후 rc=2 error (working 은 진단일 뿐)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr): def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
reg_dir = tmp_path / "jobs" reg_dir = tmp_path / "jobs"
@@ -628,6 +651,7 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
if any("agent" in c and "prompt" in c for c in st.get("calls", [])): if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
st["agents"]["agt_w2"]["agent_status"] = "working" st["agents"]["agt_w2"]["agent_status"] = "working"
state_file.write_text(json.dumps(st, indent=2)) state_file.write_text(json.dumps(st, indent=2))
_publish_started_for_registered_job(reg_dir, tmp_path)
break break
except Exception: except Exception:
pass pass
@@ -637,6 +661,10 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
t.daemon = True t.daemon = True
t.start() t.start()
# Issue #18: `working` in herdr is a diagnostic, not liveness. An agent
# that acknowledged (started) and then publishes nothing is closed after
# K x N (--idle-timeout 2) plus one grace window - well before the wall
# clock - even though herdr keeps saying `working` the whole time.
start_t = time.time() start_t = time.time()
res = _run_wrapper( res = _run_wrapper(
[ [
@@ -646,16 +674,16 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
"--prompt", "test T1-15", "--prompt", "test T1-15",
"--registry-dir", str(reg_dir), "--registry-dir", str(reg_dir),
"--workdir", str(tmp_path), "--workdir", str(tmp_path),
"--timeout", "5", "--timeout", "40",
"--idle-timeout", "2", "--idle-timeout", "2",
], ],
cwd=tmp_path, cwd=tmp_path,
timeout=60, timeout=90,
) )
stop_ev.set() stop_ev.set()
elapsed = time.time() - start_t elapsed = time.time() - start_t
assert res.returncode == 2, f"Expected rc=2 on wallclock timeout, got {res.returncode}" assert res.returncode == 2, f"Expected rc=2 (deadline_timeout), got {res.returncode}\n{res.stderr}"
assert elapsed >= 4.0, f"Must wait up to wallclock timeout, elapsed={elapsed}" assert elapsed < 39.0, f"a silent `working` agent must not be kept open to the wall clock, elapsed={elapsed}"
job_files = list(reg_dir.glob("*.json")) job_files = list(reg_dir.glob("*.json"))
assert len(job_files) == 1 assert len(job_files) == 1
@@ -663,7 +691,8 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
assert job_data.get("status") == "error" assert job_data.get("status") == "error"
final_st = json.loads(state_file.read_text()) final_st = json.loads(state_file.read_text())
agent_gets = [c for c in final_st.get("calls", []) if "agent" in c and "get" in c] agent_gets = [c for c in final_st.get("calls", []) if "agent" in c and "get" in c]
assert len(agent_gets) >= 2, "R1 loop must repeatedly query agent get during wallclock wait" assert len(agent_gets) >= 2, "the idle handshake and the one diagnostic probe query agent get"
assert "diagnostic only" in res.stderr, "Issue #18: herdr status is logged, not used as liveness"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -691,6 +720,7 @@ def test_t1_16_working_agent_done_without_event(tmp_path, mock_herdr):
if any("agent" in c and "prompt" in c for c in st.get("calls", [])): if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
st["agents"]["agt_w3"]["agent_status"] = "working" st["agents"]["agt_w3"]["agent_status"] = "working"
state_file.write_text(json.dumps(st, indent=2)) state_file.write_text(json.dumps(st, indent=2))
_publish_started_for_registered_job(reg_dir, tmp_path)
break break
except Exception: except Exception:
pass pass