feat(issue-18): decide liveness and delivery by the event protocol, not the screen
Contract (no new event types): heartbeat every N s (automatic timer), unresponsive after K x N s without an event, `started` within T s of the injection, retained per-job state, and a remote transport. - registry: heartbeat_sec / heartbeat_misses / ack_timeout_sec / transport fields; idle_timeout_sec derives K x N unless explicit (defaults 30/4/60 keep the historical 120 s idle window). - heartbeat.py + publish_event --heartbeat N: detached fixed-cadence timer publishing signed `progress` heartbeats (threaded wire publish, local record first); stopped by the terminal event via pidfile, by re-dispatch and by lifetime. Terminal events refuse --heartbeat. - publish_event: retained `<prefix>/state` (status, last_event, seq, HMAC) sent on the same connection after every event (--no-state to skip); publisher.publish_raw takes extra messages. - job_subscriber: subscribes the state topic, recovers terminal status from retained state, resets idle on any accepted message, records seq_gap and reconnected in the audit log, idle budget before the first event is T + K x N (--ack-timeout), --remote mirrors started/terminal events into the registry through the transition boundary. - lib.sh send_keys_safe: an accepted-but-unconfirmed submission is confirmed by agent_status working or the job's own `started` (_sks_job_started); pane change and busy token are no longer delivery signals. - wrapper: --heartbeat/--heartbeat-misses/--ack-timeout (MAM_* env), injected start command arms the timer, brief states the contract, _wait_started (registry / events.log / subscriber log, bounded by T and the wall clock) replaces the 10 s inject grace, R1 grace uses only the on-disk event age (herdr/pane logged once as a diagnostic), remote:<name> sessions skip preflight/idle/pane and dispatch over the broker (remote_dispatch). - remote.py (+ scripts shim): signed retained dispatch (MAM_DISPATCH_SECRET) and a runner that mirrors the job into its own registry with the orchestrator's identity, publishes started --heartbeat, runs --exec and publishes the terminal event. - loop runner: passes MAM_HEARTBEAT_SEC / MAM_HEARTBEAT_MISSES / MAM_ACK_TIMEOUT_S through; YAML rows with transport: remote delegate as remote:<name>. - docs: MESSAGING.md 2.5 contract + remote transport, delegate-job SKILL.md, registry.md, loop SKILL.md. - tests: tests/test_issue18_liveness_contract.py (16); issue 14 pane-signal tests flipped to started-based; issue 6 working-agent tests acknowledge with started; CLI snapshot/help captures refreshed. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
GodopuandClaude Fable 5.1 committed
1 parent
139bdd4b4c
commit
897b505266
21 files changed
+1989
-129
No files matched your search
+45
-22
@@ -2280,6 +2280,39 @@ except Exception:
|
||||
' <<< "$raw" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# _sks_job_started <job_id> — Issue #18 delivery signal: the job's own
|
||||
# `started` (registry status left `pending`, or a started line in the job's
|
||||
# events.log). Reads the registry the wrapper exports (SKS_REGISTRY_DIR), else
|
||||
# MAM_JOBS_DIR, else $WORKSPACE_ROOT/.mam/jobs. "adhoc" (no job) is never started.
|
||||
_sks_job_started() {
|
||||
local job_id="${1:-adhoc}" rd
|
||||
[ -n "$job_id" ] && [ "$job_id" != "adhoc" ] || return 1
|
||||
rd="${SKS_REGISTRY_DIR:-${MAM_JOBS_DIR:-${WORKSPACE_ROOT:-$PWD}/.mam/jobs}}"
|
||||
"$(_delegate_py_bin)" - "$rd" "$job_id" <<'PYEOF'
|
||||
import json, os, sys
|
||||
rd, jid = sys.argv[1], sys.argv[2]
|
||||
rec = os.path.join(rd, jid + ".json")
|
||||
try:
|
||||
st = json.load(open(rec)).get("status")
|
||||
if st and st != "pending":
|
||||
sys.exit(0)
|
||||
except Exception:
|
||||
pass
|
||||
log = os.path.join(rd, jid + ".events.log")
|
||||
try:
|
||||
with open(log) as fh:
|
||||
for line in fh:
|
||||
try:
|
||||
if json.loads(line).get("event") == "started":
|
||||
sys.exit(0)
|
||||
except Exception:
|
||||
continue
|
||||
except OSError:
|
||||
pass
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# _wait_session_gone <sess> [max_sec=5]
|
||||
# Reactive loop to wait until a herdr session goes away (happy path returns early).
|
||||
_wait_session_gone() {
|
||||
@@ -2494,9 +2527,6 @@ send_keys_safe() {
|
||||
# Native herdr 0.8+ fast path: agent prompt handles atomic submission
|
||||
if [ "${SKS_PROMPT_WAIT:-1}" = "1" ]; then
|
||||
local p_out="" p_rc=0
|
||||
# ISSUE-14: pre-submit snapshot for the delivery-confirmation poll below.
|
||||
local rpc_pre_pane=""
|
||||
[ "$_q_rc" != "2" ] && rpc_pre_pane=$(_pane_capture "$sess")
|
||||
local _p_tmp
|
||||
_p_tmp=$(mktemp 2>/dev/null || echo "/tmp/sks_p_${sess}_$$")
|
||||
if _sks_herdr agent prompt "$agent_target" "$text" --wait --until working --timeout 5000 >"$_p_tmp" 2>&1; then
|
||||
@@ -2534,29 +2564,22 @@ send_keys_safe() {
|
||||
if [ "$(_sks_agent_status "$sess" "$agent_target")" = "working" ]; then
|
||||
return 0
|
||||
fi
|
||||
# ISSUE-14: `timeout` / `agent_prompt_stalled` mean herdr ACCEPTED the
|
||||
# submission and sent the input - only its post-submit status watch came
|
||||
# up empty. An agent whose agent_status never leaves `idle` mid-turn
|
||||
# (codex under herdr 0.9.0) therefore failed every delivered prompt here:
|
||||
# the job was closed as agent_blocked and the agent's own `started` /
|
||||
# `completed`, published seconds later, were refused as late. For an
|
||||
# accepted submission keep observing for SKS_DELIVERY_GRACE_S (default
|
||||
# 15) and take any positive signal: agent_status working, a busy token
|
||||
# the pane did not show before the prompt, or the pane moving on from
|
||||
# its pre-submit snapshot. No signal at all still fails closed (rc 4).
|
||||
# ISSUE-14 / ISSUE-18: `timeout` / `agent_prompt_stalled` mean herdr
|
||||
# ACCEPTED the submission and sent the input - only its post-submit
|
||||
# status watch came up empty (codex under herdr 0.9.0 never reports
|
||||
# `working`). For an accepted submission keep observing for
|
||||
# SKS_DELIVERY_GRACE_S (default 15) and take a *protocol* signal only:
|
||||
# agent_status working, or the job's own `started` in the registry
|
||||
# (_sks_job_started). Pane changes and busy tokens are no longer
|
||||
# delivery evidence (Issue #18: they are CLI-version-specific screen
|
||||
# scraping that a headless or remote agent never produces). No signal
|
||||
# still fails closed (rc 4); the wrapper then waits for `started` up to
|
||||
# the job's ack timeout T before finalising the inject as failed.
|
||||
if printf '%s\n' "$p_out" | grep -Eq '"code": *"(timeout|agent_prompt_stalled)"'; then
|
||||
local g_deadline=$(( $(date +%s) + ${SKS_DELIVERY_GRACE_S:-15} ))
|
||||
local pre_busy_rpc=0 cur_pane
|
||||
printf '%s\n' "$rpc_pre_pane" | grep -Fq "esc to interrupt" && pre_busy_rpc=1
|
||||
while [ "$(date +%s)" -lt "$g_deadline" ]; do
|
||||
[ "$(_sks_agent_status "$sess" "$agent_target")" = "working" ] && return 0
|
||||
if [ "$_q_rc" != "2" ]; then
|
||||
cur_pane=$(_pane_capture "$sess")
|
||||
if [ "$pre_busy_rpc" -eq 0 ] && printf '%s\n' "$cur_pane" | grep -Fq "esc to interrupt"; then
|
||||
return 0
|
||||
fi
|
||||
[ -n "$cur_pane" ] && [ "$cur_pane" != "$rpc_pre_pane" ] && return 0
|
||||
fi
|
||||
_sks_job_started "$job_id" && return 0
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
@@ -35,6 +35,8 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
p_pub.add_argument("--job", required=True)
|
||||
p_pub.add_argument("--event", default="progress", choices=publish_event.VALID_EVENTS)
|
||||
p_pub.add_argument("--detail", default="")
|
||||
p_pub.add_argument("--heartbeat", type=int, default=None, metavar="N",
|
||||
help="start the automatic heartbeat timer (Issue #18)")
|
||||
p_pub.add_argument("--data", default=None)
|
||||
p_pub.add_argument("--retained", action="store_true")
|
||||
p_pub.add_argument("--registry-dir", default=None)
|
||||
@@ -74,6 +76,8 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
forward += ["--expect-dispatch-token", args.expect_dispatch_token]
|
||||
if args.attempt_id is not None:
|
||||
forward += ["--attempt-id", str(args.attempt_id)]
|
||||
if getattr(args, "heartbeat", None) is not None:
|
||||
forward += ["--heartbeat", str(args.heartbeat)]
|
||||
if args.verbose:
|
||||
forward += ["--verbose"]
|
||||
return publish_event.main(forward)
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
"""heartbeat.py — the agent-side automatic heartbeat timer (Issue #18).
|
||||
|
||||
The liveness contract obliges an agent to publish `progress` at least every N
|
||||
seconds while it works. Relying on the brief to say so meant an agent that
|
||||
forgot, or one whose turn was one long tool call, was closed as unresponsive
|
||||
while it was fine. This timer publishes the heartbeat for it.
|
||||
|
||||
`publish_event.py --event started --heartbeat N` starts one detached timer per
|
||||
job; it runs in its own session so the agent's shell may return at once. The
|
||||
timer stops on its own when
|
||||
|
||||
* the job record reaches a terminal status (completed / error / cancelled),
|
||||
* the record's dispatch_token or attempt_id changes (the job was
|
||||
re-dispatched: this attempt's heartbeats would be refused anyway),
|
||||
* the job record disappears, or
|
||||
* its lifetime exceeds the job's wall-clock timeout plus one period.
|
||||
|
||||
A terminal publish also stops it directly through the pidfile
|
||||
(`<registry>/<job_id>.heartbeat.pid`) so no heartbeat can trail a `completed`.
|
||||
|
||||
Each beat is an ordinary `publish_event` call in this process: it takes the
|
||||
next seq, is signed with the job's auth_token, carries the dispatch token and
|
||||
attempt id the timer was started with, and is recorded in events.log before
|
||||
the wire publish - exactly what the orchestrator's disk and wire liveness
|
||||
checks read.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
from lib_py.delegation import mqtt_common
|
||||
from lib_py.delegation.mqtt_common import default_registry_dir, load_job
|
||||
|
||||
logger = logging.getLogger("delegate_job.heartbeat")
|
||||
|
||||
TERMINAL_STATUSES = ("completed", "error", "cancelled")
|
||||
HEARTBEAT_DETAIL = "heartbeat"
|
||||
MAX_IN_FLIGHT = 8
|
||||
|
||||
|
||||
def pidfile_path(job_id: str, registry_dir: str) -> str:
|
||||
return os.path.join(registry_dir, "{}.heartbeat.pid".format(job_id))
|
||||
|
||||
|
||||
def _pid_alive(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def running_timer(job_id: str, registry_dir: str) -> Optional[int]:
|
||||
"""PID of a live timer for this job, or None."""
|
||||
try:
|
||||
with open(pidfile_path(job_id, registry_dir)) as fh:
|
||||
pid = int(fh.read().strip() or 0)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if pid > 0 and _pid_alive(pid):
|
||||
return pid
|
||||
return None
|
||||
|
||||
|
||||
def stop_timer(job_id: str, registry_dir: str) -> bool:
|
||||
"""Terminate the job's timer if one runs. Returns True when one was signalled."""
|
||||
pid = running_timer(job_id, registry_dir)
|
||||
path = pidfile_path(job_id, registry_dir)
|
||||
if pid is None:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def start_timer(job_id: str, registry_dir: str, interval: int,
|
||||
dispatch_token: Optional[str] = None, attempt_id: Optional[int] = None,
|
||||
python: Optional[str] = None) -> int:
|
||||
"""Spawn the detached timer process and return its pid.
|
||||
|
||||
A timer that already runs for the job is stopped first: a second `started`
|
||||
for the same job (a retry of the same command) must not double the rate.
|
||||
"""
|
||||
stop_timer(job_id, registry_dir)
|
||||
cmd: List[str] = [python or sys.executable, "-m", "lib_py.delegation.heartbeat",
|
||||
"--job", job_id, "--registry-dir", registry_dir,
|
||||
"--interval", str(int(interval))]
|
||||
if dispatch_token:
|
||||
cmd += ["--expect-dispatch-token", dispatch_token]
|
||||
if attempt_id is not None:
|
||||
cmd += ["--attempt-id", str(int(attempt_id))]
|
||||
env = dict(os.environ)
|
||||
# The timer imports lib_py from the same tree as this process.
|
||||
pkg_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
env["PYTHONPATH"] = pkg_root + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
|
||||
log_path = os.path.join(registry_dir, "{}.heartbeat.log".format(job_id))
|
||||
with open(log_path, "ab") as log:
|
||||
proc = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT,
|
||||
start_new_session=True, env=env, cwd=os.getcwd())
|
||||
with open(pidfile_path(job_id, registry_dir), "w") as fh:
|
||||
fh.write(str(proc.pid))
|
||||
logger.info("heartbeat timer pid %d for job %s every %ss", proc.pid, job_id, interval)
|
||||
return proc.pid
|
||||
|
||||
|
||||
def _should_stop(job_id: str, registry_dir: str, dispatch_token, attempt_id) -> Optional[str]:
|
||||
try:
|
||||
rec = load_job(job_id, registry_dir)
|
||||
except FileNotFoundError:
|
||||
return "job record gone"
|
||||
if rec.get("status") in TERMINAL_STATUSES:
|
||||
return "job is {}".format(rec.get("status"))
|
||||
if dispatch_token and rec.get("dispatch_token") != dispatch_token:
|
||||
return "dispatch token changed (re-dispatched)"
|
||||
if attempt_id is not None and rec.get("attempt_id", rec.get("iteration", 1)) != attempt_id:
|
||||
return "attempt changed (re-dispatched)"
|
||||
return None
|
||||
|
||||
|
||||
def run(job_id: str, registry_dir: str, interval: int, dispatch_token=None,
|
||||
attempt_id=None, max_lifetime: Optional[float] = None) -> int:
|
||||
"""The timer loop. Runs until a stop condition holds."""
|
||||
from lib_py.delegation import publish_event # late import: the CLI imports us
|
||||
|
||||
interval = max(1, int(interval))
|
||||
if max_lifetime is None:
|
||||
try:
|
||||
max_lifetime = float(load_job(job_id, registry_dir).get("timeout_sec", 3600)) + interval
|
||||
except FileNotFoundError:
|
||||
return 1
|
||||
stop = {"flag": False}
|
||||
|
||||
def _on_term(_signum, _frame):
|
||||
stop["flag"] = True
|
||||
|
||||
signal.signal(signal.SIGTERM, _on_term)
|
||||
signal.signal(signal.SIGINT, _on_term)
|
||||
|
||||
started = time.monotonic()
|
||||
own_pidfile = pidfile_path(job_id, registry_dir)
|
||||
# Fixed cadence: the next beat is due `interval` after the previous one
|
||||
# was *due*, not after its publish returned, so a slow broker (seconds per
|
||||
# ACK) does not stretch the period the contract promises. Each beat is
|
||||
# recorded on disk before its wire publish, so the on-disk cadence holds
|
||||
# even when the wire lags. Sleep first: `started` was just published.
|
||||
next_due = time.monotonic() + interval
|
||||
in_flight: List[threading.Thread] = []
|
||||
while not stop["flag"]:
|
||||
while time.monotonic() < next_due and not stop["flag"]:
|
||||
time.sleep(min(0.5, max(0.0, next_due - time.monotonic())))
|
||||
if stop["flag"]:
|
||||
break
|
||||
next_due += interval
|
||||
if next_due < time.monotonic():
|
||||
next_due = time.monotonic() + interval # fell far behind: resync
|
||||
reason = _should_stop(job_id, registry_dir, dispatch_token, attempt_id)
|
||||
if reason:
|
||||
logger.info("heartbeat timer for %s stopping: %s", job_id, reason)
|
||||
break
|
||||
if time.monotonic() - started > max_lifetime:
|
||||
logger.info("heartbeat timer for %s stopping: lifetime %.0fs exceeded", job_id, max_lifetime)
|
||||
break
|
||||
argv = ["--registry-dir", registry_dir, "--job", job_id, "--event", "progress",
|
||||
"--detail", HEARTBEAT_DETAIL, "--attempts", "2"]
|
||||
if dispatch_token:
|
||||
argv += ["--expect-dispatch-token", dispatch_token]
|
||||
if attempt_id is not None:
|
||||
argv += ["--attempt-id", str(attempt_id)]
|
||||
# Each beat publishes in its own thread: the local record (seq, HMAC,
|
||||
# events.log) is taken at once, and the wire publish - seconds per ACK
|
||||
# on a slow broker - must not delay the next beat. In-flight beats are
|
||||
# capped so a dead broker cannot pile up threads without bound.
|
||||
in_flight[:] = [t for t in in_flight if t.is_alive()]
|
||||
if len(in_flight) >= MAX_IN_FLIGHT:
|
||||
logger.warning("heartbeat for %s skipped: %d publishes still in flight", job_id, len(in_flight))
|
||||
continue
|
||||
|
||||
def _beat(argv=argv):
|
||||
try:
|
||||
rc = publish_event.main(argv)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.warning("heartbeat publish raised: %s", exc)
|
||||
rc = 2
|
||||
if rc == 4:
|
||||
logger.info("heartbeat publish refused (stale attempt); timer for %s will stop", job_id)
|
||||
stop["flag"] = True
|
||||
|
||||
t = threading.Thread(target=_beat, daemon=True)
|
||||
t.start()
|
||||
in_flight.append(t)
|
||||
for t in in_flight:
|
||||
t.join(timeout=15)
|
||||
try:
|
||||
with open(own_pidfile) as fh:
|
||||
if fh.read().strip() == str(os.getpid()):
|
||||
os.unlink(own_pidfile)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description="automatic heartbeat timer for a delegated job")
|
||||
parser.add_argument("--job", required=True)
|
||||
parser.add_argument("--registry-dir", default=default_registry_dir())
|
||||
parser.add_argument("--interval", type=int, required=True, help="heartbeat period N in seconds")
|
||||
parser.add_argument("--expect-dispatch-token", default=None)
|
||||
parser.add_argument("--attempt-id", type=int, default=None)
|
||||
parser.add_argument("--max-lifetime", type=float, default=None)
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
mqtt_common.ensure_env_loaded()
|
||||
mqtt_common.setup_logging(logging.DEBUG if args.verbose else logging.INFO)
|
||||
return run(args.job, args.registry_dir, args.interval, args.expect_dispatch_token,
|
||||
args.attempt_id, args.max_lifetime)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -44,6 +44,8 @@ from lib_py.delegation.mqtt_common import (
|
||||
logger = logging.getLogger("delegate_job.job_subscriber")
|
||||
|
||||
TERMINAL_EVENTS = ("completed", "error")
|
||||
TERMINAL_STATUSES = ("completed", "error", "cancelled")
|
||||
STATE_EVENT = "state"
|
||||
|
||||
|
||||
def _format_line(topic: str, payload: Dict[str, Any]) -> str:
|
||||
@@ -133,6 +135,46 @@ class _Watcher:
|
||||
# by the transition boundary as an idempotent ack, so a replay after
|
||||
# restart cannot double-apply either.
|
||||
self.seen_intents = {}
|
||||
# Issue #18: jobs for which at least one wire event was accepted. A seq
|
||||
# gap is only meaningful after that - the initial last_seq comes from
|
||||
# the registry and may be behind the wire (or ahead of a fresh remote).
|
||||
self.accepted_any: Set[str] = set()
|
||||
|
||||
def _note_seq_gap(self, jid: str, expected: int, got: int, via: str, topic: str) -> None:
|
||||
"""Issue #18: a seq the subscriber never saw is a detected loss."""
|
||||
logger.warning("seq gap for job %s: expected %d, got %d (via %s)", jid, expected, got, via)
|
||||
mqtt_common.append_event(jid, {
|
||||
"event": "seq_gap",
|
||||
"expected_seq": expected,
|
||||
"seq": got,
|
||||
"via": via,
|
||||
"topic": topic,
|
||||
})
|
||||
|
||||
def _on_state(self, msg, payload: Dict[str, Any], jid: str) -> None:
|
||||
"""A retained `<prefix>/state` message (Issue #18): learn the job's
|
||||
status and last seq at (re)connect without waiting for the next
|
||||
heartbeat. Signed and token-checked like an event. Delivered to the
|
||||
main loop as a state item so the idle window resets and a terminal
|
||||
status ends the wait."""
|
||||
seq = payload.get("seq")
|
||||
if not isinstance(seq, int):
|
||||
logger.warning("drop state for job %s: missing or invalid seq", jid)
|
||||
return
|
||||
last = self.last_seq.get(jid, 0)
|
||||
if jid in self.accepted_any and seq > last + 1:
|
||||
self._note_seq_gap(jid, last + 1, seq, "state", msg.topic)
|
||||
if seq > last:
|
||||
self.last_seq[jid] = seq
|
||||
mqtt_common.append_event(jid, {
|
||||
"event": "received",
|
||||
"source_event": STATE_EVENT,
|
||||
"seq": seq,
|
||||
"topic": msg.topic,
|
||||
"timestamp": payload.get("timestamp"),
|
||||
"detail": "status={} last_event={}".format(payload.get("status"), payload.get("last_event")),
|
||||
})
|
||||
self.events.put((msg.topic, payload))
|
||||
|
||||
def on_message(self, _client, _userdata, msg) -> None:
|
||||
# --- defensive parsing -------------------------------------------
|
||||
@@ -175,6 +217,9 @@ class _Watcher:
|
||||
logger.warning("drop event for job %s: attempt_id %r is older than expected %r",
|
||||
jid, ev_att, expected_att)
|
||||
return
|
||||
if payload.get("event") == STATE_EVENT:
|
||||
self._on_state(msg, payload, jid)
|
||||
return
|
||||
# --- replay attack defense: check monotonic sequence ---
|
||||
seq = payload.get("seq")
|
||||
if seq is None or not isinstance(seq, int):
|
||||
@@ -210,7 +255,10 @@ class _Watcher:
|
||||
logger.warning("drop event for job %s: seq %d is not monotonically increasing (last %d)",
|
||||
jid, seq, self.last_seq.get(jid, 0))
|
||||
return
|
||||
if jid in self.accepted_any and seq > self.last_seq.get(jid, 0) + 1:
|
||||
self._note_seq_gap(jid, self.last_seq.get(jid, 0) + 1, seq, "events", msg.topic)
|
||||
self.last_seq[jid] = seq
|
||||
self.accepted_any.add(jid)
|
||||
# Recorded only once the event is actually accepted. A late packet that the
|
||||
# sequence guard refuses must not mark its intent as seen, or the genuine
|
||||
# retry that follows would be mistaken for the replay of an event this
|
||||
@@ -232,6 +280,22 @@ class _Watcher:
|
||||
self.events.put((msg.topic, payload))
|
||||
|
||||
|
||||
def _sync_remote_status(jid: str, status: str, registry_dir: str,
|
||||
dispatch_token: Optional[str]) -> None:
|
||||
"""Issue #18: mirror a wire event into the orchestrator's registry for a
|
||||
job whose agent shares no disk with it. Goes through the same transition
|
||||
boundary as the publisher's own status sync; a refused (late/duplicate)
|
||||
transition is logged, never raised."""
|
||||
try:
|
||||
res = mqtt_common.transition_job_status(
|
||||
jid, registry_dir, _source="job_subscriber", status=status,
|
||||
expected_dispatch_token=dispatch_token)
|
||||
if not res.committed and not res.idempotent:
|
||||
logger.warning("remote status sync refused for %s -> %s: %s", jid, status, res.reason)
|
||||
except Exception as exc: # pragma: no cover - best effort
|
||||
logger.warning("remote status sync failed for %s: %s", jid, exc)
|
||||
|
||||
|
||||
def _collect_jobs(args) -> List[Dict[str, Any]]:
|
||||
"""Resolve the list of job records this invocation should watch."""
|
||||
if args.wait_any:
|
||||
@@ -276,6 +340,13 @@ def _run_subscriber(args) -> int:
|
||||
base_job = jobs[0]
|
||||
wall_timeout = args.timeout if args.timeout is not None else float(base_job.get("timeout_sec", 3600))
|
||||
idle_timeout = args.idle_timeout if args.idle_timeout is not None else float(base_job.get("idle_timeout_sec", 120))
|
||||
# Issue #18: until the first accepted event the "last event" is the
|
||||
# injection itself, which happens after this subscriber started and may
|
||||
# take up to the ack timeout T to produce `started` (a remote runner has
|
||||
# to receive the dispatch first). The idle budget before the first event
|
||||
# is therefore T + K x N; from the first event on it is K x N.
|
||||
ack_timeout = args.ack_timeout if args.ack_timeout is not None else float(base_job.get("ack_timeout_sec", 0) or 0)
|
||||
first_event_seen = False
|
||||
|
||||
# All watched jobs share a broker in practice; connect using the first
|
||||
# job's broker and subscribe to each job's events topic.
|
||||
@@ -283,16 +354,31 @@ def _run_subscriber(args) -> int:
|
||||
client = make_client("subscriber", config)
|
||||
client.on_message = watcher.on_message
|
||||
|
||||
# Issue #18: a remote job (no shared disk with the agent) has nobody else
|
||||
# to move its registry record - the subscriber does it from the wire.
|
||||
remote_sync = bool(args.remote) or any(j.get("transport") == "remote" for j in jobs)
|
||||
|
||||
subscribed_topics = []
|
||||
state_topics = []
|
||||
for job in jobs:
|
||||
prefix = job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"])
|
||||
subscribed_topics.append(f"{prefix}/events")
|
||||
state_topics.append(f"{prefix}/state")
|
||||
|
||||
connects = {"count": 0}
|
||||
|
||||
def on_connect(_c, _u, _flags, reason_code, _props):
|
||||
if mqtt_common.reason_code_value(reason_code) != 0:
|
||||
logger.error("broker connection failed: rc=%s", reason_code)
|
||||
return
|
||||
for topic in subscribed_topics:
|
||||
connects["count"] += 1
|
||||
if connects["count"] > 1:
|
||||
# Issue #18: paho reconnected on its own; re-subscribing here is
|
||||
# what makes the retained state message arrive again at once.
|
||||
logger.warning("broker reconnected (connect #%d); re-subscribing", connects["count"])
|
||||
for jid in expected_ids:
|
||||
mqtt_common.append_event(jid, {"event": "reconnected", "connect_count": connects["count"]})
|
||||
for topic in subscribed_topics + state_topics:
|
||||
_c.subscribe(topic, qos=1)
|
||||
logger.info("subscribed to %s", topic)
|
||||
|
||||
@@ -361,7 +447,7 @@ def _run_subscriber(args) -> int:
|
||||
return 2
|
||||
if not subscribed:
|
||||
last_event = now
|
||||
idle_left = idle_timeout - (now - last_event)
|
||||
idle_left = (idle_timeout if first_event_seen else idle_timeout + ack_timeout) - (now - last_event)
|
||||
if idle_left <= 0:
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
if not pending:
|
||||
@@ -386,10 +472,30 @@ def _run_subscriber(args) -> int:
|
||||
|
||||
last_event = time.monotonic()
|
||||
retention_checked = True
|
||||
print(_format_line(topic, payload), flush=True)
|
||||
first_event_seen = True
|
||||
|
||||
jid = payload["job_id"]
|
||||
event = payload.get("event")
|
||||
if event == STATE_EVENT:
|
||||
# Issue #18: retained job state. Any state resets the idle
|
||||
# window (the agent is alive enough to have published); a
|
||||
# terminal status ends the wait exactly like the terminal event.
|
||||
status = payload.get("status")
|
||||
if status in TERMINAL_STATUSES and jid not in terminal:
|
||||
synth = "completed" if status == "completed" else "error"
|
||||
payload = dict(payload, event=synth, source="state",
|
||||
detail="terminal status '{}' recovered from retained state".format(status))
|
||||
print(_format_line(topic, payload), flush=True)
|
||||
terminal[jid] = synth
|
||||
pending.discard(jid)
|
||||
if remote_sync:
|
||||
_sync_remote_status(jid, status, args.registry_dir, dispatch_tokens.get(jid))
|
||||
continue
|
||||
|
||||
print(_format_line(topic, payload), flush=True)
|
||||
if remote_sync and event in ("started", "completed", "error", "cancelled"):
|
||||
_sync_remote_status(jid, {"started": "running"}.get(event, event),
|
||||
args.registry_dir, dispatch_tokens.get(jid))
|
||||
if event in TERMINAL_EVENTS:
|
||||
if jid in terminal:
|
||||
# Already finalised: ignore duplicates / late reorders.
|
||||
@@ -421,6 +527,9 @@ def main(argv=None) -> int:
|
||||
help="wall-clock budget in seconds (default: job.timeout_sec or 3600)")
|
||||
parser.add_argument("--idle-timeout", type=float, default=None,
|
||||
help="max seconds with no new event (default: job.idle_timeout_sec or 120)")
|
||||
parser.add_argument("--ack-timeout", type=float, default=None,
|
||||
help="extra idle budget before the first event, for the injection to "
|
||||
"produce `started` (default: job.ack_timeout_sec; Issue #18)")
|
||||
parser.add_argument("--expect-retention", action="store_true",
|
||||
help="warn if no retained terminal event arrives promptly")
|
||||
parser.add_argument("--expect-dispatch-token", default=None,
|
||||
@@ -428,6 +537,10 @@ def main(argv=None) -> int:
|
||||
parser.add_argument("--attempt-id", type=int, default=None,
|
||||
help="minimum attempt id to accept events for")
|
||||
parser.add_argument("--registry-dir", default=default_registry_dir())
|
||||
parser.add_argument("--remote", action="store_true",
|
||||
help="the agent shares no disk with this registry: mirror started/"
|
||||
"terminal events into the job record (Issue #18; automatic "
|
||||
"for jobs registered with --transport remote)")
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import logging
|
||||
import sys
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from lib_py.delegation import heartbeat as heartbeat_timer
|
||||
from lib_py.delegation import mqtt_common
|
||||
from lib_py.delegation import registry
|
||||
from lib_py.delegation.mqtt_common import (
|
||||
@@ -92,15 +93,49 @@ def build_payload(
|
||||
return payload
|
||||
|
||||
|
||||
def _publish_once(config, topic: str, body: bytes, retain: bool) -> None:
|
||||
def _publish_once(config, topic: str, body: bytes, retain: bool, extra=()) -> None:
|
||||
"""Connect, publish one QoS-1 message, wait for the broker ACK, disconnect.
|
||||
|
||||
Delegates to the transport module so the CLI path and the service path share
|
||||
one implementation. Keeping a second copy here is what let the persistent
|
||||
transport's handshake and not-connected bugs exist in only one of them.
|
||||
Raises on any failure so ``with_retry`` can re-run the whole sequence.
|
||||
|
||||
``extra`` (Issue #18): further ``(topic, body, retain)`` messages sent over
|
||||
the same connection after the event - the retained job-state message, so
|
||||
it costs no second broker handshake.
|
||||
"""
|
||||
OneShotPublisher().publish_raw(config, topic, body, retain)
|
||||
OneShotPublisher().publish_raw(config, topic, body, retain, extra=extra)
|
||||
|
||||
|
||||
def state_topic_for(job: Dict[str, Any]) -> str:
|
||||
prefix = job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"])
|
||||
return f"{prefix}/state"
|
||||
|
||||
|
||||
def build_state_payload(job: Dict[str, Any], prepared: "PreparedEvent") -> Dict[str, Any]:
|
||||
"""The retained job-state message published next to every event (Issue #18).
|
||||
|
||||
A subscriber that (re)connects receives it at once and learns the job's
|
||||
status and the last seq without waiting for the next heartbeat; a seq it
|
||||
never saw in between is a detected loss, not a silent one. Signed like an
|
||||
event so a forged state cannot close a job.
|
||||
"""
|
||||
status = EVENT_TO_STATUS.get(prepared.event) or "running"
|
||||
data = {k: v for k, v in (prepared.payload.get("data") or {}).items()
|
||||
if k in ("dispatch_token", "attempt_id")}
|
||||
payload = build_payload(job_id=prepared.job_id, seq=prepared.seq, event="state",
|
||||
detail=prepared.event, data=data, auth_token=job.get("auth_token"))
|
||||
payload["status"] = status
|
||||
payload["last_event"] = prepared.event
|
||||
if job.get("auth_token"):
|
||||
# re-sign: `status`/`last_event` were added after build_payload signed
|
||||
payload["data"].pop("hmac_sig", None)
|
||||
sign_payload = {k: v for k, v in payload.items() if k != "data"}
|
||||
sign_payload["data"] = dict(payload["data"])
|
||||
msg = json.dumps(sign_payload, sort_keys=True, separators=(",", ":")).encode()
|
||||
payload["data"]["hmac_sig"] = hmac.new(job["auth_token"].encode(), msg, hashlib.sha256).hexdigest()
|
||||
return payload
|
||||
|
||||
|
||||
class PreparedEvent(object):
|
||||
@@ -256,9 +291,19 @@ def main(argv=None) -> int:
|
||||
"job's current iteration)")
|
||||
parser.add_argument("--expect-dispatch-token", default=None,
|
||||
help="expected dispatch token for CAS verification")
|
||||
parser.add_argument("--heartbeat", type=int, default=None, metavar="N",
|
||||
help="after publishing, start a detached timer that publishes a "
|
||||
"`progress` heartbeat every N seconds until the job ends "
|
||||
"(Issue #18; not with a terminal event)")
|
||||
parser.add_argument("--no-state", action="store_true",
|
||||
help="do not publish the retained <topic_prefix>/state message")
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.heartbeat is not None and (args.event in TERMINAL_EVENTS or args.heartbeat < 1):
|
||||
logger.error("--heartbeat needs a positive period and a non-terminal event")
|
||||
return 1
|
||||
|
||||
mqtt_common.ensure_env_loaded()
|
||||
mqtt_common.setup_logging(logging.DEBUG if args.verbose else logging.WARNING)
|
||||
|
||||
@@ -341,6 +386,12 @@ def main(argv=None) -> int:
|
||||
check_token, actual_tok)
|
||||
return 4
|
||||
|
||||
# Issue #18: a terminal event ends the heartbeat obligation. Stop the timer
|
||||
# before taking this event's seq so no heartbeat can follow `completed`.
|
||||
if args.event in TERMINAL_EVENTS:
|
||||
if heartbeat_timer.stop_timer(job_id, args.registry_dir):
|
||||
logger.info("stopped the heartbeat timer for job %s", job_id)
|
||||
|
||||
try:
|
||||
prepared = prepare_event(
|
||||
job_id=job_id,
|
||||
@@ -356,6 +407,12 @@ def main(argv=None) -> int:
|
||||
logger.error("%s", exc)
|
||||
return 1
|
||||
|
||||
extra_messages = []
|
||||
if not args.no_state and rec is not None:
|
||||
state_payload = build_state_payload(rec, prepared)
|
||||
extra_messages.append((state_topic_for(rec),
|
||||
json.dumps(state_payload, ensure_ascii=False).encode("utf-8"), True))
|
||||
|
||||
# Local first: the event reaches events.log before the publish can block on
|
||||
# the broker ACK and its retries (see record_local_event).
|
||||
record_local_event(prepared, args.registry_dir)
|
||||
@@ -370,7 +427,7 @@ def main(argv=None) -> int:
|
||||
publish_ok = True
|
||||
publish_error: Optional[str] = None
|
||||
try:
|
||||
publish(prepared.config, prepared.topic, prepared.body, prepared.retain)
|
||||
publish(prepared.config, prepared.topic, prepared.body, prepared.retain, extra_messages)
|
||||
except Exception as exc:
|
||||
publish_ok = False
|
||||
publish_error = str(exc)
|
||||
@@ -388,6 +445,22 @@ def main(argv=None) -> int:
|
||||
expected_attempt=sync_expected_attempt,
|
||||
local_event_recorded=True)
|
||||
|
||||
# Issue #18: the automatic heartbeat. Started whether or not the wire
|
||||
# publish succeeded - the agent is working either way, and each beat lands
|
||||
# in events.log before its own publish, which is what local liveness reads.
|
||||
if args.heartbeat is not None:
|
||||
hb_token = sync_expected_token
|
||||
hb_attempt = sync_expected_attempt
|
||||
if rec is not None:
|
||||
hb_token = hb_token or rec.get("dispatch_token")
|
||||
hb_attempt = hb_attempt if hb_attempt is not None else rec.get("attempt_id", rec.get("iteration", 1))
|
||||
try:
|
||||
pid = heartbeat_timer.start_timer(job_id, args.registry_dir, args.heartbeat,
|
||||
dispatch_token=hb_token, attempt_id=hb_attempt)
|
||||
logger.info("heartbeat timer started (pid %d, every %ss)", pid, args.heartbeat)
|
||||
except OSError as exc:
|
||||
logger.error("could not start the heartbeat timer: %s", exc)
|
||||
|
||||
if not publish_ok:
|
||||
return 2
|
||||
|
||||
|
||||
@@ -79,11 +79,13 @@ class OneShotPublisher(_BasePublisher):
|
||||
def publish(self, prepared):
|
||||
self.publish_raw(prepared.config, prepared.topic, prepared.body, prepared.retain)
|
||||
|
||||
def publish_raw(self, config, topic: str, body: bytes, retain: bool) -> None:
|
||||
def publish_raw(self, config, topic: str, body: bytes, retain: bool, extra=()) -> None:
|
||||
"""Connect, publish, wait for the ACK, disconnect.
|
||||
|
||||
The single implementation of one-shot publishing: ``publish_event``'s
|
||||
CLI path calls straight into this, so the two entry points cannot drift.
|
||||
``extra`` messages (``(topic, body, retain)`` tuples, Issue #18's retained
|
||||
job state) go out over the same connection, each awaited like the first.
|
||||
"""
|
||||
client = self.client_factory("publisher", config)
|
||||
self.connect_count += 1
|
||||
@@ -104,6 +106,8 @@ class OneShotPublisher(_BasePublisher):
|
||||
if mqtt_common.reason_code_value(connected["rc"]) != 0:
|
||||
raise ConnectionError("broker refused connection: rc={}".format(connected["rc"]))
|
||||
_send(client, topic, body, retain)
|
||||
for x_topic, x_body, x_retain in (extra or ()):
|
||||
_send(client, x_topic, x_body, x_retain)
|
||||
finally:
|
||||
client.loop_stop()
|
||||
try:
|
||||
|
||||
@@ -47,6 +47,22 @@ def generate_job_id(bits: int = 32) -> str:
|
||||
return uuid.uuid4().hex[:nibbles]
|
||||
|
||||
|
||||
# Issue #18: the liveness contract's defaults. An agent heartbeats (publishes
|
||||
# `progress`) at least every N seconds; the orchestrator treats K missed
|
||||
# heartbeats (K x N seconds since the last event) as unresponsive; `started`
|
||||
# must arrive within T seconds of the prompt injection. K x N with these
|
||||
# defaults equals the historical 120 s idle timeout, so a caller that never
|
||||
# asked for the contract sees the same idle window as before.
|
||||
DEFAULT_HEARTBEAT_SEC = 30
|
||||
DEFAULT_HEARTBEAT_MISSES = 4
|
||||
DEFAULT_ACK_TIMEOUT_SEC = 60
|
||||
|
||||
|
||||
def liveness_window(heartbeat_sec: int, heartbeat_misses: int) -> int:
|
||||
"""K x N: seconds without any event after which a job is unresponsive."""
|
||||
return max(1, int(heartbeat_sec)) * max(1, int(heartbeat_misses))
|
||||
|
||||
|
||||
def register_job(
|
||||
prompt: str,
|
||||
agent: str = "claude-code",
|
||||
@@ -54,7 +70,7 @@ def register_job(
|
||||
role: str = "Worker",
|
||||
broker: Optional[Dict[str, Any]] = None,
|
||||
timeout_sec: int = 3600,
|
||||
idle_timeout_sec: int = 120,
|
||||
idle_timeout_sec: Optional[int] = None,
|
||||
registry_dir: str = DEFAULT_REGISTRY_DIR,
|
||||
job_id: Optional[str] = None,
|
||||
expected_artifacts: Optional[List[str]] = None,
|
||||
@@ -64,13 +80,31 @@ def register_job(
|
||||
reviewer: Optional[str] = None,
|
||||
reviewer_session: Optional[str] = None,
|
||||
max_iterations: int = 5,
|
||||
heartbeat_sec: int = DEFAULT_HEARTBEAT_SEC,
|
||||
heartbeat_misses: int = DEFAULT_HEARTBEAT_MISSES,
|
||||
ack_timeout_sec: int = DEFAULT_ACK_TIMEOUT_SEC,
|
||||
dispatch_token: Optional[str] = None,
|
||||
attempt_id: int = 1,
|
||||
transport: str = "herdr",
|
||||
) -> str:
|
||||
"""Create a new ``pending`` job record and return its id.
|
||||
|
||||
``broker`` defaults to the current environment's resolved broker block, so
|
||||
the registry alone is enough for ``publish_event.py`` to connect later.
|
||||
|
||||
Issue #18: ``idle_timeout_sec`` left as None derives from the liveness
|
||||
contract (``heartbeat_misses x heartbeat_sec``); an explicit value is kept
|
||||
as given. ``dispatch_token`` / ``attempt_id`` are minted here unless a
|
||||
remote runner mirrors a job the orchestrator already registered, in which
|
||||
case it passes the orchestrator's so the events it publishes carry the
|
||||
identity the orchestrator's subscriber expects. ``transport`` records how
|
||||
the prompt reaches the agent (``herdr`` pane or ``remote`` dispatch topic).
|
||||
"""
|
||||
job_id = job_id or generate_job_id(bits)
|
||||
heartbeat_sec = max(1, int(heartbeat_sec))
|
||||
heartbeat_misses = max(1, int(heartbeat_misses))
|
||||
if idle_timeout_sec is None:
|
||||
idle_timeout_sec = liveness_window(heartbeat_sec, heartbeat_misses)
|
||||
if broker is None:
|
||||
broker = broker_config_from_env().to_registry_block()
|
||||
if auth_token is None:
|
||||
@@ -101,8 +135,13 @@ def register_job(
|
||||
"reviewer_session": reviewer_session,
|
||||
"max_iterations": int(max_iterations),
|
||||
"iteration": 1,
|
||||
"dispatch_token": uuid.uuid4().hex,
|
||||
"attempt_id": 1,
|
||||
"dispatch_token": dispatch_token or uuid.uuid4().hex,
|
||||
"attempt_id": int(attempt_id),
|
||||
# Issue #18 liveness contract
|
||||
"heartbeat_sec": heartbeat_sec,
|
||||
"heartbeat_misses": heartbeat_misses,
|
||||
"ack_timeout_sec": max(1, int(ack_timeout_sec)),
|
||||
"transport": transport,
|
||||
}
|
||||
with registry_lock(registry_dir):
|
||||
if mqtt_common._job_path(job_id, registry_dir).exists():
|
||||
@@ -276,7 +315,20 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
p_reg.add_argument("--agent-session", default="herdr:claude")
|
||||
p_reg.add_argument("--role", default="Worker", help="logical role for the delegated agent (e.g. Worker, Planner, Reviewer)")
|
||||
p_reg.add_argument("--timeout", type=int, default=3600)
|
||||
p_reg.add_argument("--idle-timeout", type=int, default=120)
|
||||
p_reg.add_argument("--idle-timeout", type=int, default=None,
|
||||
help="seconds without an event before the job is unresponsive "
|
||||
"(default: --heartbeat-misses x --heartbeat-sec)")
|
||||
p_reg.add_argument("--heartbeat-sec", type=int, default=DEFAULT_HEARTBEAT_SEC,
|
||||
help="agent heartbeat period N in seconds (Issue #18)")
|
||||
p_reg.add_argument("--heartbeat-misses", type=int, default=DEFAULT_HEARTBEAT_MISSES,
|
||||
help="missed heartbeats K before the job counts as unresponsive")
|
||||
p_reg.add_argument("--ack-timeout", type=int, default=DEFAULT_ACK_TIMEOUT_SEC,
|
||||
help="seconds T after injection within which `started` must arrive")
|
||||
p_reg.add_argument("--transport", default="herdr", choices=["herdr", "remote"],
|
||||
help="how the prompt reaches the agent")
|
||||
p_reg.add_argument("--dispatch-token", default=None, help=argparse.SUPPRESS)
|
||||
p_reg.add_argument("--attempt-id", type=int, default=1, help=argparse.SUPPRESS)
|
||||
p_reg.add_argument("--job-id", default=None, help=argparse.SUPPRESS)
|
||||
p_reg.add_argument("--bits", type=int, default=32, help="32 (PoC) or 128 (prod)")
|
||||
p_reg.add_argument("--artifact", action="append", default=[], dest="artifacts")
|
||||
p_reg.add_argument("--auth-token", default=None, help="HMAC auth token for the job (auto-generated if secure broker is detected)")
|
||||
@@ -430,6 +482,13 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
reviewer=args.reviewer,
|
||||
reviewer_session=args.reviewer_session,
|
||||
max_iterations=args.max_iterations,
|
||||
heartbeat_sec=args.heartbeat_sec,
|
||||
heartbeat_misses=args.heartbeat_misses,
|
||||
ack_timeout_sec=args.ack_timeout,
|
||||
transport=args.transport,
|
||||
dispatch_token=args.dispatch_token,
|
||||
attempt_id=args.attempt_id,
|
||||
job_id=args.job_id,
|
||||
)
|
||||
print(job_id)
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
"""remote.py — deliver a job to an agent on another host over the broker (Issue #18).
|
||||
|
||||
The herdr transport injects the prompt into a pane on this machine and reads
|
||||
the agent's `started`/`progress`/`completed` events back over the broker. A
|
||||
remote agent shares neither the pane nor the orchestrator's disk, so the
|
||||
prompt itself has to travel over the wire too:
|
||||
|
||||
orchestrator --retained `<topic_prefix>/dispatch`--> remote runner
|
||||
remote runner --`<topic_prefix>/events` + `/state`--> orchestrator's subscriber
|
||||
|
||||
`publish_dispatch` (wrapper side) publishes one retained dispatch message
|
||||
carrying the brief, the job's auth_token / dispatch_token / attempt_id and the
|
||||
liveness contract (heartbeat N, misses K, ack T). `run_runner` (remote side)
|
||||
subscribes for its session name, mirrors the job into its own registry,
|
||||
publishes `started` with the automatic heartbeat, runs the agent command, and
|
||||
publishes the terminal event. Every event it publishes is signed with the
|
||||
job's auth_token and carries the orchestrator's dispatch identity, so the
|
||||
orchestrator's unchanged subscriber accepts it; the subscriber's `--remote`
|
||||
mode mirrors those events into the orchestrator's registry.
|
||||
|
||||
Trust: a dispatch makes a runner execute a command, so it is signed with a
|
||||
pre-shared secret (`MAM_DISPATCH_SECRET`, from each host's own .mam.env) and a
|
||||
runner ignores anything unsigned or mis-signed. The auth_token travels inside
|
||||
that signed message; use this only on a private broker (see MESSAGING.md).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import shlex
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from lib_py.delegation import mqtt_common, registry
|
||||
from lib_py.delegation.mqtt_common import (
|
||||
DEFAULT_TOPIC_ROOT,
|
||||
SCHEMA_VERSION,
|
||||
broker_config_from_env,
|
||||
broker_config_from_job,
|
||||
default_registry_dir,
|
||||
load_job,
|
||||
make_client,
|
||||
)
|
||||
from lib_py.delegation.publisher import OneShotPublisher
|
||||
|
||||
logger = logging.getLogger("delegate_job.remote")
|
||||
|
||||
DISPATCH_KIND = "dispatch"
|
||||
SECRET_ENV = "MAM_DISPATCH_SECRET"
|
||||
SESSION_PREFIX = "remote:"
|
||||
|
||||
|
||||
def is_remote_session(agent_session: Optional[str]) -> bool:
|
||||
return bool(agent_session) and str(agent_session).startswith(SESSION_PREFIX)
|
||||
|
||||
|
||||
def session_name(agent_session: str) -> str:
|
||||
return agent_session[len(SESSION_PREFIX):] if is_remote_session(agent_session) else agent_session
|
||||
|
||||
|
||||
def dispatch_topic_for(job: Dict[str, Any]) -> str:
|
||||
prefix = job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"])
|
||||
return f"{prefix}/dispatch"
|
||||
|
||||
|
||||
def _canonical(payload: Dict[str, Any]) -> bytes:
|
||||
body = {k: v for k, v in payload.items() if k != "sig"}
|
||||
return json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
|
||||
|
||||
def sign_dispatch(payload: Dict[str, Any], secret: str) -> Dict[str, Any]:
|
||||
payload = dict(payload)
|
||||
payload["sig"] = hmac.new(secret.encode("utf-8"), _canonical(payload), hashlib.sha256).hexdigest()
|
||||
return payload
|
||||
|
||||
|
||||
def verify_dispatch(payload: Dict[str, Any], secret: Optional[str]) -> bool:
|
||||
"""True when the dispatch carries a valid signature for ``secret``. With
|
||||
no secret configured on the runner side nothing is accepted."""
|
||||
if not secret:
|
||||
return False
|
||||
sig = payload.get("sig")
|
||||
if not isinstance(sig, str):
|
||||
return False
|
||||
expected = hmac.new(secret.encode("utf-8"), _canonical(payload), hashlib.sha256).hexdigest()
|
||||
return hmac.compare_digest(sig, expected)
|
||||
|
||||
|
||||
def build_dispatch(job: Dict[str, Any], session: str, instructions: str) -> Dict[str, Any]:
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"kind": DISPATCH_KIND,
|
||||
"job_id": job["job_id"],
|
||||
"session": session,
|
||||
"instructions": instructions,
|
||||
"brief": job.get("prompt", ""),
|
||||
"role": job.get("role"),
|
||||
"agent": job.get("agent"),
|
||||
"auth_token": job.get("auth_token"),
|
||||
"dispatch_token": job.get("dispatch_token"),
|
||||
"attempt_id": job.get("attempt_id", job.get("iteration", 1)),
|
||||
"timeout_sec": job.get("timeout_sec", 3600),
|
||||
"idle_timeout_sec": job.get("idle_timeout_sec"),
|
||||
"heartbeat_sec": job.get("heartbeat_sec", registry.DEFAULT_HEARTBEAT_SEC),
|
||||
"heartbeat_misses": job.get("heartbeat_misses", registry.DEFAULT_HEARTBEAT_MISSES),
|
||||
"ack_timeout_sec": job.get("ack_timeout_sec", registry.DEFAULT_ACK_TIMEOUT_SEC),
|
||||
"topic_prefix": job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"]),
|
||||
"origin": socket.gethostname(),
|
||||
"timestamp": mqtt_common._utcnow(),
|
||||
}
|
||||
|
||||
|
||||
def publish_dispatch(job_id: str, registry_dir: str, session: str, instructions: str,
|
||||
secret: Optional[str] = None, allow_unsigned: bool = False) -> str:
|
||||
"""Publish the retained dispatch for ``job_id``; returns the topic."""
|
||||
job = load_job(job_id, registry_dir)
|
||||
secret = secret or os.environ.get(SECRET_ENV)
|
||||
payload = build_dispatch(job, session, instructions)
|
||||
if secret:
|
||||
payload = sign_dispatch(payload, secret)
|
||||
elif not allow_unsigned:
|
||||
raise RuntimeError("{} is not set: refusing to publish an unsigned dispatch "
|
||||
"(set it in .mam.env on both hosts, or pass --allow-unsigned)".format(SECRET_ENV))
|
||||
topic = dispatch_topic_for(job)
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
mqtt_common.with_retry(OneShotPublisher().publish_raw, attempts=3,
|
||||
exceptions=(OSError, TimeoutError, ConnectionError, ValueError))(
|
||||
broker_config_from_job(job), topic, body, True)
|
||||
mqtt_common.append_event(job_id, {"event": "dispatched", "topic": topic, "session": session,
|
||||
"attempt_id": payload["attempt_id"]})
|
||||
logger.info("dispatched job %s to remote session %s on %s", job_id, session, topic)
|
||||
return topic
|
||||
|
||||
|
||||
def clear_dispatch(config, topic: str) -> None:
|
||||
"""Retire a retained dispatch (empty retained payload) so a runner that
|
||||
restarts does not execute it again."""
|
||||
try:
|
||||
OneShotPublisher().publish_raw(config, topic, b"", True)
|
||||
except Exception as exc: # pragma: no cover - best effort
|
||||
logger.warning("could not clear dispatch %s: %s", topic, exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# remote side
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def mirror_job(d: Dict[str, Any], registry_dir: str, session: str) -> str:
|
||||
"""Create the local job record a remote agent publishes against.
|
||||
|
||||
The record carries the orchestrator's job_id, auth_token, dispatch_token
|
||||
and attempt_id, so `publish_event` on this host produces events the
|
||||
orchestrator's subscriber accepts. A previous attempt's record (a
|
||||
re-dispatch) is retired first: creation is the only write allowed here.
|
||||
"""
|
||||
jid = d["job_id"]
|
||||
path = mqtt_common._job_path(jid, registry_dir)
|
||||
if path.exists():
|
||||
old = load_job(jid, registry_dir)
|
||||
if (old.get("dispatch_token") == d.get("dispatch_token")
|
||||
and old.get("status") not in ("completed", "error", "cancelled")):
|
||||
return jid # same attempt, already mirrored (duplicate delivery)
|
||||
with mqtt_common.registry_lock(registry_dir):
|
||||
path.unlink()
|
||||
log = mqtt_common._job_path(jid, registry_dir).with_suffix(".events.log")
|
||||
if log.exists():
|
||||
log.unlink()
|
||||
registry.register_job(
|
||||
prompt=d.get("brief", ""),
|
||||
agent=d.get("agent") or "remote",
|
||||
agent_session=SESSION_PREFIX + session,
|
||||
role=d.get("role") or "Worker",
|
||||
timeout_sec=int(d.get("timeout_sec") or 3600),
|
||||
idle_timeout_sec=d.get("idle_timeout_sec"),
|
||||
registry_dir=registry_dir,
|
||||
job_id=jid,
|
||||
auth_token=d.get("auth_token"),
|
||||
heartbeat_sec=int(d.get("heartbeat_sec") or registry.DEFAULT_HEARTBEAT_SEC),
|
||||
heartbeat_misses=int(d.get("heartbeat_misses") or registry.DEFAULT_HEARTBEAT_MISSES),
|
||||
ack_timeout_sec=int(d.get("ack_timeout_sec") or registry.DEFAULT_ACK_TIMEOUT_SEC),
|
||||
dispatch_token=d.get("dispatch_token"),
|
||||
attempt_id=int(d.get("attempt_id") or 1),
|
||||
transport="remote",
|
||||
)
|
||||
brief_dir = os.path.join(registry_dir, jid)
|
||||
os.makedirs(brief_dir, exist_ok=True)
|
||||
with open(os.path.join(brief_dir, "brief.md"), "w", encoding="utf-8") as fh:
|
||||
fh.write(d.get("brief", ""))
|
||||
return jid
|
||||
|
||||
|
||||
def _publish(jid: str, registry_dir: str, event: str, detail: str, d: Dict[str, Any],
|
||||
heartbeat: Optional[int] = None) -> int:
|
||||
from lib_py.delegation import publish_event
|
||||
argv = ["--registry-dir", registry_dir, "--job", jid, "--event", event, "--detail", detail]
|
||||
if d.get("dispatch_token"):
|
||||
argv += ["--expect-dispatch-token", d["dispatch_token"]]
|
||||
if d.get("attempt_id") is not None:
|
||||
argv += ["--attempt-id", str(d["attempt_id"])]
|
||||
if heartbeat:
|
||||
argv += ["--heartbeat", str(heartbeat)]
|
||||
return publish_event.main(argv)
|
||||
|
||||
|
||||
def handle_dispatch(d: Dict[str, Any], registry_dir: str, session: str,
|
||||
exec_cmd: Optional[str], python: Optional[str] = None) -> int:
|
||||
"""Run one dispatched job to its terminal event. Returns the agent rc."""
|
||||
jid = mirror_job(d, registry_dir, session)
|
||||
heartbeat = int(d.get("heartbeat_sec") or registry.DEFAULT_HEARTBEAT_SEC)
|
||||
rc_started = _publish(jid, registry_dir, "started", "remote runner {} claimed the job".format(session),
|
||||
d, heartbeat=heartbeat)
|
||||
if rc_started not in (0, 2):
|
||||
logger.error("could not publish started for %s (rc %s)", jid, rc_started)
|
||||
return 1
|
||||
job = load_job(jid, registry_dir)
|
||||
rc = 0
|
||||
if exec_cmd:
|
||||
py = python or sys.executable
|
||||
skills_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
env = dict(os.environ)
|
||||
env["PYTHONPATH"] = skills_root + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
|
||||
env.update({
|
||||
"JOB_ID": jid,
|
||||
"MAM_JOB_ID": jid,
|
||||
"MAM_JOB_REGISTRY_DIR": registry_dir,
|
||||
"MAM_JOB_BRIEF": os.path.join(registry_dir, jid, "brief.md"),
|
||||
"MAM_JOB_INSTRUCTIONS": d.get("instructions", ""),
|
||||
"MAM_PUBLISH_CMD": " ".join(shlex.quote(x) for x in [
|
||||
py, "-m", "lib_py.delegation.publish_event", "--registry-dir", registry_dir, "--job", jid]
|
||||
+ (["--expect-dispatch-token", d["dispatch_token"]] if d.get("dispatch_token") else [])
|
||||
+ (["--attempt-id", str(d["attempt_id"])] if d.get("attempt_id") is not None else [])),
|
||||
})
|
||||
timeout = float(job.get("timeout_sec", 3600))
|
||||
try:
|
||||
res = subprocess.run(exec_cmd, shell=True, env=env, capture_output=True, text=True,
|
||||
timeout=timeout, cwd=os.getcwd())
|
||||
rc = res.returncode
|
||||
tail = (res.stdout or "").strip().splitlines()
|
||||
detail = tail[-1][:200] if tail else "exit {}".format(rc)
|
||||
if rc != 0:
|
||||
err = (res.stderr or "").strip().splitlines()
|
||||
detail = "exit {}: {}".format(rc, (err[-1] if err else detail)[:200])
|
||||
except subprocess.TimeoutExpired:
|
||||
rc, detail = 124, "agent command exceeded timeout_sec {}".format(int(timeout))
|
||||
cur = load_job(jid, registry_dir)
|
||||
if cur.get("status") not in ("completed", "error", "cancelled"):
|
||||
_publish(jid, registry_dir, "completed" if rc == 0 else "error", detail, d)
|
||||
return rc
|
||||
# No command: an agent on this host publishes the terminal event itself.
|
||||
deadline = time.monotonic() + float(job.get("timeout_sec", 3600))
|
||||
while time.monotonic() < deadline:
|
||||
cur = load_job(jid, registry_dir)
|
||||
if cur.get("status") in ("completed", "error", "cancelled"):
|
||||
return 0 if cur.get("status") == "completed" else 1
|
||||
time.sleep(1.0)
|
||||
_publish(jid, registry_dir, "error", "deadline_timeout (remote runner)", d)
|
||||
return 2
|
||||
|
||||
|
||||
def run_runner(session: str, registry_dir: str, exec_cmd: Optional[str] = None, once: bool = False,
|
||||
secret: Optional[str] = None, allow_unsigned: bool = False,
|
||||
timeout: Optional[float] = None, python: Optional[str] = None) -> int:
|
||||
secret = secret or os.environ.get(SECRET_ENV)
|
||||
if not secret and not allow_unsigned:
|
||||
logger.error("%s is not set: a runner must not execute unsigned dispatches "
|
||||
"(pass --allow-unsigned only on a private broker)", SECRET_ENV)
|
||||
return 1
|
||||
cfg = broker_config_from_env()
|
||||
client = make_client("remote-runner", cfg)
|
||||
inbox: "queue.Queue[Dict[str, Any]]" = queue.Queue()
|
||||
topic = f"{DEFAULT_TOPIC_ROOT}/+/dispatch"
|
||||
|
||||
def on_connect(_c, _u, _flags, reason_code, _props):
|
||||
if mqtt_common.reason_code_value(reason_code) != 0:
|
||||
logger.error("broker connection failed: rc=%s", reason_code)
|
||||
return
|
||||
_c.subscribe(topic, qos=1)
|
||||
print(f"SUBSCRIBED {topic}", flush=True)
|
||||
|
||||
def on_message(_c, _u, msg):
|
||||
if not msg.payload:
|
||||
return # a cleared (retired) dispatch
|
||||
try:
|
||||
d = json.loads(msg.payload.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
logger.warning("drop unparseable dispatch on %s", msg.topic)
|
||||
return
|
||||
if not isinstance(d, dict) or d.get("kind") != DISPATCH_KIND or d.get("schema_version") != SCHEMA_VERSION:
|
||||
return
|
||||
if d.get("session") != session:
|
||||
return
|
||||
if secret and not verify_dispatch(d, secret):
|
||||
logger.warning("drop dispatch for %s: bad or missing signature", d.get("job_id"))
|
||||
return
|
||||
if not secret and not allow_unsigned:
|
||||
return
|
||||
d["_topic"] = msg.topic
|
||||
inbox.put(d)
|
||||
|
||||
client.on_connect = on_connect
|
||||
client.on_message = on_message
|
||||
client.reconnect_delay_set(min_delay=1, max_delay=16)
|
||||
mqtt_common.with_retry(lambda: client.connect(cfg.host, cfg.port, cfg.keepalive),
|
||||
attempts=3, base_delay=0.2, max_delay=1.0)()
|
||||
client.loop_start()
|
||||
deadline = time.monotonic() + timeout if timeout else None
|
||||
handled = 0
|
||||
try:
|
||||
while True:
|
||||
wait = 1.0 if deadline is None else max(0.0, min(1.0, deadline - time.monotonic()))
|
||||
try:
|
||||
d = inbox.get(timeout=wait)
|
||||
except queue.Empty:
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
logger.warning("runner timeout after %d job(s)", handled)
|
||||
return 0 if handled else 2
|
||||
continue
|
||||
print("DISPATCH job={} attempt={} from={}".format(d.get("job_id"), d.get("attempt_id"), d.get("origin")),
|
||||
flush=True)
|
||||
# Retire the retained dispatch before running so a restart cannot replay it.
|
||||
clear_dispatch(cfg, d["_topic"])
|
||||
rc = handle_dispatch(d, registry_dir, session, exec_cmd, python=python)
|
||||
handled += 1
|
||||
print("DONE job={} rc={}".format(d.get("job_id"), rc), flush=True)
|
||||
if once:
|
||||
return 0
|
||||
finally:
|
||||
client.loop_stop()
|
||||
try:
|
||||
client.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description="remote transport for delegated jobs (Issue #18)")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
p_d = sub.add_parser("dispatch", help="publish the retained dispatch for a registered job")
|
||||
p_d.add_argument("--job", required=True)
|
||||
p_d.add_argument("--registry-dir", default=default_registry_dir())
|
||||
p_d.add_argument("--session", required=True, help="remote session name (without the remote: prefix)")
|
||||
p_d.add_argument("--instructions", default="")
|
||||
p_d.add_argument("--allow-unsigned", action="store_true")
|
||||
p_r = sub.add_parser("run", help="run a remote session: execute dispatches addressed to it")
|
||||
p_r.add_argument("--session", required=True)
|
||||
p_r.add_argument("--registry-dir", default=default_registry_dir())
|
||||
p_r.add_argument("--exec", dest="exec_cmd", default=None,
|
||||
help="shell command run per job (env: JOB_ID, MAM_JOB_BRIEF, MAM_PUBLISH_CMD, ...)")
|
||||
p_r.add_argument("--once", action="store_true", help="exit after one job")
|
||||
p_r.add_argument("--timeout", type=float, default=None, help="exit after this many seconds")
|
||||
p_r.add_argument("--allow-unsigned", action="store_true")
|
||||
p_r.add_argument("--python", default=None)
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
mqtt_common.ensure_env_loaded()
|
||||
mqtt_common.setup_logging(logging.DEBUG if args.verbose else logging.INFO)
|
||||
if args.command == "dispatch":
|
||||
try:
|
||||
print(publish_dispatch(args.job, args.registry_dir, args.session, args.instructions,
|
||||
allow_unsigned=args.allow_unsigned))
|
||||
except (FileNotFoundError, RuntimeError) as exc:
|
||||
logger.error("%s", exc)
|
||||
return 1
|
||||
except Exception as exc:
|
||||
logger.error("dispatch publish failed: %s", exc)
|
||||
return 2
|
||||
return 0
|
||||
return run_runner(args.session, args.registry_dir, args.exec_cmd, args.once,
|
||||
allow_unsigned=args.allow_unsigned, timeout=args.timeout, python=args.python)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -241,6 +241,32 @@ class LoopRunner:
|
||||
return "codex"
|
||||
return "claude"
|
||||
|
||||
def session_label(self, name: str) -> str:
|
||||
"""`herdr:<name>` for a pane-backed session, `remote:<name>` for a row
|
||||
registered with `transport: remote` (Issue #18: the prompt then goes
|
||||
over the broker and no herdr/pane/lsof/ps path is taken)."""
|
||||
state = getattr(self, "_sessions_state", None)
|
||||
if state is None:
|
||||
state = self.load_sessions_state()
|
||||
self._sessions_state = state
|
||||
for s in state.get("herdr_sessions", []):
|
||||
if s.get("name") == name and str(s.get("transport") or "").lower() == "remote":
|
||||
return f"remote:{name}"
|
||||
return f"herdr:{name}"
|
||||
|
||||
def contract_args(self) -> List[str]:
|
||||
"""Liveness-contract knobs passed through to every delegate job
|
||||
(Issue #18): MAM_HEARTBEAT_SEC (N), MAM_HEARTBEAT_MISSES (K),
|
||||
MAM_ACK_TIMEOUT_S (T). The wrapper derives --idle-timeout as K x N."""
|
||||
args: List[str] = []
|
||||
for env_name, flag in (("MAM_HEARTBEAT_SEC", "--heartbeat"),
|
||||
("MAM_HEARTBEAT_MISSES", "--heartbeat-misses"),
|
||||
("MAM_ACK_TIMEOUT_S", "--ack-timeout")):
|
||||
val = os.environ.get(env_name)
|
||||
if val and val.isdigit() and int(val) > 0:
|
||||
args += [flag, val]
|
||||
return args
|
||||
|
||||
def resolve_all_reviewers(self, state: Dict[str, Any]) -> List[str]:
|
||||
"""Resolve running reviewer sessions excluding creator."""
|
||||
reviewers = []
|
||||
@@ -266,12 +292,12 @@ class LoopRunner:
|
||||
def _run_cmd(p_text: str):
|
||||
cmd = [
|
||||
"bash", str(wrapper_script), "submit",
|
||||
"--agent-session", f"herdr:{agent_session}",
|
||||
"--agent-session", self.session_label(agent_session),
|
||||
"--agent", agent_type,
|
||||
"--type", "direct",
|
||||
"--role", role,
|
||||
"--prompt", p_text,
|
||||
]
|
||||
] + self.contract_args()
|
||||
env = dict(os.environ)
|
||||
if "DELEGATE_JOB_PYTHON" not in env:
|
||||
env["DELEGATE_JOB_PYTHON"] = sys.executable
|
||||
|
||||
@@ -40,7 +40,16 @@ multi-agent-mux-delegate-job submit \
|
||||
--agent-session herdr:<session_name> \
|
||||
--prompt "Task description or instructions here" \
|
||||
--role <Worker|Planner|Reviewer> \
|
||||
--timeout 3600 --idle-timeout 120
|
||||
--timeout 3600 [--heartbeat 30 --heartbeat-misses 4 --ack-timeout 60] [--idle-timeout <sec>]
|
||||
# Liveness contract (Issue #18): the agent heartbeats every N s (automatic),
|
||||
# the job is unresponsive after K x N s (--idle-timeout overrides), `started`
|
||||
# must arrive within T s of the injection or the inject fails closed.
|
||||
|
||||
# 1b) Same, for an agent on another host (no herdr pane, no shared disk):
|
||||
# the brief travels over the broker as a signed retained dispatch and a
|
||||
# `remote.py run --session <name>` runner on that host executes it.
|
||||
multi-agent-mux-delegate-job submit --agent remote --agent-session remote:<runner_session> \
|
||||
--prompt "Task description" --role Worker # needs MAM_DISPATCH_SECRET on both hosts
|
||||
|
||||
# 2) Submit a job with a feedback loop (Worker-Reviewer Loop)
|
||||
multi-agent-mux-delegate-job submit \
|
||||
@@ -77,8 +86,8 @@ Under fail-closed operation (`MAM_DELEGATE_FAIL_CLOSED=1`, default), the CLI ret
|
||||
| Exit Code | Meaning | Final Registry Status | Failure Behavior & Resilience |
|
||||
|:---:|:---:|:---:|---|
|
||||
| `0` | Success | `completed` | Job succeeded and completed event received or verified on disk with approval (`PASS`/`AGREE`). |
|
||||
| `1` | Failure | `error` | Execution failure / Agent injection error (`inject_agent`). |
|
||||
| `2` | Timeout | `error` (`deadline_timeout`) | Subscriber timeout (total budget or idle timeout reached without terminal event). **R1 Working-Agent Resilience**: If the agent in Herdr is still actively `working`, the wrapper does not immediately fail-close; it re-observes disk status at intervals up to the wall-clock `--timeout` deadline before fail-closing. |
|
||||
| `1` | Failure | `error` | Execution failure / delivery failure: no `started` within `--ack-timeout` T s after the injection (`agent_blocked`; the prompt is never re-injected), or the input never reached the pane. |
|
||||
| `2` | Timeout | `error` (`deadline_timeout`) | Subscriber timeout (total budget or idle timeout reached without terminal event). **R1 liveness grace (Issue #18)**: after the subscriber idles out the wrapper keeps re-observing the disk while the job's events.log shows an event at most K x N (`--idle-timeout`) old - the automatic heartbeat counts - up to the wall-clock `--timeout`. herdr status and the pane are logged as a diagnostic only. |
|
||||
| `3` | Broker Error | `error` (`deadline_timeout`) | Broker unavailable (`MQTT_BROKER_UNAVAILABLE`) and terminal status not found on disk during deadline. |
|
||||
| `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed (prompt never delivered, retryable). |
|
||||
|
||||
@@ -131,11 +140,12 @@ a fresh interpreter — but a test harness could.
|
||||
|
||||
Every agent participating in the delegation contract must follow the same lifecycle publishing protocol using `publish_event.py`:
|
||||
|
||||
1. **On Start**: Publish `started` event.
|
||||
`python3 .agents/skills/multi-agent-mux-delegate-job/scripts/publish_event.py --job "$JOB_ID" --event started`
|
||||
1. **On Start** (first thing, within the ack timeout T): Publish `started` and arm the heartbeat timer.
|
||||
`python3 .agents/skills/multi-agent-mux-delegate-job/scripts/publish_event.py --job "$JOB_ID" --event started --heartbeat <N>`
|
||||
The timer publishes `progress` every N s until your terminal event (Issue #18); the brief and the injected instructions carry the exact command.
|
||||
2. **On Tool/Permission Prompt**: Publish `permission_required` event.
|
||||
`python3 ... --job "$JOB_ID" --event permission_required --detail "<tool>:<reason>"`
|
||||
3. **On Progress Update (Optional)**: Publish `progress` event.
|
||||
3. **On Progress Update**: Publish `progress` at check-points (the heartbeat timer already satisfies the every-N-seconds obligation).
|
||||
`python3 ... --job "$JOB_ID" --event progress --detail "<status_update>"`
|
||||
4. **On Success**: Publish `completed` event.
|
||||
`python3 ... --job "$JOB_ID" --event completed --detail "<summary>"` (Reviewer should include `"PASS"` in the detail to approve).
|
||||
|
||||
@@ -76,11 +76,17 @@ multi-agent-mux-delegate-job <command> [options]
|
||||
|
||||
submit --agent <name> --prompt <text> [--workdir <dir>] [--agent-session <label>]
|
||||
[--timeout <sec>] [--idle-timeout <sec>] [--validate <script>]
|
||||
[--heartbeat <sec>] [--heartbeat-misses <count>] [--ack-timeout <sec>]
|
||||
[--registry-dir <dir>] [--dry-run] [--role <role_name>]
|
||||
[--type <direct|loop|discuss>] [--reviewer <reviewer_agent>]
|
||||
[--reviewer-session <reviewer_session>] [--max-iterations <count>]
|
||||
[--counterpart-role <role_name>] [--strict-role-check]
|
||||
# The skill is herdr-interactive only; --mode print was removed.
|
||||
# --agent-session herdr:<name> injects into a local herdr pane;
|
||||
# --agent-session remote:<name> dispatches over the broker (Issue #18).
|
||||
# Liveness contract: the agent heartbeats every --heartbeat N s
|
||||
# (default 30, automatic), the job is unresponsive after
|
||||
# --heartbeat-misses K x N s (default 4 x 30 = 120; --idle-timeout
|
||||
# overrides), `started` must arrive within --ack-timeout T s (60).
|
||||
status --job <id> [--registry-dir <dir>]
|
||||
list [--registry-dir <dir>]
|
||||
verify --job <id> --validate <script> [--registry-dir <dir>]
|
||||
@@ -91,7 +97,9 @@ EOF
|
||||
|
||||
# ---- arg parsing helpers --------------------------------------------------
|
||||
AGENT="claude-code"; PROMPT=""; WORKDIR="$(pwd)"; AGENT_SESSION="herdr:claude"
|
||||
TIMEOUT=3600; IDLE_TIMEOUT=120; VALIDATE=""; DRY_RUN=0
|
||||
TIMEOUT=3600; IDLE_TIMEOUT=""; VALIDATE=""; DRY_RUN=0
|
||||
# Issue #18 liveness contract (N / K / T). An unset --idle-timeout derives K x N.
|
||||
HEARTBEAT_SEC="${MAM_HEARTBEAT_SEC:-30}"; HEARTBEAT_MISSES="${MAM_HEARTBEAT_MISSES:-4}"; ACK_TIMEOUT="${MAM_ACK_TIMEOUT_S:-60}"
|
||||
JOB_ID=""; REGISTRY_DIR="$REGISTRY_DIR_DEFAULT"; DELEGATE_ROLE="Worker"
|
||||
TYPE="direct"; REVIEWER="hermes"; REVIEWER_SESSION="herdr:hermes"; MAX_ITERATIONS=5
|
||||
DEFAULT_COUNTERPART_ROLE="Reviewer"
|
||||
@@ -109,6 +117,9 @@ parse_opts() {
|
||||
--agent-session) AGENT_SESSION="$2"; shift 2;;
|
||||
--timeout) TIMEOUT="$2"; shift 2;;
|
||||
--idle-timeout) IDLE_TIMEOUT="$2"; shift 2;;
|
||||
--heartbeat) HEARTBEAT_SEC="$2"; shift 2;;
|
||||
--heartbeat-misses) HEARTBEAT_MISSES="$2"; shift 2;;
|
||||
--ack-timeout) ACK_TIMEOUT="$2"; shift 2;;
|
||||
--validate) VALIDATE="$2"; shift 2;;
|
||||
--job) JOB_ID="$2"; shift 2;;
|
||||
--registry-dir) REGISTRY_DIR="$2"; shift 2;;
|
||||
@@ -123,8 +134,14 @@ parse_opts() {
|
||||
*) echo "unknown option: $1" >&2; usage; exit 1;;
|
||||
esac
|
||||
done
|
||||
[ -n "$IDLE_TIMEOUT" ] || IDLE_TIMEOUT=$(( HEARTBEAT_SEC * HEARTBEAT_MISSES ))
|
||||
# lib.sh's send_keys_safe reads the job's own `started` from this registry.
|
||||
export SKS_REGISTRY_DIR="$REGISTRY_DIR"
|
||||
}
|
||||
|
||||
# Issue #18: `remote:<name>` sessions get the prompt over the broker, not a pane.
|
||||
_is_remote() { case "${1:-}" in remote:*) return 0;; *) return 1;; esac; }
|
||||
|
||||
_to_lower() {
|
||||
printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
@@ -140,6 +157,8 @@ cmd_submit() {
|
||||
JOB_ID="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" register \
|
||||
--prompt "$PROMPT" --agent "$AGENT" --agent-session "$AGENT_SESSION" --role "$DELEGATE_ROLE" \
|
||||
--timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
||||
--heartbeat-sec "$HEARTBEAT_SEC" --heartbeat-misses "$HEARTBEAT_MISSES" --ack-timeout "$ACK_TIMEOUT" \
|
||||
--transport "$(_is_remote "$AGENT_SESSION" && echo remote || echo herdr)" \
|
||||
--job-type "$TYPE" --reviewer "$REVIEWER" --reviewer-session "$REVIEWER_SESSION" \
|
||||
--max-iterations "$MAX_ITERATIONS")"
|
||||
echo "registered job: $JOB_ID"
|
||||
@@ -155,6 +174,7 @@ cmd_submit() {
|
||||
- **Target Agent**: $AGENT (session: $AGENT_SESSION)
|
||||
- **Role**: $DELEGATE_ROLE
|
||||
- **Timeout**: $TIMEOUT s (Idle: $IDLE_TIMEOUT s)
|
||||
- **Liveness contract**: heartbeat every $HEARTBEAT_SEC s (automatic once you publish \`started --heartbeat $HEARTBEAT_SEC\`), unresponsive after $HEARTBEAT_MISSES missed heartbeats, \`started\` expected within $ACK_TIMEOUT s
|
||||
- **Output Report Path**: .mam/jobs/$JOB_ID/$AGENT-reports/report-final.md
|
||||
|
||||
## 🔎 Task Description
|
||||
@@ -176,7 +196,7 @@ EOF
|
||||
[ -n "$_disp_tok" ] && pub_extra+=(--expect-dispatch-token "$_disp_tok")
|
||||
[ -n "$_attempt_id" ] && pub_extra+=(--attempt-id "$_attempt_id")
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${pub_extra[*]}"
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started --heartbeat $HEARTBEAT_SEC' (run it FIRST, within $ACK_TIMEOUT s; it also starts an automatic heartbeat timer), success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
run_agent "$JOB_ID" "$instructions"
|
||||
@@ -185,8 +205,8 @@ EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 1) Preflight
|
||||
if ! preflight_agent "$JOB_ID" "$AGENT_SESSION" 0; then
|
||||
# 1) Preflight (a remote session has no local pane to check: Issue #18)
|
||||
if ! _is_remote "$AGENT_SESSION" && ! preflight_agent "$JOB_ID" "$AGENT_SESSION" 0; then
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "preflight"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
@@ -194,7 +214,7 @@ EOF
|
||||
fi
|
||||
|
||||
# 2) TUI idle handshake
|
||||
if [ "$AGENT" != "human" ]; then
|
||||
if [ "$AGENT" != "human" ] && ! _is_remote "$AGENT_SESSION"; then
|
||||
local idle_rc=0
|
||||
wait_for_agent_idle "$AGENT_SESSION" || idle_rc=$?
|
||||
export _SKS_IDLE_RC="$idle_rc"
|
||||
@@ -210,6 +230,7 @@ EOF
|
||||
local logf="$REGISTRY_DIR/$JOB_ID.subscriber.out"
|
||||
local sub_extra_args=()
|
||||
[ -n "$_disp_tok" ] && sub_extra_args+=(--expect-dispatch-token "$_disp_tok")
|
||||
_is_remote "$AGENT_SESSION" && sub_extra_args+=(--remote)
|
||||
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
||||
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
||||
${sub_extra_args[@]+"${sub_extra_args[@]}"} \
|
||||
@@ -250,8 +271,8 @@ EOF
|
||||
echo "WARNING: subscriber subscribe handshake timed out — falling back to proceed" >&2
|
||||
fi
|
||||
|
||||
# 5) Inject agent
|
||||
if ! inject_agent "$JOB_ID" "$instructions" "$AGENT_SESSION"; then
|
||||
# 5) Deliver the prompt (herdr pane or remote dispatch) and wait for `started`
|
||||
if ! deliver_prompt "$JOB_ID" "$instructions" "$AGENT_SESSION" "$logf"; then
|
||||
reap_subscriber "$sub_pid"
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "inject"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
@@ -323,6 +344,7 @@ EOF
|
||||
- **Target Agent/Session**: $current_session
|
||||
- **Role**: $display_role
|
||||
- **Iteration**: $iteration
|
||||
- **Liveness contract**: heartbeat every $HEARTBEAT_SEC s (automatic once you publish \`started --heartbeat $HEARTBEAT_SEC\`), unresponsive after $HEARTBEAT_MISSES missed heartbeats, \`started\` expected within $ACK_TIMEOUT s
|
||||
- **Output Report Path**: .mam/jobs/$JOB_ID/${clean_session}-reports/report-final.md
|
||||
|
||||
## 🔎 Task Description
|
||||
@@ -341,8 +363,8 @@ EOF
|
||||
force_warn_only=1
|
||||
fi
|
||||
|
||||
# Preflight
|
||||
if ! preflight_agent "$JOB_ID" "$current_session" "$force_warn_only"; then
|
||||
# Preflight (skipped for a remote session: Issue #18)
|
||||
if ! _is_remote "$current_session" && ! preflight_agent "$JOB_ID" "$current_session" "$force_warn_only"; then
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "preflight"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
@@ -352,7 +374,7 @@ EOF
|
||||
# Idle wait
|
||||
local cur_agent="$AGENT"
|
||||
[ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER"
|
||||
if [ "$cur_agent" != "human" ]; then
|
||||
if [ "$cur_agent" != "human" ] && ! _is_remote "$current_session"; then
|
||||
local idle_rc=0
|
||||
wait_for_agent_idle "$current_session" || idle_rc=$?
|
||||
export _SKS_IDLE_RC="$idle_rc"
|
||||
@@ -381,6 +403,7 @@ EOF
|
||||
local logf="$REGISTRY_DIR/${JOB_ID}.iter_${iteration}_${display_role}.subscriber.out"
|
||||
local loop_sub_args=(--attempt-id "$attempt")
|
||||
[ -n "$_disp_tok" ] && loop_sub_args+=(--expect-dispatch-token "$_disp_tok")
|
||||
_is_remote "$current_session" && loop_sub_args+=(--remote)
|
||||
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
||||
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
||||
"${loop_sub_args[@]}" \
|
||||
@@ -425,10 +448,10 @@ EOF
|
||||
local loop_pub_extra=(--attempt-id "$attempt")
|
||||
[ -n "$_disp_tok" ] && loop_pub_extra+=(--expect-dispatch-token "$_disp_tok")
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${loop_pub_extra[*]}"
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths for iteration $iteration are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths for iteration $iteration are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started --heartbeat $HEARTBEAT_SEC' (run it FIRST, within $ACK_TIMEOUT s; it also starts an automatic heartbeat timer), success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
|
||||
# Trigger agent
|
||||
if ! inject_agent "$JOB_ID" "$instructions" "$current_session"; then
|
||||
if ! deliver_prompt "$JOB_ID" "$instructions" "$current_session" "$logf"; then
|
||||
reap_subscriber "$sub_pid"
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "inject"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
@@ -564,23 +587,37 @@ _disk_status() {
|
||||
"$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$jid.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# ISSUE-14: send_keys_safe can only report what herdr lets it observe. When it
|
||||
# gives up, the prompt may still have landed - codex under herdr 0.9.0 never
|
||||
# reports `working`, yet publishes `started` within seconds. Closing such a job
|
||||
# as agent_blocked strands a live agent whose later `completed` the CAS table
|
||||
# then refuses. Before finalising, give the registry MAM_INJECT_ACK_GRACE_S
|
||||
# (default 10 s) to advance past pending. Any other status means the job moved
|
||||
# on - normally the agent's own `started` - so the prompt is treated as
|
||||
# delivered. The status is not pinned to this dispatch and may come from an
|
||||
# external writer; the terminal handling after this still keeps such an
|
||||
# error/cancelled as it is rather than turning it into success.
|
||||
_inject_acknowledged() {
|
||||
local job_id="$1" deadline st
|
||||
deadline=$(( $(date +%s) + ${MAM_INJECT_ACK_GRACE_S:-10} ))
|
||||
# ISSUE-18: delivery is confirmed by the protocol, not the screen. After the
|
||||
# prompt was handed over (herdr accepted the submission, or the dispatch was
|
||||
# published for a remote session) the job's own `started` must arrive within
|
||||
# the ack timeout T: the registry leaves `pending` (the agent's publish_event
|
||||
# syncs it on this host), a started line lands in events.log, or the
|
||||
# subscriber prints a started line (a remote agent - no shared disk). No
|
||||
# `started` within T is a delivery failure: fail closed, never re-inject
|
||||
# (single-injection contract). The status is not pinned to this dispatch and
|
||||
# may come from an external writer; the terminal handling after this still
|
||||
# keeps such an error/cancelled as it is rather than turning it into success.
|
||||
_wait_started() {
|
||||
local job_id="$1" ack_s="${2:-$ACK_TIMEOUT}" sub_log="${3:-}" deadline wall st
|
||||
deadline=$(( $(date +%s) + ack_s ))
|
||||
# The ack wait never outlives this dispatch's wall clock (START_TIME + TIMEOUT).
|
||||
wall=$(( ${START_TIME:-$(date +%s)} + ${TIMEOUT:-3600} ))
|
||||
[ "$deadline" -gt "$wall" ] && deadline=$wall
|
||||
local ev_log="$REGISTRY_DIR/$job_id.events.log"
|
||||
while :; do
|
||||
st="$(_disk_status "$job_id")"
|
||||
if [ -n "$st" ] && [ "$st" != "pending" ]; then
|
||||
echo "inject: herdr could not confirm delivery, but job $job_id already advanced to '$st' in the registry; treating the prompt as delivered and continuing." >&2
|
||||
echo "inject: job $job_id advanced to '$st' in the registry; treating the prompt as delivered and continuing." >&2
|
||||
return 0
|
||||
fi
|
||||
# publish_event records `started` here before its wire publish and before
|
||||
# the status sync, so a slow broker cannot delay the acknowledgement.
|
||||
if [ -f "$ev_log" ] && grep -Eq '"event": *"started"' "$ev_log" 2>/dev/null; then
|
||||
echo "inject: job $job_id recorded 'started' in events.log; treating the prompt as delivered and continuing." >&2
|
||||
return 0
|
||||
fi
|
||||
if [ -n "$sub_log" ] && [ -f "$sub_log" ] && grep -Eq '^[^ ]+ job=[^ ]+ seq=[^ ]+ started( |$)' "$sub_log" 2>/dev/null; then
|
||||
echo "inject: subscriber received started for job $job_id; treating the prompt as delivered and continuing." >&2
|
||||
return 0
|
||||
fi
|
||||
[ "$(date +%s)" -ge "$deadline" ] && return 1
|
||||
@@ -588,11 +625,15 @@ _inject_acknowledged() {
|
||||
done
|
||||
}
|
||||
|
||||
# ISSUE-14: herdr's agent_status is blind for some agents (codex stays `idle`
|
||||
# for the whole turn), so "still working" must also accept the pane's own busy
|
||||
# token - the same evidence send_keys_safe uses to confirm delivery. Without it
|
||||
# a codex review longer than the idle timeout was closed as deadline_timeout
|
||||
# mid-turn and its eventual `completed` refused.
|
||||
# Legacy name retained for callers/tests of the ISSUE-14 guard.
|
||||
_inject_acknowledged() { _wait_started "$1" "${MAM_INJECT_ACK_GRACE_S:-$ACK_TIMEOUT}" "${2:-}"; }
|
||||
|
||||
# ISSUE-14 / ISSUE-18: herdr's agent_status and the pane busy token describe
|
||||
# the screen, not the protocol. They are no longer liveness *evidence* (a
|
||||
# headless or remote agent shows neither, and a CLI update can change both
|
||||
# silently); finalize_unfinished logs them as a diagnostic so an operator can
|
||||
# tell "dead" from "silent" on the same host. Liveness itself is the contract:
|
||||
# an event on disk or on the wire within K x N seconds.
|
||||
_agent_still_working() {
|
||||
local sess="$1" target="$2"
|
||||
[ "$(_sks_agent_status "$sess" "$target")" = "working" ] && return 0
|
||||
@@ -724,23 +765,27 @@ finalize_unfinished() {
|
||||
local target_agent
|
||||
target_agent=$(_sanitize_herdr_agent_name "$sess")
|
||||
|
||||
# ISSUE-14: the subscriber's idle clock only counts events it received over
|
||||
# the wire. Keep re-observing the disk while liveness evidence keeps
|
||||
# turning up - herdr `working`, the pane busy token, or a heartbeat in
|
||||
# events.log at most IDLE_TIMEOUT old (see _recent_disk_heartbeat).
|
||||
# Window policy: entering this grace opens one IDLE_TIMEOUT window and
|
||||
# every sighting re-opens it from the time of that poll. An agent that
|
||||
# leaves no evidence at all is therefore closed about 2 x IDLE_TIMEOUT
|
||||
# after its last wire event (the subscriber's idle wait plus this window;
|
||||
# it was IDLE_TIMEOUT before disk heartbeats counted), and heartbeats may
|
||||
# be up to about 2 x IDLE_TIMEOUT apart, since one still counts until it
|
||||
# is IDLE_TIMEOUT old. Evidence only ever extends the idle window, never
|
||||
# this dispatch's wall clock: the loop ends at START_TIME + TIMEOUT read
|
||||
# off the clock on every pass, and the herdr probe is cut off at that
|
||||
# deadline too (_within_deadline).
|
||||
# ISSUE-14 / ISSUE-18: the subscriber's idle clock only counts events it
|
||||
# received over the wire. Keep re-observing the disk while the *contract*
|
||||
# says the agent is alive: an event in events.log at most IDLE_TIMEOUT
|
||||
# (K x N) old (_recent_disk_heartbeat - publish_event records every event
|
||||
# there before its wire publish, the automatic heartbeat timer included).
|
||||
# herdr status and the pane are NOT evidence any more; they are logged
|
||||
# once here as a diagnostic, so an operator can tell a dead agent from a
|
||||
# silent one on the same host. Window policy: entering this grace opens
|
||||
# one IDLE_TIMEOUT window and every disk event re-opens it from the time
|
||||
# of that poll, so an agent that leaves no event at all is closed about
|
||||
# 2 x IDLE_TIMEOUT after its last wire event. Evidence only ever extends
|
||||
# the idle window, never this dispatch's wall clock: the loop ends at
|
||||
# START_TIME + TIMEOUT read off the clock on every pass.
|
||||
local deadline evidence_until left disk_st
|
||||
deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
|
||||
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
|
||||
if [ "$AGENT" != "human" ] && [ "$sess" != "human" ] && [ -n "$sess" ] && ! _is_remote "$target_sess"; then
|
||||
local diag="no"
|
||||
_within_deadline "$deadline" _agent_still_working "$sess" "$target_agent" && diag="yes"
|
||||
echo "liveness: no wire event for ${IDLE_TIMEOUT}s on job $job_id; herdr/pane says busy=$diag (diagnostic only, not evidence)" >&2
|
||||
fi
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
@@ -749,8 +794,7 @@ finalize_unfinished() {
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
break
|
||||
fi
|
||||
if { [ "$sess" != "human" ] && [ -n "$sess" ] && _within_deadline "$deadline" _agent_still_working "$sess" "$target_agent"; } \
|
||||
|| _recent_disk_heartbeat "$job_id" "$IDLE_TIMEOUT"; then
|
||||
if _recent_disk_heartbeat "$job_id" "$IDLE_TIMEOUT"; then
|
||||
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
|
||||
elif [ "$(date +%s)" -ge "$evidence_until" ]; then
|
||||
break
|
||||
@@ -876,6 +920,7 @@ else:
|
||||
|
||||
inject_agent() {
|
||||
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}"
|
||||
# $4: subscriber log (a started line there also confirms delivery)
|
||||
if [ "$AGENT" = "human" ]; then
|
||||
echo "[human agent] complete the task, then run publish_event.py --event completed"
|
||||
return 0
|
||||
@@ -889,9 +934,24 @@ inject_agent() {
|
||||
fi
|
||||
|
||||
echo "살아있는 에이전트 세션 '$sess'에 작업을 위임합니다..."
|
||||
if ! send_keys_safe "$sess" "$instructions" "$job_id" && ! _inject_acknowledged "$job_id"; then
|
||||
# ISSUE-18: send_keys_safe is the injection *means*. rc 0 (accepted, herdr
|
||||
# saw it working) and rc 4 (accepted, no screen signal) both hand over to
|
||||
# the protocol: the job's own `started` within ACK_TIMEOUT decides. Any
|
||||
# other rc means the input never reached the agent (pane never quiesced,
|
||||
# dialog, paste failure) - fail at once, never re-inject.
|
||||
local sks_rc=0
|
||||
send_keys_safe "$sess" "$instructions" "$job_id" || sks_rc=$?
|
||||
case "$sks_rc" in
|
||||
0|4) ;;
|
||||
*)
|
||||
trap - EXIT
|
||||
echo "ERROR: 프롬프트 주입 실패 — 세션 '$sess' (프롬프트 잠금 의심)" >&2
|
||||
echo "ERROR: 프롬프트 주입 실패 (send_keys_safe rc=$sks_rc) — 세션 '$sess'" >&2
|
||||
return 1 ;;
|
||||
esac
|
||||
local ack_s="${MAM_INJECT_ACK_GRACE_S:-$ACK_TIMEOUT}"
|
||||
if ! _wait_started "$job_id" "$ack_s" "${4:-}"; then
|
||||
trap - EXIT
|
||||
echo "ERROR: 프롬프트 주입 실패 — 세션 '$sess': no 'started' within ${ack_s}s (ack timeout); not re-injecting" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -900,6 +960,38 @@ inject_agent() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# remote_dispatch <job_id> <instructions> <remote:session> <subscriber_log>
|
||||
# ISSUE-18: no pane, no shared disk. Publish the retained dispatch on the job's
|
||||
# topic and wait for the agent's own `started` (seen by the subscriber) within
|
||||
# ACK_TIMEOUT. The dispatch is signed with MAM_DISPATCH_SECRET (.mam.env).
|
||||
remote_dispatch() {
|
||||
local job_id="$1" instructions="$2" target_session="$3" sub_log="${4:-}"
|
||||
local sess="${target_session#remote:}"
|
||||
echo "원격 세션 '$sess'에 작업을 dispatch 합니다 (broker)..."
|
||||
local d_args=(--job "$job_id" --registry-dir "$REGISTRY_DIR" --session "$sess" --instructions "$instructions")
|
||||
[ "${MAM_ALLOW_UNSIGNED_DISPATCH:-0}" = "1" ] && d_args+=(--allow-unsigned)
|
||||
if ! "$PY" "$SCRIPT_DIR/scripts/remote.py" dispatch "${d_args[@]}" >/dev/null; then
|
||||
echo "ERROR: dispatch publish failed for job $job_id (remote:$sess)" >&2
|
||||
return 1
|
||||
fi
|
||||
local ack_s="${MAM_INJECT_ACK_GRACE_S:-$ACK_TIMEOUT}"
|
||||
if ! _wait_started "$job_id" "$ack_s" "$sub_log"; then
|
||||
echo "ERROR: remote session '$sess' published no 'started' within ${ack_s}s (ack timeout); not re-dispatching" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "작업이 원격 세션 '$sess'에 전달되었습니다."
|
||||
return 0
|
||||
}
|
||||
|
||||
# deliver_prompt <job_id> <instructions> <session> <subscriber_log>
|
||||
deliver_prompt() {
|
||||
if _is_remote "$3"; then
|
||||
remote_dispatch "$1" "$2" "$3" "${4:-}"
|
||||
else
|
||||
inject_agent "$1" "$2" "$3" "${4:-}"
|
||||
fi
|
||||
}
|
||||
|
||||
run_agent() {
|
||||
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}" force_warn_only="${4:-0}"
|
||||
local sess="${target_session#herdr:}"
|
||||
|
||||
@@ -193,3 +193,16 @@ $PY scripts/registry.py logs --list # every job, live status
|
||||
---
|
||||
|
||||
> **구현 위치 각주 (v4.3.0)**: 본 문서가 인용하는 `scripts/*.py` 경로는 유효하며 계약도 불변이다. 실제 구현은 issue #5 phase 1에서 `.agents/skills/lib_py/delegation/`으로 이동했고, `scripts/*.py`는 그 패키지를 자기 옆 트리에 바인딩하는 호환 shim이다.
|
||||
|
||||
## Liveness contract fields (Issue #18)
|
||||
|
||||
| Field | Meaning | Default |
|
||||
|---|---|---|
|
||||
| `heartbeat_sec` | N: the agent publishes `progress` at least every N s (the `--heartbeat N` timer does it) | 30 |
|
||||
| `heartbeat_misses` | K: missed heartbeats before the job is unresponsive | 4 |
|
||||
| `idle_timeout_sec` | K x N unless `--idle-timeout` was given | 120 |
|
||||
| `ack_timeout_sec` | T: `started` must arrive within T s of the injection | 60 |
|
||||
| `transport` | `herdr` (pane injection) or `remote` (signed retained dispatch over the broker) | `herdr` |
|
||||
|
||||
`<registry>/<job>.heartbeat.pid` / `.heartbeat.log` belong to the running heartbeat timer; a terminal publish removes the pidfile.
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compatibility shim -- the implementation lives in lib_py.delegation.remote.
|
||||
|
||||
This file's path is a wire contract (agent instructions, the bash wrapper,
|
||||
SKILL.md, BOOTSTRAP.md, deploy/install.sh, tests). It binds the
|
||||
``lib_py.delegation`` *package* to the tree next to this file, once per process,
|
||||
without touching ``sys.path``; ``import remote`` then yields the very same module
|
||||
object as ``lib_py.delegation.remote``.
|
||||
|
||||
Why the package and not this one module: a per-module bind leaves a non-leaf
|
||||
module (one that imports its siblings) importing dependencies from whichever tree
|
||||
happens to be on the path, mixing a frozen copy with a live one. Why bind-once and
|
||||
never rebind: replacing a bound package would split existing references and
|
||||
break monkeypatching. See docs/contracts/issue-5/ (D2) for the context matrix.
|
||||
"""
|
||||
import importlib, importlib.util, os, sys
|
||||
_SKILLS = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
_PKG = "lib_py.delegation"
|
||||
if _PKG not in sys.modules: # bind once per process; never rebind
|
||||
try:
|
||||
import lib_py # parent resolved normally; shims never rebind it
|
||||
except ModuleNotFoundError:
|
||||
sys.path.append(_SKILLS) # standalone only: no lib_py exists anywhere to shadow
|
||||
import lib_py
|
||||
_d = os.path.join(_SKILLS, "lib_py", "delegation")
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
_PKG, os.path.join(_d, "__init__.py"), submodule_search_locations=[_d])
|
||||
_pkg = importlib.util.module_from_spec(_spec)
|
||||
sys.modules[_PKG] = _pkg # register BEFORE exec so __init__'s own imports resolve here
|
||||
_spec.loader.exec_module(_pkg)
|
||||
lib_py.delegation = _pkg
|
||||
_impl = importlib.import_module(_PKG + ".remote") # resolved through the bound package's __path__
|
||||
if __name__ != "__main__":
|
||||
sys.modules[__name__] = _impl # `import remote` is lib_py.delegation.remote
|
||||
else:
|
||||
_main = getattr(_impl, "main", None) # a module without a CLI keeps exit status 0
|
||||
sys.exit(_main() if _main is not None else 0)
|
||||
@@ -36,6 +36,27 @@ Run an autonomous planning-execution-review loop using multiple agents (Planner,
|
||||
|
||||
---
|
||||
|
||||
|
||||
## Liveness contract and remote sessions (Issue #18)
|
||||
|
||||
Every delegate job the loop submits carries the liveness contract of
|
||||
`multi-agent-mux-delegate-job`: agents heartbeat automatically once they
|
||||
publish `started --heartbeat N`, a job with no event for K x N seconds is
|
||||
closed as `deadline_timeout`, and `started` must arrive within T seconds of
|
||||
the injection. Tune it for a whole loop with environment variables, which the
|
||||
runner passes through as `--heartbeat` / `--heartbeat-misses` / `--ack-timeout`:
|
||||
|
||||
```bash
|
||||
MAM_HEARTBEAT_SEC=30 MAM_HEARTBEAT_MISSES=4 MAM_ACK_TIMEOUT_S=60 \
|
||||
bash .agents/skills/multi-agent-mux-loop/scripts/run_loop.sh --creator creator-claude-01 --all-reviewer --task "..."
|
||||
```
|
||||
|
||||
A session row in `.mam/agent-sessions.yaml` with `transport: remote` is
|
||||
delegated as `remote:<name>` (the brief goes over the broker to a
|
||||
`remote.py run --session <name>` runner on another host; no herdr, pane or
|
||||
process probing happens for it). The row still needs `role` and
|
||||
`status: running` to be picked up by `--all-reviewer`.
|
||||
|
||||
## Roles & Responsibilities
|
||||
|
||||
협업 시스템은 각 에이전트의 책임 영역을 명확히 격리하여 상호 교차 검증을 강제합니다.
|
||||
|
||||
+50
-2
@@ -154,7 +154,7 @@ Every event payload must adhere to the following schema structure:
|
||||
* **Example Detail**: `"Job 918b0612 started"`
|
||||
|
||||
#### 2. `progress`
|
||||
* **Emit Trigger**: Optional. Emitted at major check-points, long loops, or sub-task boundaries.
|
||||
* **Emit Trigger**: At least every `heartbeat_sec` while working (Issue #18 heartbeat obligation; `publish_event.py --event started --heartbeat N` automates it with `detail: heartbeat`), plus any check-point the agent wants to report.
|
||||
* **Payload Constraints**: None.
|
||||
* **Example Detail**: `"Section 1: MQTT Broker Architecture completed"`
|
||||
|
||||
@@ -185,6 +185,53 @@ To prevent unauthorized users from hijacking or spoofing events on public broker
|
||||
|
||||
---
|
||||
|
||||
### 2.5 Liveness and Delivery Contract (Issue #18)
|
||||
|
||||
No new event types: the existing events are promoted to a contract with
|
||||
obligations, periods and timeouts, so that an agent's life and the delivery of
|
||||
its prompt are decided by what the agent **publishes**, never by scraping a
|
||||
terminal (`herdr agent_status`, pane text, screen changes). This is what lets
|
||||
an agent on another host, which shares neither the pane nor the
|
||||
orchestrator's disk, take part on equal terms.
|
||||
|
||||
| Rule | Contract | Where |
|
||||
|---|---|---|
|
||||
| **Heartbeat obligation** | While working, the agent publishes `progress` at least every **N** s (`heartbeat_sec`, default 30). `publish_event.py --event started --heartbeat N` starts a detached timer that publishes `progress` (`detail: heartbeat`) every N s with the same `dispatch_token`/`attempt_id`, so the agent does not have to remember. The timer stops itself at the terminal event (which also kills it through `<registry>/<job>.heartbeat.pid`), on re-dispatch, or after `timeout_sec`. | `lib_py/delegation/heartbeat.py`, `publish_event.py --heartbeat` |
|
||||
| **Unresponsive** | The orchestrator treats a job as unresponsive after **K x N** s without any event (`heartbeat_misses`, default 4 → 120 s; `--idle-timeout` overrides). The subscriber's idle clock counts every accepted message (`progress` and retained state included); the wrapper's post-idle grace re-reads only the job's events.log (`_recent_disk_heartbeat`, written before every wire publish). herdr status and the pane are logged once as a *diagnostic* (`liveness: ... herdr/pane says busy=yes/no (diagnostic only)`) and never decide. | `multi-agent-mux-delegate-job` (`finalize_unfinished`), `job_subscriber.py` |
|
||||
| **Delivery** | After the prompt is handed over (herdr accepted the submission, or the remote dispatch was published), the agent's `started` must arrive within **T** s (`ack_timeout_sec`, default 60): the registry leaves `pending`, a `started` line lands in events.log, or the subscriber prints one. No `started` within T → the inject fails closed: rc 1, registry `error` (`agent_blocked`), and the prompt is **never re-injected**. `send_keys_safe` confirms an accepted-but-unconfirmed submission only by `agent_status == working` or the job's `started`; pane changes and busy tokens are no longer signals. | `_wait_started`, `lib.sh: _sks_job_started` |
|
||||
| **Termination** | Only `completed` / `error` (and an operator `cancelled`) end a job. Terminal immutability (Issue #6 CAS) refuses anything that arrives after `deadline_timeout` closed the job. | `engine.py` |
|
||||
| **Wire durability** | Every event (QoS 1) is followed, on the same connection, by a **retained** `<topic_prefix>/state` message (`event: state`, `status`, `last_event`, `seq`, signed like an event). A subscriber that connects or reconnects receives it at once: a terminal status ends the wait, any state resets the idle window, and a `seq` beyond the last one seen is recorded as `seq_gap` in the audit log (also for gaps between events). paho reconnects on its own; `on_connect` re-subscribes both topics and logs `reconnected`. | `publish_event.py: build_state_payload`, `job_subscriber.py: _on_state` |
|
||||
| **Credentials** | Each host connects with its own `.mam.env` (Issue #12: the password is never in a record). Every event and state message carries the job's HMAC (`auth_token`). A remote dispatch is signed with the pre-shared `MAM_DISPATCH_SECRET`. | `mqtt_common.py`, `remote.py` |
|
||||
| **What screen scraping is still for** | (a) operations that need the screen: prompt injection into a pane, trust/permission dialogs, the pre-injection idle handshake; (b) diagnostics on the same host when events stop. Neither is a liveness or delivery decision. | `lib.sh` |
|
||||
|
||||
Registry record fields: `heartbeat_sec`, `heartbeat_misses`, `ack_timeout_sec`,
|
||||
`idle_timeout_sec` (derived K x N unless explicit), `transport`
|
||||
(`herdr` / `remote`). Wrapper options: `--heartbeat`, `--heartbeat-misses`,
|
||||
`--ack-timeout` (also `MAM_HEARTBEAT_SEC`, `MAM_HEARTBEAT_MISSES`,
|
||||
`MAM_ACK_TIMEOUT_S` for the loop runner).
|
||||
|
||||
#### Remote transport (`--agent-session remote:<name>`)
|
||||
|
||||
```
|
||||
orchestrator ── retained <prefix>/dispatch (signed) ──▶ remote runner (other host)
|
||||
orchestrator ◀── <prefix>/events + retained <prefix>/state ── remote runner / agent
|
||||
```
|
||||
|
||||
`remote.py dispatch` publishes the brief, the job's `auth_token`,
|
||||
`dispatch_token`, `attempt_id` and the contract as one retained, signed
|
||||
message. `remote.py run --session <name> [--exec CMD]` on the other host
|
||||
mirrors the job into its own registry (same job id and identity), retires the
|
||||
retained dispatch, publishes `started --heartbeat N`, runs the command (env:
|
||||
`JOB_ID`, `MAM_JOB_BRIEF`, `MAM_PUBLISH_CMD`, ...) and publishes the terminal
|
||||
event unless the agent already did. The orchestrator's subscriber runs with
|
||||
`--remote` (automatic for `transport: remote` jobs) and mirrors `started` and
|
||||
terminal events into the orchestrator's registry through the transition
|
||||
boundary, because nothing else on that host will. The wrapper takes no
|
||||
herdr/pane/lsof/ps path for a remote session. Because the dispatch carries the
|
||||
job's `auth_token`, use the remote transport only on a private broker.
|
||||
|
||||
---
|
||||
|
||||
## 3. Job Lifecycle & State Transitions
|
||||
|
||||
The lifecycle of a delegated job progresses through a highly coordinated state machine, involving file-based registry claiming, asynchronous message subscription, and multi-faceted event publishing.
|
||||
@@ -291,7 +338,8 @@ The subscriber acts as the central execution watchdog:
|
||||
```
|
||||
* **Dual Timeout Semantics**:
|
||||
1. **Wall-Clock Timeout**: Calculated relative to absolute startup time (`wall_deadline = start + wall_timeout`). It acts as a hard budget limit, guarding against an agent hanging indefinitely.
|
||||
2. **Activity Idle Timeout**: Measured as the difference between the current monotonic time and the last packet arrival time (`idle_left = idle_timeout - (now - last_event)`). If the agent fails to print logs or publish progress updates for the duration of the idle window, the subscriber aborts and exits with code 2.
|
||||
2. **Activity Idle Timeout**: Measured as the difference between the current monotonic time and the last packet arrival time (`idle_left = idle_timeout - (now - last_event)`). If the agent fails to publish any event (heartbeats and retained state included) for the duration of the idle window - K x N under the Issue #18 contract - the subscriber aborts and exits with code 2.
|
||||
* **Retained state and seq gaps (Issue #18)**: the subscriber also subscribes to `<topic_prefix>/state`; a retained terminal state resolves the job at (re)connect, any state resets the idle clock, and a seq jump on either topic is written to the audit log as `seq_gap`. With `--remote` (or `transport: remote`) accepted `started`/terminal events are mirrored into the local job record.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+5
-1
@@ -2,7 +2,7 @@ usage: job_subscriber.py [-h] (--job JOB | --wait-any) [--timeout TIMEOUT]
|
||||
[--idle-timeout IDLE_TIMEOUT] [--expect-retention]
|
||||
[--expect-dispatch-token EXPECT_DISPATCH_TOKEN]
|
||||
[--attempt-id ATTEMPT_ID]
|
||||
[--registry-dir REGISTRY_DIR] [-v]
|
||||
[--registry-dir REGISTRY_DIR] [--remote] [-v]
|
||||
|
||||
Subscribe to Job events on MQTT
|
||||
|
||||
@@ -21,4 +21,8 @@ optional arguments:
|
||||
--attempt-id ATTEMPT_ID
|
||||
minimum attempt id to accept events for
|
||||
--registry-dir REGISTRY_DIR
|
||||
--remote the agent shares no disk with this registry: mirror
|
||||
started/terminal events into the job record (Issue
|
||||
#18; automatic for jobs registered with --transport
|
||||
remote)
|
||||
-v, --verbose
|
||||
+7
-1
@@ -6,7 +6,8 @@ usage: publish_event.py [-h] (--job JOB | --pick-pending)
|
||||
[--finalize-report PATH]
|
||||
[--side-effect KIND=REFERENCE]
|
||||
[--attempt-id ATTEMPT_ID]
|
||||
[--expect-dispatch-token EXPECT_DISPATCH_TOKEN] [-v]
|
||||
[--expect-dispatch-token EXPECT_DISPATCH_TOKEN]
|
||||
[--heartbeat N] [--no-state] [-v]
|
||||
|
||||
Publish a Job event to MQTT
|
||||
|
||||
@@ -33,4 +34,9 @@ optional arguments:
|
||||
job's current iteration)
|
||||
--expect-dispatch-token EXPECT_DISPATCH_TOKEN
|
||||
expected dispatch token for CAS verification
|
||||
--heartbeat N after publishing, start a detached timer that
|
||||
publishes a `progress` heartbeat every N seconds until
|
||||
the job ends (Issue #18; not with a terminal event)
|
||||
--no-state do not publish the retained <topic_prefix>/state
|
||||
message
|
||||
-v, --verbose
|
||||
@@ -85,24 +85,63 @@ def _assert_single_injection(res: subprocess.CompletedProcess):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# lib.sh: accepted submission confirmed through the pane
|
||||
# lib.sh: accepted submission confirmed by the job's own `started` (Issue #18)
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_accepted_timeout_confirmed_by_busy_token(tmp_path):
|
||||
def _job_that_starts(tmp_path, delay_s: float):
|
||||
"""A registered job whose `started` lands `delay_s` after the call - the
|
||||
protocol signal that replaced the pane (Issue #18). Returns (env, job_id)."""
|
||||
import threading
|
||||
from lib_py.delegation import registry
|
||||
|
||||
reg = tmp_path / "jobs"
|
||||
env = {"MAM_ENV_FILE": str(tmp_path / ".mam.env.absent"), "DELEGATE_JOB_LOGS_DIR": str(tmp_path / "logs"),
|
||||
"MQTT_BROKER": "127.0.0.1", "MQTT_PORT": "9", "SKS_REGISTRY_DIR": str(reg)}
|
||||
saved = {k: os.environ.get(k) for k in env}
|
||||
os.environ.update(env)
|
||||
try:
|
||||
jid = registry.register_job("issue 14 ack", registry_dir=str(reg))
|
||||
finally:
|
||||
for k, v in saved.items():
|
||||
if v is None:
|
||||
os.environ.pop(k, None)
|
||||
else:
|
||||
os.environ[k] = v
|
||||
|
||||
def start():
|
||||
time.sleep(delay_s)
|
||||
subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg), "--job", jid,
|
||||
"--event", "started", "--attempts", "1"], capture_output=True, env=_base_env(tmp_path, env))
|
||||
|
||||
threading.Thread(target=start, daemon=True).start()
|
||||
return env, jid
|
||||
|
||||
|
||||
def test_accepted_timeout_confirmed_by_started_event_busy_token_alone_is_not_enough(tmp_path):
|
||||
"""The pane shows the busy token, but that is not delivery any more: the
|
||||
job's `started` (published 2 s in) is what confirms it, and the busy token
|
||||
on its own (second run) still fails closed."""
|
||||
env, jid = _job_that_starts(tmp_path, 2.0)
|
||||
script = _codex_like_mock(CODEX_IDLE, CODEX_WORKING) + \
|
||||
'send_keys_safe "reviewer-codex-01" "review the triage report" "job_14a"'
|
||||
res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "5"})
|
||||
f'send_keys_safe "reviewer-codex-01" "review the triage report" "{jid}"'
|
||||
res = _run_lib_sh(script, tmp_path, env={**env, "SKS_DELIVERY_GRACE_S": "8"})
|
||||
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
|
||||
_assert_single_injection(res)
|
||||
assert "delivery unknown" not in res.stderr
|
||||
|
||||
script = _codex_like_mock(CODEX_IDLE, CODEX_WORKING) + \
|
||||
'send_keys_safe "reviewer-codex-01" "review the triage report" "job_14a"'
|
||||
res = _run_lib_sh(script, tmp_path, env={**env, "SKS_DELIVERY_GRACE_S": "2"})
|
||||
assert res.returncode == 4, f"busy token must not confirm delivery: rc={res.returncode}\n{res.stderr}"
|
||||
_assert_single_injection(res)
|
||||
|
||||
def test_accepted_timeout_confirmed_by_pane_change_after_fast_turn(tmp_path):
|
||||
"""A turn that already finished before the first poll shows no busy token,
|
||||
only the answer; the pane moving on from its pre-submit snapshot is enough."""
|
||||
|
||||
def test_accepted_timeout_pane_change_after_fast_turn_is_not_delivery(tmp_path):
|
||||
"""A turn that already finished shows only the answer. The pane moving on
|
||||
used to count; under the protocol contract only `started` does."""
|
||||
script = _codex_like_mock(CODEX_IDLE, CODEX_ANSWERED) + \
|
||||
'send_keys_safe "reviewer-codex-01" "Reply with OK" "job_14b"'
|
||||
res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "5"})
|
||||
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
|
||||
res = _run_lib_sh(script, tmp_path, env={"SKS_DELIVERY_GRACE_S": "2"})
|
||||
assert res.returncode == 4, f"rc={res.returncode}\n{res.stderr}"
|
||||
_assert_single_injection(res)
|
||||
|
||||
|
||||
@@ -147,7 +186,8 @@ def test_transport_error_is_not_treated_as_accepted(tmp_path):
|
||||
def test_mock_herdr_timeout_with_idle_status_is_confirmed(tmp_path, mock_herdr):
|
||||
"""Same scenario through the shared mock herdr: `timeout` from agent prompt,
|
||||
agent_status pinned to idle, the pane gaining the busy token after
|
||||
submission. Exactly one prompt call, no paste-buffer fallback."""
|
||||
submission - and the job's `started` landing 2 s in, which is what
|
||||
confirms it (Issue #18). Exactly one prompt call, no paste-buffer fallback."""
|
||||
state_file = Path(mock_herdr)
|
||||
state = json.loads(state_file.read_text())
|
||||
state["agents"]["agt_codex"] = {
|
||||
@@ -158,9 +198,10 @@ def test_mock_herdr_timeout_with_idle_status_is_confirmed(tmp_path, mock_herdr):
|
||||
}
|
||||
state_file.write_text(json.dumps(state, indent=2))
|
||||
|
||||
env, jid = _job_that_starts(tmp_path, 2.0)
|
||||
res = _run_lib_sh(
|
||||
'send_keys_safe "agt_codex" "review please" "job_14f"',
|
||||
tmp_path, env={"SKS_IDLE_HANDSHAKE": "0", "SKS_DELIVERY_GRACE_S": "5"},
|
||||
f'send_keys_safe "agt_codex" "review please" "{jid}"',
|
||||
tmp_path, env={**env, "SKS_IDLE_HANDSHAKE": "0", "SKS_DELIVERY_GRACE_S": "8"},
|
||||
)
|
||||
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
|
||||
|
||||
@@ -243,13 +284,14 @@ def test_wrapper_honours_registry_ack_over_unconfirmed_inject(tmp_path, mock_her
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# wrapper: idle-timeout grace (R1) must see a working codex through the pane
|
||||
# wrapper: idle-timeout grace (R1) is decided by the contract, not the pane
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr):
|
||||
"""herdr keeps reporting `idle` for a codex-like agent that is visibly
|
||||
working (busy token in the pane). When the subscriber idles out, the
|
||||
wrapper must keep waiting for the on-disk result instead of closing the
|
||||
job as deadline_timeout mid-turn; the agent then completes out of band."""
|
||||
def test_wrapper_idle_timeout_grace_ignores_pane_busy_token(tmp_path, mock_herdr):
|
||||
"""Issue #18 flips the Issue #14 expectation: a codex-like agent that is
|
||||
visibly working (busy token in the pane) but publishes nothing after its
|
||||
`started` is unresponsive once K x N (here --idle-timeout 3 s) has passed.
|
||||
The wrapper closes it as deadline_timeout and a `completed` written 14 s
|
||||
in is refused by the CAS table - the pane never extended the window."""
|
||||
reg_dir = tmp_path / "jobs"
|
||||
reg_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -265,7 +307,7 @@ def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr
|
||||
}
|
||||
state_file.write_text(json.dumps(state, indent=2))
|
||||
|
||||
env = _base_env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_DELIVERY_GRACE_S": "5"})
|
||||
env = _base_env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_DELIVERY_GRACE_S": "2", "MAM_INJECT_ACK_GRACE_S": "20"})
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"bash", str(WRAPPER), "submit",
|
||||
@@ -289,19 +331,29 @@ def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr
|
||||
time.sleep(0.1)
|
||||
assert jid, "job was not registered"
|
||||
|
||||
# Well past the 3 s idle timeout, the agent finishes.
|
||||
# The agent acknowledges (started) without arming a heartbeat timer ...
|
||||
time.sleep(1)
|
||||
tok = json.loads((reg_dir / f"{jid}.json").read_text())["dispatch_token"]
|
||||
ack = subprocess.run(
|
||||
[sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg_dir), "--job", jid, "--event", "started",
|
||||
"--expect-dispatch-token", tok],
|
||||
capture_output=True, text=True, cwd=str(tmp_path), env=env,
|
||||
)
|
||||
assert ack.returncode == 0, ack.stderr
|
||||
# ... and goes silent while its pane keeps showing the busy token.
|
||||
time.sleep(14)
|
||||
done = subprocess.run(
|
||||
[sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg_dir),
|
||||
"update", "--job", jid, "--status", "completed"],
|
||||
capture_output=True, text=True, cwd=str(tmp_path), env=env,
|
||||
)
|
||||
assert done.returncode == 0, done.stderr
|
||||
|
||||
stdout, stderr = proc.communicate(timeout=90)
|
||||
job_data = json.loads((reg_dir / f"{jid}.json").read_text())
|
||||
assert job_data.get("status") == "completed", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
|
||||
assert proc.returncode == 0, f"expected rc=0 after the working grace, got {proc.returncode}\n{stdout}\n{stderr}"
|
||||
assert proc.returncode == 2, f"expected deadline_timeout (rc=2): the pane is not liveness, got {proc.returncode}\n{stdout}\n{stderr}"
|
||||
assert job_data.get("status") == "error", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
|
||||
assert done.returncode != 0 or "refused" in done.stderr, "the late completed must be refused"
|
||||
assert "diagnostic only" in stderr
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -451,8 +503,9 @@ def test_wrapper_idle_grace_never_outlives_the_wall_clock(tmp_path):
|
||||
# The heartbeats were honoured: the job was not closed after the first idle
|
||||
# window, it stayed open up to the wall clock ...
|
||||
assert elapsed >= timeout_s - 1, f"closed before the wall clock: {elapsed:.1f}s\n{stderr}"
|
||||
# ... and not a moment past it beyond closing and publishing the error.
|
||||
assert elapsed <= timeout_s + 12, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}"
|
||||
# ... and not a moment past it beyond closing and publishing the error
|
||||
# (the error publish alone can take ~10 s against the public broker).
|
||||
assert elapsed <= timeout_s + 15, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -489,7 +542,7 @@ def test_publish_event_records_the_heartbeat_before_the_wire_publish(tmp_path, m
|
||||
events_log = reg_dir / f"{jid}.events.log"
|
||||
on_disk_at_publish = []
|
||||
|
||||
def slow_broker_publish(config, topic, body, retain):
|
||||
def slow_broker_publish(config, topic, body, retain, extra=()):
|
||||
on_disk_at_publish.append([e.get("event") for e in _logged_events(events_log)])
|
||||
|
||||
monkeypatch.setattr(pe, "_publish_once", slow_broker_publish)
|
||||
@@ -509,7 +562,7 @@ def test_publish_event_failed_publish_keeps_one_local_record(tmp_path, monkeypat
|
||||
events_log = reg_dir / f"{jid}.events.log"
|
||||
lines_seen = []
|
||||
|
||||
def unreachable_broker(config, topic, body, retain):
|
||||
def unreachable_broker(config, topic, body, retain, extra=()):
|
||||
lines_seen.append(len(_logged_events(events_log)))
|
||||
raise ConnectionError("broker unreachable")
|
||||
|
||||
|
||||
@@ -0,0 +1,617 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Issue #18 — liveness and delivery are decided by the event protocol, not by
|
||||
screen scraping.
|
||||
|
||||
Contract (registry fields, wrapper options):
|
||||
* heartbeat obligation - the agent publishes `progress` at least every N s;
|
||||
`publish_event --event started --heartbeat N` starts an automatic timer
|
||||
that does it for the agent and stops itself at the terminal event.
|
||||
* unresponsive - no event (disk or wire) for K x N s closes the job
|
||||
as deadline_timeout; herdr status and pane text are diagnostics only.
|
||||
* delivery - `started` must arrive within T s of the injection,
|
||||
else the inject fails closed (rc 1, registry error, single injection).
|
||||
* wire durability - every event is followed by a retained
|
||||
`<prefix>/state` message on the same connection; a subscriber that
|
||||
(re)connects recovers the status at once and records seq gaps.
|
||||
* remote transport - `--agent-session remote:<name>` delivers the brief
|
||||
over a signed retained `<prefix>/dispatch`; a runner on another host with
|
||||
its own registry and broker credentials runs it, and the orchestrator's
|
||||
subscriber mirrors its events into the orchestrator's registry.
|
||||
|
||||
Tests marked "broker" talk to the configured broker (the public default in the
|
||||
sandbox); each stays under a minute.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SKILLS = REPO_ROOT / ".agents" / "skills"
|
||||
LIB_SH = SKILLS / "lib.sh"
|
||||
WRAPPER = SKILLS / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
|
||||
SCRIPTS = SKILLS / "multi-agent-mux-delegate-job" / "scripts"
|
||||
PUBLISH_PY = SCRIPTS / "publish_event.py"
|
||||
REGISTRY_PY = SCRIPTS / "registry.py"
|
||||
REMOTE_PY = SCRIPTS / "remote.py"
|
||||
|
||||
UNREACHABLE = {"MQTT_BROKER": "127.0.0.1", "MQTT_PORT": "9", "MQTT_TLS": "0"}
|
||||
|
||||
|
||||
def _env(cwd: Path, extra=None) -> dict:
|
||||
env = dict(os.environ)
|
||||
env["DELEGATE_JOB_PYTHON"] = sys.executable
|
||||
env["AGENT_PYTHON_BIN"] = sys.executable
|
||||
env["WORKSPACE_ROOT"] = str(cwd)
|
||||
env["HOME"] = str(cwd)
|
||||
env["MAM_ENV_FILE"] = str(cwd / ".mam.env.absent")
|
||||
env["DELEGATE_JOB_LOGS_DIR"] = str(cwd / "logs")
|
||||
env["PYTHONPATH"] = str(SKILLS)
|
||||
for k in ("MQTT_BROKER", "MQTT_HOST", "MQTT_PORT", "MQTT_TLS", "MQTT_USERNAME", "MQTT_PASSWORD"):
|
||||
env.pop(k, None)
|
||||
if extra:
|
||||
env.update({k: str(v) for k, v in extra.items()})
|
||||
return env
|
||||
|
||||
|
||||
def _events(reg_dir: Path, jid: str) -> list:
|
||||
p = reg_dir / f"{jid}.events.log"
|
||||
if not p.exists():
|
||||
return []
|
||||
return [json.loads(l) for l in p.read_text().splitlines() if l.strip()]
|
||||
|
||||
|
||||
def _job(reg_dir: Path, jid: str) -> dict:
|
||||
return json.loads((reg_dir / f"{jid}.json").read_text())
|
||||
|
||||
|
||||
def _wait_job_id(reg_dir: Path, timeout=20) -> str:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
files = list(reg_dir.glob("*.json"))
|
||||
if files:
|
||||
return files[0].stem
|
||||
time.sleep(0.1)
|
||||
raise AssertionError("job was not registered")
|
||||
|
||||
|
||||
def _publish(reg_dir: Path, jid: str, event: str, env: dict, *extra) -> subprocess.CompletedProcess:
|
||||
"""Publish as the agent would: with the dispatch identity the wrapper's
|
||||
injected command carries (`--expect-dispatch-token`)."""
|
||||
tok = _job(reg_dir, jid).get("dispatch_token")
|
||||
ident = ["--expect-dispatch-token", tok] if tok else []
|
||||
return subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg_dir), "--job", jid,
|
||||
"--event", event, "--attempts", "2", *ident, *extra],
|
||||
capture_output=True, text=True, env=env, cwd=str(reg_dir.parent), timeout=120)
|
||||
|
||||
|
||||
def _timer_pid(reg_dir: Path, jid: str):
|
||||
p = reg_dir / f"{jid}.heartbeat.pid"
|
||||
try:
|
||||
return int(p.read_text().strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _alive(pid) -> bool:
|
||||
if not pid:
|
||||
return False
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
return subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True).stdout.strip() not in ("", "Z", "Z+")
|
||||
|
||||
|
||||
def _kill_timer(reg_dir: Path, jid: str):
|
||||
pid = _timer_pid(reg_dir, jid)
|
||||
if pid:
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. registry: the contract is part of the record
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_register_derives_the_idle_window_from_the_contract(tmp_path, monkeypatch):
|
||||
from lib_py.delegation import registry
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
|
||||
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
|
||||
reg = tmp_path / "jobs"
|
||||
jid = registry.register_job("contract", registry_dir=str(reg))
|
||||
rec = _job(reg, jid)
|
||||
assert (rec["heartbeat_sec"], rec["heartbeat_misses"], rec["ack_timeout_sec"]) == (30, 4, 60)
|
||||
assert rec["idle_timeout_sec"] == 120 == registry.liveness_window(30, 4)
|
||||
assert rec["transport"] == "herdr"
|
||||
|
||||
jid2 = registry.register_job("contract", registry_dir=str(reg), heartbeat_sec=5, heartbeat_misses=3)
|
||||
assert _job(reg, jid2)["idle_timeout_sec"] == 15
|
||||
jid3 = registry.register_job("contract", registry_dir=str(reg), heartbeat_sec=5, heartbeat_misses=3,
|
||||
idle_timeout_sec=40)
|
||||
assert _job(reg, jid3)["idle_timeout_sec"] == 40, "an explicit idle timeout is kept"
|
||||
|
||||
|
||||
def test_wrapper_registers_the_contract_and_derives_idle_timeout(tmp_path):
|
||||
"""--dry-run registers the job without any herdr; K x N shows up as idle."""
|
||||
reg = tmp_path / "jobs"
|
||||
reg.mkdir()
|
||||
res = subprocess.run(
|
||||
["bash", str(WRAPPER), "submit", "--agent", "human", "--prompt", "contract", "--dry-run",
|
||||
"--registry-dir", str(reg), "--workdir", str(tmp_path),
|
||||
"--heartbeat", "7", "--heartbeat-misses", "3", "--ack-timeout", "11"],
|
||||
capture_output=True, text=True, env=_env(tmp_path, UNREACHABLE), cwd=str(tmp_path), timeout=60)
|
||||
assert res.returncode == 0, res.stderr
|
||||
jid = _wait_job_id(reg)
|
||||
rec = _job(reg, jid)
|
||||
assert (rec["heartbeat_sec"], rec["heartbeat_misses"], rec["ack_timeout_sec"], rec["idle_timeout_sec"]) == (7, 3, 11, 21)
|
||||
assert "--event started --heartbeat 7" in res.stdout, "the injected start command must arm the timer"
|
||||
assert "within 11 s" in res.stdout
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. the automatic heartbeat (no broker needed: events land on disk first)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_started_with_heartbeat_beats_on_disk_until_the_terminal_event(tmp_path):
|
||||
reg = tmp_path / "jobs"
|
||||
env = _env(tmp_path, UNREACHABLE)
|
||||
jid = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "register",
|
||||
"--prompt", "hb", "--heartbeat-sec", "1", "--heartbeat-misses", "3"],
|
||||
capture_output=True, text=True, env=env, cwd=str(tmp_path)).stdout.strip()
|
||||
assert jid
|
||||
try:
|
||||
res = _publish(reg, jid, "started", env, "--heartbeat", "1", "--attempts", "1")
|
||||
assert res.returncode == 2, "the broker is unreachable, the event is still recorded locally"
|
||||
pid = _timer_pid(reg, jid)
|
||||
assert pid and _alive(pid), "a detached timer must be running"
|
||||
time.sleep(4.5)
|
||||
beats = [e for e in _events(reg, jid) if e["event"] == "progress"]
|
||||
assert len(beats) >= 3, [e["event"] for e in _events(reg, jid)]
|
||||
assert all(e["detail"] == "heartbeat" for e in beats)
|
||||
assert all(e["data"].get("dispatch_token") == _job(reg, jid)["dispatch_token"] for e in beats), \
|
||||
"heartbeats carry the dispatch identity"
|
||||
seqs = [e["seq"] for e in _events(reg, jid)]
|
||||
assert seqs == sorted(seqs) and len(set(seqs)) == len(seqs)
|
||||
|
||||
res = _publish(reg, jid, "completed", env, "--attempts", "1")
|
||||
assert res.returncode == 2
|
||||
deadline = time.time() + 5
|
||||
while time.time() < deadline and _alive(pid):
|
||||
time.sleep(0.1)
|
||||
assert not _alive(pid), "the terminal event must stop the timer"
|
||||
assert _timer_pid(reg, jid) is None
|
||||
assert _events(reg, jid)[-1]["event"] == "completed", "no heartbeat may trail the terminal event"
|
||||
assert _job(reg, jid)["status"] == "completed"
|
||||
finally:
|
||||
_kill_timer(reg, jid)
|
||||
|
||||
|
||||
def test_heartbeat_timer_stops_on_its_own_when_the_job_is_redispatched(tmp_path):
|
||||
reg = tmp_path / "jobs"
|
||||
env = _env(tmp_path, UNREACHABLE)
|
||||
jid = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "register",
|
||||
"--prompt", "hb"], capture_output=True, text=True, env=env, cwd=str(tmp_path)).stdout.strip()
|
||||
try:
|
||||
_publish(reg, jid, "started", env, "--heartbeat", "1", "--attempts", "1")
|
||||
pid = _timer_pid(reg, jid)
|
||||
assert _alive(pid)
|
||||
# An operator moves the job to the next attempt: the old attempt's timer must not keep beating.
|
||||
_publish(reg, jid, "completed", env, "--attempts", "1")
|
||||
subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), "update", "--job", jid,
|
||||
"--status", "pending", "--attempt-id", "2"], capture_output=True, text=True, env=env, cwd=str(tmp_path))
|
||||
deadline = time.time() + 5
|
||||
while time.time() < deadline and _alive(pid):
|
||||
time.sleep(0.1)
|
||||
assert not _alive(pid)
|
||||
finally:
|
||||
_kill_timer(reg, jid)
|
||||
|
||||
|
||||
def test_heartbeat_is_refused_on_a_terminal_event(tmp_path, monkeypatch):
|
||||
from lib_py.delegation import publish_event as pe, registry
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
|
||||
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
|
||||
reg = tmp_path / "jobs"
|
||||
jid = registry.register_job("hb", registry_dir=str(reg))
|
||||
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "completed", "--heartbeat", "5"]) == 1
|
||||
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "progress", "--heartbeat", "0"]) == 1
|
||||
assert _events(reg, jid) == [], "a refused call consumes no seq and records nothing"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. wire durability: retained state next to every event, seq gaps detected
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_every_event_carries_a_signed_retained_state_message(tmp_path, monkeypatch):
|
||||
from lib_py.delegation import mqtt_common, publish_event as pe, registry
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
|
||||
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
|
||||
reg = tmp_path / "jobs"
|
||||
jid = registry.register_job("state", registry_dir=str(reg), auth_token="tok-" + "a" * 20)
|
||||
sent = []
|
||||
|
||||
def fake_publish(config, topic, body, retain, extra=()):
|
||||
sent.append((topic, json.loads(body), retain, [(t, json.loads(b), r) for t, b, r in extra]))
|
||||
|
||||
monkeypatch.setattr(pe, "_publish_once", fake_publish)
|
||||
tok = _job(reg, jid)["dispatch_token"]
|
||||
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "started", "--expect-dispatch-token", tok]) == 0
|
||||
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "completed", "--detail", "PASS",
|
||||
"--expect-dispatch-token", tok]) == 0
|
||||
prefix = _job(reg, jid)["topic_prefix"]
|
||||
assert [s[0] for s in sent] == [f"{prefix}/events"] * 2
|
||||
for (topic, ev, retain, extra), status in zip(sent, ("running", "completed")):
|
||||
assert len(extra) == 1
|
||||
st_topic, st, st_retain = extra[0]
|
||||
assert st_topic == f"{prefix}/state" and st_retain is True
|
||||
assert st["event"] == "state" and st["status"] == status and st["last_event"] == ev["event"]
|
||||
assert st["seq"] == ev["seq"] and st["job_id"] == jid
|
||||
assert mqtt_common.verify_hmac(st, "tok-" + "a" * 20), "state must be signed with the job token"
|
||||
assert st["data"]["dispatch_token"] == ev["data"]["dispatch_token"]
|
||||
assert pe.main(["--registry-dir", str(reg), "--job", jid, "--event", "progress", "--no-state",
|
||||
"--expect-dispatch-token", tok]) == 0
|
||||
assert len(sent) == 3 and sent[-1][3] == [], "--no-state publishes the event alone"
|
||||
|
||||
|
||||
class _Msg:
|
||||
def __init__(self, topic, payload):
|
||||
self.topic = topic
|
||||
self.payload = json.dumps(payload).encode()
|
||||
|
||||
|
||||
def _signed(job_id, seq, event, token, **fields):
|
||||
from lib_py.delegation import publish_event as pe
|
||||
p = pe.build_payload(job_id, seq, event, "", {"dispatch_token": "d1", "attempt_id": 1}, token)
|
||||
if fields:
|
||||
# re-sign after adding state fields, as build_state_payload does
|
||||
import hashlib, hmac
|
||||
p.update(fields)
|
||||
p["data"].pop("hmac_sig", None)
|
||||
sp = {k: v for k, v in p.items() if k != "data"}
|
||||
sp["data"] = dict(p["data"])
|
||||
p["data"]["hmac_sig"] = hmac.new(token.encode(), json.dumps(sp, sort_keys=True, separators=(",", ":")).encode(),
|
||||
hashlib.sha256).hexdigest()
|
||||
return p
|
||||
|
||||
|
||||
def test_subscriber_records_seq_gaps_and_recovers_terminal_from_state(tmp_path, monkeypatch):
|
||||
from lib_py.delegation import job_subscriber as js, mqtt_common
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
|
||||
token = "t" * 24
|
||||
w = js._Watcher({"j1"}, {"j1": token}, {"j1": 0}, {"j1": "d1"}, {"j1": 1})
|
||||
w.on_message(None, None, _Msg("x/events", _signed("j1", 1, "started", token)))
|
||||
w.on_message(None, None, _Msg("x/events", _signed("j1", 2, "progress", token)))
|
||||
w.on_message(None, None, _Msg("x/events", _signed("j1", 5, "progress", token))) # 3 and 4 were lost
|
||||
audit = [e for e in mqtt_common.iter_logged_events("j1", mqtt_common.get_logs_dir()) if e["event"] == "seq_gap"]
|
||||
assert len(audit) == 1 and (audit[0]["expected_seq"], audit[0]["seq"], audit[0]["via"]) == (3, 5, "events")
|
||||
assert w.events.qsize() == 3
|
||||
|
||||
# a retained state after a reconnect: newer seq than anything seen, terminal status
|
||||
st = _signed("j1", 8, "state", token, status="completed", last_event="completed")
|
||||
w.on_message(None, None, _Msg("x/state", st))
|
||||
assert w.events.qsize() == 4
|
||||
topic, payload = list(w.events.queue)[-1]
|
||||
assert topic == "x/state" and payload["event"] == "state" and payload["status"] == "completed"
|
||||
gaps = [e for e in mqtt_common.iter_logged_events("j1", mqtt_common.get_logs_dir()) if e["event"] == "seq_gap"]
|
||||
assert len(gaps) == 2 and gaps[-1]["via"] == "state"
|
||||
# a forged state (wrong token) is dropped like a forged event
|
||||
bad = _signed("j1", 9, "state", "wrong", status="error", last_event="error")
|
||||
w.on_message(None, None, _Msg("x/state", bad))
|
||||
assert w.events.qsize() == 4
|
||||
# a state for a stale attempt (other dispatch token) is dropped
|
||||
stale = _signed("j1", 10, "state", token, status="error", last_event="error")
|
||||
stale["data"]["dispatch_token"] = "d0"
|
||||
w.on_message(None, None, _Msg("x/state", stale))
|
||||
assert w.events.qsize() == 4
|
||||
|
||||
|
||||
def test_subscriber_subscribes_to_the_state_topic(tmp_path):
|
||||
src = (SKILLS / "lib_py" / "delegation" / "job_subscriber.py").read_text()
|
||||
assert 'state_topics.append(f"{prefix}/state")' in src
|
||||
assert "for topic in subscribed_topics + state_topics:" in src
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. delivery: `started` decides, screen signals do not
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _codex_like_mock(pane_before: str, pane_after: str, code: str = "timeout") -> str:
|
||||
return f'''
|
||||
_pane_quiescent() {{ return 0; }}
|
||||
_pane_dialog_open() {{ return 1; }}
|
||||
FLAG="$WORKSPACE_ROOT/.prompt-submitted"
|
||||
rm -f "$FLAG"
|
||||
_sks_herdr() {{
|
||||
case "${{1:-}}" in
|
||||
agent)
|
||||
case "${{2:-}}" in
|
||||
prompt) touch "$FLAG"; echo '{{"error":{{"code":"{code}","message":"x"}}}}' >&2; return 1 ;;
|
||||
get) echo '{{"result":{{"agent":{{"agent_status":"idle"}}}}}}'; return 0 ;;
|
||||
esac ;;
|
||||
capture-pane)
|
||||
if [ -e "$FLAG" ]; then printf '%s\\n' "{pane_after}"; else printf '%s\\n' "{pane_before}"; fi
|
||||
return 0 ;;
|
||||
paste-buffer|set-buffer|delete-buffer|send-keys)
|
||||
echo "PASTE_PATH_USED: $*" >&2; return 0 ;;
|
||||
esac
|
||||
return 0
|
||||
}}
|
||||
'''
|
||||
|
||||
|
||||
def _run_lib_sh(snippet: str, cwd: Path, env=None, timeout=60):
|
||||
return subprocess.run(["bash", "-c", f'source "{LIB_SH}" >/dev/null 2>&1; {snippet}'],
|
||||
capture_output=True, text=True, cwd=str(cwd), env=_env(cwd, env), timeout=timeout)
|
||||
|
||||
|
||||
def test_send_keys_safe_is_confirmed_by_the_job_started_event(tmp_path, monkeypatch):
|
||||
"""herdr says `timeout`, agent_status stays idle, the pane never changes:
|
||||
the only thing that confirms delivery is the job's own `started`."""
|
||||
from lib_py.delegation import registry
|
||||
|
||||
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
|
||||
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(tmp_path / "logs"))
|
||||
reg = tmp_path / "jobs"
|
||||
jid = registry.register_job("ack", registry_dir=str(reg))
|
||||
env = _env(tmp_path, {**UNREACHABLE, "SKS_DELIVERY_GRACE_S": "14", "SKS_REGISTRY_DIR": str(reg)})
|
||||
|
||||
def agent_starts_later():
|
||||
time.sleep(2)
|
||||
subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg), "--job", jid,
|
||||
"--event", "started", "--attempts", "1"], capture_output=True, env=env, cwd=str(tmp_path))
|
||||
|
||||
t = threading.Thread(target=agent_starts_later, daemon=True)
|
||||
t.start()
|
||||
started = time.time()
|
||||
res = subprocess.run(["bash", "-c", f'source "{LIB_SH}" >/dev/null 2>&1; '
|
||||
+ _codex_like_mock("idle", "idle") + f'send_keys_safe "reviewer-codex-01" "prompt" "{jid}"'],
|
||||
capture_output=True, text=True, cwd=str(tmp_path), env=env, timeout=60)
|
||||
t.join()
|
||||
assert res.returncode == 0, f"rc={res.returncode}\n{res.stderr}"
|
||||
assert time.time() - started < 11, "confirmed once started landed, not at the end of the 14 s grace"
|
||||
assert "PASTE_PATH_USED" not in res.stderr
|
||||
|
||||
|
||||
def test_send_keys_safe_ignores_pane_change_and_busy_token(tmp_path):
|
||||
"""The screen moving on, or showing a busy token, is not delivery any more:
|
||||
with no `started` the accepted submission still fails closed (rc 4)."""
|
||||
for after in ("Working (0s • esc to interrupt)\\n› Ask Codex", "• OK\\n› Ask Codex"):
|
||||
res = _run_lib_sh(_codex_like_mock("idle", after) + 'send_keys_safe "reviewer-codex-01" "prompt" "adhoc"',
|
||||
tmp_path, env={**UNREACHABLE, "SKS_DELIVERY_GRACE_S": "2"})
|
||||
assert res.returncode == 4, f"rc={res.returncode}\n{res.stderr}"
|
||||
assert "delivery unknown/failed" in res.stderr
|
||||
assert "PASTE_PATH_USED" not in res.stderr
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. wrapper acceptance criteria (broker)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _headless_agent(state_file: Path, name: str, agent: str = "codex") -> None:
|
||||
state = json.loads(state_file.read_text())
|
||||
# agent_status pinned idle, prompt `timeout`, empty pane: nothing to scrape
|
||||
state["agents"][name] = {"agent": agent, "agent_status": "idle", "prompt_wait_result": "timeout", "buffer": ""}
|
||||
state_file.write_text(json.dumps(state, indent=2))
|
||||
|
||||
|
||||
def _submit(tmp_path: Path, reg: Path, session: str, agent: str, extra_args, env_extra=None, timeout_s=60):
|
||||
env = _env(tmp_path, {"SKS_IDLE_TIMEOUT_MS": "500", "SKS_EMPTY_GIVEUP": "1", **(env_extra or {})})
|
||||
proc = subprocess.Popen(
|
||||
["bash", str(WRAPPER), "submit", "--agent", agent, "--agent-session", session, "--prompt", "issue 18",
|
||||
"--registry-dir", str(reg), "--workdir", str(tmp_path), "--timeout", str(timeout_s), *extra_args],
|
||||
cwd=str(tmp_path), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
return proc, env
|
||||
|
||||
|
||||
def _prompt_calls(state_file: Path) -> int:
|
||||
"""`agent prompt` invocations; the isolated wrapper prefixes `--session <name>`."""
|
||||
n = 0
|
||||
for c in json.loads(state_file.read_text()).get("calls", []):
|
||||
c = list(c)
|
||||
if len(c) >= 2 and c[0] == "--session":
|
||||
c = c[2:]
|
||||
if len(c) >= 2 and c[0] == "agent" and c[1] == "prompt":
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def test_acceptance1_headless_agent_survives_on_heartbeats_alone(tmp_path, mock_herdr):
|
||||
"""herdr idle throughout, empty pane: `started --heartbeat 2` keeps the job
|
||||
alive past several K x N windows (2 x 2 = 4 s) and `completed` yields rc 0."""
|
||||
reg = tmp_path / "jobs"
|
||||
reg.mkdir()
|
||||
_headless_agent(Path(mock_herdr), "headless-codex")
|
||||
proc, env = _submit(tmp_path, reg, "herdr:headless-codex", "codex",
|
||||
["--heartbeat", "2", "--heartbeat-misses", "2", "--ack-timeout", "40"], timeout_s=90)
|
||||
jid = _wait_job_id(reg)
|
||||
try:
|
||||
time.sleep(1)
|
||||
assert _publish(reg, jid, "started", env, "--heartbeat", "2").returncode == 0
|
||||
time.sleep(16) # four idle windows without any agent-published event
|
||||
assert _job(reg, jid)["status"] == "running", "heartbeats alone must keep the job open"
|
||||
assert _publish(reg, jid, "completed", env, "--detail", "PASS").returncode == 0
|
||||
stdout, stderr = proc.communicate(timeout=120)
|
||||
finally:
|
||||
_kill_timer(reg, jid)
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
assert proc.returncode == 0, f"rc={proc.returncode}\n{stdout}\n{stderr}"
|
||||
assert _job(reg, jid)["status"] == "completed"
|
||||
assert _prompt_calls(Path(mock_herdr)) == 1
|
||||
beats = [e for e in _events(reg, jid) if e["event"] == "progress"]
|
||||
assert len(beats) >= 5, len(beats)
|
||||
brief = (reg / jid / "brief.md").read_text()
|
||||
assert "heartbeat every 2 s" in brief and "after 2 missed heartbeats" in brief and "within 40 s" in brief
|
||||
|
||||
|
||||
def test_acceptance2_no_started_within_T_fails_closed_without_reinjection(tmp_path, mock_herdr):
|
||||
reg = tmp_path / "jobs"
|
||||
reg.mkdir()
|
||||
_headless_agent(Path(mock_herdr), "mute-codex")
|
||||
started = time.time()
|
||||
proc, env = _submit(tmp_path, reg, "herdr:mute-codex", "codex",
|
||||
["--ack-timeout", "4", "--heartbeat", "2", "--heartbeat-misses", "2"],
|
||||
env_extra={"SKS_DELIVERY_GRACE_S": "1"}, timeout_s=60)
|
||||
stdout, stderr = proc.communicate(timeout=120)
|
||||
jid = _wait_job_id(reg)
|
||||
assert proc.returncode == 1, f"rc={proc.returncode}\n{stdout}\n{stderr}"
|
||||
assert "no 'started' within 4s" in stderr
|
||||
assert _job(reg, jid)["status"] == "error"
|
||||
assert _prompt_calls(Path(mock_herdr)) == 1, "single injection: never re-injected"
|
||||
assert time.time() - started < 45, "the ack wait is bounded by T, not the wall clock"
|
||||
ev = [e for e in _events(reg, jid) if e["event"] == "error"]
|
||||
assert ev and ev[-1]["detail"] == "agent_blocked"
|
||||
|
||||
|
||||
def test_acceptance3_lost_heartbeats_close_the_job_and_a_late_completed_is_refused(tmp_path, mock_herdr):
|
||||
reg = tmp_path / "jobs"
|
||||
reg.mkdir()
|
||||
_headless_agent(Path(mock_herdr), "dying-codex")
|
||||
proc, env = _submit(tmp_path, reg, "herdr:dying-codex", "codex",
|
||||
["--heartbeat", "1", "--heartbeat-misses", "3", "--ack-timeout", "40"], timeout_s=60)
|
||||
jid = _wait_job_id(reg)
|
||||
t0 = None
|
||||
try:
|
||||
time.sleep(1)
|
||||
assert _publish(reg, jid, "started", env, "--heartbeat", "1").returncode == 0
|
||||
time.sleep(4)
|
||||
_kill_timer(reg, jid) # the agent dies: no more heartbeats
|
||||
t0 = time.time()
|
||||
stdout, stderr = proc.communicate(timeout=120)
|
||||
finally:
|
||||
_kill_timer(reg, jid)
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
assert proc.returncode == 2, f"rc={proc.returncode}\n{stdout}\n{stderr}"
|
||||
assert _job(reg, jid)["status"] == "error"
|
||||
assert "herdr/pane says busy=" in stderr, "screen state is logged as a diagnostic only"
|
||||
# K x N = 3 s idle + one grace window: closed long before the 60 s wall clock
|
||||
assert time.time() - t0 < 30, f"took {time.time() - t0:.0f}s"
|
||||
late = _publish(reg, jid, "completed", env, "--detail", "too late")
|
||||
assert late.returncode in (0, 2, 4)
|
||||
assert _job(reg, jid)["status"] == "error", "terminal immutability (CAS) refuses the late completed"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. remote transport (broker): no shared disk, no pane, only the wire
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_acceptance5_remote_runner_completes_a_job_over_the_broker_only(tmp_path):
|
||||
orch = tmp_path / "orchestrator"
|
||||
remote = tmp_path / "remote-host"
|
||||
for d in (orch, remote):
|
||||
(d / "jobs").mkdir(parents=True)
|
||||
secret = "issue18-shared-secret"
|
||||
runner_env = _env(remote, {"MAM_DISPATCH_SECRET": secret, "MAM_JOBS_DIR": str(remote / "jobs")})
|
||||
runner_log = remote / "runner.log"
|
||||
with open(runner_log, "wb") as log:
|
||||
runner = subprocess.Popen(
|
||||
[sys.executable, str(REMOTE_PY), "run", "--session", "sim-worker", "--registry-dir", str(remote / "jobs"),
|
||||
"--exec", 'echo "remote work done for $JOB_ID"', "--once", "--timeout", "150"],
|
||||
cwd=str(remote), env=runner_env, stdout=log, stderr=subprocess.STDOUT)
|
||||
try:
|
||||
deadline = time.time() + 45
|
||||
while time.time() < deadline and "SUBSCRIBED" not in runner_log.read_text(errors="replace"):
|
||||
time.sleep(0.2)
|
||||
if "SUBSCRIBED" not in runner_log.read_text(errors="replace"):
|
||||
pytest.skip("broker unreachable: " + runner_log.read_text(errors="replace")[-300:])
|
||||
|
||||
reg = orch / "jobs"
|
||||
proc, env = _submit(orch, reg, "remote:sim-worker", "remote",
|
||||
["--heartbeat", "2", "--heartbeat-misses", "3", "--ack-timeout", "60"],
|
||||
env_extra={"MAM_DISPATCH_SECRET": secret}, timeout_s=120)
|
||||
stdout, stderr = proc.communicate(timeout=180)
|
||||
jid = _wait_job_id(reg)
|
||||
assert proc.returncode == 0, f"rc={proc.returncode}\n{stdout}\n{stderr}\nrunner:\n{runner_log.read_text()}"
|
||||
assert "subscriber received started" in stderr or "advanced to 'running'" in stderr
|
||||
assert _job(reg, jid)["status"] == "completed", \
|
||||
f"the subscriber mirrors the remote terminal into the registry\n{stdout}\n{stderr}\nrunner:\n{runner_log.read_text()}"
|
||||
assert _job(reg, jid)["transport"] == "remote"
|
||||
# The runner's own status sync follows its wire publish, so the
|
||||
# orchestrator can observe `completed` a moment before the remote
|
||||
# record says so: wait for the runner before reading its registry.
|
||||
runner.wait(timeout=60)
|
||||
# the remote side kept its own record with the orchestrator's identity
|
||||
mirror = _job(remote / "jobs", jid)
|
||||
assert mirror["status"] == "completed" and mirror["transport"] == "remote"
|
||||
assert mirror["dispatch_token"] == _job(reg, jid)["dispatch_token"]
|
||||
assert mirror["auth_token"] == _job(reg, jid)["auth_token"]
|
||||
assert (remote / "jobs" / jid / "brief.md").read_text().startswith("issue 18")
|
||||
# the agent command's last stdout line became the completed detail
|
||||
assert "remote work done for " + jid in stdout, stdout
|
||||
assert runner.returncode == 0
|
||||
assert "살아있는 에이전트 세션" not in stdout, "no pane injection path for a remote session"
|
||||
finally:
|
||||
if runner.poll() is None:
|
||||
runner.kill()
|
||||
|
||||
|
||||
def test_remote_runner_refuses_unsigned_dispatches(tmp_path, monkeypatch):
|
||||
from lib_py.delegation import remote
|
||||
|
||||
d = {"kind": "dispatch", "job_id": "j", "session": "s", "schema_version": 1}
|
||||
assert not remote.verify_dispatch(d, "secret")
|
||||
signed = remote.sign_dispatch(d, "secret")
|
||||
assert remote.verify_dispatch(signed, "secret")
|
||||
assert not remote.verify_dispatch(signed, "other")
|
||||
assert not remote.verify_dispatch(dict(signed, session="hijacked"), "secret")
|
||||
assert not remote.verify_dispatch(signed, None)
|
||||
monkeypatch.delenv("MAM_DISPATCH_SECRET", raising=False)
|
||||
assert remote.run_runner("s", str(tmp_path), allow_unsigned=False) == 1, "a runner without a secret refuses to run"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. loop runner: contract knobs pass through; remote rows get remote:<name>
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_loop_runner_passes_the_contract_and_remote_label(tmp_path, monkeypatch):
|
||||
from lib_py.loop import runner as loop_runner
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("WORKSPACE_ROOT", str(tmp_path))
|
||||
monkeypatch.setenv("MAM_STATE_JSON", json.dumps({"herdr_sessions": [
|
||||
{"name": "creator-claude-01", "agent": "claude", "role": "creator", "status": "running"},
|
||||
{"name": "reviewer-far-01", "agent": "claude", "role": "reviewer", "status": "running", "transport": "remote"},
|
||||
]}))
|
||||
monkeypatch.setenv("MAM_HEARTBEAT_SEC", "20")
|
||||
monkeypatch.setenv("MAM_HEARTBEAT_MISSES", "3")
|
||||
monkeypatch.setenv("MAM_ACK_TIMEOUT_S", "45")
|
||||
r = loop_runner.LoopRunner(["--creator", "creator-claude-01", "--task", "t"])
|
||||
r.parse_args()
|
||||
assert r.session_label("creator-claude-01") == "herdr:creator-claude-01"
|
||||
assert r.session_label("reviewer-far-01") == "remote:reviewer-far-01"
|
||||
assert r.contract_args() == ["--heartbeat", "20", "--heartbeat-misses", "3", "--ack-timeout", "45"]
|
||||
seen = []
|
||||
|
||||
def fake_run(cmd, **_kw):
|
||||
seen.append(cmd)
|
||||
class P: # noqa: D401 - tiny stand-in
|
||||
returncode = 0
|
||||
stdout = "registered job: deadbeef\n"
|
||||
stderr = ""
|
||||
return P()
|
||||
|
||||
monkeypatch.setattr(loop_runner.subprocess, "run", fake_run)
|
||||
assert r.delegate_job("reviewer-far-01", "claude", "Reviewer", "review") == "deadbeef"
|
||||
cmd = seen[0]
|
||||
assert cmd[cmd.index("--agent-session") + 1] == "remote:reviewer-far-01"
|
||||
assert cmd[-6:] == ["--heartbeat", "20", "--heartbeat-misses", "3", "--ack-timeout", "45"]
|
||||
@@ -338,7 +338,14 @@ def test_registry_cli_contract_snapshot(mam_sandbox):
|
||||
assert reg["--agent-session"][2] == "herdr:claude"
|
||||
assert reg["--role"][2] == "Worker"
|
||||
assert reg["--timeout"][2] == 3600
|
||||
assert reg["--idle-timeout"][2] == 120
|
||||
# Issue #18: the idle window derives from the liveness contract (K x N =
|
||||
# 4 x 30 = 120 s, the historical default) unless given explicitly.
|
||||
assert reg["--idle-timeout"][2] is None
|
||||
assert reg["--heartbeat-sec"][2] == 30
|
||||
assert reg["--heartbeat-misses"][2] == 4
|
||||
assert reg["--ack-timeout"][2] == 60
|
||||
assert reg["--transport"][1] == ("herdr", "remote")
|
||||
assert registry.liveness_window(reg["--heartbeat-sec"][2], reg["--heartbeat-misses"][2]) == 120
|
||||
assert reg["--bits"][2] == 32
|
||||
assert reg["--job-type"][1] == ("direct", "loop", "discuss")
|
||||
assert reg["--max-iterations"][2] == 5
|
||||
|
||||
@@ -603,8 +603,31 @@ def test_t1_14_working_agent_reobserve_completed(tmp_path, mock_herdr):
|
||||
assert len(agent_gets) >= 1, "R1 loop must query agent get"
|
||||
|
||||
|
||||
def _publish_started_for_registered_job(reg_dir, cwd, env=None):
|
||||
"""Issue #18: a working agent that never publishes `started` is a delivery
|
||||
failure (rc 1) under the liveness contract. The R1 re-observation cases
|
||||
below are about an agent that *did* start and then went silent, so the
|
||||
watcher threads acknowledge the job the way the injected command does."""
|
||||
deadline = time.time() + 20
|
||||
while time.time() < deadline:
|
||||
files = list(Path(reg_dir).glob("*.json"))
|
||||
if files:
|
||||
jid = files[0].stem
|
||||
tok = json.loads(files[0].read_text()).get("dispatch_token") or ""
|
||||
args = [sys.executable, str(SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"),
|
||||
"--registry-dir", str(reg_dir), "--job", jid, "--event", "started", "--attempts", "1"]
|
||||
if tok:
|
||||
args += ["--expect-dispatch-token", tok]
|
||||
run_env = dict(os.environ)
|
||||
run_env.update(env or {})
|
||||
subprocess.run(args, capture_output=True, text=True, cwd=str(cwd), env=run_env, timeout=60)
|
||||
return jid
|
||||
time.sleep(0.1)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# T1-15: R1 working 고정 + 미발행 -> 벽시계 --timeout 만료 후 rc=2 error
|
||||
# T1-15: R1 working 고정 + 미발행 -> Issue #18: K×N 후 rc=2 error (working 은 진단일 뿐)
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
reg_dir = tmp_path / "jobs"
|
||||
@@ -628,6 +651,7 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
|
||||
st["agents"]["agt_w2"]["agent_status"] = "working"
|
||||
state_file.write_text(json.dumps(st, indent=2))
|
||||
_publish_started_for_registered_job(reg_dir, tmp_path)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
@@ -637,6 +661,10 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
# Issue #18: `working` in herdr is a diagnostic, not liveness. An agent
|
||||
# that acknowledged (started) and then publishes nothing is closed after
|
||||
# K x N (--idle-timeout 2) plus one grace window - well before the wall
|
||||
# clock - even though herdr keeps saying `working` the whole time.
|
||||
start_t = time.time()
|
||||
res = _run_wrapper(
|
||||
[
|
||||
@@ -646,16 +674,16 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
"--prompt", "test T1-15",
|
||||
"--registry-dir", str(reg_dir),
|
||||
"--workdir", str(tmp_path),
|
||||
"--timeout", "5",
|
||||
"--timeout", "40",
|
||||
"--idle-timeout", "2",
|
||||
],
|
||||
cwd=tmp_path,
|
||||
timeout=60,
|
||||
timeout=90,
|
||||
)
|
||||
stop_ev.set()
|
||||
elapsed = time.time() - start_t
|
||||
assert res.returncode == 2, f"Expected rc=2 on wallclock timeout, got {res.returncode}"
|
||||
assert elapsed >= 4.0, f"Must wait up to wallclock timeout, elapsed={elapsed}"
|
||||
assert res.returncode == 2, f"Expected rc=2 (deadline_timeout), got {res.returncode}\n{res.stderr}"
|
||||
assert elapsed < 39.0, f"a silent `working` agent must not be kept open to the wall clock, elapsed={elapsed}"
|
||||
|
||||
job_files = list(reg_dir.glob("*.json"))
|
||||
assert len(job_files) == 1
|
||||
@@ -663,7 +691,8 @@ def test_t1_15_working_agent_reobserve_wallclock_timeout(tmp_path, mock_herdr):
|
||||
assert job_data.get("status") == "error"
|
||||
final_st = json.loads(state_file.read_text())
|
||||
agent_gets = [c for c in final_st.get("calls", []) if "agent" in c and "get" in c]
|
||||
assert len(agent_gets) >= 2, "R1 loop must repeatedly query agent get during wallclock wait"
|
||||
assert len(agent_gets) >= 2, "the idle handshake and the one diagnostic probe query agent get"
|
||||
assert "diagnostic only" in res.stderr, "Issue #18: herdr status is logged, not used as liveness"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -691,6 +720,7 @@ def test_t1_16_working_agent_done_without_event(tmp_path, mock_herdr):
|
||||
if any("agent" in c and "prompt" in c for c in st.get("calls", [])):
|
||||
st["agents"]["agt_w3"]["agent_status"] = "working"
|
||||
state_file.write_text(json.dumps(st, indent=2))
|
||||
_publish_started_for_registered_job(reg_dir, tmp_path)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
Reference in new issue
Block a user