fix(issue-6): fail-closed registry timeout and TUI turn handshake
- Introduce finalize_unfinished in multi-agent-mux-delegate-job for fail-closed atomic transitions - Enforce CAS dispatch_token and attempt_id pinning across engine, CLI, and lifecycle scripts - Integrate Herdr wait_for_agent_idle turn handshake to prevent prompt loss and race conditions - Resolve bash 3.2 set -u empty array expansion compatibility in lib.sh - Preserve Issue #5 R2 contract in CAS transition engine (engine.py, publish_event.py, service.py) - Add comprehensive regression suites (test_issue6_fail_closed.py, test_issue6_turn_handshake.py) - Bump lockstep framework and skills to v4.4.1 across lib.sh, eager.py, VERSIONS.md, 8 SKILL.md - Synchronize documentation (IMPROVEMENTS.md, MESSAGING.md, state-transitions.md, task.md, phase5-e2e.md) - Unanimous PASS achieved across 4 reviewer team leaders (Claude, Codex, OpenCode, Grok) Closes #6
This commit is contained in:
1 parent
7b761be7b8
commit
ec61511699
51 files changed
+3477
-293
No files matched your search
+187
-12
@@ -29,7 +29,7 @@ export WORKSPACE_ROOT
|
||||
# Framework semantic version. Single runtime source of truth; kept in lockstep
|
||||
# with VERSIONS.md and the 8 SKILL.md frontmatters by tests/test_version_consistency.py.
|
||||
# NOTE: unlike other MAM_* variables this one is intentionally NOT env-overridable.
|
||||
MAM_VERSION="4.4.0"
|
||||
MAM_VERSION="4.4.1"
|
||||
export MAM_VERSION
|
||||
AGENT_SESSIONS_YAML="${AGENT_SESSIONS_YAML:-$WORKSPACE_ROOT/.mam/agent-sessions.yaml}"
|
||||
|
||||
@@ -237,7 +237,11 @@ _real_herdr() {
|
||||
session=""
|
||||
fi
|
||||
if [ -n "$session" ]; then
|
||||
"$REAL_HERDR" --session "$session" "$@"
|
||||
if [ "${1:-}" = "--session" ] || [ "${1:-}" = "-s" ]; then
|
||||
"$REAL_HERDR" "$@"
|
||||
else
|
||||
"$REAL_HERDR" --session "$session" "$@"
|
||||
fi
|
||||
else
|
||||
"$REAL_HERDR" "$@"
|
||||
fi
|
||||
@@ -1896,12 +1900,17 @@ _delegate_submit_job_legacy() {
|
||||
# a missing script, or a broker failure never aborts the caller.
|
||||
_delegate_publish_event_legacy() {
|
||||
local job_id="$1" event="$2" detail="${3:-}"
|
||||
local tok="${4:-}" att="${5:-}"
|
||||
[ -n "$job_id" ] || return 0
|
||||
local py_bin pub
|
||||
py_bin="$(_delegate_py_bin)"
|
||||
pub="$(_delegate_script publish_event.py)"
|
||||
[ -n "$pub" ] && [ -f "$pub" ] || return 0
|
||||
"$py_bin" "$pub" --job "$job_id" --event "$event" --detail "$detail" || true
|
||||
local -a extra=()
|
||||
[ -n "${MAM_JOBS_DIR:-}" ] && extra+=(--registry-dir "$MAM_JOBS_DIR")
|
||||
[ -n "$tok" ] && extra+=(--expect-dispatch-token "$tok")
|
||||
[ -n "$att" ] && extra+=(--attempt-id "$att")
|
||||
"$py_bin" "$pub" --job "$job_id" --event "$event" --detail "$detail" ${extra[@]+"${extra[@]}"} || true
|
||||
}
|
||||
|
||||
# start_watchdog <job_id> [workdir]
|
||||
@@ -1985,12 +1994,17 @@ delegate_submit_job() {
|
||||
# failure must never abort the caller.
|
||||
delegate_publish_event() {
|
||||
local job_id="$1" event="$2" detail="${3:-}"
|
||||
local tok="${4:-}" att="${5:-}"
|
||||
[ -n "$job_id" ] || return 0
|
||||
if [ "${MAM_DELEGATION_PY:-1}" = "0" ]; then _delegate_publish_event_legacy "$@"; return $?; fi
|
||||
local pub
|
||||
pub="$(_delegate_script publish_event.py)"
|
||||
[ -n "$pub" ] && [ -f "$pub" ] || return 0
|
||||
_delegation_py publish --job "$job_id" --event "$event" --detail "$detail" || true
|
||||
local -a extra=()
|
||||
[ -n "${MAM_JOBS_DIR:-}" ] && extra+=(--registry-dir "$MAM_JOBS_DIR")
|
||||
[ -n "$tok" ] && extra+=(--expect-dispatch-token "$tok")
|
||||
[ -n "$att" ] && extra+=(--attempt-id "$att")
|
||||
_delegation_py publish --job "$job_id" --event "$event" --detail "$detail" ${extra[@]+"${extra[@]}"} || true
|
||||
}
|
||||
|
||||
# ensure_subscriber_daemon <job_id> [workdir]
|
||||
@@ -2219,6 +2233,91 @@ _pane_dialog_open() {
|
||||
_pane_tail "$1" "${MAM_DIALOG_TAIL_LINES:-8}" | grep -Eq "$pat"
|
||||
}
|
||||
|
||||
# wait_for_agent_idle <sess> [timeout_ms]
|
||||
# Phase 2 사전 TUI 유휴 핸드셰이크:
|
||||
# 0: idle / done (성공)
|
||||
# 1: agent still working (최종 타임아웃)
|
||||
# 2: blocked (대화창/모달 차단)
|
||||
# 3: unknown (휴리스틱 폴백 신호)
|
||||
wait_for_agent_idle() {
|
||||
local sess="${1#herdr:}"
|
||||
local timeout_ms="${2:-${SKS_IDLE_TIMEOUT_MS:-120000}}"
|
||||
|
||||
if [ "${SKS_IDLE_HANDSHAKE:-1}" = "0" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local target
|
||||
target=$(_sanitize_herdr_agent_name "$sess")
|
||||
|
||||
local raw_status="" agent_st=""
|
||||
raw_status="$(_sks_herdr agent get "$target" 2>/dev/null || _sks_herdr agent get "$sess" 2>/dev/null || true)"
|
||||
if [ -n "$raw_status" ]; then
|
||||
agent_st="$("$(_delegate_py_bin)" -c '
|
||||
import sys, json
|
||||
try:
|
||||
d = json.loads(sys.stdin.read())
|
||||
print((d.get("result", {}).get("agent", {}) or {}).get("agent_status", "") or "")
|
||||
except Exception:
|
||||
print("")
|
||||
' <<< "$raw_status" 2>/dev/null || true)"
|
||||
fi
|
||||
|
||||
if [ -z "$agent_st" ] || [ "$agent_st" = "unknown" ]; then
|
||||
return 3
|
||||
fi
|
||||
if [ "$agent_st" = "blocked" ]; then
|
||||
return 2
|
||||
fi
|
||||
if [ "$agent_st" = "idle" ] || [ "$agent_st" = "done" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local max_retries="${SKS_IDLE_RETRIES:-2}"
|
||||
local backoff_s="${SKS_IDLE_RETRY_BACKOFF_S:-10}"
|
||||
local try=0
|
||||
while [ "$try" -le "$max_retries" ]; do
|
||||
local watcher_out="" watcher_rc=0
|
||||
watcher_out="$("$(_delegate_py_bin)" -m lib_py.loop.watcher wait "$target" \
|
||||
--session "${HERDR_SESSION_NAME:-default}" \
|
||||
--until idle --until done --until blocked \
|
||||
--timeout-ms "$timeout_ms" 2>/dev/null)" || watcher_rc=$?
|
||||
|
||||
local parsed_st=""
|
||||
if [ -n "$watcher_out" ]; then
|
||||
parsed_st="$("$(_delegate_py_bin)" -c '
|
||||
import sys, json
|
||||
try:
|
||||
d = json.loads(sys.stdin.read())
|
||||
print(d.get("agent_status", "") or "")
|
||||
except Exception:
|
||||
print("")
|
||||
' <<< "$watcher_out" 2>/dev/null || true)"
|
||||
fi
|
||||
|
||||
if [ $watcher_rc -eq 0 ]; then
|
||||
if [ "$parsed_st" = "blocked" ]; then
|
||||
return 2
|
||||
fi
|
||||
return 0
|
||||
elif [ $watcher_rc -eq 2 ]; then
|
||||
if [ "$parsed_st" = "blocked" ]; then
|
||||
return 2
|
||||
fi
|
||||
if [ "$try" -lt "$max_retries" ]; then
|
||||
sleep "$backoff_s"
|
||||
try=$((try + 1))
|
||||
continue
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
return 3
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# send_keys_safe <sess> <text> [job_id]
|
||||
# 1. Wait for renderer quiescence (RC-A). If unobservable (headless), bypass visual checks.
|
||||
# 2. Refuse to paste while a dialog is open (RC-B/RC-C): wait up to
|
||||
@@ -2232,8 +2331,7 @@ send_keys_safe() {
|
||||
local sess="$1" text="$2" job_id="${3:-adhoc}"
|
||||
local pre_submit deadline try
|
||||
local MAM_MODAL_TOKENS="${MAM_MODAL_TOKENS:-}"
|
||||
local MAM_AGENT_NAME MAM_OWN_KEY MAM_INPUT_PROMPT MAM_INPUT_PLACEHOLDER MAM_INPUT_RULE_PATTERN
|
||||
local MAM_READY_TOKENS MAM_STRONG_READY_TOKENS MAM_WEAK_READY_TOKENS MAM_EXIT_KEY MAM_DELEGATE_AGENT_KEY
|
||||
local MAM_AGENT_NAME MAM_OWN_KEY MAM_INPUT_PROMPT MAM_INPUT_PLACEHOLDER MAM_INPUT_RULE_PATTERN MAM_READY_TOKENS MAM_STRONG_READY_TOKENS MAM_WEAK_READY_TOKENS MAM_EXIT_KEY MAM_DELEGATE_AGENT_KEY
|
||||
|
||||
if [ -z "${MAM_MODAL_TOKENS:-}" ]; then
|
||||
local _sks_agent=""
|
||||
@@ -2274,10 +2372,83 @@ send_keys_safe() {
|
||||
|
||||
local agent_target
|
||||
agent_target=$(_sanitize_herdr_agent_name "$sess")
|
||||
# Native herdr 0.8+ fast path: agent prompt handles atomic text + enter submission
|
||||
# Gated behind quiescence and dialog checks; returns 0 on RPC success to prevent duplicate input
|
||||
if _sks_herdr agent prompt "$agent_target" "$text" >/dev/null 2>&1 || _sks_herdr agent prompt "$sess" "$text" >/dev/null 2>&1; then
|
||||
return 0
|
||||
|
||||
# Busy gate: wait up to SKS_BUSY_TIMEOUT only if wait_for_agent_idle returned 3 (unknown)
|
||||
if [ "${_SKS_IDLE_RC:-0}" = "3" ]; then
|
||||
local b_deadline=$(( $(date +%s) + ${SKS_BUSY_TIMEOUT:-120} ))
|
||||
local still_busy=1
|
||||
while [ "$(date +%s)" -lt "$b_deadline" ]; do
|
||||
local tail_pane
|
||||
tail_pane=$(_pane_tail "$sess" 8)
|
||||
if ! printf '%s\n' "$tail_pane" | grep -Fq "esc to interrupt" && ! printf '%s\n' "$tail_pane" | grep -Eq "● |✽ |[A-Za-z]+ing"; then
|
||||
still_busy=0
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$still_busy" -eq 1 ]; then
|
||||
echo "send_keys_safe: pane still busy after SKS_BUSY_TIMEOUT ($sess)" >&2
|
||||
return 4
|
||||
fi
|
||||
fi
|
||||
|
||||
# Native herdr 0.8+ fast path: agent prompt handles atomic submission
|
||||
if [ "${SKS_PROMPT_WAIT:-1}" = "1" ]; then
|
||||
local p_out="" p_rc=0
|
||||
local _p_tmp
|
||||
_p_tmp=$(mktemp 2>/dev/null || echo "/tmp/sks_p_${sess}_$$")
|
||||
if _sks_herdr agent prompt "$agent_target" "$text" --wait --until working --timeout 5000 >"$_p_tmp" 2>&1; then
|
||||
rm -f "$_p_tmp" 2>/dev/null || true
|
||||
return 0
|
||||
fi
|
||||
p_rc=$?
|
||||
p_out=$(cat "$_p_tmp" 2>/dev/null || true)
|
||||
rm -f "$_p_tmp" 2>/dev/null || true
|
||||
|
||||
if printf '%s\n' "$p_out" | grep -Fq "agent_not_found"; then
|
||||
local _p_tmp2
|
||||
_p_tmp2=$(mktemp 2>/dev/null || echo "/tmp/sks_p2_${sess}_$$")
|
||||
if _sks_herdr agent prompt "$sess" "$text" --wait --until working --timeout 5000 >"$_p_tmp2" 2>&1; then
|
||||
rm -f "$_p_tmp2" 2>/dev/null || true
|
||||
return 0
|
||||
fi
|
||||
p_rc=$?
|
||||
p_out=$(cat "$_p_tmp2" 2>/dev/null || true)
|
||||
rm -f "$_p_tmp2" 2>/dev/null || true
|
||||
fi
|
||||
if printf '%s\n' "$p_out" | grep -Fq "agent_blocked"; then
|
||||
echo "send_keys_safe: agent blocked ($sess)" >&2
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Only fall through to legacy paste-buffer if agent prompt command itself is not supported
|
||||
# or if target agent is not found (generic pane / terminal)
|
||||
if printf '%s\n' "$p_out" | grep -Eq "unknown command|unrecognized command|no such command|unexpected argument|unrecognized option|agent_not_found"; then
|
||||
: # Fall through to legacy paste buffer
|
||||
else
|
||||
# Prompt was dispatched via RPC; on any error (empty output, timeout, stalled, transport error, etc.),
|
||||
# do NOT resubmit via paste buffer (single injection contract). Check agent get after 2s.
|
||||
sleep 2
|
||||
local check_raw check_st
|
||||
check_raw="$(_sks_herdr agent get "$agent_target" 2>/dev/null || _sks_herdr agent get "$sess" 2>/dev/null || true)"
|
||||
check_st="$("$(_delegate_py_bin)" -c '
|
||||
import sys, json
|
||||
try:
|
||||
d = json.loads(sys.stdin.read())
|
||||
print((d.get("result", {}).get("agent", {}) or {}).get("agent_status", "") or "")
|
||||
except Exception:
|
||||
print("")
|
||||
' <<< "$check_raw" 2>/dev/null || true)"
|
||||
if [ "$check_st" = "working" ]; then
|
||||
return 0
|
||||
fi
|
||||
echo "send_keys_safe: delivery unknown/failed ($sess)" >&2
|
||||
return 4
|
||||
fi
|
||||
else
|
||||
if _sks_herdr agent prompt "$agent_target" "$text" >/dev/null 2>&1 || _sks_herdr agent prompt "$sess" "$text" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Fallback: paste buffer submission. Compute verification markers on demand.
|
||||
@@ -2328,8 +2499,12 @@ send_keys_safe() {
|
||||
sleep "$try"
|
||||
local cur_content
|
||||
cur_content=$(_pane_capture "$sess")
|
||||
# Hardened Submission Checks
|
||||
if printf '%s\n' "$cur_content" | grep -Fq "esc to interrupt" || printf '%s\n' "$cur_content" | grep -Eq "● |✽ |[A-Za-z]+ing"; then
|
||||
# Hardened Submission Checks: exclude busy tokens if already present in pre_submit
|
||||
local pre_busy=0
|
||||
if printf '%s\n' "$pre_submit" | grep -Fq "esc to interrupt" || printf '%s\n' "$pre_submit" | grep -Eq "● |✽ |[A-Za-z]+ing"; then
|
||||
pre_busy=1
|
||||
fi
|
||||
if [ "$pre_busy" -eq 0 ] && ( printf '%s\n' "$cur_content" | grep -Fq "esc to interrupt" || printf '%s\n' "$cur_content" | grep -Eq "● |✽ |[A-Za-z]+ing" ); then
|
||||
return 0
|
||||
fi
|
||||
if [ "$was_popup" = "0" ] && ! _pane_tail "$sess" 3 | tr -d '[:space:]' | grep -Fq "$marker_norm" && [ "$cur_content" != "$pre_submit" ]; then
|
||||
|
||||
@@ -39,6 +39,8 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
p_pub.add_argument("--retained", action="store_true")
|
||||
p_pub.add_argument("--registry-dir", default=None)
|
||||
p_pub.add_argument("--attempts", type=int, default=3)
|
||||
p_pub.add_argument("--expect-dispatch-token", default=None)
|
||||
p_pub.add_argument("--attempt-id", type=int, default=None)
|
||||
p_pub.add_argument("-v", "--verbose", action="store_true")
|
||||
|
||||
p_daemon = sub.add_parser("ensure-daemon", help="start the subscriber daemon if absent")
|
||||
@@ -68,6 +70,10 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
forward += ["--retained"]
|
||||
if args.registry_dir:
|
||||
forward += ["--registry-dir", args.registry_dir]
|
||||
if args.expect_dispatch_token:
|
||||
forward += ["--expect-dispatch-token", args.expect_dispatch_token]
|
||||
if args.attempt_id is not None:
|
||||
forward += ["--attempt-id", str(args.attempt_id)]
|
||||
if args.verbose:
|
||||
forward += ["--verbose"]
|
||||
return publish_event.main(forward)
|
||||
|
||||
@@ -88,7 +88,7 @@ ATTEMPT_STARTING_REQUESTS = ("claim", "redispatch")
|
||||
|
||||
# Only these sources may move a record out of a terminal state. An agent can
|
||||
# never walk back its own terminal; that is the whole point of the CAS table.
|
||||
OPERATOR_SOURCES = ("registry_cli", "operator")
|
||||
OPERATOR_SOURCES = ("registry_cli", "operator", "delegate_wrapper")
|
||||
OPERATOR_REQUESTS = ("redispatch", "finalize")
|
||||
|
||||
# A caller that merges `status=<value>` is making the request that owns that
|
||||
@@ -178,20 +178,27 @@ def _decide(record: Dict[str, Any], request: str, expected_status, expected_atte
|
||||
"stale attempt: expected {!r}, record has {!r}".format(
|
||||
expected_attempt, actual))
|
||||
|
||||
requested = REQUEST_TO_STATUS.get(request, None)
|
||||
|
||||
if expected_dispatch_token is not None:
|
||||
actual_token = record.get(DISPATCH_TOKEN_FIELD)
|
||||
if actual_token != expected_dispatch_token:
|
||||
return _Decision(False, current,
|
||||
"dispatch token changed: the attempt this decision was "
|
||||
"made for is no longer the current one")
|
||||
elif expected_attempt is None and source not in OPERATOR_SOURCES and requested in TERMINAL_STATUSES:
|
||||
actual_token = record.get(DISPATCH_TOKEN_FIELD)
|
||||
rec_attempt = record.get("attempt_id", record.get("iteration", 1))
|
||||
if actual_token is not None and rec_attempt > 1:
|
||||
return _Decision(False, current,
|
||||
"dispatch token required: redispatched job carries a dispatch token "
|
||||
"but none was provided")
|
||||
|
||||
if expected_status is not None and current != expected_status:
|
||||
return _Decision(False, current,
|
||||
"precondition failed: expected status {!r}, found {!r}".format(
|
||||
expected_status, current))
|
||||
|
||||
requested = REQUEST_TO_STATUS.get(request, None)
|
||||
|
||||
# Authority first, so a refusal says *why* rather than blaming the state.
|
||||
if request in OPERATOR_REQUESTS and source not in OPERATOR_SOURCES:
|
||||
return _Decision(False, current,
|
||||
@@ -228,6 +235,15 @@ def _decide(record: Dict[str, Any], request: str, expected_status, expected_atte
|
||||
if operator and request == "redispatch":
|
||||
# Re-dispatching a finished job to the next iteration. The attempt
|
||||
# must move forward, so a replayed redispatch cannot resurrect it.
|
||||
new_attempt = fields.get("attempt_id")
|
||||
if new_attempt is not None:
|
||||
old_attempt = record.get("attempt_id", record.get("iteration"))
|
||||
if old_attempt is not None and int(new_attempt) <= int(old_attempt):
|
||||
return _Decision(False, current,
|
||||
"redispatch attempt_id {!r} does not advance past {!r}".format(
|
||||
new_attempt, old_attempt), conflict=True)
|
||||
return _Decision(True, "pending", "{} -> pending (redispatch)".format(current))
|
||||
|
||||
new_iteration = fields.get("iteration")
|
||||
old_iteration = record.get("iteration")
|
||||
if new_iteration is None:
|
||||
@@ -298,6 +314,15 @@ def transition(job_id, request="merge", *, registry_dir=None, expected_status=No
|
||||
# A new attempt begins here, so anything decided about the
|
||||
# previous one stops being committable from this moment on.
|
||||
record[DISPATCH_TOKEN_FIELD] = uuid.uuid4().hex
|
||||
if "attempt_id" in fields:
|
||||
record["attempt_id"] = fields["attempt_id"]
|
||||
elif "iteration" in fields:
|
||||
record["attempt_id"] = fields["iteration"]
|
||||
else:
|
||||
record.setdefault("attempt_id", record.get("iteration", 1))
|
||||
elif record.get(DISPATCH_TOKEN_FIELD) is None:
|
||||
record[DISPATCH_TOKEN_FIELD] = uuid.uuid4().hex
|
||||
record.setdefault("attempt_id", record.get("iteration", 1))
|
||||
if notify is not None:
|
||||
# One atomic commit records both the new state and the fact that
|
||||
# the world has not been told yet. If the process dies before the
|
||||
|
||||
@@ -61,15 +61,21 @@ def _check_disk_fallback(
|
||||
pending: Set[str],
|
||||
registry_dir: str,
|
||||
terminal: Dict[str, str],
|
||||
expected_dispatch_tokens: Optional[Dict[str, Optional[str]]] = None,
|
||||
expected_attempts: Optional[Dict[str, Optional[int]]] = None,
|
||||
) -> None:
|
||||
"""Check local on-disk registry and audit logs for terminal status (B-15)."""
|
||||
for jid in list(pending):
|
||||
disk_status = None
|
||||
rec_token = None
|
||||
rec_attempt = None
|
||||
try:
|
||||
job_rec = load_job(jid, registry_dir)
|
||||
disk_status = job_rec.get("status")
|
||||
rec_token = job_rec.get("dispatch_token")
|
||||
rec_attempt = job_rec.get("attempt_id", job_rec.get("iteration"))
|
||||
except Exception:
|
||||
pass
|
||||
job_rec = {}
|
||||
if not disk_status or disk_status not in ("completed", "error", "cancelled"):
|
||||
try:
|
||||
disk_status = mqtt_common.read_logged_status(jid, mqtt_common.get_logs_dir())
|
||||
@@ -77,6 +83,18 @@ def _check_disk_fallback(
|
||||
pass
|
||||
|
||||
if disk_status in ("completed", "error", "cancelled"):
|
||||
if expected_dispatch_tokens:
|
||||
exp_token = expected_dispatch_tokens.get(jid)
|
||||
if exp_token is not None and rec_token != exp_token:
|
||||
logger.warning("disk fallback ignored terminal status for %s due to dispatch_token mismatch (%r != %r)",
|
||||
jid, rec_token, exp_token)
|
||||
continue
|
||||
if expected_attempts:
|
||||
exp_att = expected_attempts.get(jid)
|
||||
if exp_att is not None and rec_attempt is not None and rec_attempt < exp_att:
|
||||
logger.warning("disk fallback ignored terminal status for %s due to attempt_id older than expected (%r < %r)",
|
||||
jid, rec_attempt, exp_att)
|
||||
continue
|
||||
synth_event = "completed" if disk_status == "completed" else "error"
|
||||
synth_payload = {
|
||||
"job_id": jid,
|
||||
@@ -95,11 +113,15 @@ def _check_disk_fallback(
|
||||
class _Watcher:
|
||||
"""Holds the shared queue + the set of job_ids we accept events for."""
|
||||
|
||||
def __init__(self, expected_job_ids: Set[str], expected_tokens: Dict[str, Optional[str]], expected_seqs: Dict[str, int]):
|
||||
def __init__(self, expected_job_ids: Set[str], expected_tokens: Dict[str, Optional[str]], expected_seqs: Dict[str, int],
|
||||
expected_dispatch_tokens: Optional[Dict[str, Optional[str]]] = None,
|
||||
expected_attempts: Optional[Dict[str, Optional[int]]] = None):
|
||||
self.events: "queue.Queue[Tuple[str, Dict[str, Any]]]" = queue.Queue()
|
||||
self.expected = set(expected_job_ids)
|
||||
self.tokens = expected_tokens # job_id -> expected auth_token (or None)
|
||||
self.last_seq = dict(expected_seqs)
|
||||
self.dispatch_tokens = dict(expected_dispatch_tokens or {})
|
||||
self.attempts = dict(expected_attempts or {})
|
||||
# Intent identities already accepted, per job. A replayed notification is
|
||||
# re-published with a *fresh* wire sequence, so the monotonic sequence
|
||||
# guard below cannot recognise it: to that guard a retry looks like a new
|
||||
@@ -135,6 +157,24 @@ class _Watcher:
|
||||
if not mqtt_common.verify_hmac(payload, expected_token):
|
||||
logger.warning("drop event for job %s: HMAC verify failed", jid)
|
||||
return
|
||||
# --- CAS dispatch token & attempt check: drop stale attempt events ---
|
||||
expected_disp_tok = self.dispatch_tokens.get(jid)
|
||||
if expected_disp_tok is not None:
|
||||
data = payload.get("data")
|
||||
ev_tok = data.get("dispatch_token") if isinstance(data, dict) else None
|
||||
if ev_tok != expected_disp_tok:
|
||||
logger.warning("drop event for job %s: dispatch_token %r does not match expected %r",
|
||||
jid, ev_tok, expected_disp_tok)
|
||||
return
|
||||
|
||||
expected_att = self.attempts.get(jid)
|
||||
if expected_att is not None:
|
||||
data = payload.get("data")
|
||||
ev_att = data.get("attempt_id") if isinstance(data, dict) else None
|
||||
if ev_att is not None and isinstance(ev_att, int) and ev_att < expected_att:
|
||||
logger.warning("drop event for job %s: attempt_id %r is older than expected %r",
|
||||
jid, ev_att, expected_att)
|
||||
return
|
||||
# --- replay attack defense: check monotonic sequence ---
|
||||
seq = payload.get("seq")
|
||||
if seq is None or not isinstance(seq, int):
|
||||
@@ -216,11 +256,21 @@ def _run_subscriber(args) -> int:
|
||||
expected_ids: Set[str] = {j["job_id"] for j in jobs}
|
||||
tokens = {j["job_id"]: j.get("auth_token") for j in jobs}
|
||||
seqs = {}
|
||||
dispatch_tokens = {}
|
||||
attempts = {}
|
||||
for j in jobs:
|
||||
jid = j["job_id"]
|
||||
last_seq = int(j.get("last_seq", 0))
|
||||
seqs[jid] = max(0, last_seq - 1)
|
||||
watcher = _Watcher(expected_ids, tokens, seqs)
|
||||
dispatch_tokens[jid] = j.get("dispatch_token")
|
||||
attempts[jid] = j.get("attempt_id", j.get("iteration", 1))
|
||||
|
||||
if args.expect_dispatch_token is not None and args.job:
|
||||
dispatch_tokens[args.job] = args.expect_dispatch_token
|
||||
if args.attempt_id is not None and args.job:
|
||||
attempts[args.job] = args.attempt_id
|
||||
|
||||
watcher = _Watcher(expected_ids, tokens, seqs, dispatch_tokens, attempts)
|
||||
|
||||
# Resolve timeouts from CLI, falling back to the (first) job's settings.
|
||||
base_job = jobs[0]
|
||||
@@ -251,7 +301,18 @@ def _run_subscriber(args) -> int:
|
||||
if rc != 0:
|
||||
logger.warning("broker disconnected (rc=%s); will retry reconnect", reason_code)
|
||||
|
||||
start = time.monotonic()
|
||||
wall_deadline = start + wall_timeout
|
||||
last_event = start
|
||||
last_disk_check = 0.0
|
||||
retention_checked = not args.expect_retention
|
||||
connected = False
|
||||
subscribed = False
|
||||
|
||||
def on_subscribe(_c, _u, mid, granted_qos, _props=None):
|
||||
nonlocal last_event, subscribed
|
||||
subscribed = True
|
||||
last_event = time.monotonic()
|
||||
for topic in subscribed_topics:
|
||||
print(f"SUBSCRIBED {topic}", flush=True)
|
||||
|
||||
@@ -262,15 +323,9 @@ def _run_subscriber(args) -> int:
|
||||
|
||||
terminal: Dict[str, str] = {} # job_id -> "completed"/"error"
|
||||
pending: Set[str] = set(expected_ids)
|
||||
start = time.monotonic()
|
||||
wall_deadline = start + wall_timeout
|
||||
last_event = start
|
||||
last_disk_check = 0.0
|
||||
retention_checked = not args.expect_retention
|
||||
connected = False
|
||||
|
||||
# Check if all jobs are already in terminal state on disk before connecting
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
if not pending:
|
||||
if any(state == "error" for state in terminal.values()):
|
||||
return 1
|
||||
@@ -283,9 +338,10 @@ def _run_subscriber(args) -> int:
|
||||
)()
|
||||
client.loop_start()
|
||||
connected = True
|
||||
last_event = time.monotonic()
|
||||
except Exception as exc:
|
||||
logger.warning("broker connection failed (%s); checking disk fallback", exc)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
if not pending:
|
||||
if any(state == "error" for state in terminal.values()):
|
||||
return 1
|
||||
@@ -297,15 +353,17 @@ def _run_subscriber(args) -> int:
|
||||
while pending:
|
||||
now = time.monotonic()
|
||||
if now >= wall_deadline:
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
if not pending:
|
||||
break
|
||||
logger.error("wall-clock timeout (%.0fs); still pending: %s",
|
||||
wall_timeout, ", ".join(sorted(pending)))
|
||||
return 2
|
||||
if not subscribed:
|
||||
last_event = now
|
||||
idle_left = idle_timeout - (now - last_event)
|
||||
if idle_left <= 0:
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
if not pending:
|
||||
break
|
||||
logger.error("idle timeout (%.0fs, no events); still pending: %s",
|
||||
@@ -323,7 +381,7 @@ def _run_subscriber(args) -> int:
|
||||
# Step 2 (B-15): Local disk fallback throttled to every 3.0s
|
||||
if (now - last_disk_check) >= 3.0:
|
||||
last_disk_check = now
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
continue
|
||||
|
||||
last_event = time.monotonic()
|
||||
@@ -365,6 +423,10 @@ def main(argv=None) -> int:
|
||||
help="max seconds with no new event (default: job.idle_timeout_sec or 120)")
|
||||
parser.add_argument("--expect-retention", action="store_true",
|
||||
help="warn if no retained terminal event arrives promptly")
|
||||
parser.add_argument("--expect-dispatch-token", default=None,
|
||||
help="expected dispatch token for CAS verification")
|
||||
parser.add_argument("--attempt-id", type=int, default=None,
|
||||
help="minimum attempt id to accept events for")
|
||||
parser.add_argument("--registry-dir", default=DEFAULT_REGISTRY_DIR)
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
@@ -240,7 +240,7 @@ def broker_config_from_env(overrides: Optional[Dict[str, Any]] = None) -> Broker
|
||||
for any key it specifies with a non-None value.
|
||||
"""
|
||||
cfg = BrokerConfig(
|
||||
host=os.environ.get("MQTT_BROKER", "broker.hivemq.com"),
|
||||
host=os.environ.get("MQTT_BROKER") or os.environ.get("MQTT_HOST", "broker.hivemq.com"),
|
||||
port=_env_int("MQTT_PORT", 1883),
|
||||
tls=_env_bool("MQTT_TLS", False),
|
||||
username=os.environ.get("MQTT_USERNAME") or None,
|
||||
@@ -428,12 +428,16 @@ def transition_job_status(job_id: str, registry_dir: str = DEFAULT_REGISTRY_DIR,
|
||||
from lib_py.delegation import engine # same package: no sibling hop (see __init__)
|
||||
|
||||
request = "merge"
|
||||
expected_dispatch_token = fields.pop("expected_dispatch_token", None)
|
||||
expected_attempt = fields.pop("expected_attempt", None)
|
||||
if "status" in fields:
|
||||
request = engine.STATUS_TO_REQUEST.get(fields["status"], "merge")
|
||||
if request != "merge":
|
||||
fields = {k: v for k, v in fields.items() if k != "status"}
|
||||
return engine.transition(job_id, request, registry_dir=registry_dir,
|
||||
fields=fields, source=_source)
|
||||
fields=fields, source=_source,
|
||||
expected_dispatch_token=expected_dispatch_token,
|
||||
expected_attempt=expected_attempt)
|
||||
|
||||
|
||||
def _update_job_status_legacy(job_id: str, registry_dir: str = DEFAULT_REGISTRY_DIR, **fields: Any) -> Dict[str, Any]:
|
||||
|
||||
@@ -127,7 +127,8 @@ class PreparedEvent(object):
|
||||
|
||||
|
||||
def prepare_event(job_id, event, detail="", data=None,
|
||||
registry_dir=DEFAULT_REGISTRY_DIR, retained=False) -> PreparedEvent:
|
||||
registry_dir=DEFAULT_REGISTRY_DIR, retained=False,
|
||||
dispatch_token=None, attempt_id=None) -> PreparedEvent:
|
||||
"""Load the job, consume the next seq, and build the signed payload.
|
||||
|
||||
Consuming the seq here (before any transport runs) is deliberate: a failed
|
||||
@@ -139,12 +140,21 @@ def prepare_event(job_id, event, detail="", data=None,
|
||||
topic = job.get("topic_prefix")
|
||||
topic = f"{topic}/events" if topic else events_topic_for(job_id)
|
||||
seq = next_seq(job_id, registry_dir)
|
||||
|
||||
payload_data = dict(data) if data else {}
|
||||
eff_dispatch_token = dispatch_token
|
||||
if eff_dispatch_token is not None and "dispatch_token" not in payload_data:
|
||||
payload_data["dispatch_token"] = eff_dispatch_token
|
||||
eff_attempt_id = attempt_id
|
||||
if eff_attempt_id is not None and "attempt_id" not in payload_data:
|
||||
payload_data["attempt_id"] = eff_attempt_id
|
||||
|
||||
payload = build_payload(
|
||||
job_id=job_id,
|
||||
seq=seq,
|
||||
event=event,
|
||||
detail=detail,
|
||||
data=data,
|
||||
data=payload_data,
|
||||
auth_token=job.get("auth_token"),
|
||||
)
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
@@ -153,7 +163,9 @@ def prepare_event(job_id, event, detail="", data=None,
|
||||
|
||||
|
||||
def record_outcome(prepared: PreparedEvent, registry_dir: str,
|
||||
publish_ok: bool, publish_error: Optional[str]) -> None:
|
||||
publish_ok: bool, publish_error: Optional[str],
|
||||
expected_dispatch_token: Optional[str] = None,
|
||||
expected_attempt: Optional[int] = None) -> None:
|
||||
"""Persist the audit record, the registry event log and the status sync.
|
||||
|
||||
All three are best effort and all three happen whether or not the publish
|
||||
@@ -176,10 +188,19 @@ def record_outcome(prepared: PreparedEvent, registry_dir: str,
|
||||
new_status = EVENT_TO_STATUS.get(prepared.event)
|
||||
if new_status:
|
||||
try:
|
||||
exp_token = expected_dispatch_token
|
||||
if exp_token is None and isinstance(prepared.payload.get("data"), dict):
|
||||
exp_token = prepared.payload["data"].get("dispatch_token")
|
||||
exp_att = expected_attempt
|
||||
if exp_att is None and isinstance(prepared.payload.get("data"), dict):
|
||||
exp_att = prepared.payload["data"].get("attempt_id")
|
||||
|
||||
# Agent authority on purpose: an agent may never move a terminal
|
||||
# record, which is what the operator-only requests are gated for.
|
||||
mqtt_common.update_job_status(prepared.job_id, registry_dir,
|
||||
_source="publish_event", status=new_status)
|
||||
_source="publish_event", status=new_status,
|
||||
expected_dispatch_token=exp_token,
|
||||
expected_attempt=exp_att)
|
||||
except Exception as exc: # pragma: no cover - best effort
|
||||
logger.warning("status sync failed: %s", exc)
|
||||
|
||||
@@ -214,6 +235,8 @@ def main(argv=None) -> int:
|
||||
parser.add_argument("--attempt-id", type=int, default=None,
|
||||
help="attempt this finalization belongs to (default: the "
|
||||
"job's current iteration)")
|
||||
parser.add_argument("--expect-dispatch-token", default=None,
|
||||
help="expected dispatch token for CAS verification")
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
@@ -250,7 +273,8 @@ def main(argv=None) -> int:
|
||||
attempt_id = args.attempt_id
|
||||
if attempt_id is None:
|
||||
try:
|
||||
attempt_id = load_job(job_id, args.registry_dir).get("iteration", 1)
|
||||
rec = load_job(job_id, args.registry_dir)
|
||||
attempt_id = rec.get("attempt_id", rec.get("iteration", 1))
|
||||
except FileNotFoundError:
|
||||
attempt_id = 1
|
||||
try:
|
||||
@@ -263,6 +287,41 @@ def main(argv=None) -> int:
|
||||
logger.info("finalized %s (sha256 %s...)", manifest["report_path"],
|
||||
manifest["sha256"][:12])
|
||||
|
||||
check_token = args.expect_dispatch_token
|
||||
if check_token is None and isinstance(data, dict) and "dispatch_token" in data:
|
||||
check_token = data.get("dispatch_token")
|
||||
try:
|
||||
rec = load_job(job_id, args.registry_dir)
|
||||
actual_tok = rec.get("dispatch_token")
|
||||
rec_attempt = rec.get("attempt_id", rec.get("iteration", 1))
|
||||
except FileNotFoundError:
|
||||
rec = None
|
||||
actual_tok = None
|
||||
rec_attempt = 1
|
||||
|
||||
check_attempt = args.attempt_id
|
||||
if check_attempt is None and isinstance(data, dict) and "attempt_id" in data:
|
||||
check_attempt = data.get("attempt_id")
|
||||
|
||||
if rec is not None and actual_tok is not None:
|
||||
if check_token is None and check_attempt is None:
|
||||
if args.event in ("completed", "error", "cancelled") and rec_attempt > 1:
|
||||
logger.error("dispatch token required: redispatched job %s carries a dispatch token "
|
||||
"but none was provided (aborting terminal publish)", job_id)
|
||||
return 4
|
||||
elif check_token is not None and actual_tok != check_token:
|
||||
logger.error("dispatch token mismatch: expected %s, got %s (aborting publish)",
|
||||
check_token, actual_tok)
|
||||
return 4
|
||||
elif check_attempt is not None and rec_attempt != check_attempt:
|
||||
logger.error("attempt mismatch: expected %s, got %s (aborting publish)",
|
||||
check_attempt, rec_attempt)
|
||||
return 4
|
||||
elif check_token is not None and actual_tok != check_token:
|
||||
logger.error("dispatch token mismatch: expected %s, got %s (aborting publish)",
|
||||
check_token, actual_tok)
|
||||
return 4
|
||||
|
||||
try:
|
||||
prepared = prepare_event(
|
||||
job_id=job_id,
|
||||
@@ -271,6 +330,8 @@ def main(argv=None) -> int:
|
||||
data=data,
|
||||
registry_dir=args.registry_dir,
|
||||
retained=args.retained,
|
||||
dispatch_token=args.expect_dispatch_token,
|
||||
attempt_id=args.attempt_id,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
logger.error("%s", exc)
|
||||
@@ -292,7 +353,16 @@ def main(argv=None) -> int:
|
||||
publish_error = str(exc)
|
||||
logger.error("publish failed after %d attempts: %s", args.attempts, exc)
|
||||
|
||||
record_outcome(prepared, args.registry_dir, publish_ok, publish_error)
|
||||
sync_expected_token = args.expect_dispatch_token
|
||||
if sync_expected_token is None and isinstance(data, dict):
|
||||
sync_expected_token = data.get("dispatch_token")
|
||||
sync_expected_attempt = args.attempt_id
|
||||
if sync_expected_attempt is None and isinstance(data, dict):
|
||||
sync_expected_attempt = data.get("attempt_id")
|
||||
|
||||
record_outcome(prepared, args.registry_dir, publish_ok, publish_error,
|
||||
expected_dispatch_token=sync_expected_token,
|
||||
expected_attempt=sync_expected_attempt)
|
||||
|
||||
if not publish_ok:
|
||||
return 2
|
||||
|
||||
@@ -100,6 +100,8 @@ def register_job(
|
||||
"reviewer_session": reviewer_session,
|
||||
"max_iterations": int(max_iterations),
|
||||
"iteration": 1,
|
||||
"dispatch_token": uuid.uuid4().hex,
|
||||
"attempt_id": 1,
|
||||
}
|
||||
with registry_lock(registry_dir):
|
||||
if mqtt_common._job_path(job_id, registry_dir).exists():
|
||||
@@ -299,8 +301,15 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
p_update.add_argument("--agent-session", default=None)
|
||||
p_update.add_argument("--prompt", default=None)
|
||||
p_update.add_argument("--iteration", type=int, default=None)
|
||||
p_update.add_argument("--attempt-id", type=int, default=None)
|
||||
p_update.add_argument("--role", default=None)
|
||||
|
||||
p_close = sub.add_parser("close", help="close a job with deadline_timeout or cancelled")
|
||||
p_close.add_argument("--job", required=True)
|
||||
p_close.add_argument("--request", required=True, choices=["deadline_timeout", "cancelled"])
|
||||
p_close.add_argument("--detail", default=None)
|
||||
p_close.add_argument("--expect-dispatch-token", default=None)
|
||||
|
||||
p_feedback = sub.add_parser("get-feedback", help="get the last feedback detail (completed/error) for a job")
|
||||
p_feedback.add_argument("--job", required=True)
|
||||
|
||||
@@ -430,6 +439,8 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
fields["prompt"] = args.prompt
|
||||
if args.iteration is not None:
|
||||
fields["iteration"] = args.iteration
|
||||
if args.attempt_id is not None:
|
||||
fields["attempt_id"] = args.attempt_id
|
||||
if args.role is not None:
|
||||
fields["role"] = args.role
|
||||
try:
|
||||
@@ -439,6 +450,47 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
return 1
|
||||
return rc
|
||||
|
||||
if args.command == "close":
|
||||
from lib_py.delegation import engine
|
||||
fields = {}
|
||||
if args.detail is not None:
|
||||
fields["detail"] = args.detail
|
||||
|
||||
if not engine.cas_enabled():
|
||||
req_status = engine.REQUEST_TO_STATUS.get(args.request, "error")
|
||||
fields["status"] = req_status
|
||||
try:
|
||||
mqtt_common.update_job_status(args.job, rd, **fields)
|
||||
except FileNotFoundError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
try:
|
||||
res = engine.transition(
|
||||
args.job,
|
||||
args.request,
|
||||
registry_dir=rd,
|
||||
source="delegate_wrapper",
|
||||
fields=fields,
|
||||
expected_dispatch_token=args.expect_dispatch_token,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if res.committed:
|
||||
return 0
|
||||
if res.idempotent:
|
||||
if res.reason:
|
||||
print(res.reason, file=sys.stderr)
|
||||
return 0
|
||||
if res.reason and "conflicting" in res.reason and "after terminal" in res.reason:
|
||||
print(res.reason, file=sys.stderr)
|
||||
return 0
|
||||
print(f"refused: {res.reason}", file=sys.stderr)
|
||||
return 4
|
||||
|
||||
if args.command == "get-feedback":
|
||||
print(get_feedback(args.job, rd))
|
||||
return 0
|
||||
|
||||
@@ -46,18 +46,54 @@ class DelegationService(object):
|
||||
registry_dir=self.registry_dir, **kw
|
||||
)
|
||||
|
||||
def publish_event(self, job_id, event, detail="", data=None, attempts=3):
|
||||
"""Publish one event. Returns the same exit status as the CLI (0 / 2)."""
|
||||
def publish_event(self, job_id, event, detail="", data=None, attempts=3,
|
||||
expect_dispatch_token=None, attempt_id=None):
|
||||
"""Publish one event. Returns the same exit status as the CLI (0 / 2 / 4)."""
|
||||
eff_dispatch_token = expect_dispatch_token
|
||||
eff_attempt_id = attempt_id
|
||||
if data and isinstance(data, dict):
|
||||
if eff_dispatch_token is None:
|
||||
eff_dispatch_token = data.get("dispatch_token")
|
||||
if eff_attempt_id is None:
|
||||
eff_attempt_id = data.get("attempt_id")
|
||||
|
||||
try:
|
||||
rec = self.load_job(job_id)
|
||||
actual_tok = rec.get("dispatch_token")
|
||||
rec_attempt = rec.get("attempt_id", rec.get("iteration", 1))
|
||||
except FileNotFoundError:
|
||||
rec = None
|
||||
actual_tok = None
|
||||
rec_attempt = 1
|
||||
|
||||
if rec is not None and actual_tok is not None:
|
||||
if eff_dispatch_token is None and eff_attempt_id is None:
|
||||
if event in ("completed", "error", "cancelled") and rec_attempt > 1:
|
||||
logger.error("dispatch token required: redispatched job %s carries a dispatch token "
|
||||
"but none was provided", job_id)
|
||||
return 4
|
||||
elif eff_dispatch_token is not None and actual_tok != eff_dispatch_token:
|
||||
return 4
|
||||
elif eff_attempt_id is not None and rec_attempt != eff_attempt_id:
|
||||
return 4
|
||||
elif eff_dispatch_token is not None and actual_tok != eff_dispatch_token:
|
||||
return 4
|
||||
|
||||
prepared = publish_event.prepare_event(
|
||||
job_id=job_id, event=event, detail=detail,
|
||||
data=data, registry_dir=self.registry_dir,
|
||||
dispatch_token=eff_dispatch_token, attempt_id=eff_attempt_id,
|
||||
)
|
||||
publisher = self._publisher
|
||||
if publisher is None:
|
||||
from lib_py.delegation.publisher import OneShotPublisher
|
||||
publisher = OneShotPublisher()
|
||||
ok, error = publisher.publish_with_retry(prepared, attempts=attempts)
|
||||
publish_event.record_outcome(prepared, self.registry_dir, ok, error)
|
||||
publish_event.record_outcome(
|
||||
prepared, self.registry_dir, ok, error,
|
||||
expected_dispatch_token=eff_dispatch_token,
|
||||
expected_attempt=eff_attempt_id,
|
||||
)
|
||||
return 0 if ok else 2
|
||||
|
||||
def ensure_subscriber_daemon(self, job_id, workdir=None):
|
||||
|
||||
@@ -13,7 +13,7 @@ import sys
|
||||
from typing import List, Sequence
|
||||
|
||||
# Framework version locked with framework constants
|
||||
FRAMEWORK_VERSION = "4.4.0"
|
||||
FRAMEWORK_VERSION = "4.4.1"
|
||||
|
||||
CORE_MODULES: Sequence[str] = (
|
||||
# Standard library modules
|
||||
|
||||
@@ -59,6 +59,7 @@ class LoopRunner:
|
||||
self.mam_dir = self.workspace_root / ".mam"
|
||||
self.checkpoint_file = self.mam_dir / "loop-checkpoint.json"
|
||||
self.created_jobs: List[str] = []
|
||||
self.failed_submit_jobs: Set[str] = set()
|
||||
self.current_plan = ""
|
||||
self.plan_job_id: Optional[str] = None
|
||||
self.base_commit = ""
|
||||
@@ -262,27 +263,49 @@ class LoopRunner:
|
||||
if not wrapper_script.exists():
|
||||
wrapper_script = Path(__file__).resolve().parent.parent / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
|
||||
|
||||
cmd = [
|
||||
"bash", str(wrapper_script), "submit",
|
||||
"--agent-session", f"herdr:{agent_session}",
|
||||
"--agent", agent_type,
|
||||
"--type", "direct",
|
||||
"--role", role,
|
||||
"--prompt", prompt,
|
||||
]
|
||||
env = dict(os.environ)
|
||||
if "DELEGATE_JOB_PYTHON" not in env:
|
||||
env["DELEGATE_JOB_PYTHON"] = sys.executable
|
||||
def _run_cmd(p_text: str):
|
||||
cmd = [
|
||||
"bash", str(wrapper_script), "submit",
|
||||
"--agent-session", f"herdr:{agent_session}",
|
||||
"--agent", agent_type,
|
||||
"--type", "direct",
|
||||
"--role", role,
|
||||
"--prompt", p_text,
|
||||
]
|
||||
env = dict(os.environ)
|
||||
if "DELEGATE_JOB_PYTHON" not in env:
|
||||
env["DELEGATE_JOB_PYTHON"] = sys.executable
|
||||
return subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.workspace_root), env=env)
|
||||
|
||||
res = _run_cmd(prompt)
|
||||
match = re.search(r"registered job:\s*([A-Za-z0-9_-]+)", res.stdout)
|
||||
job_id = match.group(1) if match else None
|
||||
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.workspace_root), env=env)
|
||||
if res.returncode != 0:
|
||||
log_error(f"delegate_job submit failed (exit {res.returncode}):\n{res.stderr or res.stdout}")
|
||||
if res.returncode == 5:
|
||||
# rc=5 (session_busy): MAM_SESSION_BUSY_RETRY_S wait, then 1 retry
|
||||
retry_s = int(os.environ.get("MAM_SESSION_BUSY_RETRY_S", "30"))
|
||||
if retry_s > 0:
|
||||
time.sleep(retry_s)
|
||||
res_retry = _run_cmd(prompt)
|
||||
match_retry = re.search(r"registered job:\s*([A-Za-z0-9_-]+)", res_retry.stdout)
|
||||
if match_retry:
|
||||
job_id_retry = match_retry.group(1)
|
||||
self.created_jobs.append(job_id_retry)
|
||||
self._record_checkpoint()
|
||||
if res_retry.returncode != 0:
|
||||
self.failed_submit_jobs.add(job_id_retry)
|
||||
log_error(f"delegate_job retry failed (exit {res_retry.returncode}):\n{res_retry.stderr or res_retry.stdout}")
|
||||
return job_id_retry
|
||||
if job_id:
|
||||
self.created_jobs.append(job_id)
|
||||
self.failed_submit_jobs.add(job_id)
|
||||
self._record_checkpoint()
|
||||
return job_id
|
||||
return None
|
||||
|
||||
# Extract Job ID (pattern: 'registered job: [A-Za-z0-9]*')
|
||||
match = re.search(r"registered job:\s*([A-Za-z0-9]+)", res.stdout)
|
||||
if match:
|
||||
job_id = match.group(1)
|
||||
if job_id:
|
||||
self.created_jobs.append(job_id)
|
||||
self._record_checkpoint()
|
||||
return job_id
|
||||
@@ -290,11 +313,17 @@ class LoopRunner:
|
||||
|
||||
def wait_for_job(self, job_id: str, max_wait: int = 3900, check_interval: float = 1.0) -> bool:
|
||||
"""Blocking wait for job completion or error with safety timeout."""
|
||||
if job_id in self.failed_submit_jobs:
|
||||
# Submit command for this job failed (rc!=0). Cap wait to 5s to avoid 3900s hang
|
||||
max_wait = min(max_wait, 5.0)
|
||||
|
||||
if self.verbose:
|
||||
log_info(f"Monitoring job '{job_id}' for status changes (timeout: {max_wait}s)...")
|
||||
|
||||
job_file = self.mam_dir / "jobs" / f"{job_id}.json"
|
||||
start_time = time.time()
|
||||
missing_count = 0
|
||||
unknown_count = 0
|
||||
|
||||
while time.time() - start_time < max_wait:
|
||||
if self.interrupted:
|
||||
@@ -306,6 +335,17 @@ class LoopRunner:
|
||||
status = json.load(f).get("status", "unknown")
|
||||
except Exception:
|
||||
status = "unknown"
|
||||
else:
|
||||
missing_count += 1
|
||||
if missing_count >= 5 and (time.time() - start_time) >= min(max_wait, 5.0):
|
||||
log_error(f"Job record '{job_file}' not found on disk; aborting wait.")
|
||||
return False
|
||||
|
||||
if status == "unknown":
|
||||
unknown_count += 1
|
||||
if unknown_count >= 5 and (time.time() - start_time) >= min(max_wait, 5.0):
|
||||
log_error(f"Job '{job_id}' record unreadable / status unknown; aborting wait.")
|
||||
return False
|
||||
|
||||
if status == "completed":
|
||||
if self.verbose:
|
||||
|
||||
@@ -59,10 +59,14 @@ class WaitOutcome(object):
|
||||
|
||||
|
||||
def build_argv(target: str, until: Sequence[str] = DEFAULT_UNTIL,
|
||||
timeout_ms: Optional[int] = None) -> list:
|
||||
timeout_ms: Optional[int] = None,
|
||||
session: Optional[str] = None) -> list:
|
||||
if not target:
|
||||
raise ValueError("wait target is required")
|
||||
argv = ["herdr", "agent", "wait", target]
|
||||
if session and session != "default":
|
||||
argv = ["herdr", "--session", session, "agent", "wait", target]
|
||||
else:
|
||||
argv = ["herdr", "agent", "wait", target]
|
||||
for state in until:
|
||||
if state not in VALID_STATES:
|
||||
raise ValueError("unsupported --until state: {!r}".format(state))
|
||||
@@ -83,9 +87,10 @@ def _parse(stream: str) -> Optional[Dict[str, Any]]:
|
||||
|
||||
|
||||
def wait_once(target: str, until: Sequence[str] = DEFAULT_UNTIL,
|
||||
timeout_ms: Optional[int] = None, run=subprocess.run) -> WaitOutcome:
|
||||
timeout_ms: Optional[int] = None, session: Optional[str] = None,
|
||||
run=subprocess.run) -> WaitOutcome:
|
||||
"""One barrier call. `run` is injectable so tests never touch a real herdr."""
|
||||
argv = build_argv(target, until, timeout_ms)
|
||||
argv = build_argv(target, until, timeout_ms, session=session)
|
||||
try:
|
||||
proc = run(argv, capture_output=True, text=True)
|
||||
except Exception as exc:
|
||||
@@ -164,3 +169,42 @@ class ReconnectBackoff(object):
|
||||
return [self.record_failure() for _ in range(n)]
|
||||
finally:
|
||||
self.current, self.consecutive_failures, self.cached_snapshot = saved
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
parser = argparse.ArgumentParser(description="watcher CLI for herdr agent wait")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
p_wait = sub.add_parser("wait", help="wait for agent state")
|
||||
p_wait.add_argument("target", help="target agent name")
|
||||
p_wait.add_argument("--session", default=None, help="herdr session name")
|
||||
p_wait.add_argument("--until", action="append", default=[], help="until state(s)")
|
||||
p_wait.add_argument("--timeout-ms", type=int, default=None, help="timeout in ms")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
session = args.session or os.environ.get("HERDR_SESSION_NAME") or None
|
||||
until = args.until or DEFAULT_UNTIL
|
||||
outcome = wait_once(args.target, until=until, timeout_ms=args.timeout_ms, session=session)
|
||||
agent_st = None
|
||||
if outcome.agent:
|
||||
agent_st = outcome.agent.get("agent_status")
|
||||
print(json.dumps({
|
||||
"kind": outcome.kind,
|
||||
"agent_status": agent_st,
|
||||
"error_code": outcome.error_code,
|
||||
}))
|
||||
rc_map = {
|
||||
MATCHED: 0,
|
||||
TIMEOUT: 2,
|
||||
AGENT_NOT_FOUND: 3,
|
||||
TRANSPORT_ERROR: 4,
|
||||
}
|
||||
return rc_map.get(outcome.kind, 4)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.exit(main())
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: multi-agent-mux-create
|
||||
description: "Create a new agent session (claude, antigravity/agy) in a dedicated herdr session for context-preserving long-running work. Always creates a herdr session — never backgrounds with nohup/disown. Writes the new session to .mam/agent-sessions.yaml. Use when you want to start a fresh agent (no prior UUID) for a new project workspace."
|
||||
version: 4.4.0
|
||||
version: 4.4.1
|
||||
author: godopu
|
||||
license: MIT
|
||||
platforms: [linux, macos]
|
||||
|
||||
@@ -344,6 +344,29 @@ if [ -n "$SUBMIT_JOB_PROMPT" ]; then
|
||||
agent_session="herdr:$SESSION_NAME"
|
||||
DELEGATE_JOB_ID=$(delegate_submit_job "$SUBMIT_JOB_PROMPT" "$delegate_agent" "$agent_session")
|
||||
echo "Submitted delegated job: $DELEGATE_JOB_ID"
|
||||
DELEGATE_JOB_TOKEN=""
|
||||
DELEGATE_JOB_ATTEMPT=""
|
||||
if [ -n "$DELEGATE_JOB_ID" ]; then
|
||||
local_info=$(python3 -c "
|
||||
import sys, json, os
|
||||
jid = '$DELEGATE_JOB_ID'
|
||||
reg_dir = os.environ.get('MAM_JOBS_DIR', '.mam/jobs')
|
||||
p = os.path.join(reg_dir, f'{jid}.json')
|
||||
if os.path.exists(p):
|
||||
try:
|
||||
with open(p) as f:
|
||||
d = json.load(f)
|
||||
tok = d.get('dispatch_token', '')
|
||||
att = d.get('attempt_id', d.get('iteration', 1))
|
||||
print(f'{tok} {att}')
|
||||
except Exception:
|
||||
pass
|
||||
" 2>/dev/null || true)
|
||||
if [ -n "$local_info" ]; then
|
||||
DELEGATE_JOB_TOKEN=$(echo "$local_info" | awk '{print $1}')
|
||||
DELEGATE_JOB_ATTEMPT=$(echo "$local_info" | awk '{print $2}')
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f "$AGENT_SESSIONS_YAML" ]; then
|
||||
@@ -494,11 +517,11 @@ Task: $SUBMIT_JOB_PROMPT"
|
||||
rc=0
|
||||
inject_instructions "$SESSION_NAME" "$instructions" "$DELEGATE_JOB_ID" || rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" error "instruction injection failed (prompt-lock, rc=$rc)"
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" error "instruction injection failed (prompt-lock, rc=$rc)" "$DELEGATE_JOB_TOKEN" "$DELEGATE_JOB_ATTEMPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" started "multi-agent-mux session created and instructions injected"
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" started "multi-agent-mux session created and instructions injected" "$DELEGATE_JOB_TOKEN" "$DELEGATE_JOB_ATTEMPT"
|
||||
# Trigger immediate priority reconcile cycle asynchronously
|
||||
(bash "$WORKSPACE/.agents/skills/multi-agent-mux-monitor/scripts/reconcile.sh" --once >/dev/null 2>&1 &) || true
|
||||
WD_PID=$(start_watchdog "$DELEGATE_JOB_ID" "$WORKSPACE")
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: multi-agent-mux-delegate-job
|
||||
description: "Delegate a unit of work to any autonomous agent (claude-code, hermes, agy, grok-build, codex, or a human) and observe it asynchronously over an MQTT event channel. Supported roles include orchestrator, worker, and reviewer."
|
||||
version: 4.4.0
|
||||
version: 4.4.1
|
||||
author: godopu
|
||||
license: MIT
|
||||
platforms: [linux, macos, windows]
|
||||
@@ -70,6 +70,22 @@ For detailed state machine diagrams and configurations, see [DELEGATION_TYPES.md
|
||||
|
||||
---
|
||||
|
||||
## Exit Codes & Failure Handling
|
||||
|
||||
Under fail-closed operation (`MAM_DELEGATE_FAIL_CLOSED=1`, default), the CLI returns deterministic exit codes:
|
||||
|
||||
| Exit Code | Meaning | Final Registry Status | Failure Behavior & Resilience |
|
||||
|:---:|:---:|:---:|---|
|
||||
| `0` | Success | `completed` | Job succeeded and completed event received or verified on disk with approval (`PASS`/`AGREE`). |
|
||||
| `1` | Failure | `error` | Execution failure / Agent injection error (`inject_agent`). |
|
||||
| `2` | Timeout | `error` (`deadline_timeout`) | Subscriber timeout (total budget or idle timeout reached without terminal event). **R1 Working-Agent Resilience**: If the agent in Herdr is still actively `working`, the wrapper does not immediately fail-close; it re-observes disk status at intervals up to the wall-clock `--timeout` deadline before fail-closing. |
|
||||
| `3` | Broker Error | `error` (`deadline_timeout`) | Broker unavailable (`MQTT_BROKER_UNAVAILABLE`) and terminal status not found on disk during deadline. |
|
||||
| `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed (prompt never delivered, retryable). |
|
||||
|
||||
In all execution paths, the final line of standard output prints the persistent audit log directory path (`<logs_root>/<job_id>`), fulfilling the parsing contract for orchestrator runners.
|
||||
|
||||
---
|
||||
|
||||
## Where the Implementation Lives (v4.3.0)
|
||||
|
||||
The implementation moved to the `lib_py.delegation` package in issue #5 phase 1.
|
||||
|
||||
@@ -125,6 +125,10 @@ parse_opts() {
|
||||
done
|
||||
}
|
||||
|
||||
_to_lower() {
|
||||
printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
|
||||
cmd_submit() {
|
||||
parse_opts "$@"
|
||||
[[ -n "$PROMPT" ]] || { echo "submit requires --prompt" >&2; exit 1; }
|
||||
@@ -159,12 +163,56 @@ EOF
|
||||
echo "provisioned job directory: $job_dir"
|
||||
fi
|
||||
|
||||
local START_TIME
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
if [[ "$TYPE" == "direct" ]]; then
|
||||
# 2) START THE SUBSCRIBER FIRST (ordering dependency — MQTT does not queue
|
||||
# non-retained messages for absent subscribers).
|
||||
# 1) Read dispatch token and attempt id for this dispatch
|
||||
local _disp_tok=""
|
||||
local _attempt_id=""
|
||||
_disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
||||
_attempt_id="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('attempt_id','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
||||
local pub_extra=()
|
||||
[ -n "$_disp_tok" ] && pub_extra+=(--expect-dispatch-token "$_disp_tok")
|
||||
[ -n "$_attempt_id" ] && pub_extra+=(--attempt-id "$_attempt_id")
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${pub_extra[*]}"
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
run_agent "$JOB_ID" "$instructions"
|
||||
local logs_root_dry="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root_dry/$JOB_ID"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 1) Preflight
|
||||
if ! preflight_agent "$JOB_ID" "$AGENT_SESSION" 0; then
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "preflight"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 5
|
||||
fi
|
||||
|
||||
# 2) TUI idle handshake
|
||||
if [ "$AGENT" != "human" ]; then
|
||||
local idle_rc=0
|
||||
wait_for_agent_idle "$AGENT_SESSION" || idle_rc=$?
|
||||
export _SKS_IDLE_RC="$idle_rc"
|
||||
if [ $idle_rc -eq 1 ] || [ $idle_rc -eq 2 ]; then
|
||||
finalize_unfinished "$JOB_ID" "error" "$idle_rc" "idle_wait"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 5
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3) START THE SUBSCRIBER
|
||||
local logf="$REGISTRY_DIR/$JOB_ID.subscriber.out"
|
||||
local sub_extra_args=()
|
||||
[ -n "$_disp_tok" ] && sub_extra_args+=(--expect-dispatch-token "$_disp_tok")
|
||||
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
||||
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
||||
${sub_extra_args[@]+"${sub_extra_args[@]}"} \
|
||||
>"$logf" 2>&1 &
|
||||
local sub_pid=$!
|
||||
echo "subscriber pid: $sub_pid (log: $logf)"
|
||||
@@ -177,14 +225,23 @@ EOF
|
||||
break
|
||||
fi
|
||||
else
|
||||
wait "$sub_pid" 2>/dev/null
|
||||
local sub_exit=$?
|
||||
local sub_exit=0
|
||||
wait "$sub_pid" 2>/dev/null || sub_exit=$?
|
||||
if [ $sub_exit -eq 0 ] || [ $sub_exit -eq 3 ]; then
|
||||
sub_ready=1
|
||||
break
|
||||
else
|
||||
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
|
||||
exit 1
|
||||
local mapped_st
|
||||
mapped_st="$(_map_sub_rc "$sub_exit" "$JOB_ID")"
|
||||
finalize_unfinished "$JOB_ID" "$mapped_st" "$sub_exit" "subscriber"
|
||||
mapped_st="${FINAL_JOB_STATUS:-$mapped_st}"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$mapped_st")"
|
||||
fi
|
||||
fi
|
||||
sleep 0.2
|
||||
@@ -193,17 +250,16 @@ EOF
|
||||
echo "WARNING: subscriber subscribe handshake timed out — falling back to proceed" >&2
|
||||
fi
|
||||
|
||||
# 3) run the agent (or print the command for dry-run / missing binary)
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID"
|
||||
# NOTE: the agent MUST use --job "$JOB_ID" (the one we just minted). Hard-coding
|
||||
# an id from an earlier session is the #1 reason a delegated job sits idle and
|
||||
# times out (see SKILL.md "Wrong job_id propagated to the agent"). We make the
|
||||
# freshness explicit in the instruction header.
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
# 5) Inject agent
|
||||
if ! inject_agent "$JOB_ID" "$instructions" "$AGENT_SESSION"; then
|
||||
reap_subscriber "$sub_pid"
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "inject"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_agent "$JOB_ID" "$instructions"
|
||||
|
||||
# 4) optional validation hook
|
||||
# 6) optional validation hook
|
||||
if [[ -n "$VALIDATE" ]]; then
|
||||
echo "running validation: $VALIDATE"
|
||||
if JOB_ID="$JOB_ID" REGISTRY_DIR="$REGISTRY_DIR" bash "$VALIDATE"; then
|
||||
@@ -214,23 +270,24 @@ EOF
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
# In dry-run we never started a real subscriber (the wrapper short-circuits
|
||||
# before launching one), but the wait below would still try to join the
|
||||
# background sub_pid from cmd_submit. Skip both the wait and the subscriber
|
||||
# log dump; the user just wants to see the instruction that would have run.
|
||||
local logs_root_dry="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root_dry/$JOB_ID"
|
||||
return 0
|
||||
fi
|
||||
|
||||
wait "$sub_pid" || true
|
||||
local sub_rc=0
|
||||
wait "$sub_pid" || sub_rc=$?
|
||||
echo "subscriber output:"; cat "$logf" || true
|
||||
|
||||
local job_status
|
||||
job_status="$(_map_sub_rc "$sub_rc" "$JOB_ID")"
|
||||
|
||||
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "direct"
|
||||
job_status="${FINAL_JOB_STATUS:-$job_status}"
|
||||
|
||||
# Last stdout line: the persistent audit-log dir for this job (see SKILL.md
|
||||
# "Audit Logs"). Callers can scrape `tail -n1` to find it.
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$job_status")"
|
||||
else
|
||||
# Implement loop/discuss orchestrator
|
||||
local iteration=1
|
||||
@@ -238,6 +295,8 @@ EOF
|
||||
local current_session="$AGENT_SESSION"
|
||||
local _phase="worker"
|
||||
local display_role="$DELEGATE_ROLE"
|
||||
local attempt=0
|
||||
local final_loop_rc=0
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
echo "[dry-run] orchestrator loop would start for job: $JOB_ID type: $TYPE"
|
||||
@@ -256,9 +315,8 @@ EOF
|
||||
# 1-1) Provision job directory and write iteration brief.md (MAM Job Restructuring)
|
||||
local job_dir="$REGISTRY_DIR/$JOB_ID"
|
||||
local clean_session="${current_session#herdr:}"
|
||||
if [[ "$DRY_RUN" != "1" ]]; then
|
||||
mkdir -p "$job_dir"
|
||||
cat <<EOF > "$job_dir/brief.md"
|
||||
mkdir -p "$job_dir"
|
||||
cat <<EOF > "$job_dir/brief.md"
|
||||
# 📋 Brief: Job $JOB_ID Delegation (Iteration $iteration)
|
||||
|
||||
- **Job ID**: $JOB_ID
|
||||
@@ -270,7 +328,40 @@ EOF
|
||||
## 🔎 Task Description
|
||||
$current_prompt
|
||||
EOF
|
||||
echo "provisioned iteration brief: $job_dir/brief.md"
|
||||
echo "provisioned iteration brief: $job_dir/brief.md"
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
|
||||
local force_warn_only=0
|
||||
local cp_lower def_lower
|
||||
cp_lower=$(_to_lower "$COUNTERPART_ROLE")
|
||||
def_lower=$(_to_lower "$DEFAULT_COUNTERPART_ROLE")
|
||||
if [[ "$_phase" == "reviewer" && "$COUNTERPART_ROLE_EXPLICIT" -eq 1 \
|
||||
&& "$cp_lower" != "$def_lower" ]]; then
|
||||
force_warn_only=1
|
||||
fi
|
||||
|
||||
# Preflight
|
||||
if ! preflight_agent "$JOB_ID" "$current_session" "$force_warn_only"; then
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "preflight"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 5
|
||||
fi
|
||||
|
||||
# Idle wait
|
||||
local cur_agent="$AGENT"
|
||||
[ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER"
|
||||
if [ "$cur_agent" != "human" ]; then
|
||||
local idle_rc=0
|
||||
wait_for_agent_idle "$current_session" || idle_rc=$?
|
||||
export _SKS_IDLE_RC="$idle_rc"
|
||||
if [ $idle_rc -eq 1 ] || [ $idle_rc -eq 2 ]; then
|
||||
finalize_unfinished "$JOB_ID" "error" "$idle_rc" "idle_wait"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 5
|
||||
fi
|
||||
fi
|
||||
|
||||
# Update job details in registry
|
||||
@@ -279,13 +370,20 @@ EOF
|
||||
--agent-session "$current_session" \
|
||||
--prompt "$current_prompt" \
|
||||
--iteration "$iteration" \
|
||||
--attempt-id "$attempt" \
|
||||
--role "$display_role" \
|
||||
--status "pending"
|
||||
|
||||
local _disp_tok=""
|
||||
_disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
||||
|
||||
# Start subscriber
|
||||
local logf="$REGISTRY_DIR/${JOB_ID}.iter_${iteration}_${display_role}.subscriber.out"
|
||||
local loop_sub_args=(--attempt-id "$attempt")
|
||||
[ -n "$_disp_tok" ] && loop_sub_args+=(--expect-dispatch-token "$_disp_tok")
|
||||
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
||||
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
||||
"${loop_sub_args[@]}" \
|
||||
>"$logf" 2>&1 &
|
||||
local sub_pid=$!
|
||||
echo "subscriber pid: $sub_pid (log: $logf)"
|
||||
@@ -298,14 +396,23 @@ EOF
|
||||
break
|
||||
fi
|
||||
else
|
||||
wait "$sub_pid" 2>/dev/null
|
||||
local sub_exit=$?
|
||||
if [ $sub_exit -eq 0 ]; then
|
||||
local sub_exit=0
|
||||
wait "$sub_pid" 2>/dev/null || sub_exit=$?
|
||||
if [ $sub_exit -eq 0 ] || [ $sub_exit -eq 3 ]; then
|
||||
sub_ready=1
|
||||
break
|
||||
else
|
||||
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
|
||||
exit 1
|
||||
local mapped_st
|
||||
mapped_st="$(_map_sub_rc "$sub_exit" "$JOB_ID")"
|
||||
finalize_unfinished "$JOB_ID" "$mapped_st" "$sub_exit" "subscriber"
|
||||
mapped_st="${FINAL_JOB_STATUS:-$mapped_st}"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$mapped_st")"
|
||||
fi
|
||||
fi
|
||||
sleep 0.2
|
||||
@@ -315,38 +422,27 @@ EOF
|
||||
fi
|
||||
|
||||
# Format instruction block
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID"
|
||||
local loop_pub_extra=(--attempt-id "$attempt")
|
||||
[ -n "$_disp_tok" ] && loop_pub_extra+=(--expect-dispatch-token "$_disp_tok")
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${loop_pub_extra[*]}"
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths for iteration $iteration are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
|
||||
# Trigger agent
|
||||
local force_warn_only=0
|
||||
if [[ "$_phase" == "reviewer" && "$COUNTERPART_ROLE_EXPLICIT" -eq 1 \
|
||||
&& "${COUNTERPART_ROLE,,}" != "${DEFAULT_COUNTERPART_ROLE,,}" ]]; then
|
||||
force_warn_only=1
|
||||
if ! inject_agent "$JOB_ID" "$instructions" "$current_session"; then
|
||||
reap_subscriber "$sub_pid"
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "inject"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 1
|
||||
fi
|
||||
run_agent "$JOB_ID" "$instructions" "$current_session" "$force_warn_only"
|
||||
|
||||
# Wait for subscriber
|
||||
local sub_rc=0
|
||||
wait "$sub_pid" || sub_rc=$?
|
||||
echo "subscriber output:"; cat "$logf" || true
|
||||
|
||||
# Check job status based on subscriber exit code
|
||||
local job_status="running"
|
||||
if [[ $sub_rc -eq 0 ]]; then
|
||||
job_status="completed"
|
||||
elif [[ $sub_rc -eq 1 ]]; then
|
||||
job_status="error"
|
||||
elif [[ $sub_rc -eq 3 ]]; then
|
||||
job_status="broker_unavailable"
|
||||
local disk_st
|
||||
disk_st="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
||||
if [[ "$disk_st" == "completed" || "$disk_st" == "error" ]]; then
|
||||
job_status="$disk_st"
|
||||
fi
|
||||
else
|
||||
job_status="timeout"
|
||||
fi
|
||||
local job_status
|
||||
job_status="$(_map_sub_rc "$sub_rc" "$JOB_ID")"
|
||||
|
||||
echo "Job role $display_role finished with status: $job_status"
|
||||
|
||||
@@ -354,11 +450,22 @@ EOF
|
||||
local feedback
|
||||
feedback="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get-feedback --job "$JOB_ID")"
|
||||
echo "Feedback/Detail: $feedback"
|
||||
|
||||
if [[ "$_phase" == "worker" ]]; then
|
||||
if [[ "$job_status" != "completed" ]]; then
|
||||
echo "Worker did not complete successfully (status: $job_status). Terminating workflow."
|
||||
break
|
||||
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "worker" "$current_session"
|
||||
job_status="${FINAL_JOB_STATUS:-$job_status}"
|
||||
if [[ "$job_status" == "completed" ]]; then
|
||||
echo "Worker completed during grace period. Continuing to reviewer."
|
||||
feedback="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get-feedback --job "$JOB_ID")"
|
||||
else
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$job_status")"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Worker completed successfully, now switch to reviewer
|
||||
@@ -374,30 +481,46 @@ EOF
|
||||
else
|
||||
if [[ "$job_status" != "completed" ]]; then
|
||||
echo "Reviewer did not complete successfully (status: $job_status). Terminating workflow."
|
||||
break
|
||||
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "reviewer" "$current_session"
|
||||
job_status="${FINAL_JOB_STATUS:-$job_status}"
|
||||
if [[ "$job_status" == "completed" ]]; then
|
||||
echo "Reviewer completed during grace period. Continuing evaluation."
|
||||
feedback="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get-feedback --job "$JOB_ID")"
|
||||
else
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$job_status")"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
# Reviewer finished. Check if pass/agree
|
||||
local success=0
|
||||
local fb_lower
|
||||
fb_lower=$(_to_lower "$feedback")
|
||||
if [[ "$TYPE" == "loop" ]]; then
|
||||
if [[ "${feedback,,}" == *"pass"* ]]; then
|
||||
if [[ "$fb_lower" == *"pass"* ]]; then
|
||||
success=1
|
||||
fi
|
||||
elif [[ "$TYPE" == "discuss" ]]; then
|
||||
if [[ "${feedback,,}" == *"agree"* ]]; then
|
||||
if [[ "$fb_lower" == *"agree"* ]]; then
|
||||
success=1
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
if [[ "$success" == "1" ]]; then
|
||||
echo "Reviewer approved the work. Finalizing job as completed."
|
||||
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" status --job "$JOB_ID" --set "completed"
|
||||
final_loop_rc=0
|
||||
break
|
||||
else
|
||||
# Reviewer rejected/provided feedback. Increment & check max iterations
|
||||
if [[ $iteration -ge $MAX_ITERATIONS ]]; then
|
||||
echo "Max iterations ($MAX_ITERATIONS) reached without approval. Terminating workflow."
|
||||
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" status --job "$JOB_ID" --set "error"
|
||||
final_loop_rc=1
|
||||
break
|
||||
fi
|
||||
|
||||
@@ -418,40 +541,216 @@ EOF
|
||||
else
|
||||
local rc=$?
|
||||
echo "validation: FAIL (exit $rc)"
|
||||
final_loop_rc=$rc
|
||||
fi
|
||||
fi
|
||||
|
||||
# Last stdout line: the persistent audit-log dir
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit "$final_loop_rc"
|
||||
fi
|
||||
}
|
||||
|
||||
run_agent() {
|
||||
local job_id="$1"; local instructions="$2"; local target_session="${3:-$AGENT_SESSION}"; local force_warn_only="${4:-0}"
|
||||
# The skill is INTERACTIVE-ONLY. We never invoke `claude -p` or any other
|
||||
# one-shot print mode, because:
|
||||
# - claude -p exits the moment stdin is drained, so there's nothing to
|
||||
# `herdr session attach` to afterwards.
|
||||
# - fire-and-forget via wrapper defeats the whole point of the audit log
|
||||
# (you can't tell what happened if the agent crashes mid-turn).
|
||||
# - the job registry already gives us an authoritative completion signal,
|
||||
# so we don't need a wrapper-side exit code to know "done".
|
||||
# The user attaches with `herdr session attach <session>` and types follow-up
|
||||
# prompts themselves. We pre-load the first prompt via stdin and `read`
|
||||
# keeps the pane open after the agent exits so the user can review.
|
||||
reap_subscriber() {
|
||||
if [ -n "${1:-}" ]; then
|
||||
kill "$1" 2>/dev/null || true
|
||||
wait "$1" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
_disk_status() {
|
||||
local jid="$1"
|
||||
"$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$jid.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true
|
||||
}
|
||||
|
||||
_map_sub_rc() {
|
||||
local sub_rc="$1" job_id="$2"
|
||||
local job_status="running"
|
||||
if [[ $sub_rc -eq 0 ]]; then
|
||||
job_status="completed"
|
||||
elif [[ $sub_rc -eq 1 ]]; then
|
||||
job_status="error"
|
||||
elif [[ $sub_rc -eq 3 ]]; then
|
||||
job_status="broker_unavailable"
|
||||
local disk_st
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [[ "$disk_st" == "completed" || "$disk_st" == "error" || "$disk_st" == "cancelled" ]]; then
|
||||
job_status="$disk_st"
|
||||
fi
|
||||
else
|
||||
job_status="timeout"
|
||||
fi
|
||||
echo "$job_status"
|
||||
}
|
||||
|
||||
_rc_for() {
|
||||
case "$1" in
|
||||
completed) echo 0 ;;
|
||||
error) echo 1 ;;
|
||||
timeout) echo 2 ;;
|
||||
broker_unavailable) echo 3 ;;
|
||||
cancelled) echo 5 ;;
|
||||
*) echo 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
finalize_unfinished() {
|
||||
local job_id="$1" job_status="$2" sub_rc="$3" phase="${4:-}"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "$job_status" = "completed" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local cur_disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$cur_disk_st" = "completed" ] || [ "$cur_disk_st" = "error" ] || [ "$cur_disk_st" = "cancelled" ]; then
|
||||
FINAL_JOB_STATUS="$cur_disk_st"
|
||||
[ "$cur_disk_st" = "completed" ] && return 0
|
||||
else
|
||||
FINAL_JOB_STATUS="$job_status"
|
||||
fi
|
||||
# broker_unavailable: re-observe disk for remaining deadline at 3s intervals
|
||||
if [ "$job_status" = "broker_unavailable" ]; then
|
||||
local now elapsed remaining
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - ${START_TIME:-now} ))
|
||||
remaining=$(( TIMEOUT - elapsed ))
|
||||
[ $remaining -lt 0 ] && remaining=0
|
||||
while [ $remaining -gt 0 ]; do
|
||||
local wait_interval=3
|
||||
[ $remaining -lt $wait_interval ] && wait_interval=$remaining
|
||||
sleep "$wait_interval"
|
||||
remaining=$(( remaining - wait_interval ))
|
||||
local disk_st
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
FINAL_JOB_STATUS="$disk_st"
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# timeout: if subscriber timed out, check if agent is still actively working in herdr
|
||||
if [ "$job_status" = "timeout" ]; then
|
||||
local target_sess="${5:-${current_session:-${target_session:-$AGENT_SESSION}}}"
|
||||
local sess="${target_sess#herdr:}"
|
||||
local target_agent
|
||||
target_agent=$(_sanitize_herdr_agent_name "$sess")
|
||||
local agent_st=""
|
||||
if [ "$sess" != "human" ] && [ -n "$sess" ]; then
|
||||
local raw_st
|
||||
raw_st="$(_sks_herdr agent get "$target_agent" 2>/dev/null || _sks_herdr agent get "$sess" 2>/dev/null || true)"
|
||||
if [ -n "$raw_st" ]; then
|
||||
agent_st="$("$(_delegate_py_bin)" -c '
|
||||
import sys, json
|
||||
try:
|
||||
d = json.loads(sys.stdin.read())
|
||||
print((d.get("result", {}).get("agent", {}) or {}).get("agent_status", "") or "")
|
||||
except Exception:
|
||||
print("")
|
||||
' <<< "$raw_st" 2>/dev/null || true)"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$agent_st" = "working" ]; then
|
||||
local now elapsed remaining
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - ${START_TIME:-now} ))
|
||||
remaining=$(( TIMEOUT - elapsed ))
|
||||
[ $remaining -lt 0 ] && remaining=0
|
||||
while [ $remaining -gt 0 ]; do
|
||||
local disk_st
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
FINAL_JOB_STATUS="$disk_st"
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
break
|
||||
fi
|
||||
local cur_raw cur_st=""
|
||||
cur_raw="$(_sks_herdr agent get "$target_agent" 2>/dev/null || _sks_herdr agent get "$sess" 2>/dev/null || true)"
|
||||
if [ -n "$cur_raw" ]; then
|
||||
cur_st="$("$(_delegate_py_bin)" -c '
|
||||
import sys, json
|
||||
try:
|
||||
d = json.loads(sys.stdin.read())
|
||||
print((d.get("result", {}).get("agent", {}) or {}).get("agent_status", "") or "")
|
||||
except Exception:
|
||||
print("")
|
||||
' <<< "$cur_raw" 2>/dev/null || true)"
|
||||
fi
|
||||
if [ "$cur_st" != "working" ]; then
|
||||
sleep 1
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
FINAL_JOB_STATUS="$disk_st"
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
fi
|
||||
break
|
||||
fi
|
||||
local wait_interval=1
|
||||
[ $remaining -lt $wait_interval ] && wait_interval=$remaining
|
||||
sleep "$wait_interval"
|
||||
remaining=$(( remaining - wait_interval ))
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
local close_req="deadline_timeout"
|
||||
local detail=""
|
||||
if [ "$phase" = "preflight" ] || [ "$phase" = "idle_wait" ]; then
|
||||
close_req="cancelled"
|
||||
detail="session_busy"
|
||||
fi
|
||||
|
||||
local close_args=(--job "$job_id" --request "$close_req")
|
||||
[ -n "$detail" ] && close_args+=(--detail "$detail")
|
||||
[ -n "${_disp_tok:-}" ] && close_args+=(--expect-dispatch-token "$_disp_tok")
|
||||
|
||||
local close_rc=0
|
||||
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" close "${close_args[@]}" 2>&1 || close_rc=$?
|
||||
|
||||
# If close failed due to dispatch token mismatch (rc=4), suppress secondary event emission
|
||||
# to prevent CAS/ABA bypass
|
||||
if [ $close_rc -eq 4 ]; then
|
||||
echo "finalize_unfinished: dispatch token mismatch (stale attempt); suppressing event emission" >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check if job completed on disk (universal, for all TYPEs)
|
||||
local cur_disk_st
|
||||
cur_disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$cur_disk_st" = "completed" ]; then
|
||||
FINAL_JOB_STATUS="completed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# direct path only: publish MQTT error/cancelled event (loop/discuss forbidden: O-1)
|
||||
if [ "$TYPE" = "direct" ]; then
|
||||
local disp_arg=()
|
||||
[ -n "${_disp_tok:-}" ] && disp_arg=(--expect-dispatch-token "$_disp_tok")
|
||||
if [ "$close_req" = "cancelled" ]; then
|
||||
"$PY" "$SCRIPT_DIR/scripts/publish_event.py" --registry-dir "$REGISTRY_DIR" --job "$job_id" --event cancelled --detail "session_busy" ${disp_arg[@]+"${disp_arg[@]}"} 2>/dev/null || true
|
||||
elif [ "$sub_rc" -ne 1 ] || [ "$phase" = "inject" ]; then
|
||||
local err_detail="deadline_timeout"
|
||||
[ "$phase" = "inject" ] && err_detail="agent_blocked"
|
||||
"$PY" "$SCRIPT_DIR/scripts/publish_event.py" --registry-dir "$REGISTRY_DIR" --job "$job_id" --event error --detail "$err_detail" ${disp_arg[@]+"${disp_arg[@]}"} 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
preflight_agent() {
|
||||
local job_id="$1" target_session="${2:-$AGENT_SESSION}" force_warn_only="${3:-0}"
|
||||
if [ "$AGENT" = "human" ]; then
|
||||
echo "[human agent] complete the task, then run publish_event.py --event completed"
|
||||
return
|
||||
return 0
|
||||
fi
|
||||
local sess="${target_session#herdr:}"
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
echo "[dry-run] would delegate task to running agent '$AGENT' in herdr session '$sess' with instructions:"
|
||||
echo "----"; echo "$instructions"; echo "----"
|
||||
return
|
||||
fi
|
||||
|
||||
# B-3: must probe for the real binary, not the herdr() function / shim wrapper.
|
||||
if ! has_real_herdr; then
|
||||
echo "ERROR: this skill requires herdr (interactive agent sessions)." >&2
|
||||
@@ -459,10 +758,6 @@ run_agent() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Auto-resolve isolation the same way resume/stop/create do — don't rely on
|
||||
# the caller having exported HERDR_SERVER_NAME by hand. This is what lets
|
||||
# delegation reach an agent living in an isolated herdr session (e.g. one
|
||||
# created with --herdr-server) instead of silently looking in "default".
|
||||
export HERDR_SESSION_NAME="$(resolve_herdr_session "$sess" "$WORKDIR")"
|
||||
|
||||
if ! herdr has-session -t "$sess" 2>/dev/null; then
|
||||
@@ -516,25 +811,45 @@ else:
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
inject_agent() {
|
||||
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}"
|
||||
if [ "$AGENT" = "human" ]; then
|
||||
echo "[human agent] complete the task, then run publish_event.py --event completed"
|
||||
return 0
|
||||
fi
|
||||
local sess="${target_session#herdr:}"
|
||||
|
||||
# Before launching the agent, set up error trap to publish error event
|
||||
if [ -n "${job_id:-}" ] && [ -n "${PY:-}" ]; then
|
||||
pub_script="$SCRIPT_DIR/scripts/publish_event.py"
|
||||
trap "rc=\$?; if [ \$rc -ne 0 ]; then \"$PY\" \"$pub_script\" --job '$job_id' --event error --detail 'agent bootstrap failed (exit '\$rc')'; fi" EXIT
|
||||
trap "rc=\$?; if [ \$rc -ne 0 ]; then \"$PY\" \"$pub_script\" --registry-dir '$REGISTRY_DIR' --job '$job_id' --event error --detail 'agent bootstrap failed (exit '\$rc')'; fi" EXIT
|
||||
fi
|
||||
|
||||
echo "살아있는 에이전트 세션 '$sess'에 작업을 위임합니다..."
|
||||
if ! send_keys_safe "$sess" "$instructions" "$job_id"; then
|
||||
trap - EXIT
|
||||
echo "ERROR: 프롬프트 주입 실패 — 세션 '$sess' (프롬프트 잠금 의심)" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# NOTE: `herdr session attach` operates on whole herdr *sessions* (server
|
||||
# instances), not an individual agent by its MAM name — `agent attach` is
|
||||
# the real command for that. HERDR_SESSION_NAME is inlined so the printed
|
||||
# command is copy-pasteable in a fresh shell that hasn't sourced lib.sh.
|
||||
|
||||
echo "작업이 세션 '$sess'에 전송되었습니다. (연결하려면: HERDR_SESSION_NAME=$HERDR_SESSION_NAME herdr agent attach $sess — lib.sh를 source한 셸에서 실행)"
|
||||
trap - EXIT
|
||||
return 0
|
||||
}
|
||||
|
||||
run_agent() {
|
||||
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}" force_warn_only="${4:-0}"
|
||||
local sess="${target_session#herdr:}"
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
echo "[dry-run] would delegate task to running agent '$AGENT' in herdr session '$sess' with instructions:"
|
||||
echo "----"; echo "$instructions"; echo "----"
|
||||
return 0
|
||||
fi
|
||||
preflight_agent "$job_id" "$target_session" "$force_warn_only" || return $?
|
||||
inject_agent "$job_id" "$instructions" "$target_session" || return $?
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: multi-agent-mux-loop
|
||||
description: "Run an autonomous planning-execution-review loop using multiple agents (Planner, Creator, Reviewers) in the workspace. Automatically orchestrates plan discussion, code changes, and peer reviews until a unanimous PASS is achieved or the maximum iteration limit is reached."
|
||||
version: 4.4.0
|
||||
version: 4.4.1
|
||||
author: godopu
|
||||
license: MIT
|
||||
platforms: [linux, macos]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: multi-agent-mux-monitor
|
||||
description: "Run a long-lived reconciler that watches .mam/agent-sessions.yaml against the actual herdr/agent runtime state and reconciles them. Use when you want live visibility into which agent sessions are running, which are dead, which have stale YAML entries, and which have new session ids that haven't been recorded yet. Runs as a persistent loop (`reconcile.sh --subscribe`) that keeps going until it times out, idles out, or is interrupted."
|
||||
version: 4.4.0
|
||||
version: 4.4.1
|
||||
author: godopu
|
||||
license: MIT
|
||||
platforms: [linux, macos]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: multi-agent-mux-orc-onboard
|
||||
description: "Register current or specified orchestrator session UUID into agent-sessions.yaml orchestrator_uuids list to prevent sub-agent discovery capture."
|
||||
version: 4.4.0
|
||||
version: 4.4.1
|
||||
author: godopu
|
||||
license: MIT
|
||||
platforms: [linux, macos]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: multi-agent-mux-resume
|
||||
description: "Resume an existing agent (claude, antigravity/agy) conversation by UUID into a herdr session. Reads .mam/agent-sessions.yaml for the saved session/conversation id, spawns (or reuses) a herdr session of the matching name, and runs `claude -r <id>` or `agy --conversation <id>` inside. Use when you want to reattach to a previous session's context, or revive a session whose herdr died but the agent's conversation is still on disk."
|
||||
version: 4.4.0
|
||||
version: 4.4.1
|
||||
author: godopu
|
||||
license: MIT
|
||||
platforms: [linux, macos]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: multi-agent-mux-status
|
||||
description: "Read-only instant snapshot of all agent herdr sessions — name, YAML status, herdr alive, pane cmd/cwd, resume UUID on disk, and any drift. No mutation. Reuses reconcile.sh --dry-run for the diff logic. Use when you want to know 'what's running RIGHT NOW' without spinning up the monitor loop."
|
||||
version: 4.4.0
|
||||
version: 4.4.1
|
||||
author: godopu
|
||||
license: MIT
|
||||
platforms: [linux, macos]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: multi-agent-mux-stop
|
||||
description: "Stop an agent herdr session (claude, antigravity/agy) and update .mam/agent-sessions.yaml. Default stops gracefully and marks status=stopped with conversation preserved for resume. Does NOT delete on-disk conversation artifacts (jsonl/db) — those are preserved unless --purge-conversation is passed. Use when ending a work session, switching to a different one, or cleaning up before a fresh start."
|
||||
version: 4.4.0
|
||||
version: 4.4.1
|
||||
author: godopu
|
||||
license: MIT
|
||||
platforms: [linux, macos]
|
||||
|
||||
@@ -142,6 +142,29 @@ sys.exit(7)
|
||||
|
||||
TARGET_CWD=$(printf '%s' "$MAPPED_DATA" | python3 -c 'import sys,json; print(json.load(sys.stdin).get("cwd",""))')
|
||||
DELEGATE_JOB_ID=$(printf '%s' "$MAPPED_DATA" | python3 -c 'import sys,json; print(json.load(sys.stdin).get("job_id",""))')
|
||||
DELEGATE_JOB_TOKEN=""
|
||||
DELEGATE_JOB_ATTEMPT=""
|
||||
if [ -n "$DELEGATE_JOB_ID" ]; then
|
||||
local_info=$(python3 -c "
|
||||
import sys, json, os
|
||||
jid = '$DELEGATE_JOB_ID'
|
||||
reg_dir = os.environ.get('MAM_JOBS_DIR', '.mam/jobs')
|
||||
p = os.path.join(reg_dir, f'{jid}.json')
|
||||
if os.path.exists(p):
|
||||
try:
|
||||
with open(p) as f:
|
||||
d = json.load(f)
|
||||
tok = d.get('dispatch_token', '')
|
||||
att = d.get('attempt_id', d.get('iteration', 1))
|
||||
print(f'{tok} {att}')
|
||||
except Exception:
|
||||
pass
|
||||
" 2>/dev/null || true)
|
||||
if [ -n "$local_info" ]; then
|
||||
DELEGATE_JOB_TOKEN=$(echo "$local_info" | awk '{print $1}')
|
||||
DELEGATE_JOB_ATTEMPT=$(echo "$local_info" | awk '{print $2}')
|
||||
fi
|
||||
fi
|
||||
|
||||
# 멱등성: STOP 모드에서 이미 stopped 인 세션이고 purge 가 아닐 때만 no-op + exit 0
|
||||
if [ "$STOP_MODE" = "1" ] && [ "$PURGE" != "1" ]; then
|
||||
@@ -190,7 +213,7 @@ if [ "$CAPTURE_ID" = "1" ] && [ -n "$TARGET_CWD" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" progress "terminating"
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" progress "terminating" "$DELEGATE_JOB_TOKEN" "$DELEGATE_JOB_ATTEMPT"
|
||||
|
||||
# graceful 종료: send-keys 로 정상 종료 유도 → 폴백 체인 (SIGTERM → SIGKILL).
|
||||
graceful_stop() {
|
||||
@@ -239,7 +262,7 @@ if [ "$PURGE" = "1" ] && [ "$HERDR_ALIVE" = "1" ]; then
|
||||
echo " Refusing registry removal — records preserved (no state was modified)." >&2
|
||||
echo " Diagnose the stuck TUI (herdr session attach '$SESSION_NAME'), then re-run" >&2
|
||||
echo " stop_session.sh --purge-conversation --yes (retry is safe/idempotent)." >&2
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" cancelled "purge aborted: session still alive; no state was modified"
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" cancelled "purge aborted: session still alive; no state was modified" "$DELEGATE_JOB_TOKEN" "$DELEGATE_JOB_ATTEMPT"
|
||||
exit 4
|
||||
fi
|
||||
fi
|
||||
@@ -319,7 +342,7 @@ else:
|
||||
print(f"updated: {name} status={target['status']}", flush=True)
|
||||
PYEOF
|
||||
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" cancelled "session stopped by operator before job completion"
|
||||
delegate_publish_event "$DELEGATE_JOB_ID" cancelled "session stopped by operator before job completion" "$DELEGATE_JOB_TOKEN" "$DELEGATE_JOB_ATTEMPT"
|
||||
|
||||
echo
|
||||
echo "=== stop complete ==="
|
||||
|
||||
Reference in new issue
Block a user