GodopuandClaude Opus 5.5
ed293dcef4
- mam_herdr: for name lookups (agent / has-session / kill-session) a HERDR_WORKSPACE_ID that is the caller pane's own workspace (it matches the HERDR_PANE_ID prefix) is dropped. Live agent names are unique per server (herdr --skill), so this cannot address a same-named agent of another workspace; an id the caller chose, or one set outside a pane, still scopes (H-18/H-21/H-23). The generated shim is unchanged (S11). - wrapper: a preflight that cannot see the target closes the job cancelled with detail target_not_found instead of session_busy; the loop's sub-job table counts it as NOT-DISPATCHED. - tests: T10-T12 with the measured pane env (same-server socket, caller w6, target w2); T8 follows the sub-job-armed guard. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1161 lines
49 KiB
Bash
Executable File
1161 lines
49 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# multi-agent-mux-delegate-job — user-facing orchestrator for the multi-agent-mux-delegate-job skill.
|
|
#
|
|
# Subcommands:
|
|
# submit register a job, start the subscriber FIRST, then run the agent,
|
|
# then (optionally) run a validation script.
|
|
# status show one job record.
|
|
# list list all jobs.
|
|
# verify run a user-supplied --validate script against a job's artifacts.
|
|
# wait block until all running/pending jobs reach a terminal state.
|
|
#
|
|
# This is a reference wrapper: it shells out to the python scripts that live
|
|
# next to it. Copy it into your project and customise as needed. It never hard
|
|
# fails if `claude`/`codex`/`herdr` are missing — it prints what it would run.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# Load local env file (.mam.env preferred, .env fallback) from workspace root or explicit MAM_ENV_FILE
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
|
TARGET_ENV="${MAM_ENV_FILE:-}"
|
|
if [[ -z "$TARGET_ENV" ]]; then
|
|
if [[ -f "$REPO_ROOT/.mam.env" ]]; then
|
|
TARGET_ENV="$REPO_ROOT/.mam.env"
|
|
elif [[ -f "$REPO_ROOT/.env" ]]; then
|
|
TARGET_ENV="$REPO_ROOT/.env"
|
|
echo "WARNING: Loading deprecated config file '$TARGET_ENV'. Please migrate to '.mam.env'." >&2
|
|
elif [[ -f .mam.env ]]; then
|
|
TARGET_ENV=".mam.env"
|
|
echo "WARNING: Loading config from cwd relative path '$TARGET_ENV'." >&2
|
|
elif [[ -f .env ]]; then
|
|
TARGET_ENV=".env"
|
|
echo "WARNING: Loading deprecated config from cwd relative path '$TARGET_ENV'. Please migrate to '.mam.env'." >&2
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "$TARGET_ENV" && -f "$TARGET_ENV" ]]; then
|
|
set -a; source "$TARGET_ENV"; set +a
|
|
fi
|
|
|
|
# Source EARLY (before any herdr usage in run_agent) — this is what turns
|
|
# plain `herdr` into the tmux-compat shim (herdr() function) and provides
|
|
# resolve_herdr_session/send_keys_safe. Sourcing it late meant the
|
|
# has-session pre-flight check below used to hit the real herdr binary with
|
|
# a nonexistent subcommand and always fail.
|
|
source "$SCRIPT_DIR/../lib.sh"
|
|
|
|
# Pick an interpreter: prefer a project .venv, else python3.
|
|
pick_python() {
|
|
local py_bin
|
|
if [[ -n "${DELEGATE_JOB_PYTHON:-}" ]]; then
|
|
py_bin="$DELEGATE_JOB_PYTHON"
|
|
elif [[ -n "${AGENT_PYTHON_BIN:-}" ]] && [[ -x "$AGENT_PYTHON_BIN" ]]; then
|
|
py_bin="$AGENT_PYTHON_BIN"
|
|
elif [[ -x "${WORKDIR:-.}/.venv/bin/python" ]]; then
|
|
py_bin="${WORKDIR}/.venv/bin/python"
|
|
elif [[ -x ".venv/bin/python" ]]; then
|
|
py_bin="$(pwd)/.venv/bin/python"
|
|
else
|
|
py_bin="python3"
|
|
fi
|
|
if ! "$py_bin" -c "import paho.mqtt" 2>/dev/null; then
|
|
echo "ERROR: paho-mqtt package is missing for $py_bin." >&2
|
|
echo " Please create a virtual environment and install it:" >&2
|
|
echo " python3 -m venv .venv && .venv/bin/pip install -r \"$SCRIPT_DIR/requirements.txt\"" >&2
|
|
exit 1
|
|
fi
|
|
echo "$py_bin"
|
|
}
|
|
|
|
REGISTRY_DIR_DEFAULT="${MAM_JOBS_DIR:-.mam/jobs}"
|
|
|
|
# Issue #21: the injected brief path, the start/success/error commands and the
|
|
# Output Report Path must work from any cwd - a creator in a git worktree runs
|
|
# them from there. Called right after `cd "$WORKDIR"`, so a relative registry
|
|
# still resolves against WORKDIR, as before.
|
|
_abs_registry_dir() {
|
|
mkdir -p "$REGISTRY_DIR"
|
|
REGISTRY_DIR="$(cd -P "$REGISTRY_DIR" && pwd -P)"
|
|
export SKS_REGISTRY_DIR="$REGISTRY_DIR"
|
|
}
|
|
|
|
# The audit-log root this wrapper's own subscriber resolves (same resolver),
|
|
# pinned into the injected commands so an agent publishing from another cwd
|
|
# does not fall back to <cwd>/.mam/delegate_job_logs.
|
|
_logs_dir_abs() {
|
|
"$PY" -c 'import os, sys; sys.path.insert(0, sys.argv[1]); from lib_py.delegation.mqtt_common import get_logs_dir; print(os.path.abspath(get_logs_dir()))' "$SCRIPT_DIR/.."
|
|
}
|
|
|
|
# _pub_cmd <extra...>: the publish command as one shell-quoted string.
|
|
_pub_cmd() {
|
|
local s
|
|
s="$(
|
|
if [ -n "${LOGS_DIR_ABS:-}" ]; then printf 'DELEGATE_JOB_LOGS_DIR=%q ' "$LOGS_DIR_ABS"; fi
|
|
printf '%q ' "$PY" "$SCRIPT_DIR/scripts/publish_event.py" --registry-dir "$REGISTRY_DIR" --job "$JOB_ID" "$@"
|
|
)"
|
|
printf '%s' "${s% }"
|
|
}
|
|
|
|
# _instructions <pub> [iteration]: the prompt injected into the agent (direct
|
|
# and loop jobs share it; test18 pins parts of this wording, so Issue #34's
|
|
# sub-delegation sentence goes at the end).
|
|
_instructions() {
|
|
local pub="$1" iter="${2:-}" brief="$REGISTRY_DIR/$JOB_ID/brief.md"
|
|
printf 'Your job_id is "%s". Detailed task requirements, instructions, and target output paths%s are documented in the task brief file at: %s. Please READ and follow %s to complete your work. Commands: start='"'"'%s --event started --heartbeat %s'"'"' (run it FIRST, within %s s; it also starts an automatic heartbeat timer), success='"'"'%s --event completed --detail <summary>'"'"', error='"'"'%s --event error --detail <reason>'"'"'. If you delegate any part of this work with multi-agent-mux-delegate-job submit, add --parent-job %s so the orchestrator waits for it.' \
|
|
"$JOB_ID" "${iter:+ for iteration $iter}" "$brief" "$brief" "$pub" "$HEARTBEAT_SEC" "$ACK_TIMEOUT" "$pub" "$pub" "$JOB_ID"
|
|
}
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
multi-agent-mux-delegate-job <command> [options]
|
|
|
|
submit --agent <name> (--prompt <text> | --prompt-file <path>) [--workdir <dir>] [--agent-session <label>]
|
|
[--timeout <sec>] [--idle-timeout <sec>] [--validate <script>]
|
|
[--heartbeat <sec>] [--heartbeat-misses <count>] [--ack-timeout <sec>]
|
|
[--registry-dir <dir>] [--dry-run] [--role <role_name>]
|
|
[--type <direct|loop|discuss>] [--reviewer <reviewer_agent>]
|
|
[--reviewer-session <reviewer_session>] [--max-iterations <count>]
|
|
[--counterpart-role <role_name>] [--strict-role-check] [--parent-job <id>]
|
|
# --parent-job names the job this one is delegated from, so a loop
|
|
# waiting on that job waits for this one too (Issue #34). It must
|
|
# exist in the registry. Default: $MAM_JOB_ID (set by the remote
|
|
# runner), ignored with a warning when the registry lacks it.
|
|
# --prompt-file passes the prompt by file: one argv string is capped
|
|
# at 131,072 bytes on Linux (Issue #36). The file is read, not kept.
|
|
# --agent-session herdr:<name> injects into a local herdr pane;
|
|
# --agent-session remote:<name> dispatches over the broker (Issue #18).
|
|
# Liveness contract: the agent heartbeats every --heartbeat N s
|
|
# (default 30, automatic), the job is unresponsive after
|
|
# --heartbeat-misses K x N s (default 4 x 30 = 120; --idle-timeout
|
|
# overrides), `started` must arrive within --ack-timeout T s (180).
|
|
status --job <id> [--registry-dir <dir>]
|
|
list [--registry-dir <dir>]
|
|
verify --job <id> --validate <script> [--registry-dir <dir>]
|
|
wait [--job <id>] [--timeout <sec>] [--registry-dir <dir>]
|
|
logs <job_id> | --list # persistent audit log (delegate_job_logs/)
|
|
EOF
|
|
}
|
|
|
|
# ---- arg parsing helpers --------------------------------------------------
|
|
AGENT="claude-code"; PROMPT=""; PROMPT_FILE=""; WORKDIR="$(pwd)"; AGENT_SESSION="herdr:claude"
|
|
TIMEOUT=3600; IDLE_TIMEOUT=""; VALIDATE=""; DRY_RUN=0
|
|
# Issue #18 liveness contract (N / K / T). An unset --idle-timeout derives K x N.
|
|
HEARTBEAT_SEC="${MAM_HEARTBEAT_SEC:-30}"; HEARTBEAT_MISSES="${MAM_HEARTBEAT_MISSES:-4}"; ACK_TIMEOUT="${MAM_ACK_TIMEOUT_S:-180}"
|
|
JOB_ID=""; REGISTRY_DIR="$REGISTRY_DIR_DEFAULT"; DELEGATE_ROLE="Worker"
|
|
TYPE="direct"; REVIEWER="hermes"; REVIEWER_SESSION="herdr:hermes"; MAX_ITERATIONS=5
|
|
DEFAULT_COUNTERPART_ROLE="Reviewer"
|
|
COUNTERPART_ROLE="$DEFAULT_COUNTERPART_ROLE"
|
|
STRICT_ROLE_CHECK=0
|
|
ROLE_ALIASES_JSON='{"worker": ["worker", "creator"], "planner": ["planner"], "reviewer": ["reviewer"]}'
|
|
COUNTERPART_ROLE_EXPLICIT=0
|
|
PARENT_JOB=""
|
|
|
|
parse_opts() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--agent) AGENT="$2"; shift 2;;
|
|
--prompt) PROMPT="$2"; shift 2;;
|
|
--prompt-file) PROMPT_FILE="$2"; shift 2;;
|
|
--workdir) WORKDIR="$2"; shift 2;;
|
|
--agent-session) AGENT_SESSION="$2"; shift 2;;
|
|
--timeout) TIMEOUT="$2"; shift 2;;
|
|
--idle-timeout) IDLE_TIMEOUT="$2"; shift 2;;
|
|
--heartbeat) HEARTBEAT_SEC="$2"; shift 2;;
|
|
--heartbeat-misses) HEARTBEAT_MISSES="$2"; shift 2;;
|
|
--ack-timeout) ACK_TIMEOUT="$2"; shift 2;;
|
|
--validate) VALIDATE="$2"; shift 2;;
|
|
--job) JOB_ID="$2"; shift 2;;
|
|
--registry-dir) REGISTRY_DIR="$2"; shift 2;;
|
|
--dry-run) DRY_RUN=1; shift;;
|
|
--role) DELEGATE_ROLE="$2"; shift 2;;
|
|
--type) TYPE="$2"; shift 2;;
|
|
--reviewer) REVIEWER="$2"; shift 2;;
|
|
--reviewer-session) REVIEWER_SESSION="$2"; shift 2;;
|
|
--max-iterations) MAX_ITERATIONS="$2"; shift 2;;
|
|
--counterpart-role) COUNTERPART_ROLE="$2"; COUNTERPART_ROLE_EXPLICIT=1; shift 2;;
|
|
--strict-role-check) STRICT_ROLE_CHECK=1; shift;;
|
|
--parent-job) PARENT_JOB="$2"; shift 2;;
|
|
*) echo "unknown option: $1" >&2; usage; exit 1;;
|
|
esac
|
|
done
|
|
[ -n "$IDLE_TIMEOUT" ] || IDLE_TIMEOUT=$(( HEARTBEAT_SEC * HEARTBEAT_MISSES ))
|
|
# lib.sh's send_keys_safe reads the job's own `started` from this registry.
|
|
export SKS_REGISTRY_DIR="$REGISTRY_DIR"
|
|
}
|
|
|
|
# Issue #18: `remote:<name>` sessions get the prompt over the broker, not a pane.
|
|
_is_remote() { case "${1:-}" in remote:*) return 0;; *) return 1;; esac; }
|
|
|
|
_to_lower() {
|
|
printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
|
|
}
|
|
|
|
cmd_submit() {
|
|
parse_opts "$@"
|
|
if [[ -n "$PROMPT_FILE" ]]; then
|
|
[[ -z "$PROMPT" ]] || { echo "submit takes --prompt or --prompt-file, not both" >&2; exit 1; }
|
|
[[ "$PROMPT_FILE" == /* ]] || PROMPT_FILE="$(pwd)/$PROMPT_FILE"
|
|
[[ -f "$PROMPT_FILE" && -r "$PROMPT_FILE" ]] || { echo "submit: cannot read --prompt-file $PROMPT_FILE" >&2; exit 1; }
|
|
[[ -s "$PROMPT_FILE" ]] || { echo "submit: --prompt-file $PROMPT_FILE is empty" >&2; exit 1; }
|
|
PROMPT="$(<"$PROMPT_FILE")"
|
|
fi
|
|
[[ -n "$PROMPT" || -n "$PROMPT_FILE" ]] || { echo "submit requires --prompt or --prompt-file" >&2; exit 1; }
|
|
# Issue #36: the prompt reaches the registry by file when it came by file,
|
|
# never as one argv string (capped at 131,072 bytes).
|
|
local reg_prompt=(--prompt "$PROMPT")
|
|
[[ -n "$PROMPT_FILE" ]] && reg_prompt=(--prompt-file "$PROMPT_FILE")
|
|
PY="$(pick_python)"
|
|
[[ "$PY" == */* && "$PY" != /* ]] && PY="$(pwd)/$PY"
|
|
cd "$WORKDIR"
|
|
_abs_registry_dir
|
|
LOGS_DIR_ABS="$(_logs_dir_abs 2>/dev/null || true)"
|
|
|
|
# Issue #34: an explicit --parent-job fails closed in the registry; the
|
|
# $MAM_JOB_ID default may name a job of another host's registry (a remote
|
|
# mirror), so it is only used when this registry has it.
|
|
local reg_parent=()
|
|
if [[ -n "$PARENT_JOB" ]]; then
|
|
reg_parent=(--parent-job "$PARENT_JOB")
|
|
elif [[ -n "${MAM_JOB_ID:-}" ]]; then
|
|
if [[ -f "$REGISTRY_DIR/$MAM_JOB_ID.json" ]]; then
|
|
PARENT_JOB="$MAM_JOB_ID"; reg_parent=(--parent-job "$PARENT_JOB")
|
|
else
|
|
echo "WARNING: MAM_JOB_ID=$MAM_JOB_ID is not in $REGISTRY_DIR; registering without a parent job" >&2
|
|
fi
|
|
fi
|
|
|
|
# 1) register job (prints the new job id)
|
|
JOB_ID="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" register \
|
|
"${reg_prompt[@]}" --agent "$AGENT" --agent-session "$AGENT_SESSION" --role "$DELEGATE_ROLE" \
|
|
--timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
|
--heartbeat-sec "$HEARTBEAT_SEC" --heartbeat-misses "$HEARTBEAT_MISSES" --ack-timeout "$ACK_TIMEOUT" \
|
|
--transport "$(_is_remote "$AGENT_SESSION" && echo remote || echo herdr)" \
|
|
--job-type "$TYPE" --reviewer "$REVIEWER" --reviewer-session "$REVIEWER_SESSION" \
|
|
--max-iterations "$MAX_ITERATIONS" ${reg_parent[@]+"${reg_parent[@]}"})"
|
|
echo "registered job: $JOB_ID"
|
|
|
|
# 1-1) Provision job directory and write direct brief.md (MAM Job Restructuring)
|
|
local job_dir="$REGISTRY_DIR/$JOB_ID"
|
|
if [[ "$DRY_RUN" != "1" ]]; then
|
|
mkdir -p "$job_dir"
|
|
cat <<EOF > "$job_dir/brief.md"
|
|
# 📋 Brief: Job $JOB_ID Delegation
|
|
|
|
- **Job ID**: $JOB_ID
|
|
- **Target Agent**: $AGENT (session: $AGENT_SESSION)
|
|
- **Role**: $DELEGATE_ROLE
|
|
- **Timeout**: $TIMEOUT s (Idle: $IDLE_TIMEOUT s)
|
|
- **Liveness contract**: heartbeat every $HEARTBEAT_SEC s (automatic once you publish \`started --heartbeat $HEARTBEAT_SEC\`), unresponsive after $HEARTBEAT_MISSES missed heartbeats, \`started\` expected within $ACK_TIMEOUT s
|
|
- **Output Report Path**: $REGISTRY_DIR/$JOB_ID/$AGENT-reports/report-final.md
|
|
- **Parent for sub-delegations**: pass \`--parent-job $JOB_ID\` to \`multi-agent-mux-delegate-job submit\`
|
|
|
|
## 🔎 Task Description
|
|
$PROMPT
|
|
EOF
|
|
echo "provisioned job directory: $job_dir"
|
|
fi
|
|
|
|
local START_TIME
|
|
START_TIME=$(date +%s)
|
|
|
|
if [[ "$TYPE" == "direct" ]]; then
|
|
# 1) Read dispatch token and attempt id for this dispatch
|
|
local _disp_tok=""
|
|
local _attempt_id=""
|
|
_disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
|
_attempt_id="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('attempt_id','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
|
local pub_extra=()
|
|
[ -n "$_disp_tok" ] && pub_extra+=(--expect-dispatch-token "$_disp_tok")
|
|
[ -n "$_attempt_id" ] && pub_extra+=(--attempt-id "$_attempt_id")
|
|
local pub instructions
|
|
pub="$(_pub_cmd ${pub_extra[@]+"${pub_extra[@]}"})"
|
|
instructions="$(_instructions "$pub")"
|
|
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
run_agent "$JOB_ID" "$instructions"
|
|
local logs_root_dry="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root_dry/$JOB_ID"
|
|
return 0
|
|
fi
|
|
|
|
# 1) Preflight (a remote session has no local pane to check: Issue #18)
|
|
if ! _is_remote "$AGENT_SESSION" && ! preflight_agent "$JOB_ID" "$AGENT_SESSION" 0; then
|
|
finalize_unfinished "$JOB_ID" "error" 1 "preflight"
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
exit 5
|
|
fi
|
|
|
|
# 2) TUI idle handshake
|
|
if [ "$AGENT" != "human" ] && ! _is_remote "$AGENT_SESSION"; then
|
|
local idle_rc=0
|
|
wait_for_agent_idle "$AGENT_SESSION" || idle_rc=$?
|
|
export _SKS_IDLE_RC="$idle_rc"
|
|
if [ $idle_rc -eq 1 ] || [ $idle_rc -eq 2 ]; then
|
|
finalize_unfinished "$JOB_ID" "error" "$idle_rc" "idle_wait"
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
exit 5
|
|
fi
|
|
fi
|
|
|
|
# 3) START THE SUBSCRIBER
|
|
local logf="$REGISTRY_DIR/$JOB_ID.subscriber.out"
|
|
# Before the first event the subscriber is bounded by the wall clock only:
|
|
# delivery (T from the injection, which happens after the subscriber is
|
|
# up and can take a while itself) is judged by _wait_started, and a failed
|
|
# injection reaps the subscriber. From the first event on, K x N applies.
|
|
local sub_extra_args=(--ack-timeout "$TIMEOUT")
|
|
[ -n "$_disp_tok" ] && sub_extra_args+=(--expect-dispatch-token "$_disp_tok")
|
|
_is_remote "$AGENT_SESSION" && sub_extra_args+=(--remote)
|
|
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
|
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
|
${sub_extra_args[@]+"${sub_extra_args[@]}"} \
|
|
>"$logf" 2>&1 &
|
|
local sub_pid=$!
|
|
echo "subscriber pid: $sub_pid (log: $logf)"
|
|
# Wait for the subscriber to CONNACK + SUBSCRIBE (reactive handshake)
|
|
local sub_ready=0
|
|
for ((i = 0; i < 25; i++)); do
|
|
if kill -0 "$sub_pid" 2>/dev/null; then
|
|
if grep -q '^SUBSCRIBED ' "$logf" 2>/dev/null; then
|
|
sub_ready=1
|
|
break
|
|
fi
|
|
else
|
|
local sub_exit=0
|
|
wait "$sub_pid" 2>/dev/null || sub_exit=$?
|
|
if [ $sub_exit -eq 0 ] || [ $sub_exit -eq 3 ]; then
|
|
sub_ready=1
|
|
break
|
|
else
|
|
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
|
|
local mapped_st
|
|
mapped_st="$(_map_sub_rc "$sub_exit" "$JOB_ID")"
|
|
finalize_unfinished "$JOB_ID" "$mapped_st" "$sub_exit" "subscriber"
|
|
mapped_st="${FINAL_JOB_STATUS:-$mapped_st}"
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
|
exit 0
|
|
fi
|
|
exit "$(_rc_for "$mapped_st")"
|
|
fi
|
|
fi
|
|
sleep 0.2
|
|
done
|
|
if [ "$sub_ready" -ne 1 ]; then
|
|
echo "WARNING: subscriber subscribe handshake timed out — falling back to proceed" >&2
|
|
fi
|
|
|
|
# 5) Deliver the prompt (herdr pane or remote dispatch) and wait for `started`
|
|
if ! deliver_prompt "$JOB_ID" "$instructions" "$AGENT_SESSION" "$logf"; then
|
|
reap_subscriber "$sub_pid"
|
|
finalize_unfinished "$JOB_ID" "error" 1 "inject"
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
exit 1
|
|
fi
|
|
|
|
# 6) optional validation hook
|
|
if [[ -n "$VALIDATE" ]]; then
|
|
echo "running validation: $VALIDATE"
|
|
if JOB_ID="$JOB_ID" REGISTRY_DIR="$REGISTRY_DIR" bash "$VALIDATE"; then
|
|
echo "validation: PASS"
|
|
else
|
|
local rc=$?
|
|
echo "validation: FAIL (exit $rc)"
|
|
fi
|
|
fi
|
|
|
|
local sub_rc=0
|
|
wait "$sub_pid" || sub_rc=$?
|
|
echo "subscriber output:"; cat "$logf" || true
|
|
|
|
local job_status
|
|
job_status="$(_map_sub_rc "$sub_rc" "$JOB_ID")"
|
|
|
|
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "direct"
|
|
job_status="${FINAL_JOB_STATUS:-$job_status}"
|
|
|
|
# Last stdout line: the persistent audit-log dir for this job (see SKILL.md
|
|
# "Audit Logs"). Callers can scrape `tail -n1` to find it.
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
|
exit 0
|
|
fi
|
|
exit "$(_rc_for "$job_status")"
|
|
else
|
|
# Implement loop/discuss orchestrator
|
|
local iteration=1
|
|
local current_prompt="$PROMPT"
|
|
local current_session="$AGENT_SESSION"
|
|
local _phase="worker"
|
|
local display_role="$DELEGATE_ROLE"
|
|
local attempt=0
|
|
local final_loop_rc=0
|
|
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
echo "[dry-run] orchestrator loop would start for job: $JOB_ID type: $TYPE"
|
|
echo "worker session: $AGENT_SESSION, reviewer session: $REVIEWER_SESSION"
|
|
local logs_root_dry="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root_dry/$JOB_ID"
|
|
return 0
|
|
fi
|
|
|
|
while true; do
|
|
echo "=================================================="
|
|
echo "Iteration $iteration - Role: $display_role"
|
|
echo "Session: $current_session"
|
|
echo "=================================================="
|
|
|
|
# 1-1) Provision job directory and write iteration brief.md (MAM Job Restructuring)
|
|
local job_dir="$REGISTRY_DIR/$JOB_ID"
|
|
local clean_session="${current_session#herdr:}"
|
|
mkdir -p "$job_dir"
|
|
cat <<EOF > "$job_dir/brief.md"
|
|
# 📋 Brief: Job $JOB_ID Delegation (Iteration $iteration)
|
|
|
|
- **Job ID**: $JOB_ID
|
|
- **Target Agent/Session**: $current_session
|
|
- **Role**: $display_role
|
|
- **Iteration**: $iteration
|
|
- **Liveness contract**: heartbeat every $HEARTBEAT_SEC s (automatic once you publish \`started --heartbeat $HEARTBEAT_SEC\`), unresponsive after $HEARTBEAT_MISSES missed heartbeats, \`started\` expected within $ACK_TIMEOUT s
|
|
- **Output Report Path**: $REGISTRY_DIR/$JOB_ID/${clean_session}-reports/report-final.md
|
|
- **Parent for sub-delegations**: pass \`--parent-job $JOB_ID\` to \`multi-agent-mux-delegate-job submit\`
|
|
|
|
## 🔎 Task Description
|
|
$current_prompt
|
|
EOF
|
|
# Issue #36: the registry gets the prompt by file, not as one argv string
|
|
printf '%s' "$current_prompt" > "$job_dir/prompt.md"
|
|
echo "provisioned iteration brief: $job_dir/brief.md"
|
|
|
|
attempt=$((attempt + 1))
|
|
|
|
local force_warn_only=0
|
|
local cp_lower def_lower
|
|
cp_lower=$(_to_lower "$COUNTERPART_ROLE")
|
|
def_lower=$(_to_lower "$DEFAULT_COUNTERPART_ROLE")
|
|
if [[ "$_phase" == "reviewer" && "$COUNTERPART_ROLE_EXPLICIT" -eq 1 \
|
|
&& "$cp_lower" != "$def_lower" ]]; then
|
|
force_warn_only=1
|
|
fi
|
|
|
|
# Preflight (skipped for a remote session: Issue #18)
|
|
if ! _is_remote "$current_session" && ! preflight_agent "$JOB_ID" "$current_session" "$force_warn_only"; then
|
|
finalize_unfinished "$JOB_ID" "error" 1 "preflight"
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
exit 5
|
|
fi
|
|
|
|
# Idle wait
|
|
local cur_agent="$AGENT"
|
|
[ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER"
|
|
if [ "$cur_agent" != "human" ] && ! _is_remote "$current_session"; then
|
|
local idle_rc=0
|
|
wait_for_agent_idle "$current_session" || idle_rc=$?
|
|
export _SKS_IDLE_RC="$idle_rc"
|
|
if [ $idle_rc -eq 1 ] || [ $idle_rc -eq 2 ]; then
|
|
finalize_unfinished "$JOB_ID" "error" "$idle_rc" "idle_wait"
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
exit 5
|
|
fi
|
|
fi
|
|
|
|
# Update job details in registry
|
|
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" update \
|
|
--job "$JOB_ID" \
|
|
--agent-session "$current_session" \
|
|
--prompt-file "$job_dir/prompt.md" \
|
|
--iteration "$iteration" \
|
|
--attempt-id "$attempt" \
|
|
--role "$display_role" \
|
|
--status "pending"
|
|
|
|
local _disp_tok=""
|
|
_disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
|
|
|
# Start subscriber
|
|
local logf="$REGISTRY_DIR/${JOB_ID}.iter_${iteration}_${display_role}.subscriber.out"
|
|
local loop_sub_args=(--attempt-id "$attempt" --ack-timeout "$TIMEOUT")
|
|
[ -n "$_disp_tok" ] && loop_sub_args+=(--expect-dispatch-token "$_disp_tok")
|
|
_is_remote "$current_session" && loop_sub_args+=(--remote)
|
|
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
|
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
|
"${loop_sub_args[@]}" \
|
|
>"$logf" 2>&1 &
|
|
local sub_pid=$!
|
|
echo "subscriber pid: $sub_pid (log: $logf)"
|
|
# Wait for the subscriber to CONNACK + SUBSCRIBE (reactive handshake)
|
|
local sub_ready=0
|
|
for ((i = 0; i < 25; i++)); do
|
|
if kill -0 "$sub_pid" 2>/dev/null; then
|
|
if grep -q '^SUBSCRIBED ' "$logf" 2>/dev/null; then
|
|
sub_ready=1
|
|
break
|
|
fi
|
|
else
|
|
local sub_exit=0
|
|
wait "$sub_pid" 2>/dev/null || sub_exit=$?
|
|
if [ $sub_exit -eq 0 ] || [ $sub_exit -eq 3 ]; then
|
|
sub_ready=1
|
|
break
|
|
else
|
|
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
|
|
local mapped_st
|
|
mapped_st="$(_map_sub_rc "$sub_exit" "$JOB_ID")"
|
|
finalize_unfinished "$JOB_ID" "$mapped_st" "$sub_exit" "subscriber"
|
|
mapped_st="${FINAL_JOB_STATUS:-$mapped_st}"
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
|
exit 0
|
|
fi
|
|
exit "$(_rc_for "$mapped_st")"
|
|
fi
|
|
fi
|
|
sleep 0.2
|
|
done
|
|
if [ "$sub_ready" -ne 1 ]; then
|
|
echo "WARNING: subscriber subscribe handshake timed out — falling back to proceed" >&2
|
|
fi
|
|
|
|
# Format instruction block
|
|
local loop_pub_extra=(--attempt-id "$attempt")
|
|
[ -n "$_disp_tok" ] && loop_pub_extra+=(--expect-dispatch-token "$_disp_tok")
|
|
local pub instructions
|
|
pub="$(_pub_cmd "${loop_pub_extra[@]}")"
|
|
instructions="$(_instructions "$pub" "$iteration")"
|
|
|
|
# Trigger agent
|
|
if ! deliver_prompt "$JOB_ID" "$instructions" "$current_session" "$logf"; then
|
|
reap_subscriber "$sub_pid"
|
|
finalize_unfinished "$JOB_ID" "error" 1 "inject"
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
exit 1
|
|
fi
|
|
|
|
# Wait for subscriber
|
|
local sub_rc=0
|
|
wait "$sub_pid" || sub_rc=$?
|
|
echo "subscriber output:"; cat "$logf" || true
|
|
|
|
local job_status
|
|
job_status="$(_map_sub_rc "$sub_rc" "$JOB_ID")"
|
|
|
|
echo "Job role $display_role finished with status: $job_status"
|
|
|
|
# Retrieve feedback from the last event
|
|
local feedback
|
|
feedback="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get-feedback --job "$JOB_ID")"
|
|
echo "Feedback/Detail: $feedback"
|
|
if [[ "$_phase" == "worker" ]]; then
|
|
if [[ "$job_status" != "completed" ]]; then
|
|
echo "Worker did not complete successfully (status: $job_status). Terminating workflow."
|
|
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "worker" "$current_session"
|
|
job_status="${FINAL_JOB_STATUS:-$job_status}"
|
|
if [[ "$job_status" == "completed" ]]; then
|
|
echo "Worker completed during grace period. Continuing to reviewer."
|
|
feedback="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get-feedback --job "$JOB_ID")"
|
|
else
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
|
exit 0
|
|
fi
|
|
exit "$(_rc_for "$job_status")"
|
|
fi
|
|
fi
|
|
|
|
# Worker completed successfully, now switch to reviewer
|
|
_phase="reviewer"
|
|
display_role="$COUNTERPART_ROLE"
|
|
current_session="$REVIEWER_SESSION"
|
|
# Build reviewer prompt based on type
|
|
if [[ "$TYPE" == "loop" ]]; then
|
|
current_prompt="Review the changes/artifacts generated for job $JOB_ID. Check if they meet the requirements. If correct, publish completed event with 'PASS'. If there are issues, publish error event with detailed feedback/nits. CRITICAL: When raising issues or giving a review, you MUST include the exact reason for the issue and a clear direction for improvement (문제 제시에 대한 이유와 확실한 개선 방향을 반드시 포함해야 합니다)."
|
|
elif [[ "$TYPE" == "discuss" ]]; then
|
|
current_prompt="Read draft/documents generated for job $JOB_ID. Review the feasibility and content. Write your feedback/objections. If you agree with the plan, reply with 'AGREE'."
|
|
fi
|
|
else
|
|
if [[ "$job_status" != "completed" ]]; then
|
|
echo "Reviewer did not complete successfully (status: $job_status). Terminating workflow."
|
|
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "reviewer" "$current_session"
|
|
job_status="${FINAL_JOB_STATUS:-$job_status}"
|
|
if [[ "$job_status" == "completed" ]]; then
|
|
echo "Reviewer completed during grace period. Continuing evaluation."
|
|
feedback="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get-feedback --job "$JOB_ID")"
|
|
else
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
|
exit 0
|
|
fi
|
|
exit "$(_rc_for "$job_status")"
|
|
fi
|
|
fi
|
|
|
|
# Reviewer finished. Check if pass/agree
|
|
local success=0
|
|
local fb_lower
|
|
fb_lower=$(_to_lower "$feedback")
|
|
if [[ "$TYPE" == "loop" ]]; then
|
|
if [[ "$fb_lower" == *"pass"* ]]; then
|
|
success=1
|
|
fi
|
|
elif [[ "$TYPE" == "discuss" ]]; then
|
|
if [[ "$fb_lower" == *"agree"* ]]; then
|
|
success=1
|
|
fi
|
|
fi
|
|
|
|
if [[ "$success" == "1" ]]; then
|
|
echo "Reviewer approved the work. Finalizing job as completed."
|
|
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" status --job "$JOB_ID" --set "completed"
|
|
final_loop_rc=0
|
|
break
|
|
else
|
|
# Reviewer rejected/provided feedback. Increment & check max iterations
|
|
if [[ $iteration -ge $MAX_ITERATIONS ]]; then
|
|
echo "Max iterations ($MAX_ITERATIONS) reached without approval. Terminating workflow."
|
|
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" status --job "$JOB_ID" --set "error"
|
|
final_loop_rc=1
|
|
break
|
|
fi
|
|
|
|
iteration=$((iteration + 1))
|
|
_phase="worker"
|
|
display_role="$DELEGATE_ROLE"
|
|
current_session="$AGENT_SESSION"
|
|
current_prompt="The reviewer provided the following feedback for job $JOB_ID: $feedback. Please modify the code/artifacts to address these comments. CRITICAL: As the Developer Team Leader, you must thoroughly review the suggested modifications, verify their validity, adopt/implement them if valid, and if you judge any recommendation to be invalid, do NOT implement it but instead explain your reasons clearly in your response and send it back to the reviewer (수정안을 최대한 꼼꼼히 검토하여 타당성을 검증하고, 타당하다면 수렴하여 수정을 진행하되, 타당하지 않다고 판단되는 부분이 있다면 그 이유를 명확히 밝혀 리뷰어에게 전달하십시오)."
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# 4) optional validation hook
|
|
if [[ -n "$VALIDATE" ]]; then
|
|
echo "running validation: $VALIDATE"
|
|
if JOB_ID="$JOB_ID" REGISTRY_DIR="$REGISTRY_DIR" bash "$VALIDATE"; then
|
|
echo "validation: PASS"
|
|
else
|
|
local rc=$?
|
|
echo "validation: FAIL (exit $rc)"
|
|
final_loop_rc=$rc
|
|
fi
|
|
fi
|
|
|
|
# Last stdout line: the persistent audit-log dir
|
|
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
|
echo "$logs_root/$JOB_ID"
|
|
exit "$final_loop_rc"
|
|
fi
|
|
}
|
|
|
|
reap_subscriber() {
|
|
if [ -n "${1:-}" ]; then
|
|
kill "$1" 2>/dev/null || true
|
|
wait "$1" 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
_disk_status() {
|
|
local jid="$1"
|
|
"$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$jid.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true
|
|
}
|
|
|
|
# ISSUE-18: delivery is confirmed by the protocol, not the screen. After the
|
|
# prompt was handed over (herdr accepted the submission, or the dispatch was
|
|
# published for a remote session) the job's own `started` must arrive within
|
|
# the ack timeout T. The evidence is lib.sh's delegate_started_recorded - the
|
|
# same predicate create_session's onboarding ack uses, so the two waits share
|
|
# one contract (ack-finding req. 2, review R1): the local publisher's line in
|
|
# events.log, or the subscriber's HMAC-verified, dispatch-attributed audit
|
|
# record of a `started` it received (a remote agent has no shared disk). The
|
|
# registry status is not evidence - a record leaves `pending` when another
|
|
# writer closes it (error, cancelled) or another dispatch claims it, none of
|
|
# which says this agent saw this prompt. No `started` within T is a delivery
|
|
# failure: fail closed, never re-inject (single-injection contract). A job
|
|
# that is already terminal is handled by the terminal logic after this, not
|
|
# turned into a success here.
|
|
_wait_started() {
|
|
local job_id="$1" ack_s="${2:-$ACK_TIMEOUT}" sub_log="${3:-}" deadline wall
|
|
# sub_log is retained for callers' signatures; the subscriber's evidence is
|
|
# read from its audit record, which is attributable, not from its console.
|
|
deadline=$(( $(date +%s) + ack_s ))
|
|
# The ack wait never outlives this dispatch's wall clock (START_TIME + TIMEOUT).
|
|
wall=$(( ${START_TIME:-$(date +%s)} + ${TIMEOUT:-3600} ))
|
|
[ "$deadline" -gt "$wall" ] && deadline=$wall
|
|
local budget=$(( deadline - $(date +%s) ))
|
|
while :; do
|
|
# publish_event records `started` locally before its wire publish and
|
|
# before the status sync, so a slow broker cannot delay the acknowledgement.
|
|
if MAM_JOBS_DIR="$REGISTRY_DIR" delegate_started_recorded "$job_id" "${_disp_tok:-}" "${LOGS_DIR_ABS:-}"; then
|
|
echo "inject: job $job_id recorded 'started' in events.log; treating the prompt as delivered and continuing." >&2
|
|
return 0
|
|
fi
|
|
if [ "$(date +%s)" -ge "$deadline" ]; then
|
|
ACK_WAITED_S="$budget"
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
# Legacy name retained for callers/tests of the ISSUE-14 guard.
|
|
_inject_acknowledged() { _wait_started "$1" "${MAM_INJECT_ACK_GRACE_S:-$ACK_TIMEOUT}" "${2:-}"; }
|
|
|
|
# ISSUE-14 / ISSUE-18: herdr's agent_status and the pane busy token describe
|
|
# the screen, not the protocol. They are no longer liveness *evidence* (a
|
|
# headless or remote agent shows neither, and a CLI update can change both
|
|
# silently); finalize_unfinished logs them as a diagnostic so an operator can
|
|
# tell "dead" from "silent" on the same host. Liveness itself is the contract:
|
|
# an event on disk or on the wire within K x N seconds.
|
|
_agent_still_working() {
|
|
local sess="$1" target="$2"
|
|
[ "$(_sks_agent_status "$sess" "$target")" = "working" ] && return 0
|
|
_pane_tail "$sess" 8 2>/dev/null | grep -Fq "esc to interrupt"
|
|
}
|
|
|
|
# _within_deadline <deadline_epoch> <cmd...>
|
|
# Run a probe that can block (a herdr RPC, a pane capture) without letting it
|
|
# outlive a wall-clock deadline: once the clock reaches <deadline> the probe
|
|
# and its children are killed and the result is rc 124, i.e. "no evidence".
|
|
_within_deadline() {
|
|
local deadline="$1" pid
|
|
shift
|
|
"$@" &
|
|
pid=$!
|
|
while kill -0 "$pid" 2>/dev/null; do
|
|
if [ "$(date +%s)" -ge "$deadline" ]; then
|
|
pkill -P "$pid" 2>/dev/null || true
|
|
kill "$pid" 2>/dev/null || true
|
|
wait "$pid" 2>/dev/null || true
|
|
return 124
|
|
fi
|
|
sleep 0.1
|
|
done
|
|
wait "$pid"
|
|
}
|
|
|
|
# ISSUE-14: agent-agnostic liveness. publish_event appends every event to the
|
|
# job's on-disk events.log BEFORE its wire publish (record_local_event), so a
|
|
# `progress` heartbeat the subscriber never saw (it had already idled out, or
|
|
# the broker dropped it) is visible here the moment it is sent - a slow or
|
|
# failing broker no longer delays it by its publish retries. An agent running
|
|
# its tests in a background shell looks idle to herdr and shows no busy
|
|
# token, yet keeps heartbeating - this is the only evidence that covers that
|
|
# case.
|
|
_recent_disk_heartbeat() {
|
|
local job_id="$1" window="$2" f="$REGISTRY_DIR/$job_id.events.log"
|
|
[ -f "$f" ] || return 1
|
|
"$PY" - "$f" "$window" <<'PYEOF'
|
|
import calendar, json, sys, time
|
|
path, window = sys.argv[1], float(sys.argv[2])
|
|
last = ""
|
|
with open(path) as fh:
|
|
for line in fh:
|
|
if line.strip():
|
|
last = line
|
|
try:
|
|
stamp = json.loads(last)["timestamp"]
|
|
seen = calendar.timegm(time.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ"))
|
|
except Exception:
|
|
sys.exit(1)
|
|
sys.exit(0 if time.time() - seen <= window else 1)
|
|
PYEOF
|
|
}
|
|
|
|
_map_sub_rc() {
|
|
local sub_rc="$1" job_id="$2"
|
|
local job_status="running"
|
|
if [[ $sub_rc -eq 0 ]]; then
|
|
job_status="completed"
|
|
elif [[ $sub_rc -eq 1 ]]; then
|
|
job_status="error"
|
|
elif [[ $sub_rc -eq 3 ]]; then
|
|
job_status="broker_unavailable"
|
|
local disk_st
|
|
disk_st="$(_disk_status "$job_id")"
|
|
if [[ "$disk_st" == "completed" || "$disk_st" == "error" || "$disk_st" == "cancelled" ]]; then
|
|
job_status="$disk_st"
|
|
fi
|
|
else
|
|
job_status="timeout"
|
|
fi
|
|
echo "$job_status"
|
|
}
|
|
|
|
_rc_for() {
|
|
case "$1" in
|
|
completed) echo 0 ;;
|
|
error) echo 1 ;;
|
|
timeout) echo 2 ;;
|
|
broker_unavailable) echo 3 ;;
|
|
cancelled) echo 5 ;;
|
|
*) echo 1 ;;
|
|
esac
|
|
}
|
|
|
|
finalize_unfinished() {
|
|
local job_id="$1" job_status="$2" sub_rc="$3" phase="${4:-}"
|
|
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
|
return 0
|
|
fi
|
|
if [ "$job_status" = "completed" ]; then
|
|
return 0
|
|
fi
|
|
|
|
local cur_disk_st="$(_disk_status "$job_id")"
|
|
if [ "$cur_disk_st" = "completed" ] || [ "$cur_disk_st" = "error" ] || [ "$cur_disk_st" = "cancelled" ]; then
|
|
FINAL_JOB_STATUS="$cur_disk_st"
|
|
[ "$cur_disk_st" = "completed" ] && return 0
|
|
else
|
|
FINAL_JOB_STATUS="$job_status"
|
|
fi
|
|
# broker_unavailable: re-observe the disk at 3 s intervals until this
|
|
# dispatch's wall-clock deadline (START_TIME + TIMEOUT). The deadline is read
|
|
# off the clock on every pass; summing the sleeps alone ignores the time the
|
|
# disk probes themselves take and overruns TIMEOUT.
|
|
if [ "$job_status" = "broker_unavailable" ]; then
|
|
local deadline left disk_st
|
|
deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
|
|
while :; do
|
|
left=$(( deadline - $(date +%s) ))
|
|
[ "$left" -le 0 ] && break
|
|
[ "$left" -gt 3 ] && left=3
|
|
sleep "$left"
|
|
disk_st="$(_disk_status "$job_id")"
|
|
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
|
job_status="$disk_st"
|
|
FINAL_JOB_STATUS="$disk_st"
|
|
[ "$disk_st" = "completed" ] && return 0
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# timeout: if subscriber timed out, check if agent is still actively working in herdr
|
|
if [ "$job_status" = "timeout" ]; then
|
|
local target_sess="${5:-${current_session:-${target_session:-$AGENT_SESSION}}}"
|
|
local sess="${target_sess#herdr:}"
|
|
local target_agent
|
|
target_agent=$(_sanitize_herdr_agent_name "$sess")
|
|
|
|
# ISSUE-14 / ISSUE-18: the subscriber's idle clock only counts events it
|
|
# received over the wire. Keep re-observing the disk while the *contract*
|
|
# says the agent is alive: an event in events.log at most IDLE_TIMEOUT
|
|
# (K x N) old (_recent_disk_heartbeat - publish_event records every event
|
|
# there before its wire publish, the automatic heartbeat timer included).
|
|
# herdr status and the pane are NOT evidence any more; they are logged
|
|
# once here as a diagnostic, so an operator can tell a dead agent from a
|
|
# silent one on the same host. Window policy: entering this grace opens
|
|
# one IDLE_TIMEOUT window and every disk event re-opens it from the time
|
|
# of that poll, so an agent that leaves no event at all is closed about
|
|
# 2 x IDLE_TIMEOUT after its last wire event. Evidence only ever extends
|
|
# the idle window, never this dispatch's wall clock: the loop ends at
|
|
# START_TIME + TIMEOUT read off the clock on every pass.
|
|
local deadline evidence_until left disk_st
|
|
deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
|
|
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
|
|
if [ "$AGENT" != "human" ] && [ "$sess" != "human" ] && [ -n "$sess" ] && ! _is_remote "$target_sess"; then
|
|
local diag="no"
|
|
_within_deadline "$deadline" _agent_still_working "$sess" "$target_agent" && diag="yes"
|
|
echo "liveness: no wire event for ${IDLE_TIMEOUT}s on job $job_id; herdr/pane says busy=$diag (diagnostic only, not evidence)" >&2
|
|
fi
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
disk_st="$(_disk_status "$job_id")"
|
|
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
|
job_status="$disk_st"
|
|
FINAL_JOB_STATUS="$disk_st"
|
|
[ "$disk_st" = "completed" ] && return 0
|
|
break
|
|
fi
|
|
if _recent_disk_heartbeat "$job_id" "$IDLE_TIMEOUT"; then
|
|
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
|
|
elif [ "$(date +%s)" -ge "$evidence_until" ]; then
|
|
break
|
|
fi
|
|
left=$(( deadline - $(date +%s) ))
|
|
[ "$left" -le 0 ] && break
|
|
sleep 1
|
|
done
|
|
fi
|
|
|
|
local close_req="deadline_timeout"
|
|
local detail=""
|
|
if [ "$phase" = "preflight" ] || [ "$phase" = "idle_wait" ]; then
|
|
close_req="cancelled"
|
|
detail="session_busy"
|
|
# Issue #34: a target the preflight could not see was never busy
|
|
[ "$phase" = "preflight" ] && [ -n "${PREFLIGHT_REASON:-}" ] && detail="$PREFLIGHT_REASON"
|
|
fi
|
|
|
|
local close_args=(--job "$job_id" --request "$close_req")
|
|
[ -n "$detail" ] && close_args+=(--detail "$detail")
|
|
[ -n "${_disp_tok:-}" ] && close_args+=(--expect-dispatch-token "$_disp_tok")
|
|
|
|
local close_rc=0
|
|
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" close "${close_args[@]}" 2>&1 || close_rc=$?
|
|
|
|
# If close failed due to dispatch token mismatch (rc=4), suppress secondary event emission
|
|
# to prevent CAS/ABA bypass
|
|
if [ $close_rc -eq 4 ]; then
|
|
echo "finalize_unfinished: dispatch token mismatch (stale attempt); suppressing event emission" >&2
|
|
return 0
|
|
fi
|
|
|
|
# Check if job completed on disk (universal, for all TYPEs)
|
|
local cur_disk_st
|
|
cur_disk_st="$(_disk_status "$job_id")"
|
|
if [ "$cur_disk_st" = "completed" ]; then
|
|
FINAL_JOB_STATUS="completed"
|
|
return 0
|
|
fi
|
|
|
|
# direct path only: publish MQTT error/cancelled event (loop/discuss forbidden: O-1)
|
|
if [ "$TYPE" = "direct" ]; then
|
|
local disp_arg=()
|
|
[ -n "${_disp_tok:-}" ] && disp_arg=(--expect-dispatch-token "$_disp_tok")
|
|
if [ "$close_req" = "cancelled" ]; then
|
|
"$PY" "$SCRIPT_DIR/scripts/publish_event.py" --registry-dir "$REGISTRY_DIR" --job "$job_id" --event cancelled --detail "$detail" ${disp_arg[@]+"${disp_arg[@]}"} 2>/dev/null || true
|
|
elif [ "$sub_rc" -ne 1 ] || [ "$phase" = "inject" ]; then
|
|
local err_detail="deadline_timeout"
|
|
[ "$phase" = "inject" ] && err_detail="agent_blocked"
|
|
"$PY" "$SCRIPT_DIR/scripts/publish_event.py" --registry-dir "$REGISTRY_DIR" --job "$job_id" --event error --detail "$err_detail" ${disp_arg[@]+"${disp_arg[@]}"} 2>/dev/null || true
|
|
fi
|
|
fi
|
|
}
|
|
|
|
preflight_agent() {
|
|
local job_id="$1" target_session="${2:-$AGENT_SESSION}" force_warn_only="${3:-0}"
|
|
if [ "$AGENT" = "human" ]; then
|
|
return 0
|
|
fi
|
|
local sess="${target_session#herdr:}"
|
|
|
|
# B-3: must probe for the real binary, not the herdr() function / shim wrapper.
|
|
if ! has_real_herdr; then
|
|
echo "ERROR: this skill requires herdr (interactive agent sessions)." >&2
|
|
echo " Ensure herdr is installed and executable." >&2
|
|
return 1
|
|
fi
|
|
|
|
export HERDR_SESSION_NAME="$(resolve_herdr_session "$sess" "$WORKDIR")"
|
|
|
|
if ! herdr has-session -t "$sess" 2>/dev/null; then
|
|
PREFLIGHT_REASON="target_not_found" # Issue #34: not a busy session
|
|
echo "ERROR: 에이전트 세션 '$sess'이 존재하지 않습니다. 작업을 위임하기 전에 먼저 에이전트 세션을 기동해 주세요." >&2
|
|
echo " 팁: 'multi-agent-mux-resume' 또는 'multi-agent-mux-create'를 통해 에이전트를 먼저 생성할 수 있습니다." >&2
|
|
return 1
|
|
fi
|
|
|
|
# Check role suitability
|
|
local sess_role job_role
|
|
sess_role=$(SESS_NAME="$sess" MAM_STATE_JSON="$(load_state_json)" "$PY" -c "
|
|
import os, json
|
|
d = json.loads(os.environ.get('MAM_STATE_JSON', '{}'))
|
|
name = os.environ.get('SESS_NAME')
|
|
for s in d.get('herdr_sessions', []):
|
|
if s.get('name') == name:
|
|
print(s.get('role', ''))
|
|
break
|
|
" 2>/dev/null || echo "")
|
|
|
|
job_role=$("$PY" -c "
|
|
import json
|
|
try:
|
|
with open('$REGISTRY_DIR/$job_id.json') as f:
|
|
print(json.load(f).get('role', ''))
|
|
except Exception:
|
|
pass
|
|
" 2>/dev/null || echo "")
|
|
|
|
if [[ -n "$job_role" && -n "$sess_role" ]]; then
|
|
local check_result
|
|
check_result=$(JOB_ROLE="$job_role" SESS_ROLE="$sess_role" ROLE_ALIASES_JSON="$ROLE_ALIASES_JSON" "$PY" -c "
|
|
import os, json
|
|
job = os.environ.get('JOB_ROLE', '').lower()
|
|
sess = os.environ.get('SESS_ROLE', '').lower()
|
|
aliases = json.loads(os.environ.get('ROLE_ALIASES_JSON', '{}'))
|
|
candidates = aliases.get(job, [job])
|
|
if any(c in sess for c in candidates):
|
|
print('OK')
|
|
else:
|
|
print('MISMATCH')
|
|
" 2>/dev/null || echo "OK")
|
|
|
|
if [[ "$check_result" == "MISMATCH" ]]; then
|
|
local mismatch_msg="Target session '$sess' has role '$sess_role' which does not match job role '$job_role'."
|
|
if [[ "$STRICT_ROLE_CHECK" -eq 1 && "$force_warn_only" -ne 1 ]]; then
|
|
echo "ERROR: role suitability mismatch. $mismatch_msg" >&2
|
|
return 1
|
|
else
|
|
echo "WARNING: $mismatch_msg" >&2
|
|
fi
|
|
fi
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
inject_agent() {
|
|
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}"
|
|
# $4: subscriber log (a started line there also confirms delivery)
|
|
if [ "$AGENT" = "human" ]; then
|
|
echo "[human agent] complete the task, then run publish_event.py --event completed"
|
|
return 0
|
|
fi
|
|
local sess="${target_session#herdr:}"
|
|
|
|
# Before launching the agent, set up error trap to publish error event
|
|
if [ -n "${job_id:-}" ] && [ -n "${PY:-}" ]; then
|
|
pub_script="$SCRIPT_DIR/scripts/publish_event.py"
|
|
trap "rc=\$?; if [ \$rc -ne 0 ]; then \"$PY\" \"$pub_script\" --registry-dir '$REGISTRY_DIR' --job '$job_id' --event error --detail 'agent bootstrap failed (exit '\$rc')'; fi" EXIT
|
|
fi
|
|
|
|
echo "살아있는 에이전트 세션 '$sess'에 작업을 위임합니다..."
|
|
# ISSUE-18: send_keys_safe is the injection *means*. rc 0 (accepted, herdr
|
|
# saw it working) and rc 4 (accepted, no screen signal) both hand over to
|
|
# the protocol: the job's own `started` within ACK_TIMEOUT decides. Any
|
|
# other rc means the input never reached the agent (pane never quiesced,
|
|
# dialog, paste failure) - fail at once, never re-inject.
|
|
local sks_rc=0
|
|
send_keys_safe "$sess" "$instructions" "$job_id" || sks_rc=$?
|
|
case "$sks_rc" in
|
|
0|4) ;;
|
|
*)
|
|
trap - EXIT
|
|
echo "ERROR: 프롬프트 주입 실패 (send_keys_safe rc=$sks_rc) — 세션 '$sess'" >&2
|
|
return 1 ;;
|
|
esac
|
|
local ack_s="${MAM_INJECT_ACK_GRACE_S:-$ACK_TIMEOUT}"
|
|
if ! _wait_started "$job_id" "$ack_s" "${4:-}"; then
|
|
trap - EXIT
|
|
echo "ERROR: 프롬프트 주입 실패 — 세션 '$sess': no 'started' within ${ACK_WAITED_S:-$ack_s}s (ack timeout ${ack_s}s, bounded by the wall clock); not re-injecting" >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "작업이 세션 '$sess'에 전송되었습니다. (연결하려면: HERDR_SESSION_NAME=$HERDR_SESSION_NAME herdr agent attach $sess — lib.sh를 source한 셸에서 실행)"
|
|
trap - EXIT
|
|
return 0
|
|
}
|
|
|
|
# remote_dispatch <job_id> <instructions> <remote:session> <subscriber_log>
|
|
# ISSUE-18: no pane, no shared disk. Publish the retained dispatch on the job's
|
|
# topic and wait for the agent's own `started` (seen by the subscriber) within
|
|
# ACK_TIMEOUT. The dispatch is signed with MAM_DISPATCH_SECRET (.mam.env).
|
|
remote_dispatch() {
|
|
local job_id="$1" instructions="$2" target_session="$3" sub_log="${4:-}"
|
|
local sess="${target_session#remote:}"
|
|
echo "원격 세션 '$sess'에 작업을 dispatch 합니다 (broker)..."
|
|
local d_args=(--job "$job_id" --registry-dir "$REGISTRY_DIR" --session "$sess" --instructions "$instructions")
|
|
[ "${MAM_ALLOW_UNSIGNED_DISPATCH:-0}" = "1" ] && d_args+=(--allow-unsigned)
|
|
if ! "$PY" "$SCRIPT_DIR/scripts/remote.py" dispatch "${d_args[@]}" >/dev/null; then
|
|
echo "ERROR: dispatch publish failed for job $job_id (remote:$sess)" >&2
|
|
return 1
|
|
fi
|
|
local ack_s="${MAM_INJECT_ACK_GRACE_S:-$ACK_TIMEOUT}"
|
|
if ! _wait_started "$job_id" "$ack_s" "$sub_log"; then
|
|
echo "ERROR: remote session '$sess' published no 'started' within ${ACK_WAITED_S:-$ack_s}s (ack timeout ${ack_s}s, bounded by the wall clock); not re-dispatching" >&2
|
|
return 1
|
|
fi
|
|
echo "작업이 원격 세션 '$sess'에 전달되었습니다."
|
|
return 0
|
|
}
|
|
|
|
# deliver_prompt <job_id> <instructions> <session> <subscriber_log>
|
|
deliver_prompt() {
|
|
if _is_remote "$3"; then
|
|
remote_dispatch "$1" "$2" "$3" "${4:-}"
|
|
else
|
|
inject_agent "$1" "$2" "$3" "${4:-}"
|
|
fi
|
|
}
|
|
|
|
run_agent() {
|
|
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}" force_warn_only="${4:-0}"
|
|
local sess="${target_session#herdr:}"
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
echo "[dry-run] would delegate task to running agent '$AGENT' in herdr session '$sess' with instructions:"
|
|
echo "----"; echo "$instructions"; echo "----"
|
|
return 0
|
|
fi
|
|
preflight_agent "$job_id" "$target_session" "$force_warn_only" || return $?
|
|
inject_agent "$job_id" "$instructions" "$target_session" || return $?
|
|
}
|
|
|
|
cmd_status() {
|
|
parse_opts "$@"
|
|
[[ -n "$JOB_ID" ]] || { echo "status requires --job" >&2; exit 1; }
|
|
PY="$(pick_python)"
|
|
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get --job "$JOB_ID"
|
|
}
|
|
|
|
cmd_list() {
|
|
parse_opts "$@"
|
|
PY="$(pick_python)"
|
|
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" list
|
|
}
|
|
|
|
cmd_verify() {
|
|
parse_opts "$@"
|
|
[[ -n "$JOB_ID" ]] || { echo "verify requires --job" >&2; exit 1; }
|
|
[[ -n "$VALIDATE" ]] || { echo "verify requires --validate <script>" >&2; exit 1; }
|
|
echo "verifying job $JOB_ID with $VALIDATE"
|
|
if JOB_ID="$JOB_ID" REGISTRY_DIR="$REGISTRY_DIR" bash "$VALIDATE"; then
|
|
echo "verify: PASS (exit 0)"; exit 0
|
|
else
|
|
rc=$?; echo "verify: FAIL (exit $rc)"; exit "$rc"
|
|
fi
|
|
}
|
|
|
|
cmd_logs() {
|
|
# logs <job_id> | logs --list — delegates to registry.py's logs CLI, which
|
|
# reads the persistent audit log under $DELEGATE_JOB_LOGS_DIR (or
|
|
# <cwd>/delegate_job_logs). Run from your project dir so the default resolves.
|
|
PY="$(pick_python)"
|
|
if [[ "${1:-}" == "--list" ]]; then
|
|
"$PY" "$SCRIPT_DIR/scripts/registry.py" logs --list
|
|
else
|
|
local jid="${1:-}"
|
|
[[ -n "$jid" ]] || { echo "logs requires <job_id> or --list" >&2; exit 1; }
|
|
"$PY" "$SCRIPT_DIR/scripts/registry.py" logs "$jid"
|
|
fi
|
|
}
|
|
|
|
cmd_wait() {
|
|
parse_opts "$@"
|
|
PY="$(pick_python)"
|
|
if [[ -n "$JOB_ID" ]]; then
|
|
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
|
--job "$JOB_ID" --timeout "$TIMEOUT"
|
|
else
|
|
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
|
--wait-any --timeout "$TIMEOUT"
|
|
fi
|
|
}
|
|
|
|
main() {
|
|
local sub="${1:-}"; shift || true
|
|
case "$sub" in
|
|
submit) cmd_submit "$@";;
|
|
status) cmd_status "$@";;
|
|
list) cmd_list "$@";;
|
|
verify) cmd_verify "$@";;
|
|
wait) cmd_wait "$@";;
|
|
logs) cmd_logs "$@";;
|
|
""|-h|--help|help) usage;;
|
|
*) echo "unknown command: $sub" >&2; usage; exit 1;;
|
|
esac
|
|
}
|
|
|
|
# Tests source this file to exercise single helpers (MAM_DELEGATE_JOB_NO_MAIN=1).
|
|
if [ "${MAM_DELEGATE_JOB_NO_MAIN:-0}" != "1" ]; then
|
|
main "$@"
|
|
fi
|