fix(issue-6): fail-closed registry timeout and TUI turn handshake
- Introduce finalize_unfinished in multi-agent-mux-delegate-job for fail-closed atomic transitions - Enforce CAS dispatch_token and attempt_id pinning across engine, CLI, and lifecycle scripts - Integrate Herdr wait_for_agent_idle turn handshake to prevent prompt loss and race conditions - Resolve bash 3.2 set -u empty array expansion compatibility in lib.sh - Preserve Issue #5 R2 contract in CAS transition engine (engine.py, publish_event.py, service.py) - Add comprehensive regression suites (test_issue6_fail_closed.py, test_issue6_turn_handshake.py) - Bump lockstep framework and skills to v4.4.1 across lib.sh, eager.py, VERSIONS.md, 8 SKILL.md - Synchronize documentation (IMPROVEMENTS.md, MESSAGING.md, state-transitions.md, task.md, phase5-e2e.md) - Unanimous PASS achieved across 4 reviewer team leaders (Claude, Codex, OpenCode, Grok) Closes #6
This commit is contained in:
Godopu committed
1 parent
7b761be7b8
commit
ec61511699
51 files changed
+3477
-293
No files matched your search
@@ -125,6 +125,10 @@ parse_opts() {
|
||||
done
|
||||
}
|
||||
|
||||
_to_lower() {
|
||||
printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
|
||||
cmd_submit() {
|
||||
parse_opts "$@"
|
||||
[[ -n "$PROMPT" ]] || { echo "submit requires --prompt" >&2; exit 1; }
|
||||
@@ -159,12 +163,56 @@ EOF
|
||||
echo "provisioned job directory: $job_dir"
|
||||
fi
|
||||
|
||||
local START_TIME
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
if [[ "$TYPE" == "direct" ]]; then
|
||||
# 2) START THE SUBSCRIBER FIRST (ordering dependency — MQTT does not queue
|
||||
# non-retained messages for absent subscribers).
|
||||
# 1) Read dispatch token and attempt id for this dispatch
|
||||
local _disp_tok=""
|
||||
local _attempt_id=""
|
||||
_disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
||||
_attempt_id="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('attempt_id','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
||||
local pub_extra=()
|
||||
[ -n "$_disp_tok" ] && pub_extra+=(--expect-dispatch-token "$_disp_tok")
|
||||
[ -n "$_attempt_id" ] && pub_extra+=(--attempt-id "$_attempt_id")
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${pub_extra[*]}"
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
run_agent "$JOB_ID" "$instructions"
|
||||
local logs_root_dry="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root_dry/$JOB_ID"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 1) Preflight
|
||||
if ! preflight_agent "$JOB_ID" "$AGENT_SESSION" 0; then
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "preflight"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 5
|
||||
fi
|
||||
|
||||
# 2) TUI idle handshake
|
||||
if [ "$AGENT" != "human" ]; then
|
||||
local idle_rc=0
|
||||
wait_for_agent_idle "$AGENT_SESSION" || idle_rc=$?
|
||||
export _SKS_IDLE_RC="$idle_rc"
|
||||
if [ $idle_rc -eq 1 ] || [ $idle_rc -eq 2 ]; then
|
||||
finalize_unfinished "$JOB_ID" "error" "$idle_rc" "idle_wait"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 5
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3) START THE SUBSCRIBER
|
||||
local logf="$REGISTRY_DIR/$JOB_ID.subscriber.out"
|
||||
local sub_extra_args=()
|
||||
[ -n "$_disp_tok" ] && sub_extra_args+=(--expect-dispatch-token "$_disp_tok")
|
||||
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
||||
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
||||
${sub_extra_args[@]+"${sub_extra_args[@]}"} \
|
||||
>"$logf" 2>&1 &
|
||||
local sub_pid=$!
|
||||
echo "subscriber pid: $sub_pid (log: $logf)"
|
||||
@@ -177,14 +225,23 @@ EOF
|
||||
break
|
||||
fi
|
||||
else
|
||||
wait "$sub_pid" 2>/dev/null
|
||||
local sub_exit=$?
|
||||
local sub_exit=0
|
||||
wait "$sub_pid" 2>/dev/null || sub_exit=$?
|
||||
if [ $sub_exit -eq 0 ] || [ $sub_exit -eq 3 ]; then
|
||||
sub_ready=1
|
||||
break
|
||||
else
|
||||
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
|
||||
exit 1
|
||||
local mapped_st
|
||||
mapped_st="$(_map_sub_rc "$sub_exit" "$JOB_ID")"
|
||||
finalize_unfinished "$JOB_ID" "$mapped_st" "$sub_exit" "subscriber"
|
||||
mapped_st="${FINAL_JOB_STATUS:-$mapped_st}"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$mapped_st")"
|
||||
fi
|
||||
fi
|
||||
sleep 0.2
|
||||
@@ -193,17 +250,16 @@ EOF
|
||||
echo "WARNING: subscriber subscribe handshake timed out — falling back to proceed" >&2
|
||||
fi
|
||||
|
||||
# 3) run the agent (or print the command for dry-run / missing binary)
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID"
|
||||
# NOTE: the agent MUST use --job "$JOB_ID" (the one we just minted). Hard-coding
|
||||
# an id from an earlier session is the #1 reason a delegated job sits idle and
|
||||
# times out (see SKILL.md "Wrong job_id propagated to the agent"). We make the
|
||||
# freshness explicit in the instruction header.
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
# 5) Inject agent
|
||||
if ! inject_agent "$JOB_ID" "$instructions" "$AGENT_SESSION"; then
|
||||
reap_subscriber "$sub_pid"
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "inject"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_agent "$JOB_ID" "$instructions"
|
||||
|
||||
# 4) optional validation hook
|
||||
# 6) optional validation hook
|
||||
if [[ -n "$VALIDATE" ]]; then
|
||||
echo "running validation: $VALIDATE"
|
||||
if JOB_ID="$JOB_ID" REGISTRY_DIR="$REGISTRY_DIR" bash "$VALIDATE"; then
|
||||
@@ -214,23 +270,24 @@ EOF
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
# In dry-run we never started a real subscriber (the wrapper short-circuits
|
||||
# before launching one), but the wait below would still try to join the
|
||||
# background sub_pid from cmd_submit. Skip both the wait and the subscriber
|
||||
# log dump; the user just wants to see the instruction that would have run.
|
||||
local logs_root_dry="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root_dry/$JOB_ID"
|
||||
return 0
|
||||
fi
|
||||
|
||||
wait "$sub_pid" || true
|
||||
local sub_rc=0
|
||||
wait "$sub_pid" || sub_rc=$?
|
||||
echo "subscriber output:"; cat "$logf" || true
|
||||
|
||||
local job_status
|
||||
job_status="$(_map_sub_rc "$sub_rc" "$JOB_ID")"
|
||||
|
||||
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "direct"
|
||||
job_status="${FINAL_JOB_STATUS:-$job_status}"
|
||||
|
||||
# Last stdout line: the persistent audit-log dir for this job (see SKILL.md
|
||||
# "Audit Logs"). Callers can scrape `tail -n1` to find it.
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$job_status")"
|
||||
else
|
||||
# Implement loop/discuss orchestrator
|
||||
local iteration=1
|
||||
@@ -238,6 +295,8 @@ EOF
|
||||
local current_session="$AGENT_SESSION"
|
||||
local _phase="worker"
|
||||
local display_role="$DELEGATE_ROLE"
|
||||
local attempt=0
|
||||
local final_loop_rc=0
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
echo "[dry-run] orchestrator loop would start for job: $JOB_ID type: $TYPE"
|
||||
@@ -256,9 +315,8 @@ EOF
|
||||
# 1-1) Provision job directory and write iteration brief.md (MAM Job Restructuring)
|
||||
local job_dir="$REGISTRY_DIR/$JOB_ID"
|
||||
local clean_session="${current_session#herdr:}"
|
||||
if [[ "$DRY_RUN" != "1" ]]; then
|
||||
mkdir -p "$job_dir"
|
||||
cat <<EOF > "$job_dir/brief.md"
|
||||
mkdir -p "$job_dir"
|
||||
cat <<EOF > "$job_dir/brief.md"
|
||||
# 📋 Brief: Job $JOB_ID Delegation (Iteration $iteration)
|
||||
|
||||
- **Job ID**: $JOB_ID
|
||||
@@ -270,7 +328,40 @@ EOF
|
||||
## 🔎 Task Description
|
||||
$current_prompt
|
||||
EOF
|
||||
echo "provisioned iteration brief: $job_dir/brief.md"
|
||||
echo "provisioned iteration brief: $job_dir/brief.md"
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
|
||||
local force_warn_only=0
|
||||
local cp_lower def_lower
|
||||
cp_lower=$(_to_lower "$COUNTERPART_ROLE")
|
||||
def_lower=$(_to_lower "$DEFAULT_COUNTERPART_ROLE")
|
||||
if [[ "$_phase" == "reviewer" && "$COUNTERPART_ROLE_EXPLICIT" -eq 1 \
|
||||
&& "$cp_lower" != "$def_lower" ]]; then
|
||||
force_warn_only=1
|
||||
fi
|
||||
|
||||
# Preflight
|
||||
if ! preflight_agent "$JOB_ID" "$current_session" "$force_warn_only"; then
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "preflight"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 5
|
||||
fi
|
||||
|
||||
# Idle wait
|
||||
local cur_agent="$AGENT"
|
||||
[ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER"
|
||||
if [ "$cur_agent" != "human" ]; then
|
||||
local idle_rc=0
|
||||
wait_for_agent_idle "$current_session" || idle_rc=$?
|
||||
export _SKS_IDLE_RC="$idle_rc"
|
||||
if [ $idle_rc -eq 1 ] || [ $idle_rc -eq 2 ]; then
|
||||
finalize_unfinished "$JOB_ID" "error" "$idle_rc" "idle_wait"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 5
|
||||
fi
|
||||
fi
|
||||
|
||||
# Update job details in registry
|
||||
@@ -279,13 +370,20 @@ EOF
|
||||
--agent-session "$current_session" \
|
||||
--prompt "$current_prompt" \
|
||||
--iteration "$iteration" \
|
||||
--attempt-id "$attempt" \
|
||||
--role "$display_role" \
|
||||
--status "pending"
|
||||
|
||||
local _disp_tok=""
|
||||
_disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
||||
|
||||
# Start subscriber
|
||||
local logf="$REGISTRY_DIR/${JOB_ID}.iter_${iteration}_${display_role}.subscriber.out"
|
||||
local loop_sub_args=(--attempt-id "$attempt")
|
||||
[ -n "$_disp_tok" ] && loop_sub_args+=(--expect-dispatch-token "$_disp_tok")
|
||||
"$PY" "$SCRIPT_DIR/scripts/job_subscriber.py" --registry-dir "$REGISTRY_DIR" \
|
||||
--job "$JOB_ID" --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
||||
"${loop_sub_args[@]}" \
|
||||
>"$logf" 2>&1 &
|
||||
local sub_pid=$!
|
||||
echo "subscriber pid: $sub_pid (log: $logf)"
|
||||
@@ -298,14 +396,23 @@ EOF
|
||||
break
|
||||
fi
|
||||
else
|
||||
wait "$sub_pid" 2>/dev/null
|
||||
local sub_exit=$?
|
||||
if [ $sub_exit -eq 0 ]; then
|
||||
local sub_exit=0
|
||||
wait "$sub_pid" 2>/dev/null || sub_exit=$?
|
||||
if [ $sub_exit -eq 0 ] || [ $sub_exit -eq 3 ]; then
|
||||
sub_ready=1
|
||||
break
|
||||
else
|
||||
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
|
||||
exit 1
|
||||
local mapped_st
|
||||
mapped_st="$(_map_sub_rc "$sub_exit" "$JOB_ID")"
|
||||
finalize_unfinished "$JOB_ID" "$mapped_st" "$sub_exit" "subscriber"
|
||||
mapped_st="${FINAL_JOB_STATUS:-$mapped_st}"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$mapped_st")"
|
||||
fi
|
||||
fi
|
||||
sleep 0.2
|
||||
@@ -315,38 +422,27 @@ EOF
|
||||
fi
|
||||
|
||||
# Format instruction block
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID"
|
||||
local loop_pub_extra=(--attempt-id "$attempt")
|
||||
[ -n "$_disp_tok" ] && loop_pub_extra+=(--expect-dispatch-token "$_disp_tok")
|
||||
local pub="$PY $SCRIPT_DIR/scripts/publish_event.py --registry-dir $REGISTRY_DIR --job $JOB_ID ${loop_pub_extra[*]}"
|
||||
local instructions="Your job_id is \"$JOB_ID\". Detailed task requirements, instructions, and target output paths for iteration $iteration are documented in the task brief file at: .mam/jobs/$JOB_ID/brief.md. Please READ and follow .mam/jobs/$JOB_ID/brief.md to complete your work. Commands: start='$pub --event started', success='$pub --event completed --detail <summary>', error='$pub --event error --detail <reason>'."
|
||||
|
||||
# Trigger agent
|
||||
local force_warn_only=0
|
||||
if [[ "$_phase" == "reviewer" && "$COUNTERPART_ROLE_EXPLICIT" -eq 1 \
|
||||
&& "${COUNTERPART_ROLE,,}" != "${DEFAULT_COUNTERPART_ROLE,,}" ]]; then
|
||||
force_warn_only=1
|
||||
if ! inject_agent "$JOB_ID" "$instructions" "$current_session"; then
|
||||
reap_subscriber "$sub_pid"
|
||||
finalize_unfinished "$JOB_ID" "error" 1 "inject"
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 1
|
||||
fi
|
||||
run_agent "$JOB_ID" "$instructions" "$current_session" "$force_warn_only"
|
||||
|
||||
# Wait for subscriber
|
||||
local sub_rc=0
|
||||
wait "$sub_pid" || sub_rc=$?
|
||||
echo "subscriber output:"; cat "$logf" || true
|
||||
|
||||
# Check job status based on subscriber exit code
|
||||
local job_status="running"
|
||||
if [[ $sub_rc -eq 0 ]]; then
|
||||
job_status="completed"
|
||||
elif [[ $sub_rc -eq 1 ]]; then
|
||||
job_status="error"
|
||||
elif [[ $sub_rc -eq 3 ]]; then
|
||||
job_status="broker_unavailable"
|
||||
local disk_st
|
||||
disk_st="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
||||
if [[ "$disk_st" == "completed" || "$disk_st" == "error" ]]; then
|
||||
job_status="$disk_st"
|
||||
fi
|
||||
else
|
||||
job_status="timeout"
|
||||
fi
|
||||
local job_status
|
||||
job_status="$(_map_sub_rc "$sub_rc" "$JOB_ID")"
|
||||
|
||||
echo "Job role $display_role finished with status: $job_status"
|
||||
|
||||
@@ -354,11 +450,22 @@ EOF
|
||||
local feedback
|
||||
feedback="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get-feedback --job "$JOB_ID")"
|
||||
echo "Feedback/Detail: $feedback"
|
||||
|
||||
if [[ "$_phase" == "worker" ]]; then
|
||||
if [[ "$job_status" != "completed" ]]; then
|
||||
echo "Worker did not complete successfully (status: $job_status). Terminating workflow."
|
||||
break
|
||||
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "worker" "$current_session"
|
||||
job_status="${FINAL_JOB_STATUS:-$job_status}"
|
||||
if [[ "$job_status" == "completed" ]]; then
|
||||
echo "Worker completed during grace period. Continuing to reviewer."
|
||||
feedback="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get-feedback --job "$JOB_ID")"
|
||||
else
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$job_status")"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Worker completed successfully, now switch to reviewer
|
||||
@@ -374,30 +481,46 @@ EOF
|
||||
else
|
||||
if [[ "$job_status" != "completed" ]]; then
|
||||
echo "Reviewer did not complete successfully (status: $job_status). Terminating workflow."
|
||||
break
|
||||
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "reviewer" "$current_session"
|
||||
job_status="${FINAL_JOB_STATUS:-$job_status}"
|
||||
if [[ "$job_status" == "completed" ]]; then
|
||||
echo "Reviewer completed during grace period. Continuing evaluation."
|
||||
feedback="$("$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" get-feedback --job "$JOB_ID")"
|
||||
else
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit "$(_rc_for "$job_status")"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
# Reviewer finished. Check if pass/agree
|
||||
local success=0
|
||||
local fb_lower
|
||||
fb_lower=$(_to_lower "$feedback")
|
||||
if [[ "$TYPE" == "loop" ]]; then
|
||||
if [[ "${feedback,,}" == *"pass"* ]]; then
|
||||
if [[ "$fb_lower" == *"pass"* ]]; then
|
||||
success=1
|
||||
fi
|
||||
elif [[ "$TYPE" == "discuss" ]]; then
|
||||
if [[ "${feedback,,}" == *"agree"* ]]; then
|
||||
if [[ "$fb_lower" == *"agree"* ]]; then
|
||||
success=1
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
if [[ "$success" == "1" ]]; then
|
||||
echo "Reviewer approved the work. Finalizing job as completed."
|
||||
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" status --job "$JOB_ID" --set "completed"
|
||||
final_loop_rc=0
|
||||
break
|
||||
else
|
||||
# Reviewer rejected/provided feedback. Increment & check max iterations
|
||||
if [[ $iteration -ge $MAX_ITERATIONS ]]; then
|
||||
echo "Max iterations ($MAX_ITERATIONS) reached without approval. Terminating workflow."
|
||||
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" status --job "$JOB_ID" --set "error"
|
||||
final_loop_rc=1
|
||||
break
|
||||
fi
|
||||
|
||||
@@ -418,40 +541,216 @@ EOF
|
||||
else
|
||||
local rc=$?
|
||||
echo "validation: FAIL (exit $rc)"
|
||||
final_loop_rc=$rc
|
||||
fi
|
||||
fi
|
||||
|
||||
# Last stdout line: the persistent audit-log dir
|
||||
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit "$final_loop_rc"
|
||||
fi
|
||||
}
|
||||
|
||||
run_agent() {
|
||||
local job_id="$1"; local instructions="$2"; local target_session="${3:-$AGENT_SESSION}"; local force_warn_only="${4:-0}"
|
||||
# The skill is INTERACTIVE-ONLY. We never invoke `claude -p` or any other
|
||||
# one-shot print mode, because:
|
||||
# - claude -p exits the moment stdin is drained, so there's nothing to
|
||||
# `herdr session attach` to afterwards.
|
||||
# - fire-and-forget via wrapper defeats the whole point of the audit log
|
||||
# (you can't tell what happened if the agent crashes mid-turn).
|
||||
# - the job registry already gives us an authoritative completion signal,
|
||||
# so we don't need a wrapper-side exit code to know "done".
|
||||
# The user attaches with `herdr session attach <session>` and types follow-up
|
||||
# prompts themselves. We pre-load the first prompt via stdin and `read`
|
||||
# keeps the pane open after the agent exits so the user can review.
|
||||
reap_subscriber() {
|
||||
if [ -n "${1:-}" ]; then
|
||||
kill "$1" 2>/dev/null || true
|
||||
wait "$1" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
_disk_status() {
|
||||
local jid="$1"
|
||||
"$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$jid.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true
|
||||
}
|
||||
|
||||
_map_sub_rc() {
|
||||
local sub_rc="$1" job_id="$2"
|
||||
local job_status="running"
|
||||
if [[ $sub_rc -eq 0 ]]; then
|
||||
job_status="completed"
|
||||
elif [[ $sub_rc -eq 1 ]]; then
|
||||
job_status="error"
|
||||
elif [[ $sub_rc -eq 3 ]]; then
|
||||
job_status="broker_unavailable"
|
||||
local disk_st
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [[ "$disk_st" == "completed" || "$disk_st" == "error" || "$disk_st" == "cancelled" ]]; then
|
||||
job_status="$disk_st"
|
||||
fi
|
||||
else
|
||||
job_status="timeout"
|
||||
fi
|
||||
echo "$job_status"
|
||||
}
|
||||
|
||||
_rc_for() {
|
||||
case "$1" in
|
||||
completed) echo 0 ;;
|
||||
error) echo 1 ;;
|
||||
timeout) echo 2 ;;
|
||||
broker_unavailable) echo 3 ;;
|
||||
cancelled) echo 5 ;;
|
||||
*) echo 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
finalize_unfinished() {
|
||||
local job_id="$1" job_status="$2" sub_rc="$3" phase="${4:-}"
|
||||
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "$job_status" = "completed" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local cur_disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$cur_disk_st" = "completed" ] || [ "$cur_disk_st" = "error" ] || [ "$cur_disk_st" = "cancelled" ]; then
|
||||
FINAL_JOB_STATUS="$cur_disk_st"
|
||||
[ "$cur_disk_st" = "completed" ] && return 0
|
||||
else
|
||||
FINAL_JOB_STATUS="$job_status"
|
||||
fi
|
||||
# broker_unavailable: re-observe disk for remaining deadline at 3s intervals
|
||||
if [ "$job_status" = "broker_unavailable" ]; then
|
||||
local now elapsed remaining
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - ${START_TIME:-now} ))
|
||||
remaining=$(( TIMEOUT - elapsed ))
|
||||
[ $remaining -lt 0 ] && remaining=0
|
||||
while [ $remaining -gt 0 ]; do
|
||||
local wait_interval=3
|
||||
[ $remaining -lt $wait_interval ] && wait_interval=$remaining
|
||||
sleep "$wait_interval"
|
||||
remaining=$(( remaining - wait_interval ))
|
||||
local disk_st
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
FINAL_JOB_STATUS="$disk_st"
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# timeout: if subscriber timed out, check if agent is still actively working in herdr
|
||||
if [ "$job_status" = "timeout" ]; then
|
||||
local target_sess="${5:-${current_session:-${target_session:-$AGENT_SESSION}}}"
|
||||
local sess="${target_sess#herdr:}"
|
||||
local target_agent
|
||||
target_agent=$(_sanitize_herdr_agent_name "$sess")
|
||||
local agent_st=""
|
||||
if [ "$sess" != "human" ] && [ -n "$sess" ]; then
|
||||
local raw_st
|
||||
raw_st="$(_sks_herdr agent get "$target_agent" 2>/dev/null || _sks_herdr agent get "$sess" 2>/dev/null || true)"
|
||||
if [ -n "$raw_st" ]; then
|
||||
agent_st="$("$(_delegate_py_bin)" -c '
|
||||
import sys, json
|
||||
try:
|
||||
d = json.loads(sys.stdin.read())
|
||||
print((d.get("result", {}).get("agent", {}) or {}).get("agent_status", "") or "")
|
||||
except Exception:
|
||||
print("")
|
||||
' <<< "$raw_st" 2>/dev/null || true)"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$agent_st" = "working" ]; then
|
||||
local now elapsed remaining
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - ${START_TIME:-now} ))
|
||||
remaining=$(( TIMEOUT - elapsed ))
|
||||
[ $remaining -lt 0 ] && remaining=0
|
||||
while [ $remaining -gt 0 ]; do
|
||||
local disk_st
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
FINAL_JOB_STATUS="$disk_st"
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
break
|
||||
fi
|
||||
local cur_raw cur_st=""
|
||||
cur_raw="$(_sks_herdr agent get "$target_agent" 2>/dev/null || _sks_herdr agent get "$sess" 2>/dev/null || true)"
|
||||
if [ -n "$cur_raw" ]; then
|
||||
cur_st="$("$(_delegate_py_bin)" -c '
|
||||
import sys, json
|
||||
try:
|
||||
d = json.loads(sys.stdin.read())
|
||||
print((d.get("result", {}).get("agent", {}) or {}).get("agent_status", "") or "")
|
||||
except Exception:
|
||||
print("")
|
||||
' <<< "$cur_raw" 2>/dev/null || true)"
|
||||
fi
|
||||
if [ "$cur_st" != "working" ]; then
|
||||
sleep 1
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
FINAL_JOB_STATUS="$disk_st"
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
fi
|
||||
break
|
||||
fi
|
||||
local wait_interval=1
|
||||
[ $remaining -lt $wait_interval ] && wait_interval=$remaining
|
||||
sleep "$wait_interval"
|
||||
remaining=$(( remaining - wait_interval ))
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
local close_req="deadline_timeout"
|
||||
local detail=""
|
||||
if [ "$phase" = "preflight" ] || [ "$phase" = "idle_wait" ]; then
|
||||
close_req="cancelled"
|
||||
detail="session_busy"
|
||||
fi
|
||||
|
||||
local close_args=(--job "$job_id" --request "$close_req")
|
||||
[ -n "$detail" ] && close_args+=(--detail "$detail")
|
||||
[ -n "${_disp_tok:-}" ] && close_args+=(--expect-dispatch-token "$_disp_tok")
|
||||
|
||||
local close_rc=0
|
||||
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" close "${close_args[@]}" 2>&1 || close_rc=$?
|
||||
|
||||
# If close failed due to dispatch token mismatch (rc=4), suppress secondary event emission
|
||||
# to prevent CAS/ABA bypass
|
||||
if [ $close_rc -eq 4 ]; then
|
||||
echo "finalize_unfinished: dispatch token mismatch (stale attempt); suppressing event emission" >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check if job completed on disk (universal, for all TYPEs)
|
||||
local cur_disk_st
|
||||
cur_disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$cur_disk_st" = "completed" ]; then
|
||||
FINAL_JOB_STATUS="completed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# direct path only: publish MQTT error/cancelled event (loop/discuss forbidden: O-1)
|
||||
if [ "$TYPE" = "direct" ]; then
|
||||
local disp_arg=()
|
||||
[ -n "${_disp_tok:-}" ] && disp_arg=(--expect-dispatch-token "$_disp_tok")
|
||||
if [ "$close_req" = "cancelled" ]; then
|
||||
"$PY" "$SCRIPT_DIR/scripts/publish_event.py" --registry-dir "$REGISTRY_DIR" --job "$job_id" --event cancelled --detail "session_busy" ${disp_arg[@]+"${disp_arg[@]}"} 2>/dev/null || true
|
||||
elif [ "$sub_rc" -ne 1 ] || [ "$phase" = "inject" ]; then
|
||||
local err_detail="deadline_timeout"
|
||||
[ "$phase" = "inject" ] && err_detail="agent_blocked"
|
||||
"$PY" "$SCRIPT_DIR/scripts/publish_event.py" --registry-dir "$REGISTRY_DIR" --job "$job_id" --event error --detail "$err_detail" ${disp_arg[@]+"${disp_arg[@]}"} 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
preflight_agent() {
|
||||
local job_id="$1" target_session="${2:-$AGENT_SESSION}" force_warn_only="${3:-0}"
|
||||
if [ "$AGENT" = "human" ]; then
|
||||
echo "[human agent] complete the task, then run publish_event.py --event completed"
|
||||
return
|
||||
return 0
|
||||
fi
|
||||
local sess="${target_session#herdr:}"
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
echo "[dry-run] would delegate task to running agent '$AGENT' in herdr session '$sess' with instructions:"
|
||||
echo "----"; echo "$instructions"; echo "----"
|
||||
return
|
||||
fi
|
||||
|
||||
# B-3: must probe for the real binary, not the herdr() function / shim wrapper.
|
||||
if ! has_real_herdr; then
|
||||
echo "ERROR: this skill requires herdr (interactive agent sessions)." >&2
|
||||
@@ -459,10 +758,6 @@ run_agent() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Auto-resolve isolation the same way resume/stop/create do — don't rely on
|
||||
# the caller having exported HERDR_SERVER_NAME by hand. This is what lets
|
||||
# delegation reach an agent living in an isolated herdr session (e.g. one
|
||||
# created with --herdr-server) instead of silently looking in "default".
|
||||
export HERDR_SESSION_NAME="$(resolve_herdr_session "$sess" "$WORKDIR")"
|
||||
|
||||
if ! herdr has-session -t "$sess" 2>/dev/null; then
|
||||
@@ -516,25 +811,45 @@ else:
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
inject_agent() {
|
||||
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}"
|
||||
if [ "$AGENT" = "human" ]; then
|
||||
echo "[human agent] complete the task, then run publish_event.py --event completed"
|
||||
return 0
|
||||
fi
|
||||
local sess="${target_session#herdr:}"
|
||||
|
||||
# Before launching the agent, set up error trap to publish error event
|
||||
if [ -n "${job_id:-}" ] && [ -n "${PY:-}" ]; then
|
||||
pub_script="$SCRIPT_DIR/scripts/publish_event.py"
|
||||
trap "rc=\$?; if [ \$rc -ne 0 ]; then \"$PY\" \"$pub_script\" --job '$job_id' --event error --detail 'agent bootstrap failed (exit '\$rc')'; fi" EXIT
|
||||
trap "rc=\$?; if [ \$rc -ne 0 ]; then \"$PY\" \"$pub_script\" --registry-dir '$REGISTRY_DIR' --job '$job_id' --event error --detail 'agent bootstrap failed (exit '\$rc')'; fi" EXIT
|
||||
fi
|
||||
|
||||
echo "살아있는 에이전트 세션 '$sess'에 작업을 위임합니다..."
|
||||
if ! send_keys_safe "$sess" "$instructions" "$job_id"; then
|
||||
trap - EXIT
|
||||
echo "ERROR: 프롬프트 주입 실패 — 세션 '$sess' (프롬프트 잠금 의심)" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# NOTE: `herdr session attach` operates on whole herdr *sessions* (server
|
||||
# instances), not an individual agent by its MAM name — `agent attach` is
|
||||
# the real command for that. HERDR_SESSION_NAME is inlined so the printed
|
||||
# command is copy-pasteable in a fresh shell that hasn't sourced lib.sh.
|
||||
|
||||
echo "작업이 세션 '$sess'에 전송되었습니다. (연결하려면: HERDR_SESSION_NAME=$HERDR_SESSION_NAME herdr agent attach $sess — lib.sh를 source한 셸에서 실행)"
|
||||
trap - EXIT
|
||||
return 0
|
||||
}
|
||||
|
||||
run_agent() {
|
||||
local job_id="$1" instructions="$2" target_session="${3:-$AGENT_SESSION}" force_warn_only="${4:-0}"
|
||||
local sess="${target_session#herdr:}"
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
echo "[dry-run] would delegate task to running agent '$AGENT' in herdr session '$sess' with instructions:"
|
||||
echo "----"; echo "$instructions"; echo "----"
|
||||
return 0
|
||||
fi
|
||||
preflight_agent "$job_id" "$target_session" "$force_warn_only" || return $?
|
||||
inject_agent "$job_id" "$instructions" "$target_session" || return $?
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
|
||||
Reference in new issue
Block a user