tmpl/multi-agent-mux
1
0 Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(issue-17): item 4 - the delegate-job wrapper's decisions move to Python

Browse Source 
The wrapper keeps spawn / wait / kill, key injection and the herdr
diagnostic probe; what it decided in bash is now lib_py/delegation/wrapper.py,
reached through one `_decide` helper:

- finalize_unfinished: the clock choice, the disk grace and broker
  re-observation, the close request / label, the close and the event
  (finalize-begin / finalize-end; bash runs only the busy probe between them)
- start_stall_watch: whether to watch (MAM_STALL_WATCH, remote, human)
- _map_sub_rc, _rc_for (+ MAM_DELEGATE_FAIL_CLOSED), the subscriber
  early-exit check and cmd_wait's rc normalisation
- mam_int / mam_int_knob run lib_py/ints.py, which is also the runner's
  _positive_int: one implementation of the Issue #26 rule

No behaviour change: tests/test_issue17_item4_decisions.py runs the old
bash functions (copied verbatim from 7209750) and the new wrapper on the
same inputs and compares output, FINAL_JOB_STATUS and the registry /
publish calls. Three static tests that pinned the old bash literals now
pin the Python entry points instead.

Refs #17

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
GodopuandClaude Opus 5.5 committed 2026-10-06 04:59:58 +00:00
1 parent 7209750ac7
commit 63d9c205ff
13 files changed
+1131 -289
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.agents/skills/multi-agent-mux-delegate-job/multi-agent-mux-delegate-job
+49 -240
View File
@@ -94,6 +94,12 @@ _logs_dir_abs() {
"$PY" -c 'import os, sys; sys.path.insert(0, sys.argv[1]); from lib_py.delegation.mqtt_common import get_logs_dir; print(os.path.abspath(get_logs_dir()))' "$SCRIPT_DIR/.."
}
# Issue #17 item 4: the wrapper's decisions are lib_py/delegation/wrapper.py;
# bash only branches on what a subcommand prints or returns.
_decide() {
PYTHONPATH="$SCRIPT_DIR/..${PYTHONPATH:+:$PYTHONPATH}" "$PY" -m lib_py.delegation.wrapper "$@"
}
# _pub_cmd <extra...>: the publish command as one shell-quoted string.
_pub_cmd() {
local s
@@ -349,21 +355,18 @@ EOF
else
local sub_exit=0
wait "$sub_pid" 2>/dev/null || sub_exit=$?
if [ $sub_exit -eq $SUB_RC_COMPLETED ] || [ $sub_exit -eq $SUB_RC_BROKER_UNAVAILABLE ]; then
if _decide subscriber-up "$sub_exit"; then
sub_ready=1
break
else
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
local mapped_st
mapped_st="$(_map_sub_rc "$sub_exit" "$JOB_ID")"
mapped_st="$(_decide sub-status "$sub_exit" --job "$JOB_ID" --registry-dir "$REGISTRY_DIR")"
finalize_unfinished "$JOB_ID" "$mapped_st" "$sub_exit" "subscriber"
mapped_st="${FINAL_JOB_STATUS:-$mapped_st}"
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
echo "$logs_root/$JOB_ID"
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
exit 0
fi
exit "$(_rc_for "$mapped_st")"
exit "$(_decide exit-code "$mapped_st")"
fi
fi
sleep 0.2
@@ -399,7 +402,7 @@ EOF
echo "subscriber output:"; cat "$logf" || true
local job_status
job_status="$(_map_sub_rc "$sub_rc" "$JOB_ID")"
job_status="$(_decide sub-status "$sub_rc" --job "$JOB_ID" --registry-dir "$REGISTRY_DIR")"
finalize_unfinished "$JOB_ID" "$job_status" "$sub_rc" "direct"
job_status="${FINAL_JOB_STATUS:-$job_status}"
@@ -408,10 +411,7 @@ EOF
# "Audit Logs"). Callers can scrape `tail -n1` to find it.
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
echo "$logs_root/$JOB_ID"
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
exit 0
fi
exit "$(_rc_for "$job_status")"
exit "$(_decide exit-code "$job_status")"
else
# Implement loop/discuss orchestrator
local iteration=1
@@ -527,21 +527,18 @@ EOF
else
local sub_exit=0
wait "$sub_pid" 2>/dev/null || sub_exit=$?
if [ $sub_exit -eq $SUB_RC_COMPLETED ] || [ $sub_exit -eq $SUB_RC_BROKER_UNAVAILABLE ]; then
if _decide subscriber-up "$sub_exit"; then
sub_ready=1
break
else
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
local mapped_st
mapped_st="$(_map_sub_rc "$sub_exit" "$JOB_ID")"
mapped_st="$(_decide sub-status "$sub_exit" --job "$JOB_ID" --registry-dir "$REGISTRY_DIR")"
finalize_unfinished "$JOB_ID" "$mapped_st" "$sub_exit" "subscriber"
mapped_st="${FINAL_JOB_STATUS:-$mapped_st}"
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
echo "$logs_root/$JOB_ID"
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
exit 0
fi
exit "$(_rc_for "$mapped_st")"
exit "$(_decide exit-code "$mapped_st")"
fi
fi
sleep 0.2
@@ -574,7 +571,7 @@ EOF
echo "subscriber output:"; cat "$logf" || true
local job_status
job_status="$(_map_sub_rc "$sub_rc" "$JOB_ID")"
job_status="$(_decide sub-status "$sub_rc" --job "$JOB_ID" --registry-dir "$REGISTRY_DIR")"
echo "Job role $display_role finished with status: $job_status"
@@ -593,10 +590,7 @@ EOF
else
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
echo "$logs_root/$JOB_ID"
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
exit 0
fi
exit "$(_rc_for "$job_status")"
exit "$(_decide exit-code "$job_status")"
fi
fi
@@ -621,10 +615,7 @@ EOF
else
local logs_root="${DELEGATE_JOB_LOGS_DIR:-$WORKDIR/delegate_job_logs}"
echo "$logs_root/$JOB_ID"
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
exit 0
fi
exit "$(_rc_for "$job_status")"
exit "$(_decide exit-code "$job_status")"
fi
fi
@@ -692,18 +683,13 @@ reap_subscriber() {
}
# Issue #30: a notify-only stall watcher for a job with a local herdr pane
# (lib_py.stall_watch decides; it never closes the job). A remote
# or human job has no pane to watch. Sets STALL_PID ("" when none started).
# (lib_py.stall_watch decides; it never closes the job). Whether to watch
# (MAM_STALL_WATCH, remote, human) is `_decide stall-watch`. Sets STALL_PID
# ("" when none started).
start_stall_watch() {
local session="$1" agent="$2" tok="${3:-}" att="${4:-}"
STALL_PID=""
[ "${MAM_STALL_WATCH:-1}" = "0" ] && return 0
if _is_remote "$session"; then
echo "stall watch: remote job, no local pane - stall detection off" >&2; return 0
fi
if [ "$agent" = "human" ] || [ "${session#herdr:}" = "human" ]; then
echo "stall watch: human job, no local pane - stall detection off" >&2; return 0
fi
_decide stall-watch "$session" "$agent" || return 0
local args=(--job "$JOB_ID" --registry-dir "$REGISTRY_DIR" --target "${session#herdr:}"
--agent "$agent" --herdr-session "${HERDR_SESSION_NAME:-}" --watch-pid "$$")
[ -n "$tok" ] && args+=(--expect-dispatch-token "$tok")
@@ -737,11 +723,6 @@ stop_stall_watch() {
STALL_PID=""
}
_disk_status() {
local jid="$1"
"$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$jid.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true
}
# ISSUE-18: delivery is confirmed by the protocol, not the screen. After the
# prompt was handed over (herdr accepted the submission, or the dispatch was
# published for a remote session) the job's own `started` must arrive within
@@ -816,208 +797,36 @@ _within_deadline() {
wait "$pid"
}
# ISSUE-14: agent-agnostic liveness. publish_event appends every event to the
# job's on-disk events.log BEFORE its wire publish (record_local_event), so a
# `progress` heartbeat the subscriber never saw (it had already idled out, or
# the broker dropped it) is visible here the moment it is sent - a slow or
# failing broker no longer delays it by its publish retries. An agent running
# its tests in a background shell looks idle to herdr and shows no busy
# token, yet keeps heartbeating - this is the only evidence that covers that
# case.
_recent_disk_heartbeat() {
local job_id="$1" window="$2" f="$REGISTRY_DIR/$job_id.events.log"
[ -f "$f" ] || return 1
"$PY" - "$f" "$window" <<'PYEOF'
import calendar, json, sys, time
path, window = sys.argv[1], float(sys.argv[2])
last = ""
with open(path) as fh:
for line in fh:
if line.strip():
last = line
try:
stamp = json.loads(last)["timestamp"]
seen = calendar.timegm(time.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ"))
except Exception:
sys.exit(1)
sys.exit(0 if time.time() - seen <= window else 1)
PYEOF
}
_map_sub_rc() {
local sub_rc="$1" job_id="$2"
local job_status="running"
if [[ $sub_rc -eq $SUB_RC_COMPLETED ]]; then
job_status="completed"
elif [[ $sub_rc -eq $SUB_RC_ERROR ]]; then
job_status="error"
elif [[ $sub_rc -eq $SUB_RC_BROKER_UNAVAILABLE ]]; then
job_status="broker_unavailable"
local disk_st
disk_st="$(_disk_status "$job_id")"
if [[ "$disk_st" == "completed" || "$disk_st" == "error" || "$disk_st" == "cancelled" ]]; then
job_status="$disk_st"
fi
else
job_status="timeout" # SUB_RC_NO_EVENT, SUB_RC_WALL_CLOCK (and any other code)
fi
echo "$job_status"
}
_rc_for() {
case "$1" in
completed) echo 0 ;;
error) echo 1 ;;
timeout) echo 2 ;;
broker_unavailable) echo 3 ;;
cancelled) echo 5 ;;
*) echo 1 ;;
esac
}
# finalize_unfinished <job_id> <status> <sub_rc> [phase] [session]
# Closes a job that did not complete (Issue #33 labels, ISSUE-14/18 disk grace,
# Issue #34 cancel reasons) and sets FINAL_JOB_STATUS. The decisions, the disk
# waits, the close and the event are `_decide finalize-begin/-end`; bash runs
# only the herdr diagnostic probe between the two, when asked to.
finalize_unfinished() {
local job_id="$1" job_status="$2" sub_rc="$3" phase="${4:-}" FINALIZE_CLOCK=""
if [ "${MAM_DELEGATE_FAIL_CLOSED:-1}" = "0" ]; then
return 0
fi
if [ "$job_status" = "completed" ]; then
return 0
fi
# Issue #30: what the stall watcher saw goes with an unfinished job's close.
if [ -s "$REGISTRY_DIR/$job_id.stall.out" ]; then
echo "stall watch log for $job_id (last 3 lines):" >&2
tail -n 3 "$REGISTRY_DIR/$job_id.stall.out" >&2 || true
fi
local cur_disk_st="$(_disk_status "$job_id")"
if [ "$cur_disk_st" = "completed" ] || [ "$cur_disk_st" = "error" ] || [ "$cur_disk_st" = "cancelled" ]; then
FINAL_JOB_STATUS="$cur_disk_st"
[ "$cur_disk_st" = "completed" ] && return 0
else
FINAL_JOB_STATUS="$job_status"
fi
# broker_unavailable: re-observe the disk at 3 s intervals until this
# dispatch's wall-clock deadline (START_TIME + TIMEOUT). The deadline is read
# off the clock on every pass; summing the sleeps alone ignores the time the
# disk probes themselves take and overruns TIMEOUT.
if [ "$job_status" = "broker_unavailable" ]; then
local deadline left disk_st
deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
while :; do
left=$(( deadline - $(date +%s) ))
[ "$left" -le 0 ] && break
[ "$left" -gt 3 ] && left=3
sleep "$left"
disk_st="$(_disk_status "$job_id")"
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
job_status="$disk_st"
FINAL_JOB_STATUS="$disk_st"
[ "$disk_st" = "completed" ] && return 0
break
fi
done
# Issue #33 (review R3): a wait that ran into the deadline is the wall clock.
[ "$(date +%s)" -ge "$deadline" ] && FINALIZE_CLOCK="wall"
fi
# timeout: if subscriber timed out, check if agent is still actively working in herdr
if [ "$job_status" = "timeout" ]; then
local target_sess="${5:-${current_session:-${target_session:-$AGENT_SESSION}}}"
local sess="${target_sess#herdr:}"
local target_agent
local job_id="$1" job_status="$2" sub_rc="$3" phase="${4:-}"
local target_sess="${5:-${current_session:-${target_session:-$AGENT_SESSION}}}"
local step final evidence_until deadline out
read -r step final evidence_until deadline <<<"$(_decide finalize-begin --job "$job_id" \
--registry-dir "$REGISTRY_DIR" --status "$job_status" ${START_TIME:+--start-time "$START_TIME"} \
--timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" --agent "$AGENT" --session "$target_sess")"
case "$step" in
keep) return 0 ;;
done) FINAL_JOB_STATUS="$final"; return 0 ;;
esac
FINAL_JOB_STATUS="$final"
if [ "$step" = "probe" ]; then
local sess="${target_sess#herdr:}" target_agent diag="no"
target_agent=$(_sanitize_herdr_agent_name "$sess")
# ISSUE-14 / ISSUE-18: the subscriber's idle clock only counts events it
# received over the wire. Keep re-observing the disk while the *contract*
# says the agent is alive: an event in events.log at most IDLE_TIMEOUT
# (K x N) old (_recent_disk_heartbeat - publish_event records every event
# there before its wire publish, the automatic heartbeat timer included).
# herdr status and the pane are NOT evidence any more; they are logged
# once here as a diagnostic, so an operator can tell a dead agent from a
# silent one on the same host. Window policy: entering this grace opens
# one IDLE_TIMEOUT window and every disk event re-opens it from the time
# of that poll, so an agent that leaves no event at all is closed about
# 2 x IDLE_TIMEOUT after its last wire event. Evidence only ever extends
# the idle window, never this dispatch's wall clock: the loop ends at
# START_TIME + TIMEOUT read off the clock on every pass.
local deadline evidence_until left disk_st
deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
if [ "$AGENT" != "human" ] && [ "$sess" != "human" ] && [ -n "$sess" ] && ! _is_remote "$target_sess"; then
local diag="no"
_within_deadline "$deadline" _agent_still_working "$sess" "$target_agent" && diag="yes"
echo "liveness: no wire event for ${IDLE_TIMEOUT}s on job $job_id; herdr/pane says busy=$diag (diagnostic only, not evidence)" >&2
fi
while [ "$(date +%s)" -lt "$deadline" ]; do
disk_st="$(_disk_status "$job_id")"
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
job_status="$disk_st"
FINAL_JOB_STATUS="$disk_st"
[ "$disk_st" = "completed" ] && return 0
break
fi
if _recent_disk_heartbeat "$job_id" "$IDLE_TIMEOUT"; then
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
elif [ "$(date +%s)" -ge "$evidence_until" ]; then
break
fi
left=$(( deadline - $(date +%s) ))
[ "$left" -le 0 ] && break
sleep 1
done
[ "$(date +%s)" -ge "$deadline" ] && FINALIZE_CLOCK="wall"
fi
# Issue #33: the subscriber's rc says which clock ran out (4 wall clock,
# 2 no event); the disk-heartbeat grace above that ends on the deadline is
# the wall clock too. bash only carries the label.
local clock="${FINALIZE_CLOCK:-idle}"
[ "$sub_rc" = "$SUB_RC_WALL_CLOCK" ] && clock="wall"
local close_req="deadline_timeout"
[ "$clock" = "wall" ] && [ "$phase" != "inject" ] && close_req="wall_clock_timeout"
local detail=""
if [ "$phase" = "preflight" ] || [ "$phase" = "idle_wait" ]; then
close_req="cancelled"
detail="session_busy"
# Issue #34: a target the preflight could not see was never busy
[ "$phase" = "preflight" ] && [ -n "${PREFLIGHT_REASON:-}" ] && detail="$PREFLIGHT_REASON"
fi
[ "$close_req" != "cancelled" ] && echo "finalize: $job_id closed by $([ "$clock" = "wall" ] && echo wall_clock || echo no_event) clock (budget ${TIMEOUT}s / idle ${IDLE_TIMEOUT}s)" >&2
local close_args=(--job "$job_id" --request "$close_req")
[ -n "$detail" ] && close_args+=(--detail "$detail")
[ -n "${_disp_tok:-}" ] && close_args+=(--expect-dispatch-token "$_disp_tok")
local close_rc=0
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" close "${close_args[@]}" 2>&1 || close_rc=$?
# If close failed due to dispatch token mismatch (rc=4), suppress secondary event emission
# to prevent CAS/ABA bypass
if [ $close_rc -eq 4 ]; then
echo "finalize_unfinished: dispatch token mismatch (stale attempt); suppressing event emission" >&2
return 0
fi
# Check if job completed on disk (universal, for all TYPEs)
local cur_disk_st
cur_disk_st="$(_disk_status "$job_id")"
if [ "$cur_disk_st" = "completed" ]; then
FINAL_JOB_STATUS="completed"
return 0
fi
# direct path only: publish MQTT error/cancelled event (loop/discuss forbidden: O-1)
if [ "$TYPE" = "direct" ]; then
local disp_arg=()
[ -n "${_disp_tok:-}" ] && disp_arg=(--expect-dispatch-token "$_disp_tok")
if [ "$close_req" = "cancelled" ]; then
"$PY" "$SCRIPT_DIR/scripts/publish_event.py" --registry-dir "$REGISTRY_DIR" --job "$job_id" --event cancelled --detail "$detail" ${disp_arg[@]+"${disp_arg[@]}"} 2>/dev/null || true
elif [ "$sub_rc" -ne "$SUB_RC_ERROR" ] || [ "$phase" = "inject" ]; then
local err_detail="$close_req"
[ "$phase" = "inject" ] && err_detail="agent_blocked"
"$PY" "$SCRIPT_DIR/scripts/publish_event.py" --registry-dir "$REGISTRY_DIR" --job "$job_id" --event error --detail "$err_detail" ${disp_arg[@]+"${disp_arg[@]}"} 2>/dev/null || true
fi
_within_deadline "$deadline" _agent_still_working "$sess" "$target_agent" && diag="yes"
echo "liveness: no wire event for ${IDLE_TIMEOUT}s on job $job_id; herdr/pane says busy=$diag (diagnostic only, not evidence)" >&2
fi
# The answer comes back on fd 3: stdout carries registry.py's close output.
{ out="$(_decide finalize-end --job "$job_id" --registry-dir "$REGISTRY_DIR" --status "$job_status" \
${START_TIME:+--start-time "$START_TIME"} --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
--sub-rc "$sub_rc" --phase "$phase" --type "$TYPE" --final "$final" --evidence-until "$evidence_until" \
--dispatch-token "${_disp_tok:-}" --preflight-reason "${PREFLIGHT_REASON:-}" \
--python "$PY" --scripts-dir "$SCRIPT_DIR/scripts" 3>&1 1>&4 4>&-)"; } 4>&1
FINAL_JOB_STATUS="$out"
}
preflight_agent() {
@@ -1229,7 +1038,7 @@ cmd_wait() {
fi
# Issue #33: either clock is a timeout (rc 2) to callers of `wait`, as before;
# the subscriber's own log says which clock it was.
[ "$rc" -eq "$SUB_RC_WALL_CLOCK" ] && rc=$SUB_RC_NO_EVENT
rc="$(_decide wait-rc "$rc")"
return "$rc"
}