feat(issue-30): a stall watcher that only notifies
lib_py/stall_watch.py runs beside the wrapper's subscriber for a local herdr job (started after delivery, stopped when the subscriber ends; MAM_STALL_WATCH=0 turns it off). It follows `herdr agent wait` transitions and publishes only `progress` events through one publisher; it has no path that closes a job. - blocked: `stalled:blocked <detection line>` once per episode, never answered. - idle for MAM_STALL_IDLE_S (600) with no background descendants of the agent pid (pane process-info + /proc; adapter resident_children and the heartbeat timer excluded) and an empty input box: `stalled:idle` and one nudge. - input box judged from adapter facts measured on herdr 0.9.1 (codex '›', grok frame '│', opencode '┃' block + stop pattern); the whole-pane fallback is gone. No facts: withheld + `stall_watch:input_check_unavailable` once; never empty for 3 x S: `stall_watch:input_never_empty` once. - herdr failures are not "no event": warning + `stall_watch:herdr_unavailable: <code>` (or agent_not_found), backoff, rc 2 after MAM_STALL_HERDR_FAIL_S (120); the wrapper prints the watcher's last lines. - remote and human jobs are skipped with one log line. - agent_status_reliable is an adapter field; false turns idle judgement off. - the loop runner relays stalled:/stall_watch: notices once each. - scoped_env drops a pane-inherited workspace scope for name lookups (#34 rule). The module sits at the top of lib_py: delegation and loop may not import the agent adapters (their sibling-import rules). Idle pane captures of the five CLIs are fixtures; the mock herdr answers process-info in the real shape. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ec9586bb9b
commit
622c1f8a9d
18 files changed
+1245
-30
No files matched your search
@@ -365,6 +365,7 @@ EOF
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 1
|
||||
fi
|
||||
start_stall_watch "$AGENT_SESSION" "$AGENT" "$_disp_tok" "$_attempt_id"
|
||||
|
||||
# 6) optional validation hook
|
||||
if [[ -n "$VALIDATE" ]]; then
|
||||
@@ -379,6 +380,7 @@ EOF
|
||||
|
||||
local sub_rc=0
|
||||
wait "$sub_pid" || sub_rc=$?
|
||||
stop_stall_watch
|
||||
echo "subscriber output:"; cat "$logf" || true
|
||||
|
||||
local job_status
|
||||
@@ -548,10 +550,12 @@ EOF
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 1
|
||||
fi
|
||||
start_stall_watch "$current_session" "$cur_agent" "$_disp_tok" "$attempt"
|
||||
|
||||
# Wait for subscriber
|
||||
local sub_rc=0
|
||||
wait "$sub_pid" || sub_rc=$?
|
||||
stop_stall_watch
|
||||
echo "subscriber output:"; cat "$logf" || true
|
||||
|
||||
local job_status
|
||||
@@ -672,6 +676,47 @@ reap_subscriber() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Issue #30: a notify-only stall watcher for a job with a local herdr pane
|
||||
# (lib_py.stall_watch decides; it never closes the job). A remote
|
||||
# or human job has no pane to watch. Sets STALL_PID ("" when none started).
|
||||
start_stall_watch() {
|
||||
local session="$1" agent="$2" tok="${3:-}" att="${4:-}"
|
||||
STALL_PID=""
|
||||
[ "${MAM_STALL_WATCH:-1}" = "0" ] && return 0
|
||||
if _is_remote "$session"; then
|
||||
echo "stall watch: remote job, no local pane - stall detection off" >&2; return 0
|
||||
fi
|
||||
if [ "$agent" = "human" ] || [ "${session#herdr:}" = "human" ]; then
|
||||
echo "stall watch: human job, no local pane - stall detection off" >&2; return 0
|
||||
fi
|
||||
local args=(--job "$JOB_ID" --registry-dir "$REGISTRY_DIR" --target "${session#herdr:}"
|
||||
--agent "$agent" --herdr-session "${HERDR_SESSION_NAME:-}")
|
||||
[ -n "$tok" ] && args+=(--expect-dispatch-token "$tok")
|
||||
[ -n "$att" ] && args+=(--attempt-id "$att")
|
||||
PYTHONPATH="$SCRIPT_DIR/..${PYTHONPATH:+:$PYTHONPATH}" "$PY" -m lib_py.stall_watch "${args[@]}" \
|
||||
>"$REGISTRY_DIR/$JOB_ID.stall.out" 2>&1 &
|
||||
STALL_PID=$!
|
||||
echo "stall watch pid: $STALL_PID (log: $REGISTRY_DIR/$JOB_ID.stall.out)"
|
||||
}
|
||||
|
||||
# A watcher that already ended on its own with an error (herdr unavailable)
|
||||
# is not left silent: its last lines go to this wrapper's stderr.
|
||||
stop_stall_watch() {
|
||||
[ -n "${STALL_PID:-}" ] || return 0
|
||||
local rc=0
|
||||
if kill -0 "$STALL_PID" 2>/dev/null; then
|
||||
kill "$STALL_PID" 2>/dev/null || true
|
||||
wait "$STALL_PID" 2>/dev/null || true
|
||||
else
|
||||
wait "$STALL_PID" 2>/dev/null || rc=$?
|
||||
fi
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "stall watch for $JOB_ID exited rc=$rc:" >&2
|
||||
tail -n 3 "$REGISTRY_DIR/$JOB_ID.stall.out" >&2 2>/dev/null || true
|
||||
fi
|
||||
STALL_PID=""
|
||||
}
|
||||
|
||||
_disk_status() {
|
||||
local jid="$1"
|
||||
"$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$jid.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true
|
||||
@@ -818,6 +863,11 @@ finalize_unfinished() {
|
||||
if [ "$job_status" = "completed" ]; then
|
||||
return 0
|
||||
fi
|
||||
# Issue #30: what the stall watcher saw goes with an unfinished job's close.
|
||||
if [ -s "$REGISTRY_DIR/$job_id.stall.out" ]; then
|
||||
echo "stall watch log for $job_id (last 3 lines):" >&2
|
||||
tail -n 3 "$REGISTRY_DIR/$job_id.stall.out" >&2 || true
|
||||
fi
|
||||
|
||||
local cur_disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$cur_disk_st" = "completed" ] || [ "$cur_disk_st" = "error" ] || [ "$cur_disk_st" = "cancelled" ]; then
|
||||
|
||||
Reference in new issue
Block a user