fix(issue-9,14): bound the R1 grace by the wall clock and keep other ls failures unknown
Round-2 review ofd5a191c(planner-reviewer-claude-01, planner-reviewer-codex-01). - delegate wrapper: the R1 idle grace and the broker_unavailable re-check counted only their sleeps against TIMEOUT, so python/herdr probe time plus fresh disk heartbeats let a job run far past --timeout (55 s for a 20 s budget in the new test). Both loops now end at START_TIME + TIMEOUT read off the clock on every pass; heartbeats extend only the idle window, and the herdr probe is cut off at the deadline (_within_deadline) - document the window policyd5a191cactually implements: an agent with no evidence is closed ~2 x IDLE_TIMEOUT after its last wire event (Issue #6 promised IDLE_TIMEOUT + 5 s) and heartbeats may be ~2 x IDLE_TIMEOUT apart; correct the claim that publish_event records a heartbeat on disk before the wire publish - it records right after the publish attempt - shim ls: every failed `agent list` other than a structured server_not_running now exits nonzero with a "state unknown" diagnostic instead of reaching the list parser, where an error document on stdout read as an empty list and let drift A terminate live rows - tests: stdout variants for down and unrelated errors, reconcile keeping herdr_confirmed False on an unrelated failure, and a wall-clock upper bound under continuous heartbeats and slow probes; the three new checks fail ond5a191c- VERSIONS.md v4.4.2: items 3 and 6 and the test count (19) Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HCGpJXhSZasJLU38ksDghe
This commit is contained in:
1 parent
d5a191c47c
commit
952657efb4
5 files changed
+221
-77
No files matched your search
@@ -599,12 +599,35 @@ _agent_still_working() {
|
||||
_pane_tail "$sess" 8 2>/dev/null | grep -Fq "esc to interrupt"
|
||||
}
|
||||
|
||||
# _within_deadline <deadline_epoch> <cmd...>
|
||||
# Run a probe that can block (a herdr RPC, a pane capture) without letting it
|
||||
# outlive a wall-clock deadline: once the clock reaches <deadline> the probe
|
||||
# and its children are killed and the result is rc 124, i.e. "no evidence".
|
||||
_within_deadline() {
|
||||
local deadline="$1" pid
|
||||
shift
|
||||
"$@" &
|
||||
pid=$!
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
pkill -P "$pid" 2>/dev/null || true
|
||||
kill "$pid" 2>/dev/null || true
|
||||
wait "$pid" 2>/dev/null || true
|
||||
return 124
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
wait "$pid"
|
||||
}
|
||||
|
||||
# ISSUE-14: agent-agnostic liveness. publish_event appends every event to the
|
||||
# job's on-disk events.log before the wire publish, so a `progress` heartbeat
|
||||
# that the subscriber never saw (it had already idled out, or the broker
|
||||
# dropped it) is still visible here. An agent running its tests in a
|
||||
# background shell looks idle to herdr and shows no busy token, yet keeps
|
||||
# heartbeating - this is the only evidence that covers that case.
|
||||
# job's on-disk events.log right after its wire publish attempt, whether that
|
||||
# publish succeeded or not, so a `progress` heartbeat the subscriber never saw
|
||||
# (it had already idled out, or the broker dropped it) is still visible here -
|
||||
# late by at most its publish retries while the broker is failing. An agent
|
||||
# running its tests in a background shell looks idle to herdr and shows no
|
||||
# busy token, yet keeps heartbeating - this is the only evidence that covers
|
||||
# that case.
|
||||
_recent_disk_heartbeat() {
|
||||
local job_id="$1" window="$2" f="$REGISTRY_DIR/$job_id.events.log"
|
||||
[ -f "$f" ] || return 1
|
||||
@@ -672,19 +695,18 @@ finalize_unfinished() {
|
||||
else
|
||||
FINAL_JOB_STATUS="$job_status"
|
||||
fi
|
||||
# broker_unavailable: re-observe disk for remaining deadline at 3s intervals
|
||||
# broker_unavailable: re-observe the disk at 3 s intervals until this
|
||||
# dispatch's wall-clock deadline (START_TIME + TIMEOUT). The deadline is read
|
||||
# off the clock on every pass; summing the sleeps alone ignores the time the
|
||||
# disk probes themselves take and overruns TIMEOUT.
|
||||
if [ "$job_status" = "broker_unavailable" ]; then
|
||||
local now elapsed remaining
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - ${START_TIME:-now} ))
|
||||
remaining=$(( TIMEOUT - elapsed ))
|
||||
[ $remaining -lt 0 ] && remaining=0
|
||||
while [ $remaining -gt 0 ]; do
|
||||
local wait_interval=3
|
||||
[ $remaining -lt $wait_interval ] && wait_interval=$remaining
|
||||
sleep "$wait_interval"
|
||||
remaining=$(( remaining - wait_interval ))
|
||||
local disk_st
|
||||
local deadline left disk_st
|
||||
deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
|
||||
while :; do
|
||||
left=$(( deadline - $(date +%s) ))
|
||||
[ "$left" -le 0 ] && break
|
||||
[ "$left" -gt 3 ] && left=3
|
||||
sleep "$left"
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
@@ -703,21 +725,23 @@ finalize_unfinished() {
|
||||
target_agent=$(_sanitize_herdr_agent_name "$sess")
|
||||
|
||||
# ISSUE-14: the subscriber's idle clock only counts events it received over
|
||||
# the wire. Keep re-observing the disk while ANY liveness evidence is at
|
||||
# most IDLE_TIMEOUT old - herdr `working`, the pane busy token, or a fresh
|
||||
# heartbeat in events.log (see _recent_disk_heartbeat). Each sighting opens
|
||||
# a new idle window; an agent that leaves no evidence for a whole window is
|
||||
# closed, still inside the wall clock. Before this, one heartbeat gap
|
||||
# longer than the idle timeout closed a live review as deadline_timeout
|
||||
# and its later `completed` was refused.
|
||||
local now elapsed remaining evidence_until
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - ${START_TIME:-now} ))
|
||||
remaining=$(( TIMEOUT - elapsed ))
|
||||
[ $remaining -lt 0 ] && remaining=0
|
||||
evidence_until=$(( now + IDLE_TIMEOUT ))
|
||||
while [ $remaining -gt 0 ]; do
|
||||
local disk_st
|
||||
# the wire. Keep re-observing the disk while liveness evidence keeps
|
||||
# turning up - herdr `working`, the pane busy token, or a heartbeat in
|
||||
# events.log at most IDLE_TIMEOUT old (see _recent_disk_heartbeat).
|
||||
# Window policy: entering this grace opens one IDLE_TIMEOUT window and
|
||||
# every sighting re-opens it from the time of that poll. An agent that
|
||||
# leaves no evidence at all is therefore closed about 2 x IDLE_TIMEOUT
|
||||
# after its last wire event (the subscriber's idle wait plus this window;
|
||||
# it was IDLE_TIMEOUT before disk heartbeats counted), and heartbeats may
|
||||
# be up to about 2 x IDLE_TIMEOUT apart, since one still counts until it
|
||||
# is IDLE_TIMEOUT old. Evidence only ever extends the idle window, never
|
||||
# this dispatch's wall clock: the loop ends at START_TIME + TIMEOUT read
|
||||
# off the clock on every pass, and the herdr probe is cut off at that
|
||||
# deadline too (_within_deadline).
|
||||
local deadline evidence_until left disk_st
|
||||
deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
|
||||
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
@@ -725,16 +749,15 @@ finalize_unfinished() {
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
break
|
||||
fi
|
||||
if { [ "$sess" != "human" ] && [ -n "$sess" ] && _agent_still_working "$sess" "$target_agent"; } \
|
||||
if { [ "$sess" != "human" ] && [ -n "$sess" ] && _within_deadline "$deadline" _agent_still_working "$sess" "$target_agent"; } \
|
||||
|| _recent_disk_heartbeat "$job_id" "$IDLE_TIMEOUT"; then
|
||||
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
|
||||
elif [ "$(date +%s)" -ge "$evidence_until" ]; then
|
||||
break
|
||||
fi
|
||||
local wait_interval=1
|
||||
[ $remaining -lt $wait_interval ] && wait_interval=$remaining
|
||||
sleep "$wait_interval"
|
||||
remaining=$(( remaining - wait_interval ))
|
||||
left=$(( deadline - $(date +%s) ))
|
||||
[ "$left" -le 0 ] && break
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
Reference in new issue
Block a user