fix(issue-14): keep a job open on fresh disk heartbeats after the subscriber idles out
The R1 grace after a subscriber idle timeout only continued while herdr reported `working` or the pane showed a busy token, and gave up the moment that evidence vanished. Two live jobs in the v4.4.2 review loop were closed as deadline_timeout that way: a grok review whose heartbeat gap was 147 s, and the creator running the full suite in a background shell (idle to herdr, no busy token) while publishing progress every 45 s. Both later published `completed`, which the terminal record refused. - `_recent_disk_heartbeat`: publish_event appends every event to the job's events.log before the wire publish, so a heartbeat the subscriber never saw is still on disk; treat one at most IDLE_TIMEOUT old as liveness - R1 loop: any evidence (herdr working, pane busy token, disk heartbeat) opens a new IDLE_TIMEOUT window; a job with no evidence for a whole window is closed as before, still inside the wall clock - test: a human-agent job whose subscriber idles out stays open across five disk heartbeats and picks up the out-of-band completion (rc 0) - VERSIONS.md v4.4.2 item 6 and test count updated Refs #14 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
1 parent
7c40bf4930
commit
d5a191c47c
3 files changed
+126
-37
No files matched your search
@@ -599,6 +599,32 @@ _agent_still_working() {
|
||||
_pane_tail "$sess" 8 2>/dev/null | grep -Fq "esc to interrupt"
|
||||
}
|
||||
|
||||
# ISSUE-14: agent-agnostic liveness. publish_event appends every event to the
|
||||
# job's on-disk events.log before the wire publish, so a `progress` heartbeat
|
||||
# that the subscriber never saw (it had already idled out, or the broker
|
||||
# dropped it) is still visible here. An agent running its tests in a
|
||||
# background shell looks idle to herdr and shows no busy token, yet keeps
|
||||
# heartbeating - this is the only evidence that covers that case.
|
||||
_recent_disk_heartbeat() {
|
||||
local job_id="$1" window="$2" f="$REGISTRY_DIR/$job_id.events.log"
|
||||
[ -f "$f" ] || return 1
|
||||
"$PY" - "$f" "$window" <<'PYEOF'
|
||||
import calendar, json, sys, time
|
||||
path, window = sys.argv[1], float(sys.argv[2])
|
||||
last = ""
|
||||
with open(path) as fh:
|
||||
for line in fh:
|
||||
if line.strip():
|
||||
last = line
|
||||
try:
|
||||
stamp = json.loads(last)["timestamp"]
|
||||
seen = calendar.timegm(time.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ"))
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
sys.exit(0 if time.time() - seen <= window else 1)
|
||||
PYEOF
|
||||
}
|
||||
|
||||
_map_sub_rc() {
|
||||
local sub_rc="$1" job_id="$2"
|
||||
local job_status="running"
|
||||
@@ -675,42 +701,41 @@ finalize_unfinished() {
|
||||
local sess="${target_sess#herdr:}"
|
||||
local target_agent
|
||||
target_agent=$(_sanitize_herdr_agent_name "$sess")
|
||||
local still_working=0
|
||||
if [ "$sess" != "human" ] && [ -n "$sess" ]; then
|
||||
_agent_still_working "$sess" "$target_agent" && still_working=1
|
||||
fi
|
||||
|
||||
if [ "$still_working" = "1" ]; then
|
||||
local now elapsed remaining
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - ${START_TIME:-now} ))
|
||||
remaining=$(( TIMEOUT - elapsed ))
|
||||
[ $remaining -lt 0 ] && remaining=0
|
||||
while [ $remaining -gt 0 ]; do
|
||||
local disk_st
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
FINAL_JOB_STATUS="$disk_st"
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
break
|
||||
fi
|
||||
if ! _agent_still_working "$sess" "$target_agent"; then
|
||||
sleep 1
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
FINAL_JOB_STATUS="$disk_st"
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
fi
|
||||
break
|
||||
fi
|
||||
local wait_interval=1
|
||||
[ $remaining -lt $wait_interval ] && wait_interval=$remaining
|
||||
sleep "$wait_interval"
|
||||
remaining=$(( remaining - wait_interval ))
|
||||
done
|
||||
fi
|
||||
# ISSUE-14: the subscriber's idle clock only counts events it received over
|
||||
# the wire. Keep re-observing the disk while ANY liveness evidence is at
|
||||
# most IDLE_TIMEOUT old - herdr `working`, the pane busy token, or a fresh
|
||||
# heartbeat in events.log (see _recent_disk_heartbeat). Each sighting opens
|
||||
# a new idle window; an agent that leaves no evidence for a whole window is
|
||||
# closed, still inside the wall clock. Before this, one heartbeat gap
|
||||
# longer than the idle timeout closed a live review as deadline_timeout
|
||||
# and its later `completed` was refused.
|
||||
local now elapsed remaining evidence_until
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - ${START_TIME:-now} ))
|
||||
remaining=$(( TIMEOUT - elapsed ))
|
||||
[ $remaining -lt 0 ] && remaining=0
|
||||
evidence_until=$(( now + IDLE_TIMEOUT ))
|
||||
while [ $remaining -gt 0 ]; do
|
||||
local disk_st
|
||||
disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
|
||||
job_status="$disk_st"
|
||||
FINAL_JOB_STATUS="$disk_st"
|
||||
[ "$disk_st" = "completed" ] && return 0
|
||||
break
|
||||
fi
|
||||
if { [ "$sess" != "human" ] && [ -n "$sess" ] && _agent_still_working "$sess" "$target_agent"; } \
|
||||
|| _recent_disk_heartbeat "$job_id" "$IDLE_TIMEOUT"; then
|
||||
evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
|
||||
elif [ "$(date +%s)" -ge "$evidence_until" ]; then
|
||||
break
|
||||
fi
|
||||
local wait_interval=1
|
||||
[ $remaining -lt $wait_interval ] && wait_interval=$remaining
|
||||
sleep "$wait_interval"
|
||||
remaining=$(( remaining - wait_interval ))
|
||||
done
|
||||
fi
|
||||
|
||||
local close_req="deadline_timeout"
|
||||
|
||||
Reference in new issue
Block a user