fix(issue-9,14): bound the R1 grace by the wall clock and keep other ls failures unknown
Round-2 review ofd5a191c(planner-reviewer-claude-01, planner-reviewer-codex-01). - delegate wrapper: the R1 idle grace and the broker_unavailable re-check counted only their sleeps against TIMEOUT, so python/herdr probe time plus fresh disk heartbeats let a job run far past --timeout (55 s for a 20 s budget in the new test). Both loops now end at START_TIME + TIMEOUT read off the clock on every pass; heartbeats extend only the idle window, and the herdr probe is cut off at the deadline (_within_deadline) - document the window policyd5a191cactually implements: an agent with no evidence is closed ~2 x IDLE_TIMEOUT after its last wire event (Issue #6 promised IDLE_TIMEOUT + 5 s) and heartbeats may be ~2 x IDLE_TIMEOUT apart; correct the claim that publish_event records a heartbeat on disk before the wire publish - it records right after the publish attempt - shim ls: every failed `agent list` other than a structured server_not_running now exits nonzero with a "state unknown" diagnostic instead of reaching the list parser, where an error document on stdout read as an empty list and let drift A terminate live rows - tests: stdout variants for down and unrelated errors, reconcile keeping herdr_confirmed False on an unrelated failure, and a wall-clock upper bound under continuous heartbeats and slow probes; the three new checks fail ond5a191c- VERSIONS.md v4.4.2: items 3 and 6 and the test count (19) Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HCGpJXhSZasJLU38ksDghe
This commit is contained in:
1 parent
d5a191c47c
commit
952657efb4
5 files changed
+221
-77
No files matched your search
@@ -366,3 +366,77 @@ def test_wrapper_idle_timeout_grace_accepts_disk_heartbeat(tmp_path):
|
||||
job_data = json.loads((reg_dir / f"{jid}.json").read_text())
|
||||
assert job_data.get("status") == "completed", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
|
||||
assert proc.returncode == 0, f"expected rc=0 while heartbeats kept landing, got {proc.returncode}\n{stdout}\n{stderr}"
|
||||
|
||||
|
||||
def test_wrapper_idle_grace_never_outlives_the_wall_clock(tmp_path):
|
||||
"""Heartbeats may extend the idle window, never the overall --timeout.
|
||||
Fresh heartbeats land in events.log every second for the whole run, and
|
||||
every python probe the wrapper makes is slowed down (herdr-RPC-like
|
||||
latency). Counting sleeps instead of reading the clock let the grace loop
|
||||
run about twice as long as TIMEOUT; the job must instead close as
|
||||
deadline_timeout within TIMEOUT plus a small close/publish margin."""
|
||||
import threading
|
||||
|
||||
timeout_s, idle_s = 20, 4
|
||||
reg_dir = tmp_path / "jobs"
|
||||
reg_dir.mkdir(parents=True, exist_ok=True)
|
||||
slow_py = tmp_path / "slow-python"
|
||||
slow_py.write_text(f'#!/bin/sh\nsleep 0.7\nexec "{sys.executable}" "$@"\n')
|
||||
slow_py.chmod(0o755)
|
||||
env = _base_env(tmp_path, {"DELEGATE_JOB_PYTHON": slow_py})
|
||||
|
||||
started = time.time()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"bash", str(WRAPPER), "submit",
|
||||
"--agent", "human",
|
||||
"--prompt", "issue 14 wall clock",
|
||||
"--registry-dir", str(reg_dir),
|
||||
"--workdir", str(tmp_path),
|
||||
"--timeout", str(timeout_s),
|
||||
"--idle-timeout", str(idle_s),
|
||||
],
|
||||
cwd=str(tmp_path), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
)
|
||||
|
||||
jid = ""
|
||||
for _ in range(200):
|
||||
job_files = list(reg_dir.glob("*.json"))
|
||||
if job_files:
|
||||
jid = job_files[0].stem
|
||||
break
|
||||
time.sleep(0.1)
|
||||
assert jid, "job was not registered"
|
||||
events_log = reg_dir / f"{jid}.events.log"
|
||||
|
||||
stop = threading.Event()
|
||||
|
||||
def keep_heartbeating():
|
||||
# The same line shape publish_event appends; only its timestamp matters
|
||||
# to _recent_disk_heartbeat.
|
||||
seq = 10_000
|
||||
while not stop.is_set():
|
||||
seq += 1
|
||||
line = {"schema_version": 1, "seq": seq, "job_id": jid, "event": "progress",
|
||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"detail": "still working", "data": {}}
|
||||
with open(events_log, "a") as fh:
|
||||
fh.write(json.dumps(line) + "\n")
|
||||
stop.wait(1)
|
||||
|
||||
beat = threading.Thread(target=keep_heartbeating, daemon=True)
|
||||
beat.start()
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=120)
|
||||
finally:
|
||||
stop.set()
|
||||
elapsed = time.time() - started
|
||||
|
||||
job_data = json.loads((reg_dir / f"{jid}.json").read_text())
|
||||
assert proc.returncode == 2, f"expected the deadline_timeout path (rc=2), got {proc.returncode}\n{stderr}"
|
||||
assert job_data.get("status") == "error", f"status={job_data.get('status')}\n{stderr}"
|
||||
# The heartbeats were honoured: the job was not closed after the first idle
|
||||
# window, it stayed open up to the wall clock ...
|
||||
assert elapsed >= timeout_s - 1, f"closed before the wall clock: {elapsed:.1f}s\n{stderr}"
|
||||
# ... and not a moment past it beyond closing and publishing the error.
|
||||
assert elapsed <= timeout_s + 12, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}"
|
||||
Reference in new issue
Block a user