fix(issue-14): keep a job open on fresh disk heartbeats after the subscriber idles out
The R1 grace after a subscriber idle timeout only continued while herdr reported `working` or the pane showed a busy token, and gave up the moment that evidence vanished. Two live jobs in the v4.4.2 review loop were closed as deadline_timeout that way: a grok review whose heartbeat gap was 147 s, and the creator running the full suite in a background shell (idle to herdr, no busy token) while publishing progress every 45 s. Both later published `completed`, which the terminal record refused. - `_recent_disk_heartbeat`: publish_event appends every event to the job's events.log before the wire publish, so a heartbeat the subscriber never saw is still on disk; treat one at most IDLE_TIMEOUT old as liveness - R1 loop: any evidence (herdr working, pane busy token, disk heartbeat) opens a new IDLE_TIMEOUT window; a job with no evidence for a whole window is closed as before, still inside the wall clock - test: a human-agent job whose subscriber idles out stays open across five disk heartbeats and picks up the out-of-band completion (rc 0) - VERSIONS.md v4.4.2 item 6 and test count updated Refs #14 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
1 parent
7c40bf4930
commit
d5a191c47c
3 files changed
+126
-37
No files matched your search
@@ -302,3 +302,67 @@ def test_wrapper_idle_timeout_grace_accepts_pane_busy_token(tmp_path, mock_herdr
|
||||
job_data = json.loads((reg_dir / f"{jid}.json").read_text())
|
||||
assert job_data.get("status") == "completed", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
|
||||
assert proc.returncode == 0, f"expected rc=0 after the working grace, got {proc.returncode}\n{stdout}\n{stderr}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# wrapper: idle-timeout grace (R1) must honour on-disk progress heartbeats
|
||||
# ---------------------------------------------------------------------------
|
||||
PUBLISH_PY = SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"
|
||||
|
||||
|
||||
def test_wrapper_idle_timeout_grace_accepts_disk_heartbeat(tmp_path):
|
||||
"""No herdr evidence at all (a human agent, or an agent running its tests
|
||||
in a background shell): once the subscriber has idled out, the wrapper
|
||||
must keep the job open as long as fresh `progress` heartbeats keep landing
|
||||
in events.log, and pick up the eventual completion. Each heartbeat here
|
||||
arrives after the previous idle window would have expired."""
|
||||
reg_dir = tmp_path / "jobs"
|
||||
reg_dir.mkdir(parents=True, exist_ok=True)
|
||||
env = _base_env(tmp_path)
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"bash", str(WRAPPER), "submit",
|
||||
"--agent", "human",
|
||||
"--prompt", "issue 14 disk heartbeat",
|
||||
"--registry-dir", str(reg_dir),
|
||||
"--workdir", str(tmp_path),
|
||||
"--timeout", "60",
|
||||
"--idle-timeout", "6",
|
||||
],
|
||||
cwd=str(tmp_path), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
)
|
||||
|
||||
jid = ""
|
||||
for _ in range(100):
|
||||
job_files = list(reg_dir.glob("*.json"))
|
||||
if job_files:
|
||||
jid = job_files[0].stem
|
||||
break
|
||||
time.sleep(0.1)
|
||||
assert jid, "job was not registered"
|
||||
|
||||
def heartbeat(detail: str):
|
||||
subprocess.run(
|
||||
[sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg_dir),
|
||||
"--job", jid, "--event", "progress", "--detail", detail],
|
||||
capture_output=True, text=True, cwd=str(tmp_path), env=env,
|
||||
)
|
||||
|
||||
# Subscriber idles out at ~6 s (plus broker connect); heartbeats then keep
|
||||
# arriving 2 s apart, well inside the 6 s window each one re-opens (the
|
||||
# wrapper's poll iteration costs ~2 s of herdr RPCs, so the ratio matters).
|
||||
time.sleep(9)
|
||||
for i in range(5):
|
||||
heartbeat(f"still working {i}")
|
||||
time.sleep(2)
|
||||
done = subprocess.run(
|
||||
[sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg_dir),
|
||||
"update", "--job", jid, "--status", "completed"],
|
||||
capture_output=True, text=True, cwd=str(tmp_path), env=env,
|
||||
)
|
||||
assert done.returncode == 0, done.stderr
|
||||
|
||||
stdout, stderr = proc.communicate(timeout=90)
|
||||
job_data = json.loads((reg_dir / f"{jid}.json").read_text())
|
||||
assert job_data.get("status") == "completed", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
|
||||
assert proc.returncode == 0, f"expected rc=0 while heartbeats kept landing, got {proc.returncode}\n{stdout}\n{stderr}"
|
||||
Reference in new issue
Block a user