fix(issue-50): the loop path waits by the registry on its pending record (review F1-F3)

F1: from phase two on, the loop's `_registry_wait` ran before the record was
reset to pending, so the waiter held a terminal record: the engine refused its
dispatch_wait_at merge and other waiters never saw it in the FIFO. The wait
(and the idle wait after it) now follows the `--status pending` update; s1
pins the order and u3c shows why a terminal self record cannot signal.
F2: the real-herdr box teardown survives a hung stop/delete, kills a server
bootstrapped by a late client call, and asserts the server and HOME are gone.
F3: the delegate-job SKILL.md exit 5 row names the registry wait.
Also drops the dead loop in s1.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
GodopuandClaude Fable 5.1 committed 2026-10-10 17:28:09 +00:00
1 parent 05bffc6475
commit 3705bc72a8
4 files changed
+56 -23

No files matched your search

@@ -104,7 +104,7 @@ Under fail-closed operation (`MAM_DELEGATE_FAIL_CLOSED=1`, default), the CLI ret
| `2` | Timeout | `error` (`deadline_timeout`) | Subscriber idle timeout (K x N without an event; the total budget is the wall-clock row below). **R1 liveness grace (Issue #18)**: after the subscriber idles out the wrapper keeps re-observing the disk while the job's events.log shows an event at most K x N (`--idle-timeout`) old - the automatic heartbeat counts - up to the wall-clock `--timeout`. herdr status and the pane are logged as a diagnostic only. | | `2` | Timeout | `error` (`deadline_timeout`) | Subscriber idle timeout (K x N without an event; the total budget is the wall-clock row below). **R1 liveness grace (Issue #18)**: after the subscriber idles out the wrapper keeps re-observing the disk while the job's events.log shows an event at most K x N (`--idle-timeout`) old - the automatic heartbeat counts - up to the wall-clock `--timeout`. herdr status and the pane are logged as a diagnostic only. |
| `2` | Wall clock | `error` (`wall_clock_timeout`) | The budget (`--timeout`) ran out, events or not (Issue #33; was `deadline_timeout` before v4.8.0). | | `2` | Wall clock | `error` (`wall_clock_timeout`) | The budget (`--timeout`) ran out, events or not (Issue #33; was `deadline_timeout` before v4.8.0). |
| `3` | Broker Error | `error` (`wall_clock_timeout` once the budget is spent, else `deadline_timeout`) | Broker unavailable (`MQTT_BROKER_UNAVAILABLE`) and terminal status not found on disk during deadline. | | `3` | Broker Error | `error` (`wall_clock_timeout` once the budget is spent, else `deadline_timeout`) | Broker unavailable (`MQTT_BROKER_UNAVAILABLE`) and terminal status not found on disk during deadline. |
| `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed (prompt never delivered, retryable). | | `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed, or the session's open jobs in the registry did not end within `MAM_DISPATCH_WAIT_S` (Issue #50) (prompt never delivered, retryable). |
In all execution paths, the final line of standard output prints the persistent audit log directory path (`<logs_root>/<job_id>`), fulfilling the parsing contract for orchestrator runners. In all execution paths, the final line of standard output prints the persistent audit log directory path (`<logs_root>/<job_id>`), fulfilling the parsing contract for orchestrator runners.
@@ -518,7 +518,19 @@ EOF
exit 5 exit 5
fi fi
# Registry wait (Issue #50), then idle wait # Update job details in registry
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" update \
--job "$JOB_ID" \
--agent-session "$current_session" \
--prompt-file "$job_dir/prompt.md" \
--iteration "$iteration" \
--attempt-id "$attempt" \
--role "$display_role" \
--status "pending"
# Registry wait (Issue #50), then idle wait - on the record just reset to
# pending for this phase, so the waiter's signal is written and other
# waiters see this job in the FIFO (review F1)
local cur_agent="$AGENT" local cur_agent="$AGENT"
[ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER" [ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER"
if [ "$cur_agent" != "human" ] && ! _is_remote "$current_session"; then if [ "$cur_agent" != "human" ] && ! _is_remote "$current_session"; then
@@ -534,16 +546,6 @@ EOF
fi fi
fi fi
# Update job details in registry
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" update \
--job "$JOB_ID" \
--agent-session "$current_session" \
--prompt-file "$job_dir/prompt.md" \
--iteration "$iteration" \
--attempt-id "$attempt" \
--role "$display_role" \
--status "pending"
local _disp_tok="" local _disp_tok=""
_disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)" _disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
+19 -2
View File
@@ -185,6 +185,21 @@ def test_u3_the_waiter_signals_on_its_own_record(tmp_path):
assert me in {j for _c, j, _s in live} assert me in {j for _c, j, _s in live}
def test_u3c_a_terminal_self_record_cannot_carry_the_signal(tmp_path):
# why the loop path resets its record to pending before it waits (s1):
# the engine refuses a merge into a terminal record, so the waiter would
# stay invisible to every other waiter's FIFO
a = _register(tmp_path, status="running")
_event(tmp_path, a, time.time())
me = _register(tmp_path, status="completed")
assert decide.wait_open_jobs(_ns(tmp_path, me, wait_s=1)) == f"busy {a}"
after = json.loads((tmp_path / f"{me}.json").read_text())
assert after["status"] == "completed" and "dispatch_wait_at" not in after
later = _register(tmp_path)
live, _stale = decide.open_jobs(str(tmp_path), "herdr:s", later)
assert me not in {j for _c, j, _s in live}
def test_u3b_the_signal_is_a_registry_merge(tmp_path, monkeypatch): def test_u3b_the_signal_is_a_registry_merge(tmp_path, monkeypatch):
me = _register(tmp_path) me = _register(tmp_path)
seen = {} seen = {}
@@ -262,12 +277,14 @@ def test_s1_both_dispatch_paths_wait_by_the_registry_before_the_idle_wait():
src = WRAPPER.read_text() src = WRAPPER.read_text()
assert src.count('_registry_wait "$JOB_ID" "$AGENT_SESSION"') == 1 assert src.count('_registry_wait "$JOB_ID" "$AGENT_SESSION"') == 1
assert src.count('_registry_wait "$JOB_ID" "$current_session"') == 1 assert src.count('_registry_wait "$JOB_ID" "$current_session"') == 1
for block in src.split("wait_for_agent_idle ")[1:]:
pass
direct = src.index('_registry_wait "$JOB_ID" "$AGENT_SESSION"') direct = src.index('_registry_wait "$JOB_ID" "$AGENT_SESSION"')
assert direct < src.index('wait_for_agent_idle "$AGENT_SESSION"') assert direct < src.index('wait_for_agent_idle "$AGENT_SESSION"')
loop = src.index('_registry_wait "$JOB_ID" "$current_session"') loop = src.index('_registry_wait "$JOB_ID" "$current_session"')
assert loop < src.index('wait_for_agent_idle "$current_session"') assert loop < src.index('wait_for_agent_idle "$current_session"')
# the loop path waits on the record already reset to pending for this
# phase (review F1): a terminal self record cannot carry the signal (u3c)
assert src.count('--status "pending"') == 1
assert src.index('--status "pending"') < loop
body = src.split("_registry_wait() {")[1].split("\n}\n")[0] body = src.split("_registry_wait() {")[1].split("\n}\n")[0]
assert "wait-open-jobs" in body and 'finalize_unfinished "$job_id" "error" "$SUB_RC_ERROR" "registry_wait"' in body assert "wait-open-jobs" in body and 'finalize_unfinished "$job_id" "error" "$SUB_RC_ERROR" "registry_wait"' in body
assert "exit 5" in body and "mam_int_knob MAM_DISPATCH_WAIT_S 600 0" in body assert "exit 5" in body and "mam_int_knob MAM_DISPATCH_WAIT_S 600 0" in body
+23 -9
View File
@@ -209,24 +209,38 @@ def _start_pane(box: Box, name=SESSION, cwd=None) -> None:
assert box.agent_status(name) == "idle", box.herdr("agent", "list").stdout assert box.agent_status(name) == "idle", box.herdr("agent", "list").stdout
def _teardown(box: Box) -> None: def _kill_servers(box: Box) -> None:
if os.environ.get("MAM_T50_KEEP_BOX"): # debugging aid: leave the box and the server in place
print("KEEP_BOX", box.home, box.sess)
return
for sub in (("session", "stop", box.sess), ("session", "delete", box.sess)):
subprocess.run([HERDR, *sub], capture_output=True, text=True, env={"HOME": str(box.home), "PATH": os.environ["PATH"]},
timeout=30)
time.sleep(0.5)
for pid in box.server_pids(): for pid in box.server_pids():
try: try:
os.kill(pid, signal.SIGKILL) os.kill(pid, signal.SIGKILL)
except OSError: except OSError:
pass pass
def _teardown(box: Box) -> None:
if os.environ.get("MAM_T50_KEEP_BOX"): # debugging aid: leave the box and the server in place
print("KEEP_BOX", box.home, box.sess)
return
for sub in (("session", "stop", box.sess), ("session", "delete", box.sess)):
try:
subprocess.run([HERDR, *sub], capture_output=True, text=True,
env={"HOME": str(box.home), "PATH": os.environ["PATH"]}, timeout=30)
except subprocess.TimeoutExpired:
pass # the kill below is what must run (review F2)
time.sleep(0.5)
_kill_servers(box)
shutil.rmtree(box.home, ignore_errors=True) shutil.rmtree(box.home, ignore_errors=True)
# herdr writes its agent-detection state under $HOME/.local/state a moment # herdr writes its agent-detection state under $HOME/.local/state a moment
# after the server is gone: sweep once more # after the server is gone, and a late client call can bootstrap a new
# server: sweep and kill once more, then prove the server is gone
time.sleep(1) time.sleep(1)
_kill_servers(box)
shutil.rmtree(box.home, ignore_errors=True) shutil.rmtree(box.home, ignore_errors=True)
deadline = time.time() + 5
while box.server_pids() and time.time() < deadline:
time.sleep(0.2)
assert not box.server_pids(), f"a server of {box.sess} survived teardown"
assert not box.home.exists(), f"{box.home} survived teardown"
@pytest.fixture @pytest.fixture