fix(issue-50): the loop path waits by the registry on its pending record (review F1-F3)
F1: from phase two on, the loop's `_registry_wait` ran before the record was reset to pending, so the waiter held a terminal record: the engine refused its dispatch_wait_at merge and other waiters never saw it in the FIFO. The wait (and the idle wait after it) now follows the `--status pending` update; s1 pins the order and u3c shows why a terminal self record cannot signal. F2: the real-herdr box teardown survives a hung stop/delete, kills a server bootstrapped by a late client call, and asserts the server and HOME are gone. F3: the delegate-job SKILL.md exit 5 row names the registry wait. Also drops the dead loop in s1. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
05bffc6475
commit
3705bc72a8
4 files changed
+56
-23
No files matched your search
@@ -104,7 +104,7 @@ Under fail-closed operation (`MAM_DELEGATE_FAIL_CLOSED=1`, default), the CLI ret
|
|||||||
| `2` | Timeout | `error` (`deadline_timeout`) | Subscriber idle timeout (K x N without an event; the total budget is the wall-clock row below). **R1 liveness grace (Issue #18)**: after the subscriber idles out the wrapper keeps re-observing the disk while the job's events.log shows an event at most K x N (`--idle-timeout`) old - the automatic heartbeat counts - up to the wall-clock `--timeout`. herdr status and the pane are logged as a diagnostic only. |
|
| `2` | Timeout | `error` (`deadline_timeout`) | Subscriber idle timeout (K x N without an event; the total budget is the wall-clock row below). **R1 liveness grace (Issue #18)**: after the subscriber idles out the wrapper keeps re-observing the disk while the job's events.log shows an event at most K x N (`--idle-timeout`) old - the automatic heartbeat counts - up to the wall-clock `--timeout`. herdr status and the pane are logged as a diagnostic only. |
|
||||||
| `2` | Wall clock | `error` (`wall_clock_timeout`) | The budget (`--timeout`) ran out, events or not (Issue #33; was `deadline_timeout` before v4.8.0). |
|
| `2` | Wall clock | `error` (`wall_clock_timeout`) | The budget (`--timeout`) ran out, events or not (Issue #33; was `deadline_timeout` before v4.8.0). |
|
||||||
| `3` | Broker Error | `error` (`wall_clock_timeout` once the budget is spent, else `deadline_timeout`) | Broker unavailable (`MQTT_BROKER_UNAVAILABLE`) and terminal status not found on disk during deadline. |
|
| `3` | Broker Error | `error` (`wall_clock_timeout` once the budget is spent, else `deadline_timeout`) | Broker unavailable (`MQTT_BROKER_UNAVAILABLE`) and terminal status not found on disk during deadline. |
|
||||||
| `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed (prompt never delivered, retryable). |
|
| `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed, or the session's open jobs in the registry did not end within `MAM_DISPATCH_WAIT_S` (Issue #50) (prompt never delivered, retryable). |
|
||||||
|
|
||||||
In all execution paths, the final line of standard output prints the persistent audit log directory path (`<logs_root>/<job_id>`), fulfilling the parsing contract for orchestrator runners.
|
In all execution paths, the final line of standard output prints the persistent audit log directory path (`<logs_root>/<job_id>`), fulfilling the parsing contract for orchestrator runners.
|
||||||
|
|
||||||
|
|||||||
@@ -518,7 +518,19 @@ EOF
|
|||||||
exit 5
|
exit 5
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Registry wait (Issue #50), then idle wait
|
# Update job details in registry
|
||||||
|
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" update \
|
||||||
|
--job "$JOB_ID" \
|
||||||
|
--agent-session "$current_session" \
|
||||||
|
--prompt-file "$job_dir/prompt.md" \
|
||||||
|
--iteration "$iteration" \
|
||||||
|
--attempt-id "$attempt" \
|
||||||
|
--role "$display_role" \
|
||||||
|
--status "pending"
|
||||||
|
|
||||||
|
# Registry wait (Issue #50), then idle wait - on the record just reset to
|
||||||
|
# pending for this phase, so the waiter's signal is written and other
|
||||||
|
# waiters see this job in the FIFO (review F1)
|
||||||
local cur_agent="$AGENT"
|
local cur_agent="$AGENT"
|
||||||
[ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER"
|
[ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER"
|
||||||
if [ "$cur_agent" != "human" ] && ! _is_remote "$current_session"; then
|
if [ "$cur_agent" != "human" ] && ! _is_remote "$current_session"; then
|
||||||
@@ -534,16 +546,6 @@ EOF
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Update job details in registry
|
|
||||||
"$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" update \
|
|
||||||
--job "$JOB_ID" \
|
|
||||||
--agent-session "$current_session" \
|
|
||||||
--prompt-file "$job_dir/prompt.md" \
|
|
||||||
--iteration "$iteration" \
|
|
||||||
--attempt-id "$attempt" \
|
|
||||||
--role "$display_role" \
|
|
||||||
--status "pending"
|
|
||||||
|
|
||||||
local _disp_tok=""
|
local _disp_tok=""
|
||||||
_disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
_disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)"
|
||||||
|
|
||||||
|
|||||||
@@ -185,6 +185,21 @@ def test_u3_the_waiter_signals_on_its_own_record(tmp_path):
|
|||||||
assert me in {j for _c, j, _s in live}
|
assert me in {j for _c, j, _s in live}
|
||||||
|
|
||||||
|
|
||||||
|
def test_u3c_a_terminal_self_record_cannot_carry_the_signal(tmp_path):
|
||||||
|
# why the loop path resets its record to pending before it waits (s1):
|
||||||
|
# the engine refuses a merge into a terminal record, so the waiter would
|
||||||
|
# stay invisible to every other waiter's FIFO
|
||||||
|
a = _register(tmp_path, status="running")
|
||||||
|
_event(tmp_path, a, time.time())
|
||||||
|
me = _register(tmp_path, status="completed")
|
||||||
|
assert decide.wait_open_jobs(_ns(tmp_path, me, wait_s=1)) == f"busy {a}"
|
||||||
|
after = json.loads((tmp_path / f"{me}.json").read_text())
|
||||||
|
assert after["status"] == "completed" and "dispatch_wait_at" not in after
|
||||||
|
later = _register(tmp_path)
|
||||||
|
live, _stale = decide.open_jobs(str(tmp_path), "herdr:s", later)
|
||||||
|
assert me not in {j for _c, j, _s in live}
|
||||||
|
|
||||||
|
|
||||||
def test_u3b_the_signal_is_a_registry_merge(tmp_path, monkeypatch):
|
def test_u3b_the_signal_is_a_registry_merge(tmp_path, monkeypatch):
|
||||||
me = _register(tmp_path)
|
me = _register(tmp_path)
|
||||||
seen = {}
|
seen = {}
|
||||||
@@ -262,12 +277,14 @@ def test_s1_both_dispatch_paths_wait_by_the_registry_before_the_idle_wait():
|
|||||||
src = WRAPPER.read_text()
|
src = WRAPPER.read_text()
|
||||||
assert src.count('_registry_wait "$JOB_ID" "$AGENT_SESSION"') == 1
|
assert src.count('_registry_wait "$JOB_ID" "$AGENT_SESSION"') == 1
|
||||||
assert src.count('_registry_wait "$JOB_ID" "$current_session"') == 1
|
assert src.count('_registry_wait "$JOB_ID" "$current_session"') == 1
|
||||||
for block in src.split("wait_for_agent_idle ")[1:]:
|
|
||||||
pass
|
|
||||||
direct = src.index('_registry_wait "$JOB_ID" "$AGENT_SESSION"')
|
direct = src.index('_registry_wait "$JOB_ID" "$AGENT_SESSION"')
|
||||||
assert direct < src.index('wait_for_agent_idle "$AGENT_SESSION"')
|
assert direct < src.index('wait_for_agent_idle "$AGENT_SESSION"')
|
||||||
loop = src.index('_registry_wait "$JOB_ID" "$current_session"')
|
loop = src.index('_registry_wait "$JOB_ID" "$current_session"')
|
||||||
assert loop < src.index('wait_for_agent_idle "$current_session"')
|
assert loop < src.index('wait_for_agent_idle "$current_session"')
|
||||||
|
# the loop path waits on the record already reset to pending for this
|
||||||
|
# phase (review F1): a terminal self record cannot carry the signal (u3c)
|
||||||
|
assert src.count('--status "pending"') == 1
|
||||||
|
assert src.index('--status "pending"') < loop
|
||||||
body = src.split("_registry_wait() {")[1].split("\n}\n")[0]
|
body = src.split("_registry_wait() {")[1].split("\n}\n")[0]
|
||||||
assert "wait-open-jobs" in body and 'finalize_unfinished "$job_id" "error" "$SUB_RC_ERROR" "registry_wait"' in body
|
assert "wait-open-jobs" in body and 'finalize_unfinished "$job_id" "error" "$SUB_RC_ERROR" "registry_wait"' in body
|
||||||
assert "exit 5" in body and "mam_int_knob MAM_DISPATCH_WAIT_S 600 0" in body
|
assert "exit 5" in body and "mam_int_knob MAM_DISPATCH_WAIT_S 600 0" in body
|
||||||
@@ -209,24 +209,38 @@ def _start_pane(box: Box, name=SESSION, cwd=None) -> None:
|
|||||||
assert box.agent_status(name) == "idle", box.herdr("agent", "list").stdout
|
assert box.agent_status(name) == "idle", box.herdr("agent", "list").stdout
|
||||||
|
|
||||||
|
|
||||||
def _teardown(box: Box) -> None:
|
def _kill_servers(box: Box) -> None:
|
||||||
if os.environ.get("MAM_T50_KEEP_BOX"): # debugging aid: leave the box and the server in place
|
|
||||||
print("KEEP_BOX", box.home, box.sess)
|
|
||||||
return
|
|
||||||
for sub in (("session", "stop", box.sess), ("session", "delete", box.sess)):
|
|
||||||
subprocess.run([HERDR, *sub], capture_output=True, text=True, env={"HOME": str(box.home), "PATH": os.environ["PATH"]},
|
|
||||||
timeout=30)
|
|
||||||
time.sleep(0.5)
|
|
||||||
for pid in box.server_pids():
|
for pid in box.server_pids():
|
||||||
try:
|
try:
|
||||||
os.kill(pid, signal.SIGKILL)
|
os.kill(pid, signal.SIGKILL)
|
||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _teardown(box: Box) -> None:
|
||||||
|
if os.environ.get("MAM_T50_KEEP_BOX"): # debugging aid: leave the box and the server in place
|
||||||
|
print("KEEP_BOX", box.home, box.sess)
|
||||||
|
return
|
||||||
|
for sub in (("session", "stop", box.sess), ("session", "delete", box.sess)):
|
||||||
|
try:
|
||||||
|
subprocess.run([HERDR, *sub], capture_output=True, text=True,
|
||||||
|
env={"HOME": str(box.home), "PATH": os.environ["PATH"]}, timeout=30)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
pass # the kill below is what must run (review F2)
|
||||||
|
time.sleep(0.5)
|
||||||
|
_kill_servers(box)
|
||||||
shutil.rmtree(box.home, ignore_errors=True)
|
shutil.rmtree(box.home, ignore_errors=True)
|
||||||
# herdr writes its agent-detection state under $HOME/.local/state a moment
|
# herdr writes its agent-detection state under $HOME/.local/state a moment
|
||||||
# after the server is gone: sweep once more
|
# after the server is gone, and a late client call can bootstrap a new
|
||||||
|
# server: sweep and kill once more, then prove the server is gone
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
_kill_servers(box)
|
||||||
shutil.rmtree(box.home, ignore_errors=True)
|
shutil.rmtree(box.home, ignore_errors=True)
|
||||||
|
deadline = time.time() + 5
|
||||||
|
while box.server_pids() and time.time() < deadline:
|
||||||
|
time.sleep(0.2)
|
||||||
|
assert not box.server_pids(), f"a server of {box.sess} survived teardown"
|
||||||
|
assert not box.home.exists(), f"{box.home} survived teardown"
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|||||||
Reference in new issue
Block a user