diff --git a/.agents/skills/multi-agent-mux-delegate-job/SKILL.md b/.agents/skills/multi-agent-mux-delegate-job/SKILL.md index 27bccff..2cd3edc 100644 --- a/.agents/skills/multi-agent-mux-delegate-job/SKILL.md +++ b/.agents/skills/multi-agent-mux-delegate-job/SKILL.md @@ -104,7 +104,7 @@ Under fail-closed operation (`MAM_DELEGATE_FAIL_CLOSED=1`, default), the CLI ret | `2` | Timeout | `error` (`deadline_timeout`) | Subscriber idle timeout (K x N without an event; the total budget is the wall-clock row below). **R1 liveness grace (Issue #18)**: after the subscriber idles out the wrapper keeps re-observing the disk while the job's events.log shows an event at most K x N (`--idle-timeout`) old - the automatic heartbeat counts - up to the wall-clock `--timeout`. herdr status and the pane are logged as a diagnostic only. | | `2` | Wall clock | `error` (`wall_clock_timeout`) | The budget (`--timeout`) ran out, events or not (Issue #33; was `deadline_timeout` before v4.8.0). | | `3` | Broker Error | `error` (`wall_clock_timeout` once the budget is spent, else `deadline_timeout`) | Broker unavailable (`MQTT_BROKER_UNAVAILABLE`) and terminal status not found on disk during deadline. | -| `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed (prompt never delivered, retryable). | +| `5` | Session Busy | `cancelled` (`session_busy`) | Precondition failed / `session_busy`: Preflight check (`preflight_agent`) or TUI idle handshake wait failed, or the session's open jobs in the registry did not end within `MAM_DISPATCH_WAIT_S` (Issue #50) (prompt never delivered, retryable). | In all execution paths, the final line of standard output prints the persistent audit log directory path (`/`), fulfilling the parsing contract for orchestrator runners. diff --git a/.agents/skills/multi-agent-mux-delegate-job/multi-agent-mux-delegate-job b/.agents/skills/multi-agent-mux-delegate-job/multi-agent-mux-delegate-job index 444f705..99dd45c 100755 --- a/.agents/skills/multi-agent-mux-delegate-job/multi-agent-mux-delegate-job +++ b/.agents/skills/multi-agent-mux-delegate-job/multi-agent-mux-delegate-job @@ -518,7 +518,19 @@ EOF exit 5 fi - # Registry wait (Issue #50), then idle wait + # Update job details in registry + "$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" update \ + --job "$JOB_ID" \ + --agent-session "$current_session" \ + --prompt-file "$job_dir/prompt.md" \ + --iteration "$iteration" \ + --attempt-id "$attempt" \ + --role "$display_role" \ + --status "pending" + + # Registry wait (Issue #50), then idle wait - on the record just reset to + # pending for this phase, so the waiter's signal is written and other + # waiters see this job in the FIFO (review F1) local cur_agent="$AGENT" [ "$_phase" = "reviewer" ] && cur_agent="$REVIEWER" if [ "$cur_agent" != "human" ] && ! _is_remote "$current_session"; then @@ -534,16 +546,6 @@ EOF fi fi - # Update job details in registry - "$PY" "$SCRIPT_DIR/scripts/registry.py" --registry-dir "$REGISTRY_DIR" update \ - --job "$JOB_ID" \ - --agent-session "$current_session" \ - --prompt-file "$job_dir/prompt.md" \ - --iteration "$iteration" \ - --attempt-id "$attempt" \ - --role "$display_role" \ - --status "pending" - local _disp_tok="" _disp_tok="$("$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$JOB_ID.json'); print(json.load(open(p)).get('dispatch_token','') or '') if os.path.exists(p) else print('')" 2>/dev/null || true)" diff --git a/tests/test_issue50_dispatch.py b/tests/test_issue50_dispatch.py index 03b5ca7..20e8594 100644 --- a/tests/test_issue50_dispatch.py +++ b/tests/test_issue50_dispatch.py @@ -185,6 +185,21 @@ def test_u3_the_waiter_signals_on_its_own_record(tmp_path): assert me in {j for _c, j, _s in live} +def test_u3c_a_terminal_self_record_cannot_carry_the_signal(tmp_path): + # why the loop path resets its record to pending before it waits (s1): + # the engine refuses a merge into a terminal record, so the waiter would + # stay invisible to every other waiter's FIFO + a = _register(tmp_path, status="running") + _event(tmp_path, a, time.time()) + me = _register(tmp_path, status="completed") + assert decide.wait_open_jobs(_ns(tmp_path, me, wait_s=1)) == f"busy {a}" + after = json.loads((tmp_path / f"{me}.json").read_text()) + assert after["status"] == "completed" and "dispatch_wait_at" not in after + later = _register(tmp_path) + live, _stale = decide.open_jobs(str(tmp_path), "herdr:s", later) + assert me not in {j for _c, j, _s in live} + + def test_u3b_the_signal_is_a_registry_merge(tmp_path, monkeypatch): me = _register(tmp_path) seen = {} @@ -262,12 +277,14 @@ def test_s1_both_dispatch_paths_wait_by_the_registry_before_the_idle_wait(): src = WRAPPER.read_text() assert src.count('_registry_wait "$JOB_ID" "$AGENT_SESSION"') == 1 assert src.count('_registry_wait "$JOB_ID" "$current_session"') == 1 - for block in src.split("wait_for_agent_idle ")[1:]: - pass direct = src.index('_registry_wait "$JOB_ID" "$AGENT_SESSION"') assert direct < src.index('wait_for_agent_idle "$AGENT_SESSION"') loop = src.index('_registry_wait "$JOB_ID" "$current_session"') assert loop < src.index('wait_for_agent_idle "$current_session"') + # the loop path waits on the record already reset to pending for this + # phase (review F1): a terminal self record cannot carry the signal (u3c) + assert src.count('--status "pending"') == 1 + assert src.index('--status "pending"') < loop body = src.split("_registry_wait() {")[1].split("\n}\n")[0] assert "wait-open-jobs" in body and 'finalize_unfinished "$job_id" "error" "$SUB_RC_ERROR" "registry_wait"' in body assert "exit 5" in body and "mam_int_knob MAM_DISPATCH_WAIT_S 600 0" in body diff --git a/tests/test_issue50_dispatch_herdr.py b/tests/test_issue50_dispatch_herdr.py index 479592c..f5d5e14 100644 --- a/tests/test_issue50_dispatch_herdr.py +++ b/tests/test_issue50_dispatch_herdr.py @@ -209,24 +209,38 @@ def _start_pane(box: Box, name=SESSION, cwd=None) -> None: assert box.agent_status(name) == "idle", box.herdr("agent", "list").stdout -def _teardown(box: Box) -> None: - if os.environ.get("MAM_T50_KEEP_BOX"): # debugging aid: leave the box and the server in place - print("KEEP_BOX", box.home, box.sess) - return - for sub in (("session", "stop", box.sess), ("session", "delete", box.sess)): - subprocess.run([HERDR, *sub], capture_output=True, text=True, env={"HOME": str(box.home), "PATH": os.environ["PATH"]}, - timeout=30) - time.sleep(0.5) +def _kill_servers(box: Box) -> None: for pid in box.server_pids(): try: os.kill(pid, signal.SIGKILL) except OSError: pass + + +def _teardown(box: Box) -> None: + if os.environ.get("MAM_T50_KEEP_BOX"): # debugging aid: leave the box and the server in place + print("KEEP_BOX", box.home, box.sess) + return + for sub in (("session", "stop", box.sess), ("session", "delete", box.sess)): + try: + subprocess.run([HERDR, *sub], capture_output=True, text=True, + env={"HOME": str(box.home), "PATH": os.environ["PATH"]}, timeout=30) + except subprocess.TimeoutExpired: + pass # the kill below is what must run (review F2) + time.sleep(0.5) + _kill_servers(box) shutil.rmtree(box.home, ignore_errors=True) # herdr writes its agent-detection state under $HOME/.local/state a moment - # after the server is gone: sweep once more + # after the server is gone, and a late client call can bootstrap a new + # server: sweep and kill once more, then prove the server is gone time.sleep(1) + _kill_servers(box) shutil.rmtree(box.home, ignore_errors=True) + deadline = time.time() + 5 + while box.server_pids() and time.time() < deadline: + time.sleep(0.2) + assert not box.server_pids(), f"a server of {box.sess} survived teardown" + assert not box.home.exists(), f"{box.home} survived teardown" @pytest.fixture