fix(issue-50): the waiter and its hold are pinned to one attempt and session; an unsignalled pending record lives by its ack budget (review 55219be1 R1, R2)
R1: wait-open-jobs takes --dispatch-token (the token the wrapper read for this attempt) and keeps it, with the session it asked about, for its whole life; every write (signal, admission, the hold's renewal) is a guarded engine merge that re-reads the record under the registry lock and refuses once the record is not that attempt's pending one in that session. A record redispatched, moved or closed by someone else makes the waiter answer `superseded`: the bash wrapper exits 5 without a close, the record is its new owner's. hold-admission is given the waiter's pinned identity instead of reading a token from disk, so a hold that starts late holds nothing. R2: a pending record that never carried a waiter's signal (create --onboard registers and injects directly and waits ack_timeout_sec for `started`) is live for that budget from created_at, not for the 30 s signal window; the window applies only once a dispatcher has signalled. Tests: u1 rows for the two pending lifetimes, u3b asserts the guard, u3f/u3g/u3h (cross-session redispatch, in-place attempt change, token changed before the waiter started) with the real waiter subprocess, u4e/u4f (late hold, running hold released by a redispatch), s1 pins the token argument and the no-close branch; real herdr C10 (onboarding valid before and after 30 s blocks the session), C11 (expired onboarding is stale), C12 (redispatched waiter leaves its record alone, exit 5). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
9cbe08e8de
commit
30c25f193c
5 files changed
+394
-104
No files matched your search
@@ -18,6 +18,7 @@ so the herdr idle wait stays and the registry wait runs before it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -37,6 +38,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SKILLS = REPO_ROOT / ".agents" / "skills"
|
||||
WRAPPER = SKILLS / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
|
||||
PUBLISH_PY = SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"
|
||||
REGISTRY_PY = SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "registry.py"
|
||||
CREATE_SH = SKILLS / "multi-agent-mux-create" / "scripts" / "create_session.sh"
|
||||
if str(SKILLS) not in sys.path:
|
||||
sys.path.insert(0, str(SKILLS))
|
||||
@@ -503,6 +505,83 @@ def test_c9_live_waiters_with_short_ack_windows_are_never_stale_nor_co_admitted(
|
||||
assert pane.index(f'Your job_id is "{b}"') < pane.index(f'Your job_id is "{c}"'), "injected in FIFO order"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# C10 - a pending onboarding (registered and injected with no waiter, as
|
||||
# create --onboard does) holds the session for its ack budget, not for the
|
||||
# 30 s signal window (review 55219be1 R2); C11 is its expired control
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _onboarding(box: Box, ack: int) -> str:
|
||||
a = box.register(ack_timeout_sec=ack)
|
||||
prompt = box.herdr("agent", "prompt", SESSION, "Onboarding " + a)
|
||||
assert prompt.returncode == 0, prompt.stdout + prompt.stderr
|
||||
return a
|
||||
|
||||
|
||||
def test_c10_a_pending_onboarding_within_its_ack_budget_blocks_the_session_past_the_signal_window(box):
|
||||
t0 = time.time()
|
||||
a = _onboarding(box, 180)
|
||||
known = box.job_ids()
|
||||
for label in ("before", "after"):
|
||||
if label == "after":
|
||||
time.sleep(max(0.0, t0 + 31.5 - time.time())) # A is now older than the signal window
|
||||
proc = box.submit(prompt="work " + label, MAM_DISPATCH_WAIT_S="3")
|
||||
b = box.wait_new_job(known)
|
||||
known.add(b)
|
||||
rc, out, err = box.finish(proc)
|
||||
assert rc == 5, (label, out + err)
|
||||
rec = box.record(b)
|
||||
assert rec["status"] == "cancelled" and rec.get("detail") == "session_busy", (label, rec)
|
||||
assert a in err and "still has open jobs in the registry" in err and "stale open job" not in err, (label, err)
|
||||
assert b not in box.pane_text(), label
|
||||
assert box.record(a)["status"] == "pending" and "dispatch_wait_at" not in box.record(a), label
|
||||
created = calendar.timegm(time.strptime(box.record(a)["created_at"], "%Y-%m-%dT%H:%M:%SZ"))
|
||||
assert 31 <= time.time() - created < 180, "the second answer came past the window, within the budget"
|
||||
|
||||
|
||||
def test_c11_an_onboarding_never_acknowledged_within_its_ack_budget_is_stale(box):
|
||||
a = _onboarding(box, 2)
|
||||
time.sleep(3.2)
|
||||
known = box.job_ids()
|
||||
proc = box.submit(MAM_DISPATCH_WAIT_S="30")
|
||||
b = box.wait_new_job(known)
|
||||
_after(2, box.publish, b, "started")
|
||||
_after(4, box.publish, b, "completed", "b done")
|
||||
rc, out, err = box.finish(proc)
|
||||
assert rc == 0, out + err
|
||||
assert f"stale open job {a} ignored (pending, no dispatch signal and no started within 2s)" in err, err
|
||||
assert box.record(b)["status"] == "completed" and box.record(a)["status"] == "pending"
|
||||
assert b in box.pane_text()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# C12 - a waiting job redispatched by someone else (new attempt) is left to
|
||||
# that attempt: the old wrapper exits 5 without a close (review 55219be1 R1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_c12_a_waiting_job_redispatched_by_someone_else_is_left_to_its_new_attempt(box):
|
||||
a = box.register()
|
||||
box.publish(a, "started")
|
||||
known = box.job_ids()
|
||||
proc = box.submit(MAM_DISPATCH_WAIT_S="60")
|
||||
b = box.wait_new_job(known)
|
||||
box.wait_field(b, "dispatch_wait_at")
|
||||
old = box.record(b)["dispatch_token"]
|
||||
res = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(box.jobs), "update", "--job", b,
|
||||
"--status", "pending", "--attempt-id", "2"], capture_output=True, text=True, env=box.env(),
|
||||
timeout=60)
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
moved = box.record(b)
|
||||
assert moved["dispatch_token"] != old and moved["attempt_id"] == 2
|
||||
box.publish(a, "completed", "prior done")
|
||||
rc, out, err = box.finish(proc)
|
||||
assert rc == 5, out + err
|
||||
assert f"job {b} was redispatched, moved or closed by someone else" in err, err
|
||||
assert box.record(b) == moved, "not closed, not admitted, not renewed: attempt 2's dispatcher owns it"
|
||||
assert box.events(b) == [] and b not in box.pane_text()
|
||||
assert out.strip().splitlines()[-1].endswith(f"/delegate_job_logs/{b}"), out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# A - three `create --onboard` at once in one server
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in new issue
Block a user