fix(issue-50, issue-55): onboarding and dispatch wait by the registry, one pane claim per server; tests stay out of the real agent homes #56

Merged
Godopu merged 17 commits from fix/issue-50-55 into main 2026-10-12 12:38:55 +00:00
5 changed files with 363 additions and 41 deletions
Showing only changes of commit 9cbe08e8de - Show all commits

No files matched your search

+13 -5
View File
@@ -284,11 +284,16 @@ def _notification(job_id, kind, **fields) -> Dict[str, Any]:
def transition(job_id, request="merge", *, registry_dir=None, expected_status=None, def transition(job_id, request="merge", *, registry_dir=None, expected_status=None,
expected_attempt=None, fields=None, source="unknown", expected_attempt=None, fields=None, source="unknown",
notify=None, expected_notification=_UNSET, notify=None, expected_notification=_UNSET,
expected_dispatch_token=None) -> TransitionResult: expected_dispatch_token=None, guard=None) -> TransitionResult:
"""Decide and commit one state transition. The only writer of job status. """Decide and commit one state transition. The only writer of job status.
Returns a result whose `notifications` have ALREADY been flushed; the field Returns a result whose `notifications` have ALREADY been flushed; the field
is kept so callers and tests can see what the transition emitted. is kept so callers and tests can see what the transition emitted.
`guard(record)`, when given, runs under the registry lock before the
decision and refuses the commit by returning false: a precondition that
spans other records (Issue #50 dispatch admission), which the per-record
expectations cannot express. A refusal by the guard is not a conflict.
""" """
registry_dir = registry_dir or mqtt_common.DEFAULT_REGISTRY_DIR registry_dir = registry_dir or mqtt_common.DEFAULT_REGISTRY_DIR
fields = dict(fields or {}) fields = dict(fields or {})
@@ -297,10 +302,13 @@ def transition(job_id, request="merge", *, registry_dir=None, expected_status=No
with mqtt_common.registry_lock(registry_dir): with mqtt_common.registry_lock(registry_dir):
record = mqtt_common.load_job(job_id, registry_dir) record = mqtt_common.load_job(job_id, registry_dir)
from_status = record.get("status") from_status = record.get("status")
decision = _decide(record, request, expected_status, expected_attempt, if guard is not None and not guard(record):
source=source, fields=fields, decision = _Decision(False, from_status, "guard refused")
expected_notification=expected_notification, else:
expected_dispatch_token=expected_dispatch_token) decision = _decide(record, request, expected_status, expected_attempt,
source=source, fields=fields,
expected_notification=expected_notification,
expected_dispatch_token=expected_dispatch_token)
notifications: List[Dict[str, Any]] = [] notifications: List[Dict[str, Any]] = []
if decision.commit: if decision.commit:
+131 -16
View File
@@ -63,9 +63,24 @@ EXIT_CODES = {"completed": 0, "error": 1, "timeout": 2, "broker_unavailable": 3,
# 44 = the agent ended it with error (or it was cancelled), 45 = it did not end # 44 = the agent ended it with error (or it was cancelled), 45 = it did not end
# within MAM_ONBOARD_WAIT_S (or its record is gone). The session is kept either way. # within MAM_ONBOARD_WAIT_S (or its record is gone). The session is kept either way.
ONBOARD_EXIT_CODES = {"completed": 0, "error": 44, "cancelled": 44, "timeout": 45, "missing": 45} ONBOARD_EXIT_CODES = {"completed": 0, "error": 44, "cancelled": 44, "timeout": 45, "missing": 45}
DEFAULT_ACK_TIMEOUT_SEC, DEFAULT_IDLE_TIMEOUT_SEC = 180, 120 # registry.py's defaults, as a last resort DEFAULT_IDLE_TIMEOUT_SEC = 120 # registry.py's default, as a last resort
# Issue #50: a pending record's liveness is its dispatcher's signal - written
# every DISPATCH_SIGNAL_EVERY s while it waits and, once admitted, while it
# holds the session - and it stays live for EVERY x MISSES s after the last
# one (Issue #18's heartbeat x misses, on the dispatcher's side). The window
# is not ack_timeout_sec: that is the agent's budget to answer a prompt and
# goes down to 1 s, below any cadence a second-resolution signal can keep
# (review aa1bff92 F2).
DISPATCH_SIGNAL_FIELD = "dispatch_wait_at" DISPATCH_SIGNAL_FIELD = "dispatch_wait_at"
DISPATCH_SIGNAL_EVERY = 10 DISPATCH_SIGNAL_EVERY = 10
DISPATCH_SIGNAL_MISSES = 3
DISPATCH_SIGNAL_WINDOW = DISPATCH_SIGNAL_EVERY * DISPATCH_SIGNAL_MISSES
# The admission (review aa1bff92 F1): `clear` is persisted on the admitted
# record, scoped to its attempt by the dispatch token, so a later candidate -
# a lower id in the same second, a redispatched older loop job - waits for it
# until it ends or its signal goes stale.
ADMITTED_AT_FIELD = "dispatch_admitted_at"
ADMITTED_TOKEN_FIELD = "dispatch_admitted_token"
def _now() -> int: def _now() -> int:
@@ -244,7 +259,10 @@ def open_jobs(registry_dir: str, session: str, job_id: str, now: Optional[int] =
live: [(created_at, job_id, status)] - open jobs the Issue #18 contract live: [(created_at, job_id, status)] - open jobs the Issue #18 contract
still calls alive: `running` with an event in events.log within still calls alive: `running` with an event in events.log within
idle_timeout_sec; `pending` with a dispatch signal (the waiter's idle_timeout_sec; `pending` with a dispatch signal (the waiter's
`dispatch_wait_at`, else created_at) within ack_timeout_sec. `dispatch_wait_at`, else created_at) within DISPATCH_SIGNAL_WINDOW.
A live pending record whose admission names its current dispatch
token is reported as `admitted`: it holds the session as a running
job does, whatever its place in the FIFO.
stale: [(job_id, reason)] - non-terminal records nothing has touched for stale: [(job_id, reason)] - non-terminal records nothing has touched for
longer than that: a wrapper killed before or while dispatching, a longer than that: a wrapper killed before or while dispatching, a
zombie's unclosed record. They are reported, never waited for, so zombie's unclosed record. They are reported, never waited for, so
@@ -273,15 +291,98 @@ def open_jobs(registry_dir: str, session: str, job_id: str, now: Optional[int] =
else: else:
stale.append((other, f"running, no event for {idle}s")) stale.append((other, f"running, no event for {idle}s"))
elif status == "pending": elif status == "pending":
ack = _int_field(rec, "ack_timeout_sec", DEFAULT_ACK_TIMEOUT_SEC)
seen = _parse_ts(rec.get(DISPATCH_SIGNAL_FIELD) or rec.get("created_at")) seen = _parse_ts(rec.get(DISPATCH_SIGNAL_FIELD) or rec.get("created_at"))
if seen is not None and now - seen <= ack: if seen is not None and now - seen <= DISPATCH_SIGNAL_WINDOW:
live.append((str(rec.get("created_at") or ""), other, status)) admitted = bool(rec.get(ADMITTED_TOKEN_FIELD)) and \
rec.get(ADMITTED_TOKEN_FIELD) == rec.get("dispatch_token")
live.append((str(rec.get("created_at") or ""), other, "admitted" if admitted else status))
else: else:
stale.append((other, f"pending, no dispatch signal for {ack}s")) stale.append((other, f"pending, no dispatch signal for {DISPATCH_SIGNAL_WINDOW}s"))
return live, stale return live, stale
def _first_in_line(live, mine) -> bool:
"""No live running or admitted job, and `mine` earliest among the live
pending ones (FIFO by (created_at, job_id))."""
if any(st != "pending" for _c, _j, st in live):
return False
return min([(c, j) for c, j, _st in live] + [mine]) == mine
def _admit(registry_dir: str, session: str, job_id: str) -> bool:
"""`clear`, persisted: one merge through the engine whose guard re-reads
the session's open jobs under the registry lock, so two waiters cannot
both be admitted. Scoped to this attempt (its dispatch token: a
redispatch voids it); the signal is renewed with it. False when the
record is not pending any more, its attempt changed, or another waiter
was admitted first - the caller keeps waiting and sees that one."""
from lib_py.delegation import engine, mqtt_common
me = _load_record(registry_dir, job_id) or {}
token = me.get("dispatch_token")
if not token:
print(f"dispatch wait: job {job_id} carries no dispatch token; cannot be admitted", file=sys.stderr)
return False
mine = (str(me.get("created_at") or ""), job_id)
def guard(record):
if record.get("status") != "pending" or record.get("dispatch_token") != token:
return False
live, _stale = open_jobs(registry_dir, session, job_id)
return _first_in_line(live, mine)
now = mqtt_common._utcnow()
try:
return engine.transition(job_id, "merge", registry_dir=registry_dir, source="dispatch_admit", guard=guard,
fields={DISPATCH_SIGNAL_FIELD: now, ADMITTED_AT_FIELD: now,
ADMITTED_TOKEN_FIELD: token}).committed
except Exception as exc: # the record is gone or unwritable: not admitted, keep waiting
print(f"dispatch wait: could not admit {job_id}: {exc}", file=sys.stderr)
return False
def _start_hold(a: argparse.Namespace) -> None:
"""The waiter answers `clear` and ends; a detached hold keeps the admitted
record's signal fresh until the record leaves pending (the agent's
`started`, a close), its attempt changes, the wrapper is gone, or the
wall clock ends (review aa1bff92 F2). Its output is not the answer."""
cmd = [sys.executable, "-m", "lib_py.delegation.wrapper", "hold-admission", "--job", a.job,
"--registry-dir", a.registry_dir, "--timeout", str(a.timeout)]
if a.start_time is not None:
cmd += ["--start-time", str(a.start_time)]
if a.watch_pid:
cmd += ["--watch-pid", str(a.watch_pid)]
subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
start_new_session=True, env=dict(os.environ, PYTHONPATH=_pythonpath()))
def _pythonpath() -> str:
skills = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
extra = os.environ.get("PYTHONPATH", "")
return skills + (os.pathsep + extra if extra else "")
def hold_admission(a: argparse.Namespace) -> str:
"""`released` once the record is not this attempt's admitted pending one
any more, `timeout` at the wall clock; the signal is renewed every
DISPATCH_SIGNAL_EVERY s in between. OwnerGone (the wrapper) ends it."""
me = _load_record(a.registry_dir, a.job) or {}
token = me.get(ADMITTED_TOKEN_FIELD)
deadline = (a.start_time if a.start_time is not None else _now()) + a.timeout
last_signal = _now()
while True:
_sleep(a, 1)
rec = _load_record(a.registry_dir, a.job)
if token is None or rec is None or rec.get("status") != "pending" \
or rec.get("dispatch_token") != token or rec.get(ADMITTED_TOKEN_FIELD) != token:
return "released"
now = _now()
if now >= deadline:
return "timeout"
if now - last_signal >= DISPATCH_SIGNAL_EVERY:
_signal_dispatch_wait(a.registry_dir, a.job)
last_signal = now
def _signal_dispatch_wait(registry_dir: str, job_id: str) -> None: def _signal_dispatch_wait(registry_dir: str, job_id: str) -> None:
"""The waiter's liveness: a merge of dispatch_wait_at into its own pending """The waiter's liveness: a merge of dispatch_wait_at into its own pending
record through the registry's one transition boundary (status untouched, record through the registry's one transition boundary (status untouched,
@@ -295,12 +396,14 @@ def _signal_dispatch_wait(registry_dir: str, job_id: str) -> None:
def wait_open_jobs(a: argparse.Namespace) -> str: def wait_open_jobs(a: argparse.Namespace) -> str:
"""`clear` when no live open job of the session remains and this job is """`clear` when no live running or admitted job of the session remains,
the earliest live pending one (FIFO by (created_at, job_id)); `busy <ids>` this job is the earliest live pending one (FIFO by (created_at, job_id))
when the bound runs out first. The bound is min(--wait-s, this job's and its admission is committed (_admit); `busy <ids>` when the bound runs
remaining wall clock). Every DISPATCH_SIGNAL_EVERY s the waiter signals on out first. The bound is min(--wait-s, this job's remaining wall clock).
its own record so later waiters see it alive; a waiter whose wrapper is Every DISPATCH_SIGNAL_EVERY s the waiter signals on its own record so
gone stops (OwnerGone) and its signal goes stale.""" later waiters see it alive; a waiter whose wrapper is gone stops
(OwnerGone) and its signal goes stale. With --hold the admission is kept
alive after the answer (_start_hold)."""
me = _load_record(a.registry_dir, a.job) or {} me = _load_record(a.registry_dir, a.job) or {}
mine = (str(me.get("created_at") or ""), a.job) mine = (str(me.get("created_at") or ""), a.job)
remaining = (a.start_time if a.start_time is not None else _now()) + a.timeout - _now() remaining = (a.start_time if a.start_time is not None else _now()) + a.timeout - _now()
@@ -314,10 +417,10 @@ def wait_open_jobs(a: argparse.Namespace) -> str:
if other not in reported: if other not in reported:
reported.add(other) reported.add(other)
print(f"stale open job {other} ignored ({reason})", file=sys.stderr) print(f"stale open job {other} ignored ({reason})", file=sys.stderr)
if not any(st == "running" for _c, _j, st in live): if _first_in_line(live, mine) and _admit(a.registry_dir, a.agent_session, a.job):
queue = [(c, j) for c, j, st in live if st == "pending"] + [mine] if getattr(a, "hold", False):
if min(queue) == mine: _start_hold(a)
return "clear" return "clear"
if now >= deadline: if now >= deadline:
return "busy " + " ".join(j for _c, j, _st in live) return "busy " + " ".join(j for _c, j, _st in live)
if last_signal is None or now - last_signal >= DISPATCH_SIGNAL_EVERY: if last_signal is None or now - last_signal >= DISPATCH_SIGNAL_EVERY:
@@ -480,6 +583,13 @@ def _parser() -> argparse.ArgumentParser:
s.add_argument("--start-time", type=int, default=None) s.add_argument("--start-time", type=int, default=None)
s.add_argument("--timeout", type=int, required=True, help="the job's wall-clock budget") s.add_argument("--timeout", type=int, required=True, help="the job's wall-clock budget")
s.add_argument("--watch-pid", type=int, default=None, help="the wrapper's pid; gone -> stop waiting") s.add_argument("--watch-pid", type=int, default=None, help="the wrapper's pid; gone -> stop waiting")
s.add_argument("--hold", action="store_true", help="keep the admission's signal fresh after `clear`")
s = sub.add_parser("hold-admission")
s.add_argument("--job", required=True)
s.add_argument("--registry-dir", required=True)
s.add_argument("--start-time", type=int, default=None)
s.add_argument("--timeout", type=int, required=True)
s.add_argument("--watch-pid", type=int, default=None, help="the wrapper's pid; gone -> stop holding")
for name in ("finalize-begin", "finalize-end"): for name in ("finalize-begin", "finalize-end"):
s = sub.add_parser(name) s = sub.add_parser(name)
s.add_argument("--job", required=True) s.add_argument("--job", required=True)
@@ -544,6 +654,11 @@ def main(argv: Optional[List[str]] = None) -> int:
print(wait_open_jobs(a)) print(wait_open_jobs(a))
except OwnerGone: except OwnerGone:
print(f"wait-open-jobs: wrapper pid {a.watch_pid} is gone; stopped waiting on job {a.job}", file=sys.stderr) print(f"wait-open-jobs: wrapper pid {a.watch_pid} is gone; stopped waiting on job {a.job}", file=sys.stderr)
elif a.cmd == "hold-admission":
try:
print(hold_admission(a))
except OwnerGone:
print(f"hold-admission: wrapper pid {a.watch_pid} is gone; stopped holding job {a.job}", file=sys.stderr)
else: else:
try: try:
_answer(finalize_end(a)) _answer(finalize_end(a))
@@ -855,7 +855,7 @@ _registry_wait() {
local job_id="$1" sess="$2" res local job_id="$1" sess="$2" res
_decided res wait-open-jobs --job "$job_id" --agent-session "$sess" --registry-dir "$REGISTRY_DIR" \ _decided res wait-open-jobs --job "$job_id" --agent-session "$sess" --registry-dir "$REGISTRY_DIR" \
--wait-s "$(mam_int_knob MAM_DISPATCH_WAIT_S 600 0)" ${START_TIME:+--start-time "$START_TIME"} \ --wait-s "$(mam_int_knob MAM_DISPATCH_WAIT_S 600 0)" ${START_TIME:+--start-time "$START_TIME"} \
--timeout "$TIMEOUT" --watch-pid "$$" --timeout "$TIMEOUT" --watch-pid "$$" --hold
case "$res" in case "$res" in
clear) return 0 ;; clear) return 0 ;;
busy*) busy*)
+177 -15
View File
@@ -2,8 +2,10 @@
- U: the decisions in lib_py.delegation.wrapper - the open-job table (status x - U: the decisions in lib_py.delegation.wrapper - the open-job table (status x
signal age x session spelling), FIFO release among live pending waiters, signal age x session spelling), FIFO release among live pending waiters,
the waiter's own signal written through the registry, the bound, a dead the admission persisted on the released record (review aa1bff92 F1) and
wrapper, and finalize-end's `registry_wait` phase closing as session_busy. held alive after the answer (F2), the waiter's own signal written through
the registry, the bound, a dead wrapper, and finalize-end's `registry_wait`
phase closing as session_busy.
- H: against a real herdr 0.9.3 server (one-off session, fake claude in the - H: against a real herdr 0.9.3 server (one-off session, fake claude in the
pane, the real publish path through publish_event.py and the suite's local pane, the real publish path through publish_event.py and the suite's local
broker): see test_issue50_dispatch_herdr.py. broker): see test_issue50_dispatch_herdr.py.
@@ -16,6 +18,7 @@ import json
import os import os
import subprocess import subprocess
import sys import sys
import threading
import time import time
from pathlib import Path from pathlib import Path
@@ -24,6 +27,8 @@ import pytest
REPO_ROOT = Path(__file__).resolve().parent.parent REPO_ROOT = Path(__file__).resolve().parent.parent
SKILLS = REPO_ROOT / ".agents" / "skills" SKILLS = REPO_ROOT / ".agents" / "skills"
WRAPPER = SKILLS / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job" WRAPPER = SKILLS / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
REGISTRY_PY = SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "registry.py"
WINDOW = 30 # DISPATCH_SIGNAL_EVERY (10) x DISPATCH_SIGNAL_MISSES (3), pinned here
if str(SKILLS) not in sys.path: if str(SKILLS) not in sys.path:
sys.path.insert(0, str(SKILLS)) sys.path.insert(0, str(SKILLS))
@@ -49,6 +54,21 @@ def _edit(reg, jid, **fields):
path.write_text(json.dumps(rec)) path.write_text(json.dumps(rec))
def _rec(reg, jid):
return json.loads((reg / f"{jid}.json").read_text())
def _at(text):
return calendar.timegm(time.strptime(text, "%Y-%m-%dT%H:%M:%SZ"))
def _register_as(reg, jid, **over):
"""A record under a chosen id (the file name is what open_jobs keys on)."""
real = _register(reg, job_id=jid, **over)
os.rename(reg / f"{real}.json", reg / f"{jid}.json")
return jid
def _event(reg, jid, epoch, event="progress"): def _event(reg, jid, epoch, event="progress"):
with open(reg / f"{jid}.events.log", "a") as fh: with open(reg / f"{jid}.events.log", "a") as fh:
fh.write(json.dumps({"job_id": jid, "event": event, "timestamp": _iso(epoch)}) + "\n") fh.write(json.dumps({"job_id": jid, "event": event, "timestamp": _iso(epoch)}) + "\n")
@@ -67,18 +87,20 @@ def _ns(reg, job, session="herdr:s", wait_s=0, start_time=None, timeout=3600, wa
("running", 10, None, "live"), # an event idle_timeout_sec (120) ago at most ("running", 10, None, "live"), # an event idle_timeout_sec (120) ago at most
("running", 121, None, "stale"), ("running", 121, None, "stale"),
("running", None, None, "stale"), # running with no events.log at all ("running", None, None, "stale"), # running with no events.log at all
("pending", 10, None, "live"), # created within ack_timeout_sec (180) ("pending", 10, None, "live"), # created within the signal window (30)
("pending", 181, None, "stale"), ("pending", 30, None, "live"),
("pending", 31, None, "stale"),
("pending", 181, 5, "live"), # old record, fresh dispatch_wait_at ("pending", 181, 5, "live"), # old record, fresh dispatch_wait_at
("pending", 181, 181, "stale"), ("pending", 181, 31, "stale"),
("pending", 10, 181, "stale"), # the signal, once present, is what counts ("pending", 10, 31, "stale"), # the signal, once present, is what counts
("pending", 1, 2, "live"), # ack_timeout_sec (1 here) is not the window
("completed", 10, None, None), ("completed", 10, None, None),
("error", 10, None, None), ("error", 10, None, None),
("cancelled", 10, None, None), ("cancelled", 10, None, None),
]) ])
def test_u1_open_jobs_table(tmp_path, status, age, signal, want): def test_u1_open_jobs_table(tmp_path, status, age, signal, want):
NOW = int(time.time()) # the running check reads the real clock (recent_disk_heartbeat) NOW = int(time.time()) # the running check reads the real clock (recent_disk_heartbeat)
other = _register(tmp_path, status=status) other = _register(tmp_path, status=status, ack=1)
if status == "running" and age is not None: if status == "running" and age is not None:
_event(tmp_path, other, NOW - age) _event(tmp_path, other, NOW - age)
if status == "pending": if status == "pending":
@@ -97,6 +119,22 @@ def test_u1_open_jobs_table(tmp_path, status, age, signal, want):
assert got == (set(), set()) assert got == (set(), set())
@pytest.mark.parametrize("token_match,signal,want", [(True, 5, "admitted"), (False, 5, "pending"), (True, 31, "stale")])
def test_u1d_an_admission_is_scoped_to_the_attempt_and_lives_by_the_signal(tmp_path, token_match, signal, want):
NOW = int(time.time())
other = _register(tmp_path)
token = _rec(tmp_path, other)["dispatch_token"] if token_match else "a-previous-attempt"
_edit(tmp_path, other, dispatch_wait_at=_iso(NOW - signal), dispatch_admitted_at=_iso(NOW - signal),
dispatch_admitted_token=token)
me = _register(tmp_path)
live, stale = decide.open_jobs(str(tmp_path), "herdr:s", me, now=NOW)
if want == "stale":
assert (live, [j for j, _r in stale]) == ([], [other])
else:
assert [(j, st) for _c, j, st in live] == [(other, want)]
assert decide.DISPATCH_SIGNAL_WINDOW == WINDOW
@pytest.mark.parametrize("record_session,asked", [("herdr:s", "s"), ("s", "herdr:s"), ("herdr:s", "herdr:s")]) @pytest.mark.parametrize("record_session,asked", [("herdr:s", "s"), ("s", "herdr:s"), ("herdr:s", "herdr:s")])
def test_u1b_the_session_spelling_is_normalised(tmp_path, record_session, asked): def test_u1b_the_session_spelling_is_normalised(tmp_path, record_session, asked):
other = _register(tmp_path, session=record_session) other = _register(tmp_path, session=record_session)
@@ -135,20 +173,61 @@ def test_u2_clear_only_when_no_live_running_and_this_job_is_the_earliest_pending
def test_u2b_a_tie_on_created_at_is_broken_by_job_id(tmp_path): def test_u2b_a_tie_on_created_at_is_broken_by_job_id(tmp_path):
same = _iso(time.time()) same = _iso(time.time())
first = _register(tmp_path, created_at=same, job_id="aaaa0001") _register_as(tmp_path, "aaaa0001", created_at=same)
second = _register(tmp_path, created_at=same, job_id="aaaa0002") _register_as(tmp_path, "aaaa0002", created_at=same)
# records carry their job_id inside; the file name is what open_jobs keys on
for jid, real in ((first, "aaaa0001"), (second, "aaaa0002")):
os.rename(tmp_path / f"{jid}.json", tmp_path / f"{real}.json")
assert decide.wait_open_jobs(_ns(tmp_path, "aaaa0001")) == "clear" assert decide.wait_open_jobs(_ns(tmp_path, "aaaa0001")) == "clear"
assert decide.wait_open_jobs(_ns(tmp_path, "aaaa0002")) == "busy aaaa0001" assert decide.wait_open_jobs(_ns(tmp_path, "aaaa0002")) == "busy aaaa0001"
def test_u2c_a_dead_waiter_is_ignored_and_reported(tmp_path, capsys): def test_u2c_a_dead_waiter_is_ignored_and_reported(tmp_path, capsys):
b = _register(tmp_path, ack=5, created_at=_iso(time.time() - 60), dispatch_wait_at=_iso(time.time() - 6)) b = _register(tmp_path, ack=5, created_at=_iso(time.time() - 60), dispatch_wait_at=_iso(time.time() - WINDOW - 1))
me = _register(tmp_path) me = _register(tmp_path)
assert decide.wait_open_jobs(_ns(tmp_path, me)) == "clear" assert decide.wait_open_jobs(_ns(tmp_path, me)) == "clear"
assert f"stale open job {b} ignored (pending, no dispatch signal for 5s)" in capsys.readouterr().err assert f"stale open job {b} ignored (pending, no dispatch signal for {WINDOW}s)" in capsys.readouterr().err
# review aa1bff92 F1: `clear` is an admission the record carries, so a candidate
# that would sort first - a lower id in the same second, a redispatched older
# loop job - waits for the admitted job until it ends
def test_u2e_a_lower_id_arriving_after_clear_waits_for_the_admitted_job(tmp_path):
same = _iso(time.time())
_register_as(tmp_path, "ffff0001", created_at=same)
assert decide.wait_open_jobs(_ns(tmp_path, "ffff0001")) == "clear"
_register_as(tmp_path, "0000000a", created_at=same)
assert decide.wait_open_jobs(_ns(tmp_path, "0000000a")) == "busy ffff0001"
assert decide.wait_open_jobs(_ns(tmp_path, "0000000a", wait_s=2)) == "busy ffff0001", "still, at the bound"
# the admitted job's `started` keeps the hold (running, with an event)...
_edit(tmp_path, "ffff0001", status="running")
_event(tmp_path, "ffff0001", time.time(), "started")
assert decide.wait_open_jobs(_ns(tmp_path, "0000000a")) == "busy ffff0001"
# ... and its end releases the next one
_edit(tmp_path, "ffff0001", status="completed")
assert decide.wait_open_jobs(_ns(tmp_path, "0000000a")) == "clear"
def test_u2f_an_older_loop_attempt_redispatched_after_another_admission_waits(tmp_path):
old = _register(tmp_path, status="completed")
time.sleep(1.1)
cur = _register(tmp_path)
assert decide.wait_open_jobs(_ns(tmp_path, cur)) == "clear"
res = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(tmp_path), "update", "--job", old,
"--status", "pending", "--attempt-id", "2"], capture_output=True, text=True, timeout=60)
assert res.returncode == 0 and _rec(tmp_path, old)["status"] == "pending", res.stdout + res.stderr
assert _rec(tmp_path, old)["created_at"] < _rec(tmp_path, cur)["created_at"], "it would sort first"
assert decide.wait_open_jobs(_ns(tmp_path, old)) == f"busy {cur}"
_edit(tmp_path, cur, status="completed")
assert decide.wait_open_jobs(_ns(tmp_path, old)) == "clear"
def test_u2g_an_admission_whose_owner_stopped_signalling_expires(tmp_path, capsys):
a = _register(tmp_path)
assert decide.wait_open_jobs(_ns(tmp_path, a)) == "clear"
b = _register(tmp_path)
assert decide.wait_open_jobs(_ns(tmp_path, b)) == f"busy {a}"
_edit(tmp_path, a, dispatch_wait_at=_iso(time.time() - WINDOW - 1)) # its hold is gone
assert decide.wait_open_jobs(_ns(tmp_path, b)) == "clear"
assert f"stale open job {a} ignored (pending, no dispatch signal for {WINDOW}s)" in capsys.readouterr().err
def test_u2d_the_bound_is_the_remaining_wall_clock(tmp_path): def test_u2d_the_bound_is_the_remaining_wall_clock(tmp_path):
@@ -185,6 +264,45 @@ def test_u3_the_waiter_signals_on_its_own_record(tmp_path):
assert me in {j for _c, j, _s in live} assert me in {j for _c, j, _s in live}
def test_u3d_clear_is_persisted_on_the_record_scoped_to_its_attempt(tmp_path):
me = _register(tmp_path)
t0 = int(time.time())
assert decide.wait_open_jobs(_ns(tmp_path, me)) == "clear"
rec = _rec(tmp_path, me)
assert rec["status"] == "pending" and rec["dispatch_admitted_token"] == rec["dispatch_token"]
assert _at(rec["dispatch_admitted_at"]) >= t0 and _at(rec["dispatch_wait_at"]) >= t0, "the signal is renewed with it"
assert not (tmp_path / f"{me}.events.log").exists(), "an admission is not the agent's `started`"
later = _register(tmp_path)
assert decide.open_jobs(str(tmp_path), "herdr:s", later)[0] == [(rec["created_at"], me, "admitted")]
# a redispatch (new attempt, new token) voids it
res = subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(tmp_path), "update", "--job", me,
"--status", "pending", "--attempt-id", "2"], capture_output=True, text=True, timeout=60)
assert res.returncode == 0, res.stdout + res.stderr
assert _rec(tmp_path, me)["dispatch_token"] != rec["dispatch_token"]
assert decide.open_jobs(str(tmp_path), "herdr:s", later)[0] == [(rec["created_at"], me, "pending")]
def test_u3e_the_admission_is_rechecked_under_the_registry_lock(tmp_path, monkeypatch):
# the waiter's read said "first in line"; by the time its merge holds the
# lock another waiter was admitted: the guard refuses, the wait goes on
me = _register(tmp_path)
other = _register(tmp_path)
real_open_jobs = decide.open_jobs
calls = []
def open_jobs(registry_dir, session, job_id, now=None):
calls.append(job_id)
live, stale = real_open_jobs(registry_dir, session, job_id, now)
if len(calls) >= 2: # the guard's read (and every poll after it)
live = [(c, j, "admitted") for c, j, _st in live]
return live, stale
monkeypatch.setattr(decide, "open_jobs", open_jobs)
assert decide.wait_open_jobs(_ns(tmp_path, me, wait_s=1)) == f"busy {other}"
assert "dispatch_admitted_token" not in _rec(tmp_path, me)
assert len(calls) >= 2
def test_u3c_a_terminal_self_record_cannot_carry_the_signal(tmp_path): def test_u3c_a_terminal_self_record_cannot_carry_the_signal(tmp_path):
# why the loop path resets its record to pending before it waits (s1): # why the loop path resets its record to pending before it waits (s1):
# the engine refuses a merge into a terminal record, so the waiter would # the engine refuses a merge into a terminal record, so the waiter would
@@ -227,7 +345,50 @@ def test_u4_open_jobs_prints_live_and_stale(tmp_path):
me = _register(tmp_path) me = _register(tmp_path)
res = _cli(tmp_path, "open-jobs", "--job", me, "--agent-session", "herdr:s", "--registry-dir", str(tmp_path)) res = _cli(tmp_path, "open-jobs", "--job", me, "--agent-session", "herdr:s", "--registry-dir", str(tmp_path))
assert res.returncode == 0, res.stderr assert res.returncode == 0, res.stderr
assert f"{a} running" in res.stdout and f"{b} stale pending, no dispatch signal for 180s" in res.stdout assert f"{a} running" in res.stdout and f"{b} stale pending, no dispatch signal for {WINDOW}s" in res.stdout
def _hold_pids(jid):
out = subprocess.run(["pgrep", "-f", f"hold-admission --job {jid}"], capture_output=True, text=True).stdout
return [int(p) for p in out.split()]
def test_u4c_the_wait_holds_its_admission_until_the_record_leaves_pending(tmp_path):
# review aa1bff92 F2: after `clear` the waiter ends with its answer; the
# hold renews the admitted record's signal until the agent's `started`
me = _register(tmp_path)
try:
res = _cli(tmp_path, "wait-open-jobs", "--job", me, "--agent-session", "herdr:s", "--registry-dir",
str(tmp_path), "--wait-s", "0", "--timeout", "3600", "--watch-pid", str(os.getpid()), "--hold")
assert res.stdout.strip() == "clear", res.stderr
first = _rec(tmp_path, me)["dispatch_wait_at"]
assert _hold_pids(me), "a detached hold is running"
deadline = time.time() + decide.DISPATCH_SIGNAL_EVERY + 5
while time.time() < deadline and _rec(tmp_path, me)["dispatch_wait_at"] == first:
time.sleep(0.5)
assert _at(_rec(tmp_path, me)["dispatch_wait_at"]) >= _at(first) + decide.DISPATCH_SIGNAL_EVERY - 1
assert _rec(tmp_path, me)["status"] == "pending" and not (tmp_path / f"{me}.events.log").exists()
_edit(tmp_path, me, status="running") # the agent's `started`
deadline = time.time() + 5
while time.time() < deadline and _hold_pids(me):
time.sleep(0.2)
assert not _hold_pids(me), "released with the record"
finally:
for pid in _hold_pids(me):
os.kill(pid, 9)
def test_u4d_the_hold_ends_with_its_wrapper_and_without_an_admission(tmp_path):
me = _register(tmp_path)
res = _cli(tmp_path, "hold-admission", "--job", me, "--registry-dir", str(tmp_path), "--timeout", "3600")
assert res.stdout.strip() == "released", res.stderr # never admitted: nothing to hold
assert decide.wait_open_jobs(_ns(tmp_path, me)) == "clear"
gone = subprocess.Popen(["true"])
gone.wait()
t0 = time.time()
res = _cli(tmp_path, "hold-admission", "--job", me, "--registry-dir", str(tmp_path), "--timeout", "3600",
"--watch-pid", str(gone.pid))
assert res.stdout == "" and "is gone" in res.stderr and time.time() - t0 < 10, res.stdout + res.stderr
def test_u4b_wait_open_jobs_stops_with_its_wrapper(tmp_path): def test_u4b_wait_open_jobs_stops_with_its_wrapper(tmp_path):
@@ -292,3 +453,4 @@ def test_s1_both_dispatch_paths_wait_by_the_registry_before_the_idle_wait():
body = src.split("_registry_wait() {")[1].split("\n}\n")[0] body = src.split("_registry_wait() {")[1].split("\n}\n")[0]
assert "wait-open-jobs" in body and 'finalize_unfinished "$job_id" "error" "$SUB_RC_ERROR" "registry_wait"' in body assert "wait-open-jobs" in body and 'finalize_unfinished "$job_id" "error" "$SUB_RC_ERROR" "registry_wait"' in body
assert "exit 5" in body and "mam_int_knob MAM_DISPATCH_WAIT_S 600 0" in body assert "exit 5" in body and "mam_int_knob MAM_DISPATCH_WAIT_S 600 0" in body
assert "--hold" in body, "the admission is held alive after the answer (u4c)"
+41 -4
View File
@@ -329,7 +329,8 @@ def test_c3_a_dead_running_record_does_not_block_the_session(box):
assert rc == 0, out + err assert rc == 0, out + err
assert f"stale open job {a} ignored (running, no event for 2s)" in err, err assert f"stale open job {a} ignored (running, no event for 2s)" in err, err
assert box.record(b)["status"] == "completed" assert box.record(b)["status"] == "completed"
assert "dispatch_wait_at" not in box.record(b), "nothing to wait for: no signal written" rec = box.record(b) # nothing to wait for: the only signal is the admission's own
assert rec["dispatch_admitted_token"] == rec["dispatch_token"] and rec["dispatch_wait_at"] == rec["dispatch_admitted_at"]
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -434,6 +435,9 @@ def test_c7_two_waiters_run_one_after_the_other_and_neither_is_cancelled(box):
assert "cancelled" not in out_b + out_c and "stale open job" not in err_b + err_c assert "cancelled" not in out_b + out_c and "stale open job" not in err_b + err_c
assert _ts(box.events(b)[0]) >= _ts(box.events(a)[-1]) assert _ts(box.events(b)[0]) >= _ts(box.events(a)[-1])
assert _ts(box.events(c)[0]) >= _ts(box.events(b)[-1]), "C started only after B ended" assert _ts(box.events(c)[0]) >= _ts(box.events(b)[-1]), "C started only after B ended"
# the admission (review aa1bff92 F1): B's before its own `started`, C's only once B ended
assert box.record(b)["dispatch_admitted_at"] <= _ts(box.events(b)[0])
assert box.record(c)["dispatch_admitted_at"] >= _ts(box.events(b)[-1])
pane = box.pane_text() pane = box.pane_text()
assert pane.index(f'Your job_id is "{b}"') < pane.index(f'Your job_id is "{c}"'), "injected in FIFO order" assert pane.index(f'Your job_id is "{b}"') < pane.index(f'Your job_id is "{c}"'), "injected in FIFO order"
@@ -456,16 +460,49 @@ def test_c8_a_dead_waiter_is_skipped_once_its_signal_is_stale(box):
c = box.wait_new_job(known | {b}) c = box.wait_new_job(known | {b})
assert box.record(b)["created_at"] < box.record(c)["created_at"] assert box.record(b)["created_at"] < box.record(c)["created_at"]
_after(2, box.publish, a, "completed", "prior done") _after(2, box.publish, a, "completed", "prior done")
_after(14, box.publish, c, "started") # after B's 5 s ack window has long passed _after(40, box.publish, c, "started") # after B's signal window (30 s) has passed
_after(16, box.publish, c, "completed", "c done") _after(42, box.publish, c, "completed", "c done")
rc, out, err = box.finish(proc_c) rc, out, err = box.finish(proc_c)
assert rc == 0, out + err assert rc == 0, out + err
assert f"stale open job {b} ignored (pending, no dispatch signal for 5s)" in err, err assert f"stale open job {b} ignored (pending, no dispatch signal for 30s)" in err, err
assert box.record(c)["status"] == "completed", (box.record(c), box.events(c), out, err) assert box.record(c)["status"] == "completed", (box.record(c), box.events(c), out, err)
assert box.record(b)["status"] == "pending", "the dead waiter's record is left as it is" assert box.record(b)["status"] == "pending", "the dead waiter's record is left as it is"
assert _ts(box.events(c)[0]) >= _ts(box.events(a)[-1]) assert _ts(box.events(c)[0]) >= _ts(box.events(a)[-1])
# ---------------------------------------------------------------------------
# C9 - C8's live counterpart (review aa1bff92 F2): waiters with short ack
# windows, held across many of them, are never stale and never co-admitted
# ---------------------------------------------------------------------------
def test_c9_live_waiters_with_short_ack_windows_are_never_stale_nor_co_admitted(box):
a = box.register()
box.publish(a, "started")
known = box.job_ids()
proc_b = box.submit(prompt="work B", ack="5", MAM_DISPATCH_WAIT_S="60")
b = box.wait_new_job(known)
time.sleep(1.2)
proc_c = box.submit(prompt="work C", ack="5", MAM_DISPATCH_WAIT_S="60")
c = box.wait_new_job(known | {b})
box.wait_field(c, "dispatch_wait_at")
box.wait_field(b, "dispatch_wait_at")
_after(12, box.publish, a, "completed", "prior done") # both waited through two ack windows
_after(16, box.publish, b, "started") # B is injected at ~13 s
_after(19, box.publish, b, "completed", "b done")
_after(23, box.publish, c, "started") # C at ~20 s
_after(25, box.publish, c, "completed", "c done")
rc_b, out_b, err_b = box.finish(proc_b)
rc_c, out_c, err_c = box.finish(proc_c)
assert (rc_b, rc_c) == (0, 0), out_b + err_b + out_c + err_c
assert "stale open job" not in err_b + err_c, err_b + err_c
assert "cancelled" not in out_b + out_c
assert box.record(b)["status"] == box.record(c)["status"] == "completed"
assert box.record(c)["dispatch_admitted_at"] >= _ts(box.events(b)[-1]), "C admitted only once B ended"
assert _ts(box.events(c)[0]) >= _ts(box.events(b)[-1])
pane = box.pane_text()
assert pane.index(f'Your job_id is "{b}"') < pane.index(f'Your job_id is "{c}"'), "injected in FIFO order"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# A - three `create --onboard` at once in one server # A - three `create --onboard` at once in one server
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------