fix(issue-50): the registry wait admits one job per session and holds it (review aa1bff92 F1, F2)
F1: `clear` is now an admission persisted on the released record - a merge through the engine whose new `guard` re-reads the session's open jobs under the registry lock, so two waiters cannot both be admitted. It is scoped to the attempt by the dispatch token (a redispatch voids it) and reported as `admitted` to every other waiter, which holds the session as a running job does, whatever its FIFO place: a lower id in the same second or a redispatched older loop job now waits. F2: a pending record's liveness window is the dispatcher's signal cadence times three misses (10 s x 3 = 30 s), no longer ack_timeout_sec, which the agent's budget takes down to 1 s. After `clear` a detached hold renews the admitted record's signal until the record leaves pending, its attempt changes, the wrapper is gone, or the wall clock ends. Tests: u1 boundaries, u1d/u2e/u2f/u2g/u3d/u3e for the admission, u4c/u4d for the hold, s1 pins --hold; real herdr: C3/C7/C8 adjusted, C9 is C8's live-owner counterpart with short ack windows. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
9e78ff3ba3
commit
9cbe08e8de
5 files changed
+363
-41
No files matched your search
@@ -63,9 +63,24 @@ EXIT_CODES = {"completed": 0, "error": 1, "timeout": 2, "broker_unavailable": 3,
|
||||
# 44 = the agent ended it with error (or it was cancelled), 45 = it did not end
|
||||
# within MAM_ONBOARD_WAIT_S (or its record is gone). The session is kept either way.
|
||||
ONBOARD_EXIT_CODES = {"completed": 0, "error": 44, "cancelled": 44, "timeout": 45, "missing": 45}
|
||||
DEFAULT_ACK_TIMEOUT_SEC, DEFAULT_IDLE_TIMEOUT_SEC = 180, 120 # registry.py's defaults, as a last resort
|
||||
DEFAULT_IDLE_TIMEOUT_SEC = 120 # registry.py's default, as a last resort
|
||||
# Issue #50: a pending record's liveness is its dispatcher's signal - written
|
||||
# every DISPATCH_SIGNAL_EVERY s while it waits and, once admitted, while it
|
||||
# holds the session - and it stays live for EVERY x MISSES s after the last
|
||||
# one (Issue #18's heartbeat x misses, on the dispatcher's side). The window
|
||||
# is not ack_timeout_sec: that is the agent's budget to answer a prompt and
|
||||
# goes down to 1 s, below any cadence a second-resolution signal can keep
|
||||
# (review aa1bff92 F2).
|
||||
DISPATCH_SIGNAL_FIELD = "dispatch_wait_at"
|
||||
DISPATCH_SIGNAL_EVERY = 10
|
||||
DISPATCH_SIGNAL_MISSES = 3
|
||||
DISPATCH_SIGNAL_WINDOW = DISPATCH_SIGNAL_EVERY * DISPATCH_SIGNAL_MISSES
|
||||
# The admission (review aa1bff92 F1): `clear` is persisted on the admitted
|
||||
# record, scoped to its attempt by the dispatch token, so a later candidate -
|
||||
# a lower id in the same second, a redispatched older loop job - waits for it
|
||||
# until it ends or its signal goes stale.
|
||||
ADMITTED_AT_FIELD = "dispatch_admitted_at"
|
||||
ADMITTED_TOKEN_FIELD = "dispatch_admitted_token"
|
||||
|
||||
|
||||
def _now() -> int:
|
||||
@@ -244,7 +259,10 @@ def open_jobs(registry_dir: str, session: str, job_id: str, now: Optional[int] =
|
||||
live: [(created_at, job_id, status)] - open jobs the Issue #18 contract
|
||||
still calls alive: `running` with an event in events.log within
|
||||
idle_timeout_sec; `pending` with a dispatch signal (the waiter's
|
||||
`dispatch_wait_at`, else created_at) within ack_timeout_sec.
|
||||
`dispatch_wait_at`, else created_at) within DISPATCH_SIGNAL_WINDOW.
|
||||
A live pending record whose admission names its current dispatch
|
||||
token is reported as `admitted`: it holds the session as a running
|
||||
job does, whatever its place in the FIFO.
|
||||
stale: [(job_id, reason)] - non-terminal records nothing has touched for
|
||||
longer than that: a wrapper killed before or while dispatching, a
|
||||
zombie's unclosed record. They are reported, never waited for, so
|
||||
@@ -273,15 +291,98 @@ def open_jobs(registry_dir: str, session: str, job_id: str, now: Optional[int] =
|
||||
else:
|
||||
stale.append((other, f"running, no event for {idle}s"))
|
||||
elif status == "pending":
|
||||
ack = _int_field(rec, "ack_timeout_sec", DEFAULT_ACK_TIMEOUT_SEC)
|
||||
seen = _parse_ts(rec.get(DISPATCH_SIGNAL_FIELD) or rec.get("created_at"))
|
||||
if seen is not None and now - seen <= ack:
|
||||
live.append((str(rec.get("created_at") or ""), other, status))
|
||||
if seen is not None and now - seen <= DISPATCH_SIGNAL_WINDOW:
|
||||
admitted = bool(rec.get(ADMITTED_TOKEN_FIELD)) and \
|
||||
rec.get(ADMITTED_TOKEN_FIELD) == rec.get("dispatch_token")
|
||||
live.append((str(rec.get("created_at") or ""), other, "admitted" if admitted else status))
|
||||
else:
|
||||
stale.append((other, f"pending, no dispatch signal for {ack}s"))
|
||||
stale.append((other, f"pending, no dispatch signal for {DISPATCH_SIGNAL_WINDOW}s"))
|
||||
return live, stale
|
||||
|
||||
|
||||
def _first_in_line(live, mine) -> bool:
|
||||
"""No live running or admitted job, and `mine` earliest among the live
|
||||
pending ones (FIFO by (created_at, job_id))."""
|
||||
if any(st != "pending" for _c, _j, st in live):
|
||||
return False
|
||||
return min([(c, j) for c, j, _st in live] + [mine]) == mine
|
||||
|
||||
|
||||
def _admit(registry_dir: str, session: str, job_id: str) -> bool:
|
||||
"""`clear`, persisted: one merge through the engine whose guard re-reads
|
||||
the session's open jobs under the registry lock, so two waiters cannot
|
||||
both be admitted. Scoped to this attempt (its dispatch token: a
|
||||
redispatch voids it); the signal is renewed with it. False when the
|
||||
record is not pending any more, its attempt changed, or another waiter
|
||||
was admitted first - the caller keeps waiting and sees that one."""
|
||||
from lib_py.delegation import engine, mqtt_common
|
||||
me = _load_record(registry_dir, job_id) or {}
|
||||
token = me.get("dispatch_token")
|
||||
if not token:
|
||||
print(f"dispatch wait: job {job_id} carries no dispatch token; cannot be admitted", file=sys.stderr)
|
||||
return False
|
||||
mine = (str(me.get("created_at") or ""), job_id)
|
||||
|
||||
def guard(record):
|
||||
if record.get("status") != "pending" or record.get("dispatch_token") != token:
|
||||
return False
|
||||
live, _stale = open_jobs(registry_dir, session, job_id)
|
||||
return _first_in_line(live, mine)
|
||||
|
||||
now = mqtt_common._utcnow()
|
||||
try:
|
||||
return engine.transition(job_id, "merge", registry_dir=registry_dir, source="dispatch_admit", guard=guard,
|
||||
fields={DISPATCH_SIGNAL_FIELD: now, ADMITTED_AT_FIELD: now,
|
||||
ADMITTED_TOKEN_FIELD: token}).committed
|
||||
except Exception as exc: # the record is gone or unwritable: not admitted, keep waiting
|
||||
print(f"dispatch wait: could not admit {job_id}: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
|
||||
def _start_hold(a: argparse.Namespace) -> None:
|
||||
"""The waiter answers `clear` and ends; a detached hold keeps the admitted
|
||||
record's signal fresh until the record leaves pending (the agent's
|
||||
`started`, a close), its attempt changes, the wrapper is gone, or the
|
||||
wall clock ends (review aa1bff92 F2). Its output is not the answer."""
|
||||
cmd = [sys.executable, "-m", "lib_py.delegation.wrapper", "hold-admission", "--job", a.job,
|
||||
"--registry-dir", a.registry_dir, "--timeout", str(a.timeout)]
|
||||
if a.start_time is not None:
|
||||
cmd += ["--start-time", str(a.start_time)]
|
||||
if a.watch_pid:
|
||||
cmd += ["--watch-pid", str(a.watch_pid)]
|
||||
subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
start_new_session=True, env=dict(os.environ, PYTHONPATH=_pythonpath()))
|
||||
|
||||
|
||||
def _pythonpath() -> str:
|
||||
skills = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
extra = os.environ.get("PYTHONPATH", "")
|
||||
return skills + (os.pathsep + extra if extra else "")
|
||||
|
||||
|
||||
def hold_admission(a: argparse.Namespace) -> str:
|
||||
"""`released` once the record is not this attempt's admitted pending one
|
||||
any more, `timeout` at the wall clock; the signal is renewed every
|
||||
DISPATCH_SIGNAL_EVERY s in between. OwnerGone (the wrapper) ends it."""
|
||||
me = _load_record(a.registry_dir, a.job) or {}
|
||||
token = me.get(ADMITTED_TOKEN_FIELD)
|
||||
deadline = (a.start_time if a.start_time is not None else _now()) + a.timeout
|
||||
last_signal = _now()
|
||||
while True:
|
||||
_sleep(a, 1)
|
||||
rec = _load_record(a.registry_dir, a.job)
|
||||
if token is None or rec is None or rec.get("status") != "pending" \
|
||||
or rec.get("dispatch_token") != token or rec.get(ADMITTED_TOKEN_FIELD) != token:
|
||||
return "released"
|
||||
now = _now()
|
||||
if now >= deadline:
|
||||
return "timeout"
|
||||
if now - last_signal >= DISPATCH_SIGNAL_EVERY:
|
||||
_signal_dispatch_wait(a.registry_dir, a.job)
|
||||
last_signal = now
|
||||
|
||||
|
||||
def _signal_dispatch_wait(registry_dir: str, job_id: str) -> None:
|
||||
"""The waiter's liveness: a merge of dispatch_wait_at into its own pending
|
||||
record through the registry's one transition boundary (status untouched,
|
||||
@@ -295,12 +396,14 @@ def _signal_dispatch_wait(registry_dir: str, job_id: str) -> None:
|
||||
|
||||
|
||||
def wait_open_jobs(a: argparse.Namespace) -> str:
|
||||
"""`clear` when no live open job of the session remains and this job is
|
||||
the earliest live pending one (FIFO by (created_at, job_id)); `busy <ids>`
|
||||
when the bound runs out first. The bound is min(--wait-s, this job's
|
||||
remaining wall clock). Every DISPATCH_SIGNAL_EVERY s the waiter signals on
|
||||
its own record so later waiters see it alive; a waiter whose wrapper is
|
||||
gone stops (OwnerGone) and its signal goes stale."""
|
||||
"""`clear` when no live running or admitted job of the session remains,
|
||||
this job is the earliest live pending one (FIFO by (created_at, job_id))
|
||||
and its admission is committed (_admit); `busy <ids>` when the bound runs
|
||||
out first. The bound is min(--wait-s, this job's remaining wall clock).
|
||||
Every DISPATCH_SIGNAL_EVERY s the waiter signals on its own record so
|
||||
later waiters see it alive; a waiter whose wrapper is gone stops
|
||||
(OwnerGone) and its signal goes stale. With --hold the admission is kept
|
||||
alive after the answer (_start_hold)."""
|
||||
me = _load_record(a.registry_dir, a.job) or {}
|
||||
mine = (str(me.get("created_at") or ""), a.job)
|
||||
remaining = (a.start_time if a.start_time is not None else _now()) + a.timeout - _now()
|
||||
@@ -314,10 +417,10 @@ def wait_open_jobs(a: argparse.Namespace) -> str:
|
||||
if other not in reported:
|
||||
reported.add(other)
|
||||
print(f"stale open job {other} ignored ({reason})", file=sys.stderr)
|
||||
if not any(st == "running" for _c, _j, st in live):
|
||||
queue = [(c, j) for c, j, st in live if st == "pending"] + [mine]
|
||||
if min(queue) == mine:
|
||||
return "clear"
|
||||
if _first_in_line(live, mine) and _admit(a.registry_dir, a.agent_session, a.job):
|
||||
if getattr(a, "hold", False):
|
||||
_start_hold(a)
|
||||
return "clear"
|
||||
if now >= deadline:
|
||||
return "busy " + " ".join(j for _c, j, _st in live)
|
||||
if last_signal is None or now - last_signal >= DISPATCH_SIGNAL_EVERY:
|
||||
@@ -480,6 +583,13 @@ def _parser() -> argparse.ArgumentParser:
|
||||
s.add_argument("--start-time", type=int, default=None)
|
||||
s.add_argument("--timeout", type=int, required=True, help="the job's wall-clock budget")
|
||||
s.add_argument("--watch-pid", type=int, default=None, help="the wrapper's pid; gone -> stop waiting")
|
||||
s.add_argument("--hold", action="store_true", help="keep the admission's signal fresh after `clear`")
|
||||
s = sub.add_parser("hold-admission")
|
||||
s.add_argument("--job", required=True)
|
||||
s.add_argument("--registry-dir", required=True)
|
||||
s.add_argument("--start-time", type=int, default=None)
|
||||
s.add_argument("--timeout", type=int, required=True)
|
||||
s.add_argument("--watch-pid", type=int, default=None, help="the wrapper's pid; gone -> stop holding")
|
||||
for name in ("finalize-begin", "finalize-end"):
|
||||
s = sub.add_parser(name)
|
||||
s.add_argument("--job", required=True)
|
||||
@@ -544,6 +654,11 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
print(wait_open_jobs(a))
|
||||
except OwnerGone:
|
||||
print(f"wait-open-jobs: wrapper pid {a.watch_pid} is gone; stopped waiting on job {a.job}", file=sys.stderr)
|
||||
elif a.cmd == "hold-admission":
|
||||
try:
|
||||
print(hold_admission(a))
|
||||
except OwnerGone:
|
||||
print(f"hold-admission: wrapper pid {a.watch_pid} is gone; stopped holding job {a.job}", file=sys.stderr)
|
||||
else:
|
||||
try:
|
||||
_answer(finalize_end(a))
|
||||
|
||||
Reference in new issue
Block a user