feat(issue-33): the job budget travels and the two clocks carry different labels

- wrapper: TIMEOUT falls back to MAM_JOB_TIMEOUT_S (--timeout > env > 3600);
  an invalid value warns and keeps 3600.
- loop: --job-timeout <sec> (run_loop.sh passes it on; refused in the legacy
  shell loop). The runner always hands the wrapper --timeout, so the budget
  the wrapper registers is the one its wait caps come from.
- runner: wait_for_job and the MAM_CHILD_WAIT_S default are budget + 300 s
  (MAM_WAIT_GRACE_S), today's 3900 with no option; an explicit
  MAM_CHILD_WAIT_S still wins.
- subscriber: the wall-clock budget returns 4 (RC_WALL_CLOCK), silence keeps
  2 (RC_NO_EVENT). The wrapper maps both to caller rc 2 and labels the close
  from the rc: wall_clock_timeout vs deadline_timeout; the disk-heartbeat
  grace that ends on the deadline is the wall clock too.
- registry close accepts wall_clock_timeout (engine: -> error); the remote
  runner's deadline and command-timeout details lead with wall_clock_timeout;
  the sub-job table files it CLOSED-UNCONFIRMED (first token, colon stripped).

Test expectations changed only where the contract changed: the loop's
contract_args now starts with --timeout, the subscriber's exit-code docstring
names rc 2/4, and the registry --help capture lists the new close kind.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
GodopuandClaude Opus 5.5 committed 2026-09-25 11:43:10 +00:00
1 parent 8ea65bb238
commit ec9586bb9b
11 files changed
+409 -23

No files matched your search

+2 -1
View File
@@ -47,7 +47,8 @@ REQUEST_TO_STATUS = {
"completed": "completed",
"error": "error",
"cancelled": "cancelled",
"deadline_timeout": "error",
"deadline_timeout": "error", # no event for K x N (subscriber rc 2)
"wall_clock_timeout": "error", # the job budget ran out (subscriber rc 4, Issue #33)
"merge": None,
# Operator-authority requests. The wrapper drives multi-iteration delegation
# (`submit --type loop|discuss`) by re-dispatching a finished job to the next
@@ -19,7 +19,9 @@ stdout = event lines only. Diagnostics go to stderr via logging.
Exit codes:
0 all watched jobs reached ``completed``
1 any watched job reached ``error``
2 timed out (wall-clock or idle) before all jobs finished
2 no event for the idle window (K x N) - the wrapper labels it deadline_timeout
3 broker unreachable
4 the wall-clock budget (--timeout) ran out - the wrapper labels it wall_clock_timeout
"""
from __future__ import annotations
@@ -47,6 +49,12 @@ logger = logging.getLogger("delegate_job.job_subscriber")
TERMINAL_EVENTS = ("completed", "error")
TERMINAL_STATUSES = ("completed", "error", "cancelled")
STATE_EVENT = "state"
# Issue #33: which clock ended the wait. The wrapper maps both to a timeout
# (caller rc 2) and labels the close from them: rc 2 -> deadline_timeout
# (no event for the idle window, K x N), rc 4 -> wall_clock_timeout (the job
# budget, --timeout, ran out while events may still have been arriving).
RC_NO_EVENT = 2
RC_WALL_CLOCK = 4
def _format_line(topic: str, payload: Dict[str, Any]) -> str:
@@ -473,7 +481,7 @@ def _run_subscriber(args) -> int:
break
logger.error("wall-clock timeout (%.0fs); still pending: %s",
wall_timeout, ", ".join(sorted(pending)))
return 2
return RC_WALL_CLOCK
if not subscribed:
last_event = now
idle_left = (idle_timeout if first_event_seen else idle_timeout + ack_timeout) - (now - last_event)
@@ -483,7 +491,7 @@ def _run_subscriber(args) -> int:
break
logger.error("idle timeout (%.0fs, no events); still pending: %s",
idle_timeout, ", ".join(sorted(pending)))
return 2
return RC_NO_EVENT
wait = min(wall_deadline - now, idle_left, 1.0)
try:
topic, payload = watcher.events.get(timeout=wait)
+2 -2
View File
@@ -419,9 +419,9 @@ def _build_parser() -> argparse.ArgumentParser:
p_update.add_argument("--attempt-id", type=int, default=None)
p_update.add_argument("--role", default=None)
p_close = sub.add_parser("close", help="close a job with deadline_timeout or cancelled")
p_close = sub.add_parser("close", help="close a job with deadline_timeout, wall_clock_timeout or cancelled")
p_close.add_argument("--job", required=True)
p_close.add_argument("--request", required=True, choices=["deadline_timeout", "cancelled"])
p_close.add_argument("--request", required=True, choices=["deadline_timeout", "wall_clock_timeout", "cancelled"])
p_close.add_argument("--detail", default=None)
p_close.add_argument("--expect-dispatch-token", default=None)
+2 -2
View File
@@ -331,7 +331,7 @@ def handle_dispatch(d: Dict[str, Any], registry_dir: str, session: str,
timeout = float(job.get("timeout_sec", 3600))
rc, out, err, timed_out = _run_agent_command(exec_cmd, env, timeout)
if timed_out:
detail = "agent command exceeded timeout_sec {}".format(int(timeout))
detail = "wall_clock_timeout: agent command exceeded timeout_sec {}".format(int(timeout))
else:
tail = (out or "").strip().splitlines()
detail = tail[-1][:200] if tail else "exit {}".format(rc)
@@ -349,7 +349,7 @@ def handle_dispatch(d: Dict[str, Any], registry_dir: str, session: str,
if cur.get("status") in ("completed", "error", "cancelled"):
return 0 if cur.get("status") == "completed" else 1
time.sleep(1.0)
_publish(jid, registry_dir, "error", "deadline_timeout (remote runner)", d)
_publish(jid, registry_dir, "error", "wall_clock_timeout (remote runner)", d)
return 2