refactor(issue-17): item 4 - the delegate-job wrapper's decisions move to Python

The wrapper keeps spawn / wait / kill, key injection and the herdr
diagnostic probe; what it decided in bash is now lib_py/delegation/wrapper.py,
reached through one `_decide` helper:

- finalize_unfinished: the clock choice, the disk grace and broker
  re-observation, the close request / label, the close and the event
  (finalize-begin / finalize-end; bash runs only the busy probe between them)
- start_stall_watch: whether to watch (MAM_STALL_WATCH, remote, human)
- _map_sub_rc, _rc_for (+ MAM_DELEGATE_FAIL_CLOSED), the subscriber
  early-exit check and cmd_wait's rc normalisation
- mam_int / mam_int_knob run lib_py/ints.py, which is also the runner's
  _positive_int: one implementation of the Issue #26 rule

No behaviour change: tests/test_issue17_item4_decisions.py runs the old
bash functions (copied verbatim from 7209750) and the new wrapper on the
same inputs and compares output, FINAL_JOB_STATUS and the registry /
publish calls. Three static tests that pinned the old bash literals now
pin the Python entry points instead.

Refs #17

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.agents/skills/lib_py/delegation/publish_event.py

@@ -208,7 +208,7 @@ def record_local_event(prepared: PreparedEvent, registry_dir: str) -> None:
     """Append the event to the job's on-disk events.log BEFORE any wire publish.
 
     The delegate wrapper reads this log as liveness evidence once its
-    subscriber has idled out (`_recent_disk_heartbeat`). Recorded only after
+    subscriber has idled out (`wrapper.recent_disk_heartbeat`). Recorded only after
     the publish - which blocks on the broker ACK and its retries - a slow
     broker's heartbeat reached the disk too late for that grace window, and
     live jobs were closed as deadline_timeout. Best effort, like the rest of

.agents/skills/lib_py/delegation/wrapper.py

@@ -0,0 +1,296 @@
+"""Issue #17 item 4: the delegate-job wrapper's decisions.
+
+The bash wrapper (multi-agent-mux-delegate-job) spawns, waits, kills and
+injects; everything it used to decide in bash is here, and it branches only on
+what a subcommand prints or returns:
+
+  sub-status <rc> --job J --registry-dir R   subscriber rc -> job status (was _map_sub_rc)
+  subscriber-up <rc>                         exit 0: a subscriber that ended before
+                                             SUBSCRIBED still lets the job proceed
+  exit-code <status>                         the wrapper's exit code (was _rc_for, with
+                                             MAM_DELEGATE_FAIL_CLOSED=0 -> 0)
+  wait-rc <rc>                               `wait`'s rc (either clock is a timeout, 2)
+  stall-watch <session> <agent>              exit 0: start the stall watcher (Issue #30)
+  finalize-begin / finalize-end ...          finalize_unfinished, split around the one
+                                             herdr diagnostic probe bash still runs
+
+finalize-end answers on fd 3 (stdout when fd 3 is closed): its stdout carries
+registry.py's close output to the wrapper's stdout, as before.
+
+Stdlib only and Python 3.9 (macOS /usr/bin/python3); registry.py and
+publish_event.py run as the processes the wrapper ran, with its interpreter.
+"""
+from __future__ import annotations
+
+import argparse
+import calendar
+import json
+import os
+import subprocess
+import sys
+import time
+from typing import List, Optional
+
+# job_subscriber.py's RC_* (a test keeps them equal; importing it would load paho).
+SUB_RC_COMPLETED, SUB_RC_ERROR, SUB_RC_NO_EVENT, SUB_RC_BROKER_UNAVAILABLE, SUB_RC_WALL_CLOCK = 0, 1, 2, 3, 4
+
+TERMINAL = ("completed", "error", "cancelled")
+EXIT_CODES = {"completed": 0, "error": 1, "timeout": 2, "broker_unavailable": 3, "cancelled": 5}
+
+
+def _now() -> int:
+    return int(time.time())  # bash's $(date +%s)
+
+
+def disk_status(registry_dir: str, job_id: str) -> str:
+    """The record's status, "" when it is missing or unreadable."""
+    try:
+        with open(os.path.join(registry_dir, f"{job_id}.json")) as fh:
+            return str(json.load(fh).get("status", ""))
+    except Exception:
+        return ""
+
+
+def sub_status(rc: int, registry_dir: str, job_id: str) -> str:
+    if rc == SUB_RC_COMPLETED:
+        return "completed"
+    if rc == SUB_RC_ERROR:
+        return "error"
+    if rc == SUB_RC_BROKER_UNAVAILABLE:
+        disk = disk_status(registry_dir, job_id)
+        return disk if disk in TERMINAL else "broker_unavailable"
+    return "timeout"  # SUB_RC_NO_EVENT, SUB_RC_WALL_CLOCK (and any other code)
+
+
+def subscriber_up(rc: int) -> bool:
+    return rc in (SUB_RC_COMPLETED, SUB_RC_BROKER_UNAVAILABLE)
+
+
+def fail_closed() -> bool:
+    return (os.environ.get("MAM_DELEGATE_FAIL_CLOSED") or "1") != "0"
+
+
+def exit_code(status: str) -> int:
+    return EXIT_CODES.get(status, 1) if fail_closed() else 0
+
+
+def wait_rc(rc: int) -> int:
+    return SUB_RC_NO_EVENT if rc == SUB_RC_WALL_CLOCK else rc
+
+
+def _bare(session: str) -> str:
+    return session[len("herdr:"):] if session.startswith("herdr:") else session
+
+
+def stall_watch(session: str, agent: str) -> bool:
+    """A remote or human job has no local pane to watch."""
+    if (os.environ.get("MAM_STALL_WATCH") or "1") == "0":
+        return False
+    if session.startswith("remote:"):
+        print("stall watch: remote job, no local pane - stall detection off", file=sys.stderr)
+        return False
+    if agent == "human" or _bare(session) == "human":
+        print("stall watch: human job, no local pane - stall detection off", file=sys.stderr)
+        return False
+    return True
+
+
+def recent_disk_heartbeat(registry_dir: str, job_id: str, window: float) -> bool:
+    """ISSUE-14: publish_event appends every event to the job's events.log
+    before its wire publish, so a heartbeat the subscriber never saw is
+    visible here; true when the last one is at most `window` s old."""
+    try:
+        last = ""
+        with open(os.path.join(registry_dir, f"{job_id}.events.log")) as fh:
+            for line in fh:
+                if line.strip():
+                    last = line
+        seen = calendar.timegm(time.strptime(json.loads(last)["timestamp"], "%Y-%m-%dT%H:%M:%SZ"))
+    except Exception:
+        return False
+    return time.time() - seen <= window
+
+
+def _tail(path: str, n: int) -> List[str]:
+    with open(path, errors="replace") as fh:
+        return fh.read().splitlines()[-n:]
+
+
+def finalize_begin(a: argparse.Namespace) -> str:
+    """`keep` (FINAL_JOB_STATUS untouched), `done <final>`, or
+    `go|probe <final> <evidence_until> <deadline>`."""
+    if not fail_closed() or a.status == "completed":
+        return "keep"
+    stall_out = os.path.join(a.registry_dir, f"{a.job}.stall.out")
+    if os.path.isfile(stall_out) and os.path.getsize(stall_out) > 0:
+        # Issue #30: what the stall watcher saw goes with an unfinished job's close.
+        print(f"stall watch log for {a.job} (last 3 lines):", file=sys.stderr)
+        try:
+            for line in _tail(stall_out, 3):
+                print(line, file=sys.stderr)
+        except OSError:
+            pass
+    disk = disk_status(a.registry_dir, a.job)
+    if disk == "completed":
+        return "done completed"
+    final = disk if disk in TERMINAL else a.status
+    now = _now()
+    deadline = (a.start_time if a.start_time is not None else now) + a.timeout
+    evidence_until = now + a.idle_timeout
+    sess = _bare(a.session)
+    # herdr status and the pane are a diagnostic only (ISSUE-14 / ISSUE-18):
+    # bash runs that probe, bounded by the deadline, when asked to.
+    probe = (a.status == "timeout" and a.agent != "human" and sess != "human" and sess != ""
+             and not a.session.startswith("remote:"))
+    return f"{'probe' if probe else 'go'} {final} {evidence_until} {deadline}"
+
+
+def _run(a: argparse.Namespace, script: str, args: List[str], merge: bool) -> int:
+    sys.stdout.flush()
+    cmd = [a.python, os.path.join(a.scripts_dir, script), "--registry-dir", a.registry_dir, *args]
+    return subprocess.call(cmd, stderr=subprocess.STDOUT if merge else subprocess.DEVNULL)
+
+
+def finalize_end(a: argparse.Namespace) -> str:
+    """The rest of finalize_unfinished; returns FINAL_JOB_STATUS."""
+    job, reg, status, final = a.job, a.registry_dir, a.status, a.final
+    clock = ""
+    deadline = (a.start_time if a.start_time is not None else _now()) + a.timeout
+    if status == "broker_unavailable":
+        # Re-observe the disk at 3 s intervals until this dispatch's wall-clock
+        # deadline, read off the clock on every pass.
+        while True:
+            left = deadline - _now()
+            if left <= 0:
+                break
+            time.sleep(min(left, 3))
+            disk = disk_status(reg, job)
+            if disk in TERMINAL:
+                status = final = disk
+                if disk == "completed":
+                    return final
+                break
+        if _now() >= deadline:  # Issue #33 (review R3): a wait into the deadline is the wall clock
+            clock = "wall"
+    if status == "timeout":
+        # ISSUE-14 / ISSUE-18: keep re-observing the disk while the contract
+        # says the agent is alive - an event in events.log at most IDLE_TIMEOUT
+        # (K x N) old. Entering this grace opens one window and every disk event
+        # re-opens it; evidence never extends the wall clock.
+        evidence_until = a.evidence_until
+        while _now() < deadline:
+            disk = disk_status(reg, job)
+            if disk in TERMINAL:
+                status = final = disk
+                if disk == "completed":
+                    return final
+                break
+            if recent_disk_heartbeat(reg, job, a.idle_timeout):
+                evidence_until = _now() + a.idle_timeout
+            elif _now() >= evidence_until:
+                break
+            if deadline - _now() <= 0:
+                break
+            time.sleep(1)
+        if _now() >= deadline:
+            clock = "wall"
+
+    # Issue #33: the subscriber's rc says which clock ran out (4 wall clock,
+    # 2 no event); a grace above that ends on the deadline is the wall clock too.
+    clock = clock or "idle"
+    if a.sub_rc == SUB_RC_WALL_CLOCK:
+        clock = "wall"
+    close_req = "wall_clock_timeout" if clock == "wall" and a.phase != "inject" else "deadline_timeout"
+    detail = ""
+    if a.phase in ("preflight", "idle_wait"):
+        close_req, detail = "cancelled", "session_busy"
+        # Issue #34: a target the preflight could not see was never busy
+        if a.phase == "preflight" and a.preflight_reason:
+            detail = a.preflight_reason
+    if close_req != "cancelled":
+        print(f"finalize: {job} closed by {'wall_clock' if clock == 'wall' else 'no_event'} clock "
+              f"(budget {a.timeout}s / idle {a.idle_timeout}s)", file=sys.stderr)
+    tok = ["--expect-dispatch-token", a.dispatch_token] if a.dispatch_token else []
+    close_args = ["close", "--job", job, "--request", close_req] + (["--detail", detail] if detail else []) + tok
+    if _run(a, "registry.py", close_args, merge=True) == 4:
+        # a dispatch token mismatch (stale attempt): no secondary event (CAS/ABA)
+        print("finalize_unfinished: dispatch token mismatch (stale attempt); suppressing event emission",
+              file=sys.stderr)
+        return final
+    if disk_status(reg, job) == "completed":
+        return "completed"
+    # direct path only: publish the error/cancelled event (loop/discuss forbidden: O-1)
+    if a.type == "direct":
+        if close_req == "cancelled":
+            _run(a, "publish_event.py", ["--job", job, "--event", "cancelled", "--detail", detail] + tok, merge=False)
+        elif a.sub_rc != SUB_RC_ERROR or a.phase == "inject":
+            err_detail = "agent_blocked" if a.phase == "inject" else close_req
+            _run(a, "publish_event.py", ["--job", job, "--event", "error", "--detail", err_detail] + tok, merge=False)
+    return final
+
+
+def _answer(text: str) -> None:
+    try:
+        os.write(3, (text + "\n").encode())
+    except OSError:
+        print(text)
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="python -m lib_py.delegation.wrapper")
+    sub = p.add_subparsers(dest="cmd", required=True)
+    s = sub.add_parser("sub-status")
+    s.add_argument("rc", type=int)
+    s.add_argument("--job", required=True)
+    s.add_argument("--registry-dir", required=True)
+    sub.add_parser("subscriber-up").add_argument("rc", type=int)
+    sub.add_parser("exit-code").add_argument("status")
+    sub.add_parser("wait-rc").add_argument("rc", type=int)
+    s = sub.add_parser("stall-watch")
+    s.add_argument("session")
+    s.add_argument("agent")
+    for name in ("finalize-begin", "finalize-end"):
+        s = sub.add_parser(name)
+        s.add_argument("--job", required=True)
+        s.add_argument("--registry-dir", required=True)
+        s.add_argument("--status", required=True)
+        s.add_argument("--start-time", type=int, default=None)
+        s.add_argument("--timeout", type=int, required=True)
+        s.add_argument("--idle-timeout", type=int, required=True)
+        if name == "finalize-begin":
+            s.add_argument("--agent", default="")
+            s.add_argument("--session", default="")
+        else:
+            s.add_argument("--sub-rc", type=int, required=True)
+            s.add_argument("--phase", default="")
+            s.add_argument("--type", default="direct")
+            s.add_argument("--final", required=True)
+            s.add_argument("--evidence-until", type=int, default=0)
+            s.add_argument("--dispatch-token", default="")
+            s.add_argument("--preflight-reason", default="")
+            s.add_argument("--python", default=sys.executable)
+            s.add_argument("--scripts-dir", required=True)
+    return p
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    a = _parser().parse_args(argv)
+    if a.cmd == "sub-status":
+        print(sub_status(a.rc, a.registry_dir, a.job))
+    elif a.cmd == "subscriber-up":
+        return 0 if subscriber_up(a.rc) else 1
+    elif a.cmd == "exit-code":
+        print(exit_code(a.status))
+    elif a.cmd == "wait-rc":
+        print(wait_rc(a.rc))
+    elif a.cmd == "stall-watch":
+        return 0 if stall_watch(a.session, a.agent) else 1
+    elif a.cmd == "finalize-begin":
+        print(finalize_begin(a))
+    else:
+        _answer(finalize_end(a))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/lib_py/ints.py

@@ -0,0 +1,63 @@
+"""Issue #26 / #17 item 4: the one integer rule for knobs and flags.
+
+lib_int.sh's mam_int / mam_int_knob run this file, and the loop runner's
+_positive_int is positive_int below, so bash and Python share one
+implementation. Surrounding ASCII blanks are allowed, then ASCII digits only,
+read as DECIMAL (bash would read 0700 as octal 448), and
+minimum <= n <= 999999999. The per-knob minimum lives in
+tests/test_issue26_ints.py.
+
+Stdlib only and Python 3.9: lib_int.sh runs it as a script (python3 -I),
+where lib_py is not importable.
+
+  ints.py int <value> <min>                    print n, or exit 1
+  ints.py knob <name> <value> <default> <min>  print n; empty gives <default>,
+                                               invalid gives <default> + a warning
+"""
+from __future__ import annotations
+
+import sys
+from typing import List, Optional
+
+_BLANKS = " \t\n\r\v\f"
+
+
+def positive_int(text: Optional[str], minimum: int = 1) -> Optional[int]:
+    """n when `text` is a decimal integer >= minimum (and <= 999999999), else None."""
+    s = (text or "").strip(_BLANKS)
+    if not s or not s.isascii() or not s.isdigit():
+        return None
+    s = s.lstrip("0") or "0"
+    if len(s) > 9:  # bounded before int(), which refuses > 4300 digits
+        return None
+    n = int(s)
+    return n if n >= minimum else None
+
+
+def knob(name: str, raw: str, default: str, minimum: int) -> str:
+    """mam_int_knob: the value of $name by the rule, <default> when empty or invalid."""
+    if not raw:
+        return default
+    n = positive_int(raw, minimum)
+    if n is not None:
+        return str(n)
+    print(f"WARNING: {name}={raw} ignored (not a decimal integer >= {minimum}); using {default}", file=sys.stderr)
+    return default
+
+
+def main(argv: List[str]) -> int:
+    if len(argv) == 3 and argv[0] == "int":
+        n = positive_int(argv[1], int(argv[2]))
+        if n is None:
+            return 1
+        print(n)
+        return 0
+    if len(argv) == 5 and argv[0] == "knob":
+        print(knob(argv[1], argv[2], argv[3], int(argv[4])))
+        return 0
+    print("usage: ints.py int <value> <min> | knob <name> <value> <default> <min>", file=sys.stderr)
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

.agents/skills/lib_py/loop/eager.py

@@ -46,6 +46,7 @@ CORE_MODULES: Sequence[str] = (
     "lib_py.loop.store",
     "lib_py.loop.watcher",
     # Delegation and common modules
+    "lib_py.ints",
     "lib_py.paths",
     "lib_py.state",
     "lib_py.atomic_yaml",

.agents/skills/lib_py/loop/runner.py

@@ -25,6 +25,7 @@ import time
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Set, Tuple
 
+from lib_py import ints
 from lib_py.delegation import registry as job_registry
 from lib_py.loop import checkpoint, eager, parser
 
@@ -68,17 +69,8 @@ def log_error(msg: str) -> None:
 
 
 
-def _positive_int(text: Optional[str], minimum: int = 1) -> Optional[int]:
-    """Issue #26: lib.sh mam_int's rule - blanks around, digits only, decimal
-    (0700 is 700), minimum <= n <= 999999999; else None."""
-    s = (text or "").strip(" \t\n\r\v\f")
-    if not s or not s.isascii() or not s.isdigit():
-        return None
-    s = s.lstrip("0") or "0"
-    if len(s) > 9:  # like mam_int: bounded before int(), which refuses > 4300 digits
-        return None
-    n = int(s)
-    return n if n >= minimum else None
+# Issue #17 item 4: lib.sh mam_int's rule, one implementation (lib_py/ints.py).
+_positive_int = ints.positive_int
 
 class LoopRunner:
     """Orchestrates planning, code implementation, and multi-agent peer reviews."""