tmpl/multi-agent-mux
1
0 Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(issue-18): admit remote attempts atomically, end timed-out commands with their group, fence heartbeats off the terminal, renew liveness only on news

Browse Source 
Review round 2 (planner-reviewer-codex-01, NOT PASS at d09c162):

- R1 remote.mirror_job: admission under the registry lock. The same
  dispatch_token is idempotent (a finished attempt never runs again), a
  lower attempt or the same attempt under another token is refused before
  any write, and only a strictly newer attempt replaces the record
  (register_job(supersede_older_attempt=True); events.log kept, seqs stay
  monotonic).
- R2 remote.handle_dispatch: the --exec command runs in its own session;
  at timeout_sec, or when the runner is interrupted (SIGTERM now unwinds
  it), the whole process group is terminated (TERM, then KILL) before the
  error is published.
- R3 terminal barrier: a terminal publish raises an attempt-scoped barrier
  under the lock that allocates seqs, stops the heartbeat timer, gives it
  MAM_HEARTBEAT_DRAIN_S (default 1 s) to exit and then kills it, and only
  then takes its own seq. Beats are admitted under that lock
  (next_seq(admit=...)); a beat still on the wire is cancelled with the
  timer rather than holding the terminal back; and a late non-terminal
  event never publishes a running state over a terminal record.
- R4 job_subscriber: a retained state renews the idle clock only when its
  seq is newer than everything already accepted; re-delivered or older
  snapshots still recover a terminal status.
- tests: five regressions, all failing on 77f2829; T1-14's wall clock
  15 -> 30 s against public-broker latency.
- docs: MESSAGING 2.5 (barrier, state renewal, detection vs final close),
  heartbeat docstring, loop SKILL.md (remote rows need the Python runner),
  VERSIONS v4.5.0 items 1/4/5 and the test count (25).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HCGpJXhSZasJLU38ksDghe
This commit is contained in:
GodopuandClaude Opus 5 committed 2026-09-13 20:49:56 +09:00
1 parent 77f2829b8e
commit 135d76f903
11 files changed
+577 -67
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.agents/skills/lib_py/delegation/remote.py
+130 -36
View File
@@ -33,6 +33,7 @@ import logging
import os
import queue
import shlex
import signal
import socket
import subprocess
import sys
@@ -153,15 +154,39 @@ def clear_dispatch(config, topic: str) -> None:
# remote side
# ---------------------------------------------------------------------------
def _attempt_of(record: Dict[str, Any]) -> int:
try:
return int(record.get("attempt_id", record.get("iteration", 1)) or 1)
except (TypeError, ValueError):
return 1
def mirror_job(d: Dict[str, Any], registry_dir: str, session: str) -> Optional[str]:
"""Create the local job record a remote agent publishes against.
"""Create the local job record a remote agent publishes against, or refuse
the dispatch (None).
The record carries the orchestrator's job_id, auth_token, dispatch_token
and attempt_id, so `publish_event` on this host produces events the
orchestrator's subscriber accepts. A previous attempt's record (a
re-dispatch) is retired first: creation is the only write allowed here.
orchestrator's subscriber accepts. Admission (review round 2, codex R1): a
valid signature proves where a dispatch came from, not that it is current,
so
* the same dispatch_token is idempotent while its attempt is open, and is
never run again once that attempt reached a terminal;
* a lower attempt, or the same attempt under another token, is a stale or
conflicting copy: refused before anything is written, so the record and
its events.log stay as they are;
* only a strictly newer attempt replaces the record, and that check and
the write are one step under the registry lock
(``register_job(supersede_older_attempt=True)``). The events.log is kept:
the history of earlier attempts is not erased.
"""
jid = d["job_id"]
try:
incoming = int(d.get("attempt_id") or 1)
except (TypeError, ValueError):
logger.warning("drop dispatch for %s: attempt_id %r is not a number", jid, d.get("attempt_id"))
return None
path = mqtt_common._job_path(jid, registry_dir)
if path.exists():
old = load_job(jid, registry_dir)
@@ -169,28 +194,32 @@ def mirror_job(d: Dict[str, Any], registry_dir: str, session: str) -> Optional[s
if old.get("status") in ("completed", "error", "cancelled"):
return None # this attempt already ran to its end: a re-sent dispatch is not run again
return jid # same attempt, already mirrored (duplicate delivery)
with mqtt_common.registry_lock(registry_dir):
path.unlink()
log = mqtt_common._job_path(jid, registry_dir).with_suffix(".events.log")
if log.exists():
log.unlink()
registry.register_job(
prompt=d.get("brief", ""),
agent=d.get("agent") or "remote",
agent_session=SESSION_PREFIX + session,
role=d.get("role") or "Worker",
timeout_sec=int(d.get("timeout_sec") or 3600),
idle_timeout_sec=d.get("idle_timeout_sec"),
registry_dir=registry_dir,
job_id=jid,
auth_token=d.get("auth_token"),
heartbeat_sec=int(d.get("heartbeat_sec") or registry.DEFAULT_HEARTBEAT_SEC),
heartbeat_misses=int(d.get("heartbeat_misses") or registry.DEFAULT_HEARTBEAT_MISSES),
ack_timeout_sec=int(d.get("ack_timeout_sec") or registry.DEFAULT_ACK_TIMEOUT_SEC),
dispatch_token=d.get("dispatch_token"),
attempt_id=int(d.get("attempt_id") or 1),
transport="remote",
)
if incoming <= _attempt_of(old):
logger.warning("refusing dispatch for %s attempt %s: this runner already holds attempt %s "
"(a stale or conflicting copy)", jid, incoming, _attempt_of(old))
return None
try:
registry.register_job(
prompt=d.get("brief", ""),
agent=d.get("agent") or "remote",
agent_session=SESSION_PREFIX + session,
role=d.get("role") or "Worker",
timeout_sec=int(d.get("timeout_sec") or 3600),
idle_timeout_sec=d.get("idle_timeout_sec"),
registry_dir=registry_dir,
job_id=jid,
auth_token=d.get("auth_token"),
heartbeat_sec=int(d.get("heartbeat_sec") or registry.DEFAULT_HEARTBEAT_SEC),
heartbeat_misses=int(d.get("heartbeat_misses") or registry.DEFAULT_HEARTBEAT_MISSES),
ack_timeout_sec=int(d.get("ack_timeout_sec") or registry.DEFAULT_ACK_TIMEOUT_SEC),
dispatch_token=d.get("dispatch_token"),
attempt_id=incoming,
transport="remote",
supersede_older_attempt=True,
)
except FileExistsError as exc: # lost the race to a newer (or equal) attempt under the lock
logger.warning("refusing dispatch for %s: %s", jid, exc)
return None
brief_dir = os.path.join(registry_dir, jid)
os.makedirs(brief_dir, exist_ok=True)
with open(os.path.join(brief_dir, "brief.md"), "w", encoding="utf-8") as fh:
@@ -212,13 +241,68 @@ def _publish(jid: str, registry_dir: str, event: str, detail: str, d: Dict[str,
return publish_event.main(argv)
def _group_alive(pgid: int) -> bool:
try:
os.killpg(pgid, 0)
except OSError:
return False
return True
def _terminate_group(proc: "subprocess.Popen", grace: float = 5.0) -> None:
"""TERM, then KILL, the agent command's own process group (it runs in its
own session) and wait until the group is gone. Only that owned group is
signalled (review round 2, codex R2)."""
for sig, wait in ((signal.SIGTERM, grace), (signal.SIGKILL, 2.0)):
try:
os.killpg(proc.pid, sig)
except OSError:
return # the group is already gone
deadline = time.monotonic() + wait
while time.monotonic() < deadline:
proc.poll() # reap the shell itself
if not _group_alive(proc.pid):
return
time.sleep(0.1)
def _run_agent_command(exec_cmd: str, env: Dict[str, str], timeout: float):
"""Run the agent command in its own session; returns (rc, stdout, stderr,
timed_out). At the deadline - or when the runner itself is interrupted -
the whole process group is terminated before this returns, so no
descendant keeps working past the job's timeout or overlaps the next
dispatch (review round 2, codex R2: `subprocess.run(shell=True,
timeout=...)` killed only the shell)."""
proc = subprocess.Popen(exec_cmd, shell=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, cwd=os.getcwd(), start_new_session=True)
try:
out, err = proc.communicate(timeout=timeout)
return proc.returncode, out, err, False
except subprocess.TimeoutExpired:
_terminate_group(proc)
try:
out, err = proc.communicate(timeout=5)
except subprocess.TimeoutExpired:
# a descendant that left the group still holds the pipes: stop reading
for stream in (proc.stdout, proc.stderr):
try:
stream.close()
except Exception:
pass
out, err = "", ""
return 124, out, err, True
except BaseException:
_terminate_group(proc)
raise
def handle_dispatch(d: Dict[str, Any], registry_dir: str, session: str,
exec_cmd: Optional[str], python: Optional[str] = None) -> int:
"""Run one dispatched job to its terminal event. Returns the agent rc."""
jid = mirror_job(d, registry_dir, session)
if jid is None:
logger.info("dispatch for %s attempt %s already finished here; ignoring the re-sent copy",
d.get("job_id"), d.get("attempt_id"))
logger.info("dispatch for %s attempt %s not admitted (already finished here, or a stale or "
"conflicting copy); not running it", d.get("job_id"), d.get("attempt_id"))
return 0
heartbeat = int(d.get("heartbeat_sec") or registry.DEFAULT_HEARTBEAT_SEC)
rc_started = _publish(jid, registry_dir, "started", "remote runner {} claimed the job".format(session),
@@ -245,17 +329,15 @@ def handle_dispatch(d: Dict[str, Any], registry_dir: str, session: str,
+ (["--attempt-id", str(d["attempt_id"])] if d.get("attempt_id") is not None else [])),
})
timeout = float(job.get("timeout_sec", 3600))
try:
res = subprocess.run(exec_cmd, shell=True, env=env, capture_output=True, text=True,
timeout=timeout, cwd=os.getcwd())
rc = res.returncode
tail = (res.stdout or "").strip().splitlines()
rc, out, err, timed_out = _run_agent_command(exec_cmd, env, timeout)
if timed_out:
detail = "agent command exceeded timeout_sec {}".format(int(timeout))
else:
tail = (out or "").strip().splitlines()
detail = tail[-1][:200] if tail else "exit {}".format(rc)
if rc != 0:
err = (res.stderr or "").strip().splitlines()
detail = "exit {}: {}".format(rc, (err[-1] if err else detail)[:200])
except subprocess.TimeoutExpired:
rc, detail = 124, "agent command exceeded timeout_sec {}".format(int(timeout))
errs = (err or "").strip().splitlines()
detail = "exit {}: {}".format(rc, (errs[-1] if errs else detail)[:200])
cur = load_job(jid, registry_dir)
if cur.get("status") not in ("completed", "error", "cancelled"):
_publish(jid, registry_dir, "completed" if rc == 0 else "error", detail, d)
@@ -316,6 +398,16 @@ def run_runner(session: str, registry_dir: str, exec_cmd: Optional[str] = None,
client.reconnect_delay_set(min_delay=1, max_delay=16)
mqtt_common.with_retry(lambda: client.connect(cfg.host, cfg.port, cfg.keepalive),
attempts=3, base_delay=0.2, max_delay=1.0)()
# Review round 2 (codex R2): SIGTERM unwinds the runner like an interrupt,
# so a command still running is terminated together with its process group
# (_run_agent_command) instead of being left behind as an orphan.
def _on_term(signum, _frame):
raise SystemExit(128 + signum)
try:
previous_term = signal.signal(signal.SIGTERM, _on_term)
except ValueError: # not the main thread (a caller driving the runner in-process)
previous_term = None
client.loop_start()
deadline = time.monotonic() + timeout if timeout else None
handled = 0
@@ -339,6 +431,8 @@ def run_runner(session: str, registry_dir: str, exec_cmd: Optional[str] = None,
if once:
return 0
finally:
if previous_term is not None:
signal.signal(signal.SIGTERM, previous_term)
client.loop_stop()
try:
client.disconnect()