tmpl/multi-agent-mux
1
0 Fork 0
Code Issues 9 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(issue-6): fail-closed registry timeout and TUI turn handshake

Browse Source 
- Introduce finalize_unfinished in multi-agent-mux-delegate-job for fail-closed atomic transitions
- Enforce CAS dispatch_token and attempt_id pinning across engine, CLI, and lifecycle scripts
- Integrate Herdr wait_for_agent_idle turn handshake to prevent prompt loss and race conditions
- Resolve bash 3.2 set -u empty array expansion compatibility in lib.sh
- Preserve Issue #5 R2 contract in CAS transition engine (engine.py, publish_event.py, service.py)
- Add comprehensive regression suites (test_issue6_fail_closed.py, test_issue6_turn_handshake.py)
- Bump lockstep framework and skills to v4.4.1 across lib.sh, eager.py, VERSIONS.md, 8 SKILL.md
- Synchronize documentation (IMPROVEMENTS.md, MESSAGING.md, state-transitions.md, task.md, phase5-e2e.md)
- Unanimous PASS achieved across 4 reviewer team leaders (Claude, Codex, OpenCode, Grok)

Closes #6
This commit is contained in:
Godopu committed 2026-09-09 14:02:47 +09:00
1 parent 7b761be7b8
commit ec61511699
51 files changed
+3477 -293
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.agents/skills/lib_py/loop/eager.py
+1 -1
View File
@@ -13,7 +13,7 @@ import sys
from typing import List, Sequence
# Framework version locked with framework constants
FRAMEWORK_VERSION = "4.4.0"
FRAMEWORK_VERSION = "4.4.1"
CORE_MODULES: Sequence[str] = (
# Standard library modules
.agents/skills/lib_py/loop/runner.py
+56 -16
View File
@@ -59,6 +59,7 @@ class LoopRunner:
self.mam_dir = self.workspace_root / ".mam"
self.checkpoint_file = self.mam_dir / "loop-checkpoint.json"
self.created_jobs: List[str] = []
self.failed_submit_jobs: Set[str] = set()
self.current_plan = ""
self.plan_job_id: Optional[str] = None
self.base_commit = ""
@@ -262,27 +263,49 @@ class LoopRunner:
if not wrapper_script.exists():
wrapper_script = Path(__file__).resolve().parent.parent / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
cmd = [
"bash", str(wrapper_script), "submit",
"--agent-session", f"herdr:{agent_session}",
"--agent", agent_type,
"--type", "direct",
"--role", role,
"--prompt", prompt,
]
env = dict(os.environ)
if "DELEGATE_JOB_PYTHON" not in env:
env["DELEGATE_JOB_PYTHON"] = sys.executable
def _run_cmd(p_text: str):
cmd = [
"bash", str(wrapper_script), "submit",
"--agent-session", f"herdr:{agent_session}",
"--agent", agent_type,
"--type", "direct",
"--role", role,
"--prompt", p_text,
]
env = dict(os.environ)
if "DELEGATE_JOB_PYTHON" not in env:
env["DELEGATE_JOB_PYTHON"] = sys.executable
return subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.workspace_root), env=env)
res = _run_cmd(prompt)
match = re.search(r"registered job:\s*([A-Za-z0-9_-]+)", res.stdout)
job_id = match.group(1) if match else None
res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.workspace_root), env=env)
if res.returncode != 0:
log_error(f"delegate_job submit failed (exit {res.returncode}):\n{res.stderr or res.stdout}")
if res.returncode == 5:
# rc=5 (session_busy): MAM_SESSION_BUSY_RETRY_S wait, then 1 retry
retry_s = int(os.environ.get("MAM_SESSION_BUSY_RETRY_S", "30"))
if retry_s > 0:
time.sleep(retry_s)
res_retry = _run_cmd(prompt)
match_retry = re.search(r"registered job:\s*([A-Za-z0-9_-]+)", res_retry.stdout)
if match_retry:
job_id_retry = match_retry.group(1)
self.created_jobs.append(job_id_retry)
self._record_checkpoint()
if res_retry.returncode != 0:
self.failed_submit_jobs.add(job_id_retry)
log_error(f"delegate_job retry failed (exit {res_retry.returncode}):\n{res_retry.stderr or res_retry.stdout}")
return job_id_retry
if job_id:
self.created_jobs.append(job_id)
self.failed_submit_jobs.add(job_id)
self._record_checkpoint()
return job_id
return None
# Extract Job ID (pattern: 'registered job: [A-Za-z0-9]*')
match = re.search(r"registered job:\s*([A-Za-z0-9]+)", res.stdout)
if match:
job_id = match.group(1)
if job_id:
self.created_jobs.append(job_id)
self._record_checkpoint()
return job_id
@@ -290,11 +313,17 @@ class LoopRunner:
def wait_for_job(self, job_id: str, max_wait: int = 3900, check_interval: float = 1.0) -> bool:
"""Blocking wait for job completion or error with safety timeout."""
if job_id in self.failed_submit_jobs:
# Submit command for this job failed (rc!=0). Cap wait to 5s to avoid 3900s hang
max_wait = min(max_wait, 5.0)
if self.verbose:
log_info(f"Monitoring job '{job_id}' for status changes (timeout: {max_wait}s)...")
job_file = self.mam_dir / "jobs" / f"{job_id}.json"
start_time = time.time()
missing_count = 0
unknown_count = 0
while time.time() - start_time < max_wait:
if self.interrupted:
@@ -306,6 +335,17 @@ class LoopRunner:
status = json.load(f).get("status", "unknown")
except Exception:
status = "unknown"
else:
missing_count += 1
if missing_count >= 5 and (time.time() - start_time) >= min(max_wait, 5.0):
log_error(f"Job record '{job_file}' not found on disk; aborting wait.")
return False
if status == "unknown":
unknown_count += 1
if unknown_count >= 5 and (time.time() - start_time) >= min(max_wait, 5.0):
log_error(f"Job '{job_id}' record unreadable / status unknown; aborting wait.")
return False
if status == "completed":
if self.verbose:
.agents/skills/lib_py/loop/watcher.py
+48 -4
View File
@@ -59,10 +59,14 @@ class WaitOutcome(object):
def build_argv(target: str, until: Sequence[str] = DEFAULT_UNTIL,
timeout_ms: Optional[int] = None) -> list:
timeout_ms: Optional[int] = None,
session: Optional[str] = None) -> list:
if not target:
raise ValueError("wait target is required")
argv = ["herdr", "agent", "wait", target]
if session and session != "default":
argv = ["herdr", "--session", session, "agent", "wait", target]
else:
argv = ["herdr", "agent", "wait", target]
for state in until:
if state not in VALID_STATES:
raise ValueError("unsupported --until state: {!r}".format(state))
@@ -83,9 +87,10 @@ def _parse(stream: str) -> Optional[Dict[str, Any]]:
def wait_once(target: str, until: Sequence[str] = DEFAULT_UNTIL,
timeout_ms: Optional[int] = None, run=subprocess.run) -> WaitOutcome:
timeout_ms: Optional[int] = None, session: Optional[str] = None,
run=subprocess.run) -> WaitOutcome:
"""One barrier call. `run` is injectable so tests never touch a real herdr."""
argv = build_argv(target, until, timeout_ms)
argv = build_argv(target, until, timeout_ms, session=session)
try:
proc = run(argv, capture_output=True, text=True)
except Exception as exc:
@@ -164,3 +169,42 @@ class ReconnectBackoff(object):
return [self.record_failure() for _ in range(n)]
finally:
self.current, self.consecutive_failures, self.cached_snapshot = saved
def main(argv: Optional[Sequence[str]] = None) -> int:
import argparse
import os
import sys
parser = argparse.ArgumentParser(description="watcher CLI for herdr agent wait")
sub = parser.add_subparsers(dest="command", required=True)
p_wait = sub.add_parser("wait", help="wait for agent state")
p_wait.add_argument("target", help="target agent name")
p_wait.add_argument("--session", default=None, help="herdr session name")
p_wait.add_argument("--until", action="append", default=[], help="until state(s)")
p_wait.add_argument("--timeout-ms", type=int, default=None, help="timeout in ms")
args = parser.parse_args(argv)
session = args.session or os.environ.get("HERDR_SESSION_NAME") or None
until = args.until or DEFAULT_UNTIL
outcome = wait_once(args.target, until=until, timeout_ms=args.timeout_ms, session=session)
agent_st = None
if outcome.agent:
agent_st = outcome.agent.get("agent_status")
print(json.dumps({
"kind": outcome.kind,
"agent_status": agent_st,
"error_code": outcome.error_code,
}))
rc_map = {
MATCHED: 0,
TIMEOUT: 2,
AGENT_NOT_FOUND: 3,
TRANSPORT_ERROR: 4,
}
return rc_map.get(outcome.kind, 4)
if __name__ == "__main__":
import sys
sys.exit(main())