fix(issue-6): fail-closed registry timeout and TUI turn handshake

- Introduce finalize_unfinished in multi-agent-mux-delegate-job for fail-closed atomic transitions
- Enforce CAS dispatch_token and attempt_id pinning across engine, CLI, and lifecycle scripts
- Integrate Herdr wait_for_agent_idle turn handshake to prevent prompt loss and race conditions
- Resolve bash 3.2 set -u empty array expansion compatibility in lib.sh
- Preserve Issue #5 R2 contract in CAS transition engine (engine.py, publish_event.py, service.py)
- Add comprehensive regression suites (test_issue6_fail_closed.py, test_issue6_turn_handshake.py)
- Bump lockstep framework and skills to v4.4.1 across lib.sh, eager.py, VERSIONS.md, 8 SKILL.md
- Synchronize documentation (IMPROVEMENTS.md, MESSAGING.md, state-transitions.md, task.md, phase5-e2e.md)
- Unanimous PASS achieved across 4 reviewer team leaders (Claude, Codex, OpenCode, Grok)

Closes #6
This commit is contained in:
Godopu committed 2026-09-09 14:02:47 +09:00
1 parent 7b761be7b8
commit ec61511699
51 files changed
+3477 -293

No files matched your search

+56 -16
View File
@@ -59,6 +59,7 @@ class LoopRunner:
self.mam_dir = self.workspace_root / ".mam"
self.checkpoint_file = self.mam_dir / "loop-checkpoint.json"
self.created_jobs: List[str] = []
self.failed_submit_jobs: Set[str] = set()
self.current_plan = ""
self.plan_job_id: Optional[str] = None
self.base_commit = ""
@@ -262,27 +263,49 @@ class LoopRunner:
if not wrapper_script.exists():
wrapper_script = Path(__file__).resolve().parent.parent / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
cmd = [
"bash", str(wrapper_script), "submit",
"--agent-session", f"herdr:{agent_session}",
"--agent", agent_type,
"--type", "direct",
"--role", role,
"--prompt", prompt,
]
env = dict(os.environ)
if "DELEGATE_JOB_PYTHON" not in env:
env["DELEGATE_JOB_PYTHON"] = sys.executable
def _run_cmd(p_text: str):
cmd = [
"bash", str(wrapper_script), "submit",
"--agent-session", f"herdr:{agent_session}",
"--agent", agent_type,
"--type", "direct",
"--role", role,
"--prompt", p_text,
]
env = dict(os.environ)
if "DELEGATE_JOB_PYTHON" not in env:
env["DELEGATE_JOB_PYTHON"] = sys.executable
return subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.workspace_root), env=env)
res = _run_cmd(prompt)
match = re.search(r"registered job:\s*([A-Za-z0-9_-]+)", res.stdout)
job_id = match.group(1) if match else None
res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.workspace_root), env=env)
if res.returncode != 0:
log_error(f"delegate_job submit failed (exit {res.returncode}):\n{res.stderr or res.stdout}")
if res.returncode == 5:
# rc=5 (session_busy): MAM_SESSION_BUSY_RETRY_S wait, then 1 retry
retry_s = int(os.environ.get("MAM_SESSION_BUSY_RETRY_S", "30"))
if retry_s > 0:
time.sleep(retry_s)
res_retry = _run_cmd(prompt)
match_retry = re.search(r"registered job:\s*([A-Za-z0-9_-]+)", res_retry.stdout)
if match_retry:
job_id_retry = match_retry.group(1)
self.created_jobs.append(job_id_retry)
self._record_checkpoint()
if res_retry.returncode != 0:
self.failed_submit_jobs.add(job_id_retry)
log_error(f"delegate_job retry failed (exit {res_retry.returncode}):\n{res_retry.stderr or res_retry.stdout}")
return job_id_retry
if job_id:
self.created_jobs.append(job_id)
self.failed_submit_jobs.add(job_id)
self._record_checkpoint()
return job_id
return None
# Extract Job ID (pattern: 'registered job: [A-Za-z0-9]*')
match = re.search(r"registered job:\s*([A-Za-z0-9]+)", res.stdout)
if match:
job_id = match.group(1)
if job_id:
self.created_jobs.append(job_id)
self._record_checkpoint()
return job_id
@@ -290,11 +313,17 @@ class LoopRunner:
def wait_for_job(self, job_id: str, max_wait: int = 3900, check_interval: float = 1.0) -> bool:
"""Blocking wait for job completion or error with safety timeout."""
if job_id in self.failed_submit_jobs:
# Submit command for this job failed (rc!=0). Cap wait to 5s to avoid 3900s hang
max_wait = min(max_wait, 5.0)
if self.verbose:
log_info(f"Monitoring job '{job_id}' for status changes (timeout: {max_wait}s)...")
job_file = self.mam_dir / "jobs" / f"{job_id}.json"
start_time = time.time()
missing_count = 0
unknown_count = 0
while time.time() - start_time < max_wait:
if self.interrupted:
@@ -306,6 +335,17 @@ class LoopRunner:
status = json.load(f).get("status", "unknown")
except Exception:
status = "unknown"
else:
missing_count += 1
if missing_count >= 5 and (time.time() - start_time) >= min(max_wait, 5.0):
log_error(f"Job record '{job_file}' not found on disk; aborting wait.")
return False
if status == "unknown":
unknown_count += 1
if unknown_count >= 5 and (time.time() - start_time) >= min(max_wait, 5.0):
log_error(f"Job '{job_id}' record unreadable / status unknown; aborting wait.")
return False
if status == "completed":
if self.verbose: