fix(issue-6): fail-closed registry timeout and TUI turn handshake
- Introduce finalize_unfinished in multi-agent-mux-delegate-job for fail-closed atomic transitions - Enforce CAS dispatch_token and attempt_id pinning across engine, CLI, and lifecycle scripts - Integrate Herdr wait_for_agent_idle turn handshake to prevent prompt loss and race conditions - Resolve bash 3.2 set -u empty array expansion compatibility in lib.sh - Preserve Issue #5 R2 contract in CAS transition engine (engine.py, publish_event.py, service.py) - Add comprehensive regression suites (test_issue6_fail_closed.py, test_issue6_turn_handshake.py) - Bump lockstep framework and skills to v4.4.1 across lib.sh, eager.py, VERSIONS.md, 8 SKILL.md - Synchronize documentation (IMPROVEMENTS.md, MESSAGING.md, state-transitions.md, task.md, phase5-e2e.md) - Unanimous PASS achieved across 4 reviewer team leaders (Claude, Codex, OpenCode, Grok) Closes #6
This commit is contained in:
1 parent
7b761be7b8
commit
ec61511699
51 files changed
+3477
-293
No files matched your search
@@ -59,6 +59,7 @@ class LoopRunner:
|
||||
self.mam_dir = self.workspace_root / ".mam"
|
||||
self.checkpoint_file = self.mam_dir / "loop-checkpoint.json"
|
||||
self.created_jobs: List[str] = []
|
||||
self.failed_submit_jobs: Set[str] = set()
|
||||
self.current_plan = ""
|
||||
self.plan_job_id: Optional[str] = None
|
||||
self.base_commit = ""
|
||||
@@ -262,27 +263,49 @@ class LoopRunner:
|
||||
if not wrapper_script.exists():
|
||||
wrapper_script = Path(__file__).resolve().parent.parent / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
|
||||
|
||||
cmd = [
|
||||
"bash", str(wrapper_script), "submit",
|
||||
"--agent-session", f"herdr:{agent_session}",
|
||||
"--agent", agent_type,
|
||||
"--type", "direct",
|
||||
"--role", role,
|
||||
"--prompt", prompt,
|
||||
]
|
||||
env = dict(os.environ)
|
||||
if "DELEGATE_JOB_PYTHON" not in env:
|
||||
env["DELEGATE_JOB_PYTHON"] = sys.executable
|
||||
def _run_cmd(p_text: str):
|
||||
cmd = [
|
||||
"bash", str(wrapper_script), "submit",
|
||||
"--agent-session", f"herdr:{agent_session}",
|
||||
"--agent", agent_type,
|
||||
"--type", "direct",
|
||||
"--role", role,
|
||||
"--prompt", p_text,
|
||||
]
|
||||
env = dict(os.environ)
|
||||
if "DELEGATE_JOB_PYTHON" not in env:
|
||||
env["DELEGATE_JOB_PYTHON"] = sys.executable
|
||||
return subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.workspace_root), env=env)
|
||||
|
||||
res = _run_cmd(prompt)
|
||||
match = re.search(r"registered job:\s*([A-Za-z0-9_-]+)", res.stdout)
|
||||
job_id = match.group(1) if match else None
|
||||
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(self.workspace_root), env=env)
|
||||
if res.returncode != 0:
|
||||
log_error(f"delegate_job submit failed (exit {res.returncode}):\n{res.stderr or res.stdout}")
|
||||
if res.returncode == 5:
|
||||
# rc=5 (session_busy): MAM_SESSION_BUSY_RETRY_S wait, then 1 retry
|
||||
retry_s = int(os.environ.get("MAM_SESSION_BUSY_RETRY_S", "30"))
|
||||
if retry_s > 0:
|
||||
time.sleep(retry_s)
|
||||
res_retry = _run_cmd(prompt)
|
||||
match_retry = re.search(r"registered job:\s*([A-Za-z0-9_-]+)", res_retry.stdout)
|
||||
if match_retry:
|
||||
job_id_retry = match_retry.group(1)
|
||||
self.created_jobs.append(job_id_retry)
|
||||
self._record_checkpoint()
|
||||
if res_retry.returncode != 0:
|
||||
self.failed_submit_jobs.add(job_id_retry)
|
||||
log_error(f"delegate_job retry failed (exit {res_retry.returncode}):\n{res_retry.stderr or res_retry.stdout}")
|
||||
return job_id_retry
|
||||
if job_id:
|
||||
self.created_jobs.append(job_id)
|
||||
self.failed_submit_jobs.add(job_id)
|
||||
self._record_checkpoint()
|
||||
return job_id
|
||||
return None
|
||||
|
||||
# Extract Job ID (pattern: 'registered job: [A-Za-z0-9]*')
|
||||
match = re.search(r"registered job:\s*([A-Za-z0-9]+)", res.stdout)
|
||||
if match:
|
||||
job_id = match.group(1)
|
||||
if job_id:
|
||||
self.created_jobs.append(job_id)
|
||||
self._record_checkpoint()
|
||||
return job_id
|
||||
@@ -290,11 +313,17 @@ class LoopRunner:
|
||||
|
||||
def wait_for_job(self, job_id: str, max_wait: int = 3900, check_interval: float = 1.0) -> bool:
|
||||
"""Blocking wait for job completion or error with safety timeout."""
|
||||
if job_id in self.failed_submit_jobs:
|
||||
# Submit command for this job failed (rc!=0). Cap wait to 5s to avoid 3900s hang
|
||||
max_wait = min(max_wait, 5.0)
|
||||
|
||||
if self.verbose:
|
||||
log_info(f"Monitoring job '{job_id}' for status changes (timeout: {max_wait}s)...")
|
||||
|
||||
job_file = self.mam_dir / "jobs" / f"{job_id}.json"
|
||||
start_time = time.time()
|
||||
missing_count = 0
|
||||
unknown_count = 0
|
||||
|
||||
while time.time() - start_time < max_wait:
|
||||
if self.interrupted:
|
||||
@@ -306,6 +335,17 @@ class LoopRunner:
|
||||
status = json.load(f).get("status", "unknown")
|
||||
except Exception:
|
||||
status = "unknown"
|
||||
else:
|
||||
missing_count += 1
|
||||
if missing_count >= 5 and (time.time() - start_time) >= min(max_wait, 5.0):
|
||||
log_error(f"Job record '{job_file}' not found on disk; aborting wait.")
|
||||
return False
|
||||
|
||||
if status == "unknown":
|
||||
unknown_count += 1
|
||||
if unknown_count >= 5 and (time.time() - start_time) >= min(max_wait, 5.0):
|
||||
log_error(f"Job '{job_id}' record unreadable / status unknown; aborting wait.")
|
||||
return False
|
||||
|
||||
if status == "completed":
|
||||
if self.verbose:
|
||||
|
||||
Reference in new issue
Block a user