fix(issue-6): fail-closed registry timeout and TUI turn handshake
- Introduce finalize_unfinished in multi-agent-mux-delegate-job for fail-closed atomic transitions - Enforce CAS dispatch_token and attempt_id pinning across engine, CLI, and lifecycle scripts - Integrate Herdr wait_for_agent_idle turn handshake to prevent prompt loss and race conditions - Resolve bash 3.2 set -u empty array expansion compatibility in lib.sh - Preserve Issue #5 R2 contract in CAS transition engine (engine.py, publish_event.py, service.py) - Add comprehensive regression suites (test_issue6_fail_closed.py, test_issue6_turn_handshake.py) - Bump lockstep framework and skills to v4.4.1 across lib.sh, eager.py, VERSIONS.md, 8 SKILL.md - Synchronize documentation (IMPROVEMENTS.md, MESSAGING.md, state-transitions.md, task.md, phase5-e2e.md) - Unanimous PASS achieved across 4 reviewer team leaders (Claude, Codex, OpenCode, Grok) Closes #6
This commit is contained in:
Godopu committed
1 parent
7b761be7b8
commit
ec61511699
51 files changed
+3477
-293
No files matched your search
@@ -61,15 +61,21 @@ def _check_disk_fallback(
|
||||
pending: Set[str],
|
||||
registry_dir: str,
|
||||
terminal: Dict[str, str],
|
||||
expected_dispatch_tokens: Optional[Dict[str, Optional[str]]] = None,
|
||||
expected_attempts: Optional[Dict[str, Optional[int]]] = None,
|
||||
) -> None:
|
||||
"""Check local on-disk registry and audit logs for terminal status (B-15)."""
|
||||
for jid in list(pending):
|
||||
disk_status = None
|
||||
rec_token = None
|
||||
rec_attempt = None
|
||||
try:
|
||||
job_rec = load_job(jid, registry_dir)
|
||||
disk_status = job_rec.get("status")
|
||||
rec_token = job_rec.get("dispatch_token")
|
||||
rec_attempt = job_rec.get("attempt_id", job_rec.get("iteration"))
|
||||
except Exception:
|
||||
pass
|
||||
job_rec = {}
|
||||
if not disk_status or disk_status not in ("completed", "error", "cancelled"):
|
||||
try:
|
||||
disk_status = mqtt_common.read_logged_status(jid, mqtt_common.get_logs_dir())
|
||||
@@ -77,6 +83,18 @@ def _check_disk_fallback(
|
||||
pass
|
||||
|
||||
if disk_status in ("completed", "error", "cancelled"):
|
||||
if expected_dispatch_tokens:
|
||||
exp_token = expected_dispatch_tokens.get(jid)
|
||||
if exp_token is not None and rec_token != exp_token:
|
||||
logger.warning("disk fallback ignored terminal status for %s due to dispatch_token mismatch (%r != %r)",
|
||||
jid, rec_token, exp_token)
|
||||
continue
|
||||
if expected_attempts:
|
||||
exp_att = expected_attempts.get(jid)
|
||||
if exp_att is not None and rec_attempt is not None and rec_attempt < exp_att:
|
||||
logger.warning("disk fallback ignored terminal status for %s due to attempt_id older than expected (%r < %r)",
|
||||
jid, rec_attempt, exp_att)
|
||||
continue
|
||||
synth_event = "completed" if disk_status == "completed" else "error"
|
||||
synth_payload = {
|
||||
"job_id": jid,
|
||||
@@ -95,11 +113,15 @@ def _check_disk_fallback(
|
||||
class _Watcher:
|
||||
"""Holds the shared queue + the set of job_ids we accept events for."""
|
||||
|
||||
def __init__(self, expected_job_ids: Set[str], expected_tokens: Dict[str, Optional[str]], expected_seqs: Dict[str, int]):
|
||||
def __init__(self, expected_job_ids: Set[str], expected_tokens: Dict[str, Optional[str]], expected_seqs: Dict[str, int],
|
||||
expected_dispatch_tokens: Optional[Dict[str, Optional[str]]] = None,
|
||||
expected_attempts: Optional[Dict[str, Optional[int]]] = None):
|
||||
self.events: "queue.Queue[Tuple[str, Dict[str, Any]]]" = queue.Queue()
|
||||
self.expected = set(expected_job_ids)
|
||||
self.tokens = expected_tokens # job_id -> expected auth_token (or None)
|
||||
self.last_seq = dict(expected_seqs)
|
||||
self.dispatch_tokens = dict(expected_dispatch_tokens or {})
|
||||
self.attempts = dict(expected_attempts or {})
|
||||
# Intent identities already accepted, per job. A replayed notification is
|
||||
# re-published with a *fresh* wire sequence, so the monotonic sequence
|
||||
# guard below cannot recognise it: to that guard a retry looks like a new
|
||||
@@ -135,6 +157,24 @@ class _Watcher:
|
||||
if not mqtt_common.verify_hmac(payload, expected_token):
|
||||
logger.warning("drop event for job %s: HMAC verify failed", jid)
|
||||
return
|
||||
# --- CAS dispatch token & attempt check: drop stale attempt events ---
|
||||
expected_disp_tok = self.dispatch_tokens.get(jid)
|
||||
if expected_disp_tok is not None:
|
||||
data = payload.get("data")
|
||||
ev_tok = data.get("dispatch_token") if isinstance(data, dict) else None
|
||||
if ev_tok != expected_disp_tok:
|
||||
logger.warning("drop event for job %s: dispatch_token %r does not match expected %r",
|
||||
jid, ev_tok, expected_disp_tok)
|
||||
return
|
||||
|
||||
expected_att = self.attempts.get(jid)
|
||||
if expected_att is not None:
|
||||
data = payload.get("data")
|
||||
ev_att = data.get("attempt_id") if isinstance(data, dict) else None
|
||||
if ev_att is not None and isinstance(ev_att, int) and ev_att < expected_att:
|
||||
logger.warning("drop event for job %s: attempt_id %r is older than expected %r",
|
||||
jid, ev_att, expected_att)
|
||||
return
|
||||
# --- replay attack defense: check monotonic sequence ---
|
||||
seq = payload.get("seq")
|
||||
if seq is None or not isinstance(seq, int):
|
||||
@@ -216,11 +256,21 @@ def _run_subscriber(args) -> int:
|
||||
expected_ids: Set[str] = {j["job_id"] for j in jobs}
|
||||
tokens = {j["job_id"]: j.get("auth_token") for j in jobs}
|
||||
seqs = {}
|
||||
dispatch_tokens = {}
|
||||
attempts = {}
|
||||
for j in jobs:
|
||||
jid = j["job_id"]
|
||||
last_seq = int(j.get("last_seq", 0))
|
||||
seqs[jid] = max(0, last_seq - 1)
|
||||
watcher = _Watcher(expected_ids, tokens, seqs)
|
||||
dispatch_tokens[jid] = j.get("dispatch_token")
|
||||
attempts[jid] = j.get("attempt_id", j.get("iteration", 1))
|
||||
|
||||
if args.expect_dispatch_token is not None and args.job:
|
||||
dispatch_tokens[args.job] = args.expect_dispatch_token
|
||||
if args.attempt_id is not None and args.job:
|
||||
attempts[args.job] = args.attempt_id
|
||||
|
||||
watcher = _Watcher(expected_ids, tokens, seqs, dispatch_tokens, attempts)
|
||||
|
||||
# Resolve timeouts from CLI, falling back to the (first) job's settings.
|
||||
base_job = jobs[0]
|
||||
@@ -251,7 +301,18 @@ def _run_subscriber(args) -> int:
|
||||
if rc != 0:
|
||||
logger.warning("broker disconnected (rc=%s); will retry reconnect", reason_code)
|
||||
|
||||
start = time.monotonic()
|
||||
wall_deadline = start + wall_timeout
|
||||
last_event = start
|
||||
last_disk_check = 0.0
|
||||
retention_checked = not args.expect_retention
|
||||
connected = False
|
||||
subscribed = False
|
||||
|
||||
def on_subscribe(_c, _u, mid, granted_qos, _props=None):
|
||||
nonlocal last_event, subscribed
|
||||
subscribed = True
|
||||
last_event = time.monotonic()
|
||||
for topic in subscribed_topics:
|
||||
print(f"SUBSCRIBED {topic}", flush=True)
|
||||
|
||||
@@ -262,15 +323,9 @@ def _run_subscriber(args) -> int:
|
||||
|
||||
terminal: Dict[str, str] = {} # job_id -> "completed"/"error"
|
||||
pending: Set[str] = set(expected_ids)
|
||||
start = time.monotonic()
|
||||
wall_deadline = start + wall_timeout
|
||||
last_event = start
|
||||
last_disk_check = 0.0
|
||||
retention_checked = not args.expect_retention
|
||||
connected = False
|
||||
|
||||
# Check if all jobs are already in terminal state on disk before connecting
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
if not pending:
|
||||
if any(state == "error" for state in terminal.values()):
|
||||
return 1
|
||||
@@ -283,9 +338,10 @@ def _run_subscriber(args) -> int:
|
||||
)()
|
||||
client.loop_start()
|
||||
connected = True
|
||||
last_event = time.monotonic()
|
||||
except Exception as exc:
|
||||
logger.warning("broker connection failed (%s); checking disk fallback", exc)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
if not pending:
|
||||
if any(state == "error" for state in terminal.values()):
|
||||
return 1
|
||||
@@ -297,15 +353,17 @@ def _run_subscriber(args) -> int:
|
||||
while pending:
|
||||
now = time.monotonic()
|
||||
if now >= wall_deadline:
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
if not pending:
|
||||
break
|
||||
logger.error("wall-clock timeout (%.0fs); still pending: %s",
|
||||
wall_timeout, ", ".join(sorted(pending)))
|
||||
return 2
|
||||
if not subscribed:
|
||||
last_event = now
|
||||
idle_left = idle_timeout - (now - last_event)
|
||||
if idle_left <= 0:
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
if not pending:
|
||||
break
|
||||
logger.error("idle timeout (%.0fs, no events); still pending: %s",
|
||||
@@ -323,7 +381,7 @@ def _run_subscriber(args) -> int:
|
||||
# Step 2 (B-15): Local disk fallback throttled to every 3.0s
|
||||
if (now - last_disk_check) >= 3.0:
|
||||
last_disk_check = now
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal)
|
||||
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
|
||||
continue
|
||||
|
||||
last_event = time.monotonic()
|
||||
@@ -365,6 +423,10 @@ def main(argv=None) -> int:
|
||||
help="max seconds with no new event (default: job.idle_timeout_sec or 120)")
|
||||
parser.add_argument("--expect-retention", action="store_true",
|
||||
help="warn if no retained terminal event arrives promptly")
|
||||
parser.add_argument("--expect-dispatch-token", default=None,
|
||||
help="expected dispatch token for CAS verification")
|
||||
parser.add_argument("--attempt-id", type=int, default=None,
|
||||
help="minimum attempt id to accept events for")
|
||||
parser.add_argument("--registry-dir", default=DEFAULT_REGISTRY_DIR)
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
Reference in new issue
Block a user