fix(issue-6): fail-closed registry timeout and TUI turn handshake
- Introduce finalize_unfinished in multi-agent-mux-delegate-job for fail-closed atomic transitions - Enforce CAS dispatch_token and attempt_id pinning across engine, CLI, and lifecycle scripts - Integrate Herdr wait_for_agent_idle turn handshake to prevent prompt loss and race conditions - Resolve bash 3.2 set -u empty array expansion compatibility in lib.sh - Preserve Issue #5 R2 contract in CAS transition engine (engine.py, publish_event.py, service.py) - Add comprehensive regression suites (test_issue6_fail_closed.py, test_issue6_turn_handshake.py) - Bump lockstep framework and skills to v4.4.1 across lib.sh, eager.py, VERSIONS.md, 8 SKILL.md - Synchronize documentation (IMPROVEMENTS.md, MESSAGING.md, state-transitions.md, task.md, phase5-e2e.md) - Unanimous PASS achieved across 4 reviewer team leaders (Claude, Codex, OpenCode, Grok) Closes #6
This commit is contained in:
1 parent
7b761be7b8
commit
ec61511699
51 files changed
+3477
-293
No files matched your search
@@ -127,7 +127,8 @@ class PreparedEvent(object):
|
||||
|
||||
|
||||
def prepare_event(job_id, event, detail="", data=None,
|
||||
registry_dir=DEFAULT_REGISTRY_DIR, retained=False) -> PreparedEvent:
|
||||
registry_dir=DEFAULT_REGISTRY_DIR, retained=False,
|
||||
dispatch_token=None, attempt_id=None) -> PreparedEvent:
|
||||
"""Load the job, consume the next seq, and build the signed payload.
|
||||
|
||||
Consuming the seq here (before any transport runs) is deliberate: a failed
|
||||
@@ -139,12 +140,21 @@ def prepare_event(job_id, event, detail="", data=None,
|
||||
topic = job.get("topic_prefix")
|
||||
topic = f"{topic}/events" if topic else events_topic_for(job_id)
|
||||
seq = next_seq(job_id, registry_dir)
|
||||
|
||||
payload_data = dict(data) if data else {}
|
||||
eff_dispatch_token = dispatch_token
|
||||
if eff_dispatch_token is not None and "dispatch_token" not in payload_data:
|
||||
payload_data["dispatch_token"] = eff_dispatch_token
|
||||
eff_attempt_id = attempt_id
|
||||
if eff_attempt_id is not None and "attempt_id" not in payload_data:
|
||||
payload_data["attempt_id"] = eff_attempt_id
|
||||
|
||||
payload = build_payload(
|
||||
job_id=job_id,
|
||||
seq=seq,
|
||||
event=event,
|
||||
detail=detail,
|
||||
data=data,
|
||||
data=payload_data,
|
||||
auth_token=job.get("auth_token"),
|
||||
)
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
@@ -153,7 +163,9 @@ def prepare_event(job_id, event, detail="", data=None,
|
||||
|
||||
|
||||
def record_outcome(prepared: PreparedEvent, registry_dir: str,
|
||||
publish_ok: bool, publish_error: Optional[str]) -> None:
|
||||
publish_ok: bool, publish_error: Optional[str],
|
||||
expected_dispatch_token: Optional[str] = None,
|
||||
expected_attempt: Optional[int] = None) -> None:
|
||||
"""Persist the audit record, the registry event log and the status sync.
|
||||
|
||||
All three are best effort and all three happen whether or not the publish
|
||||
@@ -176,10 +188,19 @@ def record_outcome(prepared: PreparedEvent, registry_dir: str,
|
||||
new_status = EVENT_TO_STATUS.get(prepared.event)
|
||||
if new_status:
|
||||
try:
|
||||
exp_token = expected_dispatch_token
|
||||
if exp_token is None and isinstance(prepared.payload.get("data"), dict):
|
||||
exp_token = prepared.payload["data"].get("dispatch_token")
|
||||
exp_att = expected_attempt
|
||||
if exp_att is None and isinstance(prepared.payload.get("data"), dict):
|
||||
exp_att = prepared.payload["data"].get("attempt_id")
|
||||
|
||||
# Agent authority on purpose: an agent may never move a terminal
|
||||
# record, which is what the operator-only requests are gated for.
|
||||
mqtt_common.update_job_status(prepared.job_id, registry_dir,
|
||||
_source="publish_event", status=new_status)
|
||||
_source="publish_event", status=new_status,
|
||||
expected_dispatch_token=exp_token,
|
||||
expected_attempt=exp_att)
|
||||
except Exception as exc: # pragma: no cover - best effort
|
||||
logger.warning("status sync failed: %s", exc)
|
||||
|
||||
@@ -214,6 +235,8 @@ def main(argv=None) -> int:
|
||||
parser.add_argument("--attempt-id", type=int, default=None,
|
||||
help="attempt this finalization belongs to (default: the "
|
||||
"job's current iteration)")
|
||||
parser.add_argument("--expect-dispatch-token", default=None,
|
||||
help="expected dispatch token for CAS verification")
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
@@ -250,7 +273,8 @@ def main(argv=None) -> int:
|
||||
attempt_id = args.attempt_id
|
||||
if attempt_id is None:
|
||||
try:
|
||||
attempt_id = load_job(job_id, args.registry_dir).get("iteration", 1)
|
||||
rec = load_job(job_id, args.registry_dir)
|
||||
attempt_id = rec.get("attempt_id", rec.get("iteration", 1))
|
||||
except FileNotFoundError:
|
||||
attempt_id = 1
|
||||
try:
|
||||
@@ -263,6 +287,41 @@ def main(argv=None) -> int:
|
||||
logger.info("finalized %s (sha256 %s...)", manifest["report_path"],
|
||||
manifest["sha256"][:12])
|
||||
|
||||
check_token = args.expect_dispatch_token
|
||||
if check_token is None and isinstance(data, dict) and "dispatch_token" in data:
|
||||
check_token = data.get("dispatch_token")
|
||||
try:
|
||||
rec = load_job(job_id, args.registry_dir)
|
||||
actual_tok = rec.get("dispatch_token")
|
||||
rec_attempt = rec.get("attempt_id", rec.get("iteration", 1))
|
||||
except FileNotFoundError:
|
||||
rec = None
|
||||
actual_tok = None
|
||||
rec_attempt = 1
|
||||
|
||||
check_attempt = args.attempt_id
|
||||
if check_attempt is None and isinstance(data, dict) and "attempt_id" in data:
|
||||
check_attempt = data.get("attempt_id")
|
||||
|
||||
if rec is not None and actual_tok is not None:
|
||||
if check_token is None and check_attempt is None:
|
||||
if args.event in ("completed", "error", "cancelled") and rec_attempt > 1:
|
||||
logger.error("dispatch token required: redispatched job %s carries a dispatch token "
|
||||
"but none was provided (aborting terminal publish)", job_id)
|
||||
return 4
|
||||
elif check_token is not None and actual_tok != check_token:
|
||||
logger.error("dispatch token mismatch: expected %s, got %s (aborting publish)",
|
||||
check_token, actual_tok)
|
||||
return 4
|
||||
elif check_attempt is not None and rec_attempt != check_attempt:
|
||||
logger.error("attempt mismatch: expected %s, got %s (aborting publish)",
|
||||
check_attempt, rec_attempt)
|
||||
return 4
|
||||
elif check_token is not None and actual_tok != check_token:
|
||||
logger.error("dispatch token mismatch: expected %s, got %s (aborting publish)",
|
||||
check_token, actual_tok)
|
||||
return 4
|
||||
|
||||
try:
|
||||
prepared = prepare_event(
|
||||
job_id=job_id,
|
||||
@@ -271,6 +330,8 @@ def main(argv=None) -> int:
|
||||
data=data,
|
||||
registry_dir=args.registry_dir,
|
||||
retained=args.retained,
|
||||
dispatch_token=args.expect_dispatch_token,
|
||||
attempt_id=args.attempt_id,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
logger.error("%s", exc)
|
||||
@@ -292,7 +353,16 @@ def main(argv=None) -> int:
|
||||
publish_error = str(exc)
|
||||
logger.error("publish failed after %d attempts: %s", args.attempts, exc)
|
||||
|
||||
record_outcome(prepared, args.registry_dir, publish_ok, publish_error)
|
||||
sync_expected_token = args.expect_dispatch_token
|
||||
if sync_expected_token is None and isinstance(data, dict):
|
||||
sync_expected_token = data.get("dispatch_token")
|
||||
sync_expected_attempt = args.attempt_id
|
||||
if sync_expected_attempt is None and isinstance(data, dict):
|
||||
sync_expected_attempt = data.get("attempt_id")
|
||||
|
||||
record_outcome(prepared, args.registry_dir, publish_ok, publish_error,
|
||||
expected_dispatch_token=sync_expected_token,
|
||||
expected_attempt=sync_expected_attempt)
|
||||
|
||||
if not publish_ok:
|
||||
return 2
|
||||
|
||||
Reference in new issue
Block a user