tmpl/multi-agent-mux
1
0 Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(issue-6): fail-closed registry timeout and TUI turn handshake

Browse Source 
- Introduce finalize_unfinished in multi-agent-mux-delegate-job for fail-closed atomic transitions
- Enforce CAS dispatch_token and attempt_id pinning across engine, CLI, and lifecycle scripts
- Integrate Herdr wait_for_agent_idle turn handshake to prevent prompt loss and race conditions
- Resolve bash 3.2 set -u empty array expansion compatibility in lib.sh
- Preserve Issue #5 R2 contract in CAS transition engine (engine.py, publish_event.py, service.py)
- Add comprehensive regression suites (test_issue6_fail_closed.py, test_issue6_turn_handshake.py)
- Bump lockstep framework and skills to v4.4.1 across lib.sh, eager.py, VERSIONS.md, 8 SKILL.md
- Synchronize documentation (IMPROVEMENTS.md, MESSAGING.md, state-transitions.md, task.md, phase5-e2e.md)
- Unanimous PASS achieved across 4 reviewer team leaders (Claude, Codex, OpenCode, Grok)

Closes #6
This commit is contained in:
Godopu committed 2026-09-09 14:02:47 +09:00
1 parent 7b761be7b8
commit ec61511699
51 files changed
+3477 -293
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.agents/skills/lib_py/delegation/cli.py
+6
View File
@@ -39,6 +39,8 @@ def _build_parser() -> argparse.ArgumentParser:
p_pub.add_argument("--retained", action="store_true")
p_pub.add_argument("--registry-dir", default=None)
p_pub.add_argument("--attempts", type=int, default=3)
p_pub.add_argument("--expect-dispatch-token", default=None)
p_pub.add_argument("--attempt-id", type=int, default=None)
p_pub.add_argument("-v", "--verbose", action="store_true")
p_daemon = sub.add_parser("ensure-daemon", help="start the subscriber daemon if absent")
@@ -68,6 +70,10 @@ def main(argv: Optional[List[str]] = None) -> int:
forward += ["--retained"]
if args.registry_dir:
forward += ["--registry-dir", args.registry_dir]
if args.expect_dispatch_token:
forward += ["--expect-dispatch-token", args.expect_dispatch_token]
if args.attempt_id is not None:
forward += ["--attempt-id", str(args.attempt_id)]
if args.verbose:
forward += ["--verbose"]
return publish_event.main(forward)
.agents/skills/lib_py/delegation/engine.py
+28 -3
View File
@@ -88,7 +88,7 @@ ATTEMPT_STARTING_REQUESTS = ("claim", "redispatch")
# Only these sources may move a record out of a terminal state. An agent can
# never walk back its own terminal; that is the whole point of the CAS table.
OPERATOR_SOURCES = ("registry_cli", "operator")
OPERATOR_SOURCES = ("registry_cli", "operator", "delegate_wrapper")
OPERATOR_REQUESTS = ("redispatch", "finalize")
# A caller that merges `status=<value>` is making the request that owns that
@@ -178,20 +178,27 @@ def _decide(record: Dict[str, Any], request: str, expected_status, expected_atte
"stale attempt: expected {!r}, record has {!r}".format(
expected_attempt, actual))
requested = REQUEST_TO_STATUS.get(request, None)
if expected_dispatch_token is not None:
actual_token = record.get(DISPATCH_TOKEN_FIELD)
if actual_token != expected_dispatch_token:
return _Decision(False, current,
"dispatch token changed: the attempt this decision was "
"made for is no longer the current one")
elif expected_attempt is None and source not in OPERATOR_SOURCES and requested in TERMINAL_STATUSES:
actual_token = record.get(DISPATCH_TOKEN_FIELD)
rec_attempt = record.get("attempt_id", record.get("iteration", 1))
if actual_token is not None and rec_attempt > 1:
return _Decision(False, current,
"dispatch token required: redispatched job carries a dispatch token "
"but none was provided")
if expected_status is not None and current != expected_status:
return _Decision(False, current,
"precondition failed: expected status {!r}, found {!r}".format(
expected_status, current))
requested = REQUEST_TO_STATUS.get(request, None)
# Authority first, so a refusal says *why* rather than blaming the state.
if request in OPERATOR_REQUESTS and source not in OPERATOR_SOURCES:
return _Decision(False, current,
@@ -228,6 +235,15 @@ def _decide(record: Dict[str, Any], request: str, expected_status, expected_atte
if operator and request == "redispatch":
# Re-dispatching a finished job to the next iteration. The attempt
# must move forward, so a replayed redispatch cannot resurrect it.
new_attempt = fields.get("attempt_id")
if new_attempt is not None:
old_attempt = record.get("attempt_id", record.get("iteration"))
if old_attempt is not None and int(new_attempt) <= int(old_attempt):
return _Decision(False, current,
"redispatch attempt_id {!r} does not advance past {!r}".format(
new_attempt, old_attempt), conflict=True)
return _Decision(True, "pending", "{} -> pending (redispatch)".format(current))
new_iteration = fields.get("iteration")
old_iteration = record.get("iteration")
if new_iteration is None:
@@ -298,6 +314,15 @@ def transition(job_id, request="merge", *, registry_dir=None, expected_status=No
# A new attempt begins here, so anything decided about the
# previous one stops being committable from this moment on.
record[DISPATCH_TOKEN_FIELD] = uuid.uuid4().hex
if "attempt_id" in fields:
record["attempt_id"] = fields["attempt_id"]
elif "iteration" in fields:
record["attempt_id"] = fields["iteration"]
else:
record.setdefault("attempt_id", record.get("iteration", 1))
elif record.get(DISPATCH_TOKEN_FIELD) is None:
record[DISPATCH_TOKEN_FIELD] = uuid.uuid4().hex
record.setdefault("attempt_id", record.get("iteration", 1))
if notify is not None:
# One atomic commit records both the new state and the fact that
# the world has not been told yet. If the process dies before the
.agents/skills/lib_py/delegation/job_subscriber.py
+76 -14
View File
@@ -61,15 +61,21 @@ def _check_disk_fallback(
pending: Set[str],
registry_dir: str,
terminal: Dict[str, str],
expected_dispatch_tokens: Optional[Dict[str, Optional[str]]] = None,
expected_attempts: Optional[Dict[str, Optional[int]]] = None,
) -> None:
"""Check local on-disk registry and audit logs for terminal status (B-15)."""
for jid in list(pending):
disk_status = None
rec_token = None
rec_attempt = None
try:
job_rec = load_job(jid, registry_dir)
disk_status = job_rec.get("status")
rec_token = job_rec.get("dispatch_token")
rec_attempt = job_rec.get("attempt_id", job_rec.get("iteration"))
except Exception:
pass
job_rec = {}
if not disk_status or disk_status not in ("completed", "error", "cancelled"):
try:
disk_status = mqtt_common.read_logged_status(jid, mqtt_common.get_logs_dir())
@@ -77,6 +83,18 @@ def _check_disk_fallback(
pass
if disk_status in ("completed", "error", "cancelled"):
if expected_dispatch_tokens:
exp_token = expected_dispatch_tokens.get(jid)
if exp_token is not None and rec_token != exp_token:
logger.warning("disk fallback ignored terminal status for %s due to dispatch_token mismatch (%r != %r)",
jid, rec_token, exp_token)
continue
if expected_attempts:
exp_att = expected_attempts.get(jid)
if exp_att is not None and rec_attempt is not None and rec_attempt < exp_att:
logger.warning("disk fallback ignored terminal status for %s due to attempt_id older than expected (%r < %r)",
jid, rec_attempt, exp_att)
continue
synth_event = "completed" if disk_status == "completed" else "error"
synth_payload = {
"job_id": jid,
@@ -95,11 +113,15 @@ def _check_disk_fallback(
class _Watcher:
"""Holds the shared queue + the set of job_ids we accept events for."""
def __init__(self, expected_job_ids: Set[str], expected_tokens: Dict[str, Optional[str]], expected_seqs: Dict[str, int]):
def __init__(self, expected_job_ids: Set[str], expected_tokens: Dict[str, Optional[str]], expected_seqs: Dict[str, int],
expected_dispatch_tokens: Optional[Dict[str, Optional[str]]] = None,
expected_attempts: Optional[Dict[str, Optional[int]]] = None):
self.events: "queue.Queue[Tuple[str, Dict[str, Any]]]" = queue.Queue()
self.expected = set(expected_job_ids)
self.tokens = expected_tokens # job_id -> expected auth_token (or None)
self.last_seq = dict(expected_seqs)
self.dispatch_tokens = dict(expected_dispatch_tokens or {})
self.attempts = dict(expected_attempts or {})
# Intent identities already accepted, per job. A replayed notification is
# re-published with a *fresh* wire sequence, so the monotonic sequence
# guard below cannot recognise it: to that guard a retry looks like a new
@@ -135,6 +157,24 @@ class _Watcher:
if not mqtt_common.verify_hmac(payload, expected_token):
logger.warning("drop event for job %s: HMAC verify failed", jid)
return
# --- CAS dispatch token & attempt check: drop stale attempt events ---
expected_disp_tok = self.dispatch_tokens.get(jid)
if expected_disp_tok is not None:
data = payload.get("data")
ev_tok = data.get("dispatch_token") if isinstance(data, dict) else None
if ev_tok != expected_disp_tok:
logger.warning("drop event for job %s: dispatch_token %r does not match expected %r",
jid, ev_tok, expected_disp_tok)
return
expected_att = self.attempts.get(jid)
if expected_att is not None:
data = payload.get("data")
ev_att = data.get("attempt_id") if isinstance(data, dict) else None
if ev_att is not None and isinstance(ev_att, int) and ev_att < expected_att:
logger.warning("drop event for job %s: attempt_id %r is older than expected %r",
jid, ev_att, expected_att)
return
# --- replay attack defense: check monotonic sequence ---
seq = payload.get("seq")
if seq is None or not isinstance(seq, int):
@@ -216,11 +256,21 @@ def _run_subscriber(args) -> int:
expected_ids: Set[str] = {j["job_id"] for j in jobs}
tokens = {j["job_id"]: j.get("auth_token") for j in jobs}
seqs = {}
dispatch_tokens = {}
attempts = {}
for j in jobs:
jid = j["job_id"]
last_seq = int(j.get("last_seq", 0))
seqs[jid] = max(0, last_seq - 1)
watcher = _Watcher(expected_ids, tokens, seqs)
dispatch_tokens[jid] = j.get("dispatch_token")
attempts[jid] = j.get("attempt_id", j.get("iteration", 1))
if args.expect_dispatch_token is not None and args.job:
dispatch_tokens[args.job] = args.expect_dispatch_token
if args.attempt_id is not None and args.job:
attempts[args.job] = args.attempt_id
watcher = _Watcher(expected_ids, tokens, seqs, dispatch_tokens, attempts)
# Resolve timeouts from CLI, falling back to the (first) job's settings.
base_job = jobs[0]
@@ -251,7 +301,18 @@ def _run_subscriber(args) -> int:
if rc != 0:
logger.warning("broker disconnected (rc=%s); will retry reconnect", reason_code)
start = time.monotonic()
wall_deadline = start + wall_timeout
last_event = start
last_disk_check = 0.0
retention_checked = not args.expect_retention
connected = False
subscribed = False
def on_subscribe(_c, _u, mid, granted_qos, _props=None):
nonlocal last_event, subscribed
subscribed = True
last_event = time.monotonic()
for topic in subscribed_topics:
print(f"SUBSCRIBED {topic}", flush=True)
@@ -262,15 +323,9 @@ def _run_subscriber(args) -> int:
terminal: Dict[str, str] = {} # job_id -> "completed"/"error"
pending: Set[str] = set(expected_ids)
start = time.monotonic()
wall_deadline = start + wall_timeout
last_event = start
last_disk_check = 0.0
retention_checked = not args.expect_retention
connected = False
# Check if all jobs are already in terminal state on disk before connecting
_check_disk_fallback(pending, args.registry_dir, terminal)
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
if not pending:
if any(state == "error" for state in terminal.values()):
return 1
@@ -283,9 +338,10 @@ def _run_subscriber(args) -> int:
)()
client.loop_start()
connected = True
last_event = time.monotonic()
except Exception as exc:
logger.warning("broker connection failed (%s); checking disk fallback", exc)
_check_disk_fallback(pending, args.registry_dir, terminal)
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
if not pending:
if any(state == "error" for state in terminal.values()):
return 1
@@ -297,15 +353,17 @@ def _run_subscriber(args) -> int:
while pending:
now = time.monotonic()
if now >= wall_deadline:
_check_disk_fallback(pending, args.registry_dir, terminal)
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
if not pending:
break
logger.error("wall-clock timeout (%.0fs); still pending: %s",
wall_timeout, ", ".join(sorted(pending)))
return 2
if not subscribed:
last_event = now
idle_left = idle_timeout - (now - last_event)
if idle_left <= 0:
_check_disk_fallback(pending, args.registry_dir, terminal)
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
if not pending:
break
logger.error("idle timeout (%.0fs, no events); still pending: %s",
@@ -323,7 +381,7 @@ def _run_subscriber(args) -> int:
# Step 2 (B-15): Local disk fallback throttled to every 3.0s
if (now - last_disk_check) >= 3.0:
last_disk_check = now
_check_disk_fallback(pending, args.registry_dir, terminal)
_check_disk_fallback(pending, args.registry_dir, terminal, dispatch_tokens, attempts)
continue
last_event = time.monotonic()
@@ -365,6 +423,10 @@ def main(argv=None) -> int:
help="max seconds with no new event (default: job.idle_timeout_sec or 120)")
parser.add_argument("--expect-retention", action="store_true",
help="warn if no retained terminal event arrives promptly")
parser.add_argument("--expect-dispatch-token", default=None,
help="expected dispatch token for CAS verification")
parser.add_argument("--attempt-id", type=int, default=None,
help="minimum attempt id to accept events for")
parser.add_argument("--registry-dir", default=DEFAULT_REGISTRY_DIR)
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args(argv)
.agents/skills/lib_py/delegation/mqtt_common.py
+6 -2
View File
@@ -240,7 +240,7 @@ def broker_config_from_env(overrides: Optional[Dict[str, Any]] = None) -> Broker
for any key it specifies with a non-None value.
"""
cfg = BrokerConfig(
host=os.environ.get("MQTT_BROKER", "broker.hivemq.com"),
host=os.environ.get("MQTT_BROKER") or os.environ.get("MQTT_HOST", "broker.hivemq.com"),
port=_env_int("MQTT_PORT", 1883),
tls=_env_bool("MQTT_TLS", False),
username=os.environ.get("MQTT_USERNAME") or None,
@@ -428,12 +428,16 @@ def transition_job_status(job_id: str, registry_dir: str = DEFAULT_REGISTRY_DIR,
from lib_py.delegation import engine # same package: no sibling hop (see __init__)
request = "merge"
expected_dispatch_token = fields.pop("expected_dispatch_token", None)
expected_attempt = fields.pop("expected_attempt", None)
if "status" in fields:
request = engine.STATUS_TO_REQUEST.get(fields["status"], "merge")
if request != "merge":
fields = {k: v for k, v in fields.items() if k != "status"}
return engine.transition(job_id, request, registry_dir=registry_dir,
fields=fields, source=_source)
fields=fields, source=_source,
expected_dispatch_token=expected_dispatch_token,
expected_attempt=expected_attempt)
def _update_job_status_legacy(job_id: str, registry_dir: str = DEFAULT_REGISTRY_DIR, **fields: Any) -> Dict[str, Any]:
.agents/skills/lib_py/delegation/publish_event.py
+76 -6
View File
@@ -127,7 +127,8 @@ class PreparedEvent(object):
def prepare_event(job_id, event, detail="", data=None,
registry_dir=DEFAULT_REGISTRY_DIR, retained=False) -> PreparedEvent:
registry_dir=DEFAULT_REGISTRY_DIR, retained=False,
dispatch_token=None, attempt_id=None) -> PreparedEvent:
"""Load the job, consume the next seq, and build the signed payload.
Consuming the seq here (before any transport runs) is deliberate: a failed
@@ -139,12 +140,21 @@ def prepare_event(job_id, event, detail="", data=None,
topic = job.get("topic_prefix")
topic = f"{topic}/events" if topic else events_topic_for(job_id)
seq = next_seq(job_id, registry_dir)
payload_data = dict(data) if data else {}
eff_dispatch_token = dispatch_token
if eff_dispatch_token is not None and "dispatch_token" not in payload_data:
payload_data["dispatch_token"] = eff_dispatch_token
eff_attempt_id = attempt_id
if eff_attempt_id is not None and "attempt_id" not in payload_data:
payload_data["attempt_id"] = eff_attempt_id
payload = build_payload(
job_id=job_id,
seq=seq,
event=event,
detail=detail,
data=data,
data=payload_data,
auth_token=job.get("auth_token"),
)
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
@@ -153,7 +163,9 @@ def prepare_event(job_id, event, detail="", data=None,
def record_outcome(prepared: PreparedEvent, registry_dir: str,
publish_ok: bool, publish_error: Optional[str]) -> None:
publish_ok: bool, publish_error: Optional[str],
expected_dispatch_token: Optional[str] = None,
expected_attempt: Optional[int] = None) -> None:
"""Persist the audit record, the registry event log and the status sync.
All three are best effort and all three happen whether or not the publish
@@ -176,10 +188,19 @@ def record_outcome(prepared: PreparedEvent, registry_dir: str,
new_status = EVENT_TO_STATUS.get(prepared.event)
if new_status:
try:
exp_token = expected_dispatch_token
if exp_token is None and isinstance(prepared.payload.get("data"), dict):
exp_token = prepared.payload["data"].get("dispatch_token")
exp_att = expected_attempt
if exp_att is None and isinstance(prepared.payload.get("data"), dict):
exp_att = prepared.payload["data"].get("attempt_id")
# Agent authority on purpose: an agent may never move a terminal
# record, which is what the operator-only requests are gated for.
mqtt_common.update_job_status(prepared.job_id, registry_dir,
_source="publish_event", status=new_status)
_source="publish_event", status=new_status,
expected_dispatch_token=exp_token,
expected_attempt=exp_att)
except Exception as exc: # pragma: no cover - best effort
logger.warning("status sync failed: %s", exc)
@@ -214,6 +235,8 @@ def main(argv=None) -> int:
parser.add_argument("--attempt-id", type=int, default=None,
help="attempt this finalization belongs to (default: the "
"job's current iteration)")
parser.add_argument("--expect-dispatch-token", default=None,
help="expected dispatch token for CAS verification")
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args(argv)
@@ -250,7 +273,8 @@ def main(argv=None) -> int:
attempt_id = args.attempt_id
if attempt_id is None:
try:
attempt_id = load_job(job_id, args.registry_dir).get("iteration", 1)
rec = load_job(job_id, args.registry_dir)
attempt_id = rec.get("attempt_id", rec.get("iteration", 1))
except FileNotFoundError:
attempt_id = 1
try:
@@ -263,6 +287,41 @@ def main(argv=None) -> int:
logger.info("finalized %s (sha256 %s...)", manifest["report_path"],
manifest["sha256"][:12])
check_token = args.expect_dispatch_token
if check_token is None and isinstance(data, dict) and "dispatch_token" in data:
check_token = data.get("dispatch_token")
try:
rec = load_job(job_id, args.registry_dir)
actual_tok = rec.get("dispatch_token")
rec_attempt = rec.get("attempt_id", rec.get("iteration", 1))
except FileNotFoundError:
rec = None
actual_tok = None
rec_attempt = 1
check_attempt = args.attempt_id
if check_attempt is None and isinstance(data, dict) and "attempt_id" in data:
check_attempt = data.get("attempt_id")
if rec is not None and actual_tok is not None:
if check_token is None and check_attempt is None:
if args.event in ("completed", "error", "cancelled") and rec_attempt > 1:
logger.error("dispatch token required: redispatched job %s carries a dispatch token "
"but none was provided (aborting terminal publish)", job_id)
return 4
elif check_token is not None and actual_tok != check_token:
logger.error("dispatch token mismatch: expected %s, got %s (aborting publish)",
check_token, actual_tok)
return 4
elif check_attempt is not None and rec_attempt != check_attempt:
logger.error("attempt mismatch: expected %s, got %s (aborting publish)",
check_attempt, rec_attempt)
return 4
elif check_token is not None and actual_tok != check_token:
logger.error("dispatch token mismatch: expected %s, got %s (aborting publish)",
check_token, actual_tok)
return 4
try:
prepared = prepare_event(
job_id=job_id,
@@ -271,6 +330,8 @@ def main(argv=None) -> int:
data=data,
registry_dir=args.registry_dir,
retained=args.retained,
dispatch_token=args.expect_dispatch_token,
attempt_id=args.attempt_id,
)
except FileNotFoundError as exc:
logger.error("%s", exc)
@@ -292,7 +353,16 @@ def main(argv=None) -> int:
publish_error = str(exc)
logger.error("publish failed after %d attempts: %s", args.attempts, exc)
record_outcome(prepared, args.registry_dir, publish_ok, publish_error)
sync_expected_token = args.expect_dispatch_token
if sync_expected_token is None and isinstance(data, dict):
sync_expected_token = data.get("dispatch_token")
sync_expected_attempt = args.attempt_id
if sync_expected_attempt is None and isinstance(data, dict):
sync_expected_attempt = data.get("attempt_id")
record_outcome(prepared, args.registry_dir, publish_ok, publish_error,
expected_dispatch_token=sync_expected_token,
expected_attempt=sync_expected_attempt)
if not publish_ok:
return 2
.agents/skills/lib_py/delegation/registry.py
+52
View File
@@ -100,6 +100,8 @@ def register_job(
"reviewer_session": reviewer_session,
"max_iterations": int(max_iterations),
"iteration": 1,
"dispatch_token": uuid.uuid4().hex,
"attempt_id": 1,
}
with registry_lock(registry_dir):
if mqtt_common._job_path(job_id, registry_dir).exists():
@@ -299,8 +301,15 @@ def _build_parser() -> argparse.ArgumentParser:
p_update.add_argument("--agent-session", default=None)
p_update.add_argument("--prompt", default=None)
p_update.add_argument("--iteration", type=int, default=None)
p_update.add_argument("--attempt-id", type=int, default=None)
p_update.add_argument("--role", default=None)
p_close = sub.add_parser("close", help="close a job with deadline_timeout or cancelled")
p_close.add_argument("--job", required=True)
p_close.add_argument("--request", required=True, choices=["deadline_timeout", "cancelled"])
p_close.add_argument("--detail", default=None)
p_close.add_argument("--expect-dispatch-token", default=None)
p_feedback = sub.add_parser("get-feedback", help="get the last feedback detail (completed/error) for a job")
p_feedback.add_argument("--job", required=True)
@@ -430,6 +439,8 @@ def main(argv: Optional[List[str]] = None) -> int:
fields["prompt"] = args.prompt
if args.iteration is not None:
fields["iteration"] = args.iteration
if args.attempt_id is not None:
fields["attempt_id"] = args.attempt_id
if args.role is not None:
fields["role"] = args.role
try:
@@ -439,6 +450,47 @@ def main(argv: Optional[List[str]] = None) -> int:
return 1
return rc
if args.command == "close":
from lib_py.delegation import engine
fields = {}
if args.detail is not None:
fields["detail"] = args.detail
if not engine.cas_enabled():
req_status = engine.REQUEST_TO_STATUS.get(args.request, "error")
fields["status"] = req_status
try:
mqtt_common.update_job_status(args.job, rd, **fields)
except FileNotFoundError as exc:
print(str(exc), file=sys.stderr)
return 1
return 0
try:
res = engine.transition(
args.job,
args.request,
registry_dir=rd,
source="delegate_wrapper",
fields=fields,
expected_dispatch_token=args.expect_dispatch_token,
)
except FileNotFoundError as exc:
print(str(exc), file=sys.stderr)
return 1
if res.committed:
return 0
if res.idempotent:
if res.reason:
print(res.reason, file=sys.stderr)
return 0
if res.reason and "conflicting" in res.reason and "after terminal" in res.reason:
print(res.reason, file=sys.stderr)
return 0
print(f"refused: {res.reason}", file=sys.stderr)
return 4
if args.command == "get-feedback":
print(get_feedback(args.job, rd))
return 0
.agents/skills/lib_py/delegation/service.py
+39 -3
View File
@@ -46,18 +46,54 @@ class DelegationService(object):
registry_dir=self.registry_dir, **kw
)
def publish_event(self, job_id, event, detail="", data=None, attempts=3):
"""Publish one event. Returns the same exit status as the CLI (0 / 2)."""
def publish_event(self, job_id, event, detail="", data=None, attempts=3,
expect_dispatch_token=None, attempt_id=None):
"""Publish one event. Returns the same exit status as the CLI (0 / 2 / 4)."""
eff_dispatch_token = expect_dispatch_token
eff_attempt_id = attempt_id
if data and isinstance(data, dict):
if eff_dispatch_token is None:
eff_dispatch_token = data.get("dispatch_token")
if eff_attempt_id is None:
eff_attempt_id = data.get("attempt_id")
try:
rec = self.load_job(job_id)
actual_tok = rec.get("dispatch_token")
rec_attempt = rec.get("attempt_id", rec.get("iteration", 1))
except FileNotFoundError:
rec = None
actual_tok = None
rec_attempt = 1
if rec is not None and actual_tok is not None:
if eff_dispatch_token is None and eff_attempt_id is None:
if event in ("completed", "error", "cancelled") and rec_attempt > 1:
logger.error("dispatch token required: redispatched job %s carries a dispatch token "
"but none was provided", job_id)
return 4
elif eff_dispatch_token is not None and actual_tok != eff_dispatch_token:
return 4
elif eff_attempt_id is not None and rec_attempt != eff_attempt_id:
return 4
elif eff_dispatch_token is not None and actual_tok != eff_dispatch_token:
return 4
prepared = publish_event.prepare_event(
job_id=job_id, event=event, detail=detail,
data=data, registry_dir=self.registry_dir,
dispatch_token=eff_dispatch_token, attempt_id=eff_attempt_id,
)
publisher = self._publisher
if publisher is None:
from lib_py.delegation.publisher import OneShotPublisher
publisher = OneShotPublisher()
ok, error = publisher.publish_with_retry(prepared, attempts=attempts)
publish_event.record_outcome(prepared, self.registry_dir, ok, error)
publish_event.record_outcome(
prepared, self.registry_dir, ok, error,
expected_dispatch_token=eff_dispatch_token,
expected_attempt=eff_attempt_id,
)
return 0 if ok else 2
def ensure_subscriber_daemon(self, job_id, workdir=None):