feat(issue-5): single CAS transition boundary for every job-status write

engine.transition is now the only writer of job status. It implements the target
table in state-transitions.md §3: claim, non-terminal metadata, explicit
terminals, idempotent re-receipt, refusal of conflicting or late signals with an
audit entry, stale-attempt rejection, and deadline timeout.

Two write paths become one. update_job_status merged fields unconditionally;
pick_pending hand-rolled a correct pending->running swap plus its own audit
mirror. Neither was wrong alone, but a rule added to one silently missed the
other, which is the shape that produced the phase 1 transport bugs. Both keep
their signatures and their preserved legacy bodies behind MAM_ENGINE_CAS=0.

A merged `status` value is routed to the request that owns it, so a raw field
write can no longer slip past the table. Getting that wrong first was caught by
the characterization tests: a metadata merge was overwriting the caller's status
with the pre-merge value.

Lock discipline is now enforced, not described. The lock covers a decision and
one record write; audit mirroring is queued and flushed after release, carrying a
monotonic transition_seq so ordering survives the move. A test watches for
subprocess, socket and sleep calls while the lock is held and asserts none.

Structural correction to my own plan, caught by a phase 1 guard: the plan put the
engine in lib_py/loop, but delegation would then have had to import a sibling,
which is exactly what mixes trees under a frozen loop. The engine belongs with
the registry writers, so it lives in lib_py/delegation and loop consumes it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UrDNJkHmm18uu7n7JwV5Qd
This commit is contained in:
GodopuandClaude Opus 5 committed 2026-09-06 18:53:22 +09:00
1 parent 9389d7ebe2
commit 2532e78453
8 files changed
+794 -3

No files matched your search

+228
View File
@@ -0,0 +1,228 @@
"""The single state-transition authority.
Every job-status write goes through `transition`. Before phase 2a there were two
write paths: `update_job_status` merged fields unconditionally, and
`pick_pending` hand-rolled a correct pending->running swap plus its own audit
mirror. Neither was wrong on its own; having two was the risk, because a fix or
a rule added to one silently misses the other.
It lives in `lib_py.delegation`, not `lib_py.loop`, because it writes the
registry and the registry writers are here. A frozen loop can bind
`lib_py.delegation` to a different tree than the parent package, so delegation
importing a sibling would mix trees -- the rule this package states in its own
__init__, and which caught this during implementation.
Lock discipline (the reason this is a module and not a helper):
with registry_lock: decide, write the record once, QUEUE notifications
after the lock: flush notifications, publish, anything slow
Nothing that can block on another process or the network may run while the lock
is held. `pick_pending` already mirrored its audit entries after releasing the
lock; this generalises that shape to every transition.
`MAM_ENGINE_CAS=0` restores the previous unconditional-merge behaviour without a
redeploy. The legacy bodies are preserved by their callers for that reason.
"""
from __future__ import annotations
import logging
import os
from typing import Any, Dict, List, Optional
from lib_py.delegation import mqtt_common
logger = logging.getLogger("loop.engine")
TERMINAL_STATUSES = ("completed", "error", "cancelled")
NON_TERMINAL_STATUSES = ("pending", "running")
# request -> the status it asks for. `merge`/`progress` ask for nothing.
REQUEST_TO_STATUS = {
"claim": "running",
"started": "running",
"progress": None,
"permission_required": None,
"completed": "completed",
"error": "error",
"cancelled": "cancelled",
"deadline_timeout": "error",
"merge": None,
}
# A caller that merges `status=<value>` is making the request that owns that
# value; mapping it here is what stops a raw field write from sidestepping the
# CAS table. `pending` has no owning request (nothing transitions *to* pending),
# so it stays an ordinary metadata merge, exactly as before.
STATUS_TO_REQUEST = {
"running": "started",
"completed": "completed",
"error": "error",
"cancelled": "cancelled",
}
def cas_enabled() -> bool:
"""The rollback switch. Read per call so a test or operator can flip it."""
return os.environ.get("MAM_ENGINE_CAS", "1") != "0"
class TransitionResult(object):
__slots__ = ("committed", "from_status", "to_status", "reason", "record", "notifications")
def __init__(self, committed, from_status, to_status, reason, record, notifications):
self.committed = committed
self.from_status = from_status
self.to_status = to_status
self.reason = reason
self.record = record
self.notifications = notifications
def __repr__(self): # pragma: no cover - debugging aid
return "TransitionResult(committed={!r}, {!r}->{!r}, {!r})".format(
self.committed, self.from_status, self.to_status, self.reason)
class _Decision(object):
__slots__ = ("commit", "to_status", "reason", "conflict")
def __init__(self, commit, to_status, reason, conflict=False):
self.commit = commit
self.to_status = to_status
self.reason = reason
self.conflict = conflict
def _decide(record: Dict[str, Any], request: str, expected_status, expected_attempt) -> _Decision:
"""Pure CAS decision. No I/O, so it is safe to call under the lock.
Implements the target table in docs/contracts/issue-5/state-transitions.md §3.
"""
current = record.get("status")
# Stale attempt or generation loses before anything else is considered.
if expected_attempt is not None:
actual = record.get("attempt_id", record.get("iteration"))
if actual is not None and actual != expected_attempt:
return _Decision(False, current,
"stale attempt: expected {!r}, record has {!r}".format(
expected_attempt, actual))
if expected_status is not None and current != expected_status:
return _Decision(False, current,
"precondition failed: expected status {!r}, found {!r}".format(
expected_status, current))
requested = REQUEST_TO_STATUS.get(request, None)
if current in TERMINAL_STATUSES:
# Idempotent re-receipt of the same terminal is a no-op ack.
if requested == current:
return _Decision(False, current, "idempotent: already {}".format(current))
if requested is None:
return _Decision(False, current,
"late {} after terminal {}; refusing".format(request, current),
conflict=True)
return _Decision(False, current,
"conflicting {} after terminal {}; refusing".format(requested, current),
conflict=True)
if requested is None:
# progress / permission_required / metadata-only merge: the engine does
# not set a status, so any status in `fields` is applied as-is.
return _Decision(True, None, "metadata updated ({})".format(request))
return _Decision(True, requested, "{} -> {}".format(current, requested))
def _notification(job_id, kind, **fields) -> Dict[str, Any]:
payload = {"job_id": job_id, "kind": kind}
payload.update(fields)
return payload
def transition(job_id, request="merge", *, registry_dir=None, expected_status=None,
expected_attempt=None, fields=None, source="unknown") -> TransitionResult:
"""Decide and commit one state transition. The only writer of job status.
Returns a result whose `notifications` have ALREADY been flushed; the field
is kept so callers and tests can see what the transition emitted.
"""
registry_dir = registry_dir or mqtt_common.DEFAULT_REGISTRY_DIR
fields = dict(fields or {})
# ---- short transaction: decide and write, never do I/O that can block ----
with mqtt_common.registry_lock(registry_dir):
record = mqtt_common.load_job(job_id, registry_dir)
from_status = record.get("status")
decision = _decide(record, request, expected_status, expected_attempt)
notifications: List[Dict[str, Any]] = []
if decision.commit:
record.update(fields)
if decision.to_status is not None:
record["status"] = decision.to_status
record["updated_at"] = mqtt_common._utcnow()
# A monotonic counter so a late flush can still be ordered correctly
# once the audit mirror happens outside the lock.
record["transition_seq"] = int(record.get("transition_seq", 0)) + 1
mqtt_common._atomic_write_record(job_id, registry_dir, record)
to_status = record.get("status")
if to_status != from_status:
notifications.append(_notification(
job_id, "status_changed", **{
"from": from_status, "to": to_status,
"by": source, "timestamp": record["updated_at"],
"transition_seq": record["transition_seq"],
}))
else:
to_status = from_status
if decision.conflict:
notifications.append(_notification(
job_id, "conflict_refused",
attempted=request, current=from_status,
reason=decision.reason, by=source,
timestamp=mqtt_common._utcnow()))
# ---- lock released; everything below may block ----
_flush(notifications, registry_dir)
if not decision.commit and decision.conflict:
logger.warning("refused %s for job %s: %s", request, job_id, decision.reason)
return TransitionResult(decision.commit, from_status, to_status,
decision.reason, record, notifications)
def _flush(notifications, registry_dir) -> None:
"""Mirror queued notifications into the audit log, outside the lock.
Best effort by design: the registry record is already durable, and an audit
write must never be able to fail a transition that has committed.
"""
for note in notifications:
job_id = note["job_id"]
kind = note["kind"]
try:
if kind == "status_changed":
mqtt_common.update_logged_status(
job_id, note["to"], updated_at=note.get("timestamp"))
mqtt_common.append_event(job_id, {
"event": "status_changed",
"from": note.get("from"),
"to": note.get("to"),
"by": note.get("by"),
"timestamp": note.get("timestamp"),
"transition_seq": note.get("transition_seq"),
})
elif kind == "conflict_refused":
mqtt_common.append_event(job_id, {
"event": "conflict_refused",
"attempted": note.get("attempted"),
"current": note.get("current"),
"reason": note.get("reason"),
"by": note.get("by"),
"timestamp": note.get("timestamp"),
})
except Exception as exc: # pragma: no cover - best effort
logger.warning("audit mirror failed for %s: %s", job_id, exc)
@@ -393,7 +393,38 @@ def _atomic_write_record(job_id: str, registry_dir: str, record: Dict[str, Any])
def update_job_status(job_id: str, registry_dir: str = DEFAULT_REGISTRY_DIR, **fields: Any) -> Dict[str, Any]:
"""Atomically merge ``fields`` into a job record under the registry lock.
"""Merge ``fields`` into a job record through the single transition boundary.
Signature, return value and exceptions are unchanged: phase 1's G-1..G-4
assert through this function, and the shell helpers capture its effects.
What changed is who performs the write -- ``lib_py.loop.engine`` is now the
only writer of job status, so the CAS rules and the audit mirror exist once
instead of twice.
``MAM_ENGINE_CAS=0`` restores the previous unconditional merge below.
"""
from lib_py.delegation import engine # same package: no sibling hop (see __init__)
if not engine.cas_enabled():
return _update_job_status_legacy(job_id, registry_dir, **fields)
request = "merge"
if "status" in fields:
# A merged status value is really a transition request; routing it to the
# owning request is what subjects it to the CAS table instead of letting
# a raw field write slip past it.
request = engine.STATUS_TO_REQUEST.get(fields["status"], "merge")
if request != "merge":
fields = {k: v for k, v in fields.items() if k != "status"}
result = engine.transition(job_id, request, registry_dir=registry_dir,
fields=fields, source="update_job_status")
return result.record
def _update_job_status_legacy(job_id: str, registry_dir: str = DEFAULT_REGISTRY_DIR, **fields: Any) -> Dict[str, Any]:
"""Pre-phase-2a body, preserved for MAM_ENGINE_CAS=0 rollback.
Atomically merge ``fields`` into a job record under the registry lock.
Always refreshes ``updated_at``. Returns the new record. Raises
FileNotFoundError if the job does not exist.
+33 -2
View File
@@ -117,6 +117,37 @@ def pick_pending(agent_session: str, registry_dir: str = DEFAULT_REGISTRY_DIR) -
``running`` atomically under the lock. Returns the job id, or None if no
pending job matches. This is how each herdr session takes only its own work
without two sessions grabbing the same job."""
from lib_py.delegation import engine # same package: no sibling hop (see __init__)
if not engine.cas_enabled():
return _pick_pending_legacy(agent_session, registry_dir)
# Selection still happens under the lock, then the claim itself is committed
# by the single transition boundary with an explicit precondition, so two
# sessions still cannot both win.
with registry_lock(registry_dir):
candidates = [
record for record in _iter_records(registry_dir)
if record.get("status") == "pending" and record.get("agent_session") == agent_session
]
if not candidates:
return None
candidates.sort(key=lambda r: r.get("created_at", ""))
chosen_id = candidates[0]["job_id"]
result = engine.transition(chosen_id, "claim", registry_dir=registry_dir,
expected_status="pending", source=agent_session)
if not result.committed:
# Someone else claimed it between selection and commit; the precondition
# refused, which is exactly the behaviour the lock used to provide.
logger.info("job %s was claimed by another session first", chosen_id)
return None
logger.info("session %s picked job %s", agent_session, chosen_id)
return chosen_id
def _pick_pending_legacy(agent_session: str, registry_dir: str = DEFAULT_REGISTRY_DIR) -> Optional[str]:
"""Pre-phase-2a body, preserved for MAM_ENGINE_CAS=0 rollback."""
with registry_lock(registry_dir):
candidates = []
for record in _iter_records(registry_dir):
@@ -132,8 +163,8 @@ def pick_pending(agent_session: str, registry_dir: str = DEFAULT_REGISTRY_DIR) -
logger.info("session %s picked job %s", agent_session, chosen["job_id"])
job_id = chosen["job_id"]
updated_at = chosen["updated_at"]
# pick_pending writes the record directly (not via update_job_status), so it
# mirrors the pending->running transition into the audit log here. Best-effort.
# Mirrors the pending->running transition into the audit log outside the
# lock. This is the shape the engine now generalises to every transition.
mqtt_common.update_logged_status(job_id, "running", updated_at=updated_at)
mqtt_common.append_event(job_id, {
"event": "status_changed",
+16
View File
@@ -0,0 +1,16 @@
"""Loop core: the single state-transition authority and the shadow watcher.
Issue #5 phase 2a. Two things live here and they are deliberately separated:
* `lib_py.delegation.engine` decides and commits state. It is the only writer
of job status, and it lives with the other registry writers rather than here
-- see that module's docstring for why the package boundary matters under a
frozen loop.
* `watcher` observes and never decides; `policy` decides and never writes;
`shadow` runs the two together and, in this phase, only logs what it *would*
have committed.
Package rule, enforced by test: nothing here imports a `lib_py` module other
than `lib_py.delegation`. Under a frozen loop the parent package may resolve to
a different tree than a bound sibling, so a stray import silently mixes trees.
"""
+59
View File
@@ -0,0 +1,59 @@
"""Four-fold execution attribution, as a pure predicate.
A terminal state may only be inferred for the attempt that actually ran. Idle is
not evidence of finishing: a reused session sitting idle from previous work looks
identical unless job, attempt, session and pane generation all agree AND the
attempt was observed to actually execute after dispatch was acknowledged.
"""
from __future__ import annotations
REQUIRED_FIELDS = ("job_id", "attempt_id", "agent_session_uuid", "pane_process_generation")
class AttributionResult(object):
__slots__ = ("ok", "reason")
def __init__(self, ok, reason):
self.ok = ok
self.reason = reason
def check(dispatch_context, observed=None) -> AttributionResult:
"""Compare a dispatch record against what was observed.
`observed` may carry any subset of the four identifiers; every key it does
supply must match. A missing identifier is never treated as agreement.
"""
if not isinstance(dispatch_context, dict):
return AttributionResult(False, "dispatch_context missing")
for field in REQUIRED_FIELDS:
if field not in dispatch_context:
return AttributionResult(False, "missing attribution field: {}".format(field))
value = dispatch_context[field]
if value is None or value == "":
return AttributionResult(False, "empty attribution field: {}".format(field))
if not dispatch_context.get("dispatch_ack"):
return AttributionResult(False, "dispatch was never acknowledged")
# The anchor is the whole point: without it, idle is just idle.
if not dispatch_context.get("execution_anchor_observed"):
return AttributionResult(False, "no execution anchor observed for this attempt")
for field, value in (observed or {}).items():
if field in REQUIRED_FIELDS and value != dispatch_context[field]:
return AttributionResult(
False, "attribution mismatch on {}".format(field))
return AttributionResult(True, "four-fold attribution satisfied")
def pane_matches_generation(pane_id, pane_process_generation) -> bool:
"""`pane_process_generation` is '<pane_id>:<pid>:<start>'.
Compared on the whole leading segment so w2X:p1 never matches w2X:p10.
"""
if not pane_id or not pane_process_generation:
return False
return str(pane_process_generation).startswith(str(pane_id) + ":")
+56
View File
@@ -0,0 +1,56 @@
"""The one data shape that crosses the watcher -> engine boundary.
Keeping this a plain value with no I/O is what lets `policy.decide` stay a pure
function, which in turn is what lets the phase-0 scenario fixtures be fed to it
verbatim as test vectors.
"""
from __future__ import annotations
SOURCE_KINDS = ("herdr", "event", "filesystem", "harness")
# Decisions the policy may return. Same vocabulary as the phase-0 contracts, so
# a scenario's expected_policy.decision can be compared directly.
DECISIONS = ("hold", "hold_blocked", "hold_reconnect", "reject_stale", "candidate_approved")
class Observation(object):
"""One point-in-time fact from one source. Never a conclusion."""
__slots__ = ("source_kind", "payload", "captured_at", "capture_ref", "provenance")
def __init__(self, source_kind, payload, captured_at=None, capture_ref=None,
provenance="observed"):
if source_kind not in SOURCE_KINDS:
raise ValueError("unsupported source_kind: {!r}".format(source_kind))
self.source_kind = source_kind
self.payload = dict(payload or {})
self.captured_at = captured_at
self.capture_ref = capture_ref
self.provenance = provenance
def __repr__(self): # pragma: no cover - debugging aid
return "Observation({!r}, {!r})".format(self.source_kind, self.payload)
class Decision(object):
"""What the policy concluded, and why. `reason` is for the audit trail."""
__slots__ = ("decision", "reason", "attribution_ok")
def __init__(self, decision, reason, attribution_ok=False):
if decision not in DECISIONS:
raise ValueError("unknown decision: {!r}".format(decision))
self.decision = decision
self.reason = reason
self.attribution_ok = attribution_ok
@property
def would_complete(self) -> bool:
return self.decision == "candidate_approved"
def as_dict(self) -> dict:
return {"decision": self.decision, "reason": self.reason,
"attribution_ok": self.attribution_ok}
def __repr__(self): # pragma: no cover - debugging aid
return "Decision({!r}, {!r})".format(self.decision, self.reason)
+166
View File
@@ -0,0 +1,166 @@
"""Wraps `herdr agent wait`. Observes; never concludes.
Live contract (herdr 0.8.2, re-confirmed while planning):
herdr agent wait <TARGET> --until <STATUS>... --timeout <MS>
* `--timeout` is MILLISECONDS. The parameter here is named `timeout_ms` so the
unit cannot be misread at a call site; there is deliberately no seconds form.
* `--until` is always passed explicitly. Omitting it matches idle/done/blocked,
silently excluding `working`, which is a state the watcher cares about.
* success -> exit 0, snapshot JSON on STDOUT.
* timeout and unknown target -> exit 1, error JSON on STDERR, stdout empty.
The stream split is the contract: an error is never read from stdout.
A single wait is one synchronous barrier returning one final snapshot, so
continuous observation is built by repeating it, not by streaming.
"""
from __future__ import annotations
import json
import logging
import subprocess
from typing import Any, Dict, Optional, Sequence
from lib_py.loop.evidence import Observation
logger = logging.getLogger("loop.watcher")
DEFAULT_UNTIL = ("idle", "done", "blocked")
VALID_STATES = ("idle", "working", "blocked", "done", "unknown")
# Outcome kinds. `timeout` is an ordinary no-event result, not a failure.
MATCHED = "matched"
TIMEOUT = "timeout"
AGENT_NOT_FOUND = "agent_not_found"
TRANSPORT_ERROR = "transport_error"
BACKOFF_START_S = 1.0
BACKOFF_MAX_S = 16.0
class WaitOutcome(object):
__slots__ = ("kind", "agent", "error_code", "detail", "raw")
def __init__(self, kind, agent=None, error_code=None, detail="", raw=None):
self.kind = kind
self.agent = agent
self.error_code = error_code
self.detail = detail
self.raw = raw
@property
def healthy(self) -> bool:
"""A timeout is healthy: the socket answered, the state just did not change."""
return self.kind in (MATCHED, TIMEOUT, AGENT_NOT_FOUND)
def __repr__(self): # pragma: no cover - debugging aid
return "WaitOutcome({!r}, error_code={!r})".format(self.kind, self.error_code)
def build_argv(target: str, until: Sequence[str] = DEFAULT_UNTIL,
timeout_ms: Optional[int] = None) -> list:
if not target:
raise ValueError("wait target is required")
argv = ["herdr", "agent", "wait", target]
for state in until:
if state not in VALID_STATES:
raise ValueError("unsupported --until state: {!r}".format(state))
argv += ["--until", state]
if timeout_ms is not None:
if int(timeout_ms) <= 0:
raise ValueError("timeout_ms must be positive milliseconds")
argv += ["--timeout", str(int(timeout_ms))]
return argv
def _parse(stream: str) -> Optional[Dict[str, Any]]:
try:
payload = json.loads(stream)
return payload if isinstance(payload, dict) else None
except Exception:
return None
def wait_once(target: str, until: Sequence[str] = DEFAULT_UNTIL,
timeout_ms: Optional[int] = None, run=subprocess.run) -> WaitOutcome:
"""One barrier call. `run` is injectable so tests never touch a real herdr."""
argv = build_argv(target, until, timeout_ms)
try:
proc = run(argv, capture_output=True, text=True)
except Exception as exc:
return WaitOutcome(TRANSPORT_ERROR, detail="herdr could not be executed: {}".format(exc))
if proc.returncode == 0:
payload = _parse(proc.stdout or "")
agent = ((payload or {}).get("result") or {}).get("agent")
if not isinstance(agent, dict):
return WaitOutcome(TRANSPORT_ERROR,
detail="exit 0 without a parseable agent snapshot",
raw=proc.stdout)
return WaitOutcome(MATCHED, agent=agent, raw=payload)
# Non-zero: the payload is on stderr by contract, and stdout is empty.
payload = _parse(proc.stderr or "")
code = ((payload or {}).get("error") or {}).get("code")
if code == "timeout":
return WaitOutcome(TIMEOUT, error_code=code, raw=payload)
if code == "agent_not_found":
return WaitOutcome(AGENT_NOT_FOUND, error_code=code, raw=payload)
return WaitOutcome(TRANSPORT_ERROR, error_code=code,
detail="unrecognised failure (rc={})".format(proc.returncode),
raw=proc.stderr)
def observation_from(outcome: WaitOutcome, capture_ref=None) -> Optional[Observation]:
"""Turn a matched snapshot into evidence. Non-matches carry no agent state."""
if outcome.kind != MATCHED or not outcome.agent:
return None
agent = outcome.agent
return Observation(
"herdr",
{
"agent_status": agent.get("agent_status"),
"interactive_ready": agent.get("interactive_ready"),
"pane_id": agent.get("pane_id"),
"state_change_seq": agent.get("state_change_seq"),
},
capture_ref=capture_ref,
)
class ReconnectBackoff(object):
"""Doubling backoff for transport failures, reset by any healthy answer.
The snapshot cache is dropped on failure on purpose: after a socket gap the
watcher must re-observe rather than reason from a state that may have moved
while it was blind. That is what makes `hold_reconnect` the honest decision.
"""
def __init__(self, start=BACKOFF_START_S, maximum=BACKOFF_MAX_S):
self.start = start
self.maximum = maximum
self.current = 0.0
self.consecutive_failures = 0
self.cached_snapshot = None
def record_success(self, snapshot=None) -> None:
self.consecutive_failures = 0
self.current = 0.0
if snapshot is not None:
self.cached_snapshot = snapshot
def record_failure(self) -> float:
self.consecutive_failures += 1
self.cached_snapshot = None # stale evidence must not survive a gap
self.current = self.start if self.current == 0.0 else min(self.current * 2.0, self.maximum)
return self.current
def sequence(self, n: int) -> list:
"""The first n delays, for documentation and for tests to pin."""
saved = (self.current, self.consecutive_failures, self.cached_snapshot)
try:
self.current, self.consecutive_failures = 0.0, 0
return [self.record_failure() for _ in range(n)]
finally:
self.current, self.consecutive_failures, self.cached_snapshot = saved