GodopuandClaude Opus 5
8d7e010f9f
The subscriber marked a notification intent as seen before the monotonic sequence guard ran. A late packet was then dropped by that guard with its intent already burned, so the genuine retry of a notification this subscriber had never delivered was mistaken for a replay and dropped too. Identity comparison stays ahead of the sequence guard -- a duplicate must not consume a sequence number -- but the bookkeeping write moves behind it, so "seen" means "accepted" rather than "arrived". An intent that cannot yield a stable identity (not an object, or holding an unhashable value) is now refused instead of raising out of the callback: a paho callback that raises takes the network loop down for every job that subscriber is watching. Six regressions; three of them fail on the unfixed tree, and one drives a real notifier retry through the real consumer with no hand-built payload. Full suite: 732 passed, no skips. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UrDNJkHmm18uu7n7JwV5Qd
1086 lines
48 KiB
Python
1086 lines
48 KiB
Python
#!/usr/bin/env python3
|
|
"""Issue #5 Phase 2b — flag-gated auto-completion.
|
|
|
|
Characterize first, as in 2a: the interesting property here is what the system
|
|
*refuses* to do, so the refusals are pinned before anything is added that could
|
|
weaken them.
|
|
|
|
Sections
|
|
1. X-01..X-05 the fail-closed preconditions as they stand today.
|
|
2. finalize the completion manifest: written by the agent, re-derived here.
|
|
3. autocomplete the gate, and everything that must stop it.
|
|
4. notifier durable pending notification and restart recovery.
|
|
5. flags default off, evidence-backed allowlist, per-job opt-in.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
SKILLS = REPO_ROOT / ".agents" / "skills"
|
|
WATCHER_FIXTURES = REPO_ROOT / "tests" / "fixtures" / "issue_5" / "watcher"
|
|
|
|
|
|
def _reg(tmp_path) -> str:
|
|
d = tmp_path / ".mam" / "jobs"
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
return str(d)
|
|
|
|
|
|
def _make_job(registry_dir, **over):
|
|
from lib_py.delegation import registry
|
|
return registry.register_job(over.pop("prompt", "p"), registry_dir=registry_dir, **over)
|
|
|
|
|
|
def _load(registry_dir, job_id):
|
|
from lib_py.delegation import mqtt_common
|
|
return mqtt_common.load_job(job_id, registry_dir)
|
|
|
|
|
|
def _fingerprint(path: Path) -> dict:
|
|
out = {}
|
|
for p in sorted(Path(path).rglob("*")):
|
|
if p.is_file():
|
|
out[str(p.relative_to(path))] = hashlib.sha256(p.read_bytes()).hexdigest()
|
|
return out
|
|
|
|
|
|
def _scenario(name="finalized_candidate"):
|
|
return json.loads((WATCHER_FIXTURES / (name + ".json")).read_text(encoding="utf-8"))
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _default_flags(monkeypatch):
|
|
"""Auto-completion is off unless a test turns it on. That is the product
|
|
default too, and every test that does not opt in is therefore also a check
|
|
that the default really is off."""
|
|
monkeypatch.delenv("MAM_AUTO_COMPLETE", raising=False)
|
|
monkeypatch.delenv("MAM_AUTO_COMPLETE_BACKENDS", raising=False)
|
|
monkeypatch.setenv("MAM_ENGINE_CAS", "1")
|
|
|
|
|
|
# ==============================================================================
|
|
# 1. X-01..X-05 — the fail-closed preconditions, before anything is added
|
|
# ==============================================================================
|
|
|
|
def test_x01_a_job_without_a_manifest_is_never_approved():
|
|
"""The property that makes unsupported agents safe without any new branch."""
|
|
import copy
|
|
from lib_py.loop import policy
|
|
|
|
scenario = _scenario()
|
|
assert policy.decide(scenario).decision == "candidate_approved"
|
|
stripped = copy.deepcopy(scenario)
|
|
for item in stripped["timeline"]:
|
|
(item.get("payload") or {}).pop("report_manifest", None)
|
|
stripped["filesystem"]["manifest_committed"] = False
|
|
decision = policy.decide(stripped)
|
|
assert decision.decision == "hold" and "manifest" in decision.reason
|
|
|
|
|
|
def test_x02_no_production_code_writes_a_completion_manifest_yet():
|
|
"""Pins the starting point: the protocol is entirely new in phase 2b."""
|
|
hits = []
|
|
for path in sorted((SKILLS / "lib_py").rglob("*.py")):
|
|
if path.name in ("finalize.py", "autocomplete.py"):
|
|
continue
|
|
text = path.read_text(encoding="utf-8")
|
|
if "report-manifest" in text:
|
|
hits.append(path.name)
|
|
assert not hits, "manifest writers already exist: {}".format(hits)
|
|
|
|
|
|
def test_x03_the_record_carries_iteration_not_an_attempt_id(mam_sandbox, tmp_path, monkeypatch):
|
|
"""Why auto-completion does not use the engine's expected_attempt.
|
|
|
|
The engine compares expected_attempt against `attempt_id` falling back to
|
|
`iteration`. A record has only the latter, and the two mean different things,
|
|
so the attempt guard is applied where attempt identity actually lives: the
|
|
dispatch record in the shadow store, compared against observed evidence."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
record = _load(reg, job_id)
|
|
assert "iteration" in record
|
|
assert "attempt_id" not in record, (
|
|
"if this ever changes, revisit the layering decision in the phase 2b report"
|
|
)
|
|
|
|
|
|
def test_x04_policy_rejects_evidence_from_a_superseded_attempt():
|
|
"""The attempt guard that auto-completion actually relies on."""
|
|
import copy
|
|
from lib_py.loop import policy
|
|
|
|
scenario = _scenario()
|
|
superseded = copy.deepcopy(scenario)
|
|
superseded["dispatch_context"]["attempt_id"] = 2 # evidence still says 1
|
|
decision = policy.decide(superseded)
|
|
assert decision.decision == "hold" and "attempt_id" in decision.reason
|
|
|
|
|
|
def test_x05_the_explicit_event_path_is_untouched_by_default(mam_sandbox, tmp_path, monkeypatch):
|
|
"""An explicitly completed job looks exactly as it always did."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import mqtt_common
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
mqtt_common.update_job_status(job_id, reg, _source="publish_event", status="running")
|
|
mqtt_common.update_job_status(job_id, reg, _source="publish_event", status="completed")
|
|
record = _load(reg, job_id)
|
|
assert record["status"] == "completed"
|
|
assert "completed_by" not in record, "an explicit completion carries no auto marker"
|
|
|
|
|
|
# ==============================================================================
|
|
# 2. finalize — declared by the agent, re-derived by the reader
|
|
# ==============================================================================
|
|
|
|
def _report(tmp_path, body="done\n", staged=False):
|
|
d = tmp_path / "claude-reports"
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
path = d / "report-final.md"
|
|
(path.with_name(path.name + ".tmp") if staged else path).write_text(body, encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def test_finalize_promotes_a_staged_report_atomically(tmp_path):
|
|
"""A reader must never see a half-written report as finalized."""
|
|
from lib_py.delegation import finalize
|
|
|
|
path = _report(tmp_path, "final body\n", staged=True)
|
|
assert not path.exists() and path.with_name(path.name + ".tmp").exists()
|
|
manifest = finalize.finalize_report(path, "job1", 1)
|
|
assert path.exists() and not path.with_name(path.name + ".tmp").exists()
|
|
assert manifest["sha256"] == finalize.sha256_of(path)
|
|
|
|
|
|
def test_finalize_verifies_against_the_file_not_the_declaration(tmp_path):
|
|
"""The hash is recomputed, so a forged manifest cannot certify itself."""
|
|
from lib_py.delegation import finalize
|
|
|
|
path = _report(tmp_path)
|
|
finalize.finalize_report(path, "job1", 1)
|
|
assert finalize.verify_manifest(path.parent, "job1").ok
|
|
|
|
manifest_file = finalize.manifest_in(path.parent)
|
|
forged = json.loads(manifest_file.read_text(encoding="utf-8"))
|
|
forged["sha256"] = "0" * 64
|
|
manifest_file.write_text(json.dumps(forged), encoding="utf-8")
|
|
verdict = finalize.verify_manifest(path.parent, "job1")
|
|
assert not verdict.ok and "changed after finalization" in verdict.reason
|
|
|
|
|
|
def test_finalize_is_invalidated_by_a_rewritten_report(tmp_path):
|
|
"""A new body needs a new finalization; the old hash is void immediately."""
|
|
from lib_py.delegation import finalize
|
|
|
|
path = _report(tmp_path, "first\n")
|
|
finalize.finalize_report(path, "job1", 1)
|
|
assert finalize.verify_manifest(path.parent, "job1").ok
|
|
path.write_text("rewritten\n", encoding="utf-8")
|
|
verdict = finalize.verify_manifest(path.parent, "job1")
|
|
assert not verdict.ok and "changed after finalization" in verdict.reason
|
|
|
|
|
|
@pytest.mark.parametrize("kwargs,fragment", [
|
|
({"job_id": "other"}, "belongs to job"),
|
|
({"expected_attempt": 9}, "belongs to attempt"),
|
|
])
|
|
def test_finalize_rejects_a_manifest_for_a_different_job_or_attempt(tmp_path, kwargs, fragment):
|
|
from lib_py.delegation import finalize
|
|
path = _report(tmp_path)
|
|
finalize.finalize_report(path, "job1", 1)
|
|
args = {"job_id": "job1"}
|
|
args.update(kwargs)
|
|
verdict = finalize.verify_manifest(path.parent, **args)
|
|
assert not verdict.ok and fragment in verdict.reason
|
|
|
|
|
|
def test_finalize_requires_evidence_for_every_declared_side_effect(tmp_path):
|
|
"""Partial evidence is no evidence: the work is not known to be complete."""
|
|
from lib_py.delegation import finalize
|
|
|
|
path = _report(tmp_path)
|
|
finalize.finalize_report(path, "job1", 1,
|
|
side_effects=finalize.parse_side_effects(["git_push=abc123"]))
|
|
ok = finalize.verify_manifest(path.parent, "job1", expected_artifacts=["git_push"])
|
|
assert ok.ok
|
|
missing = finalize.verify_manifest(path.parent, "job1",
|
|
expected_artifacts=["git_push", "api_call"])
|
|
assert not missing.ok and "api_call" in missing.reason
|
|
|
|
|
|
def test_finalize_rejects_a_malformed_side_effect_pair():
|
|
from lib_py.delegation import finalize
|
|
with pytest.raises(ValueError):
|
|
finalize.parse_side_effects(["nokey"])
|
|
with pytest.raises(ValueError):
|
|
finalize.parse_side_effects(["kind="])
|
|
|
|
|
|
def test_finalize_missing_manifest_is_a_refusal_not_a_crash(tmp_path):
|
|
from lib_py.delegation import finalize
|
|
d = tmp_path / "empty"
|
|
d.mkdir()
|
|
verdict = finalize.verify_manifest(d, "job1")
|
|
assert not verdict.ok and "no completion manifest" in verdict.reason
|
|
|
|
|
|
def test_publish_event_finalizes_before_it_publishes(mam_sandbox, tmp_path, monkeypatch):
|
|
"""The CLI surface an agent actually calls."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import publish_event
|
|
from lib_py.delegation import finalize
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
record = _load(reg, job_id)
|
|
record["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
|
Path(reg, job_id + ".json").write_text(json.dumps(record), encoding="utf-8")
|
|
|
|
path = _report(tmp_path, "the report\n", staged=True)
|
|
rc = publish_event.main([
|
|
"--registry-dir", reg, "--job", job_id, "--event", "completed",
|
|
"--attempts", "1", "--finalize-report", str(path),
|
|
"--side-effect", "git_push=deadbeef",
|
|
])
|
|
assert rc == 2, "the broker is unreachable, but finalization still happened"
|
|
assert path.exists(), "the staged report was promoted"
|
|
verdict = finalize.verify_manifest(path.parent, job_id)
|
|
assert verdict.ok
|
|
assert verdict.manifest["side_effects"][0]["kind"] == "git_push"
|
|
|
|
|
|
def test_publish_event_reports_a_bad_side_effect_instead_of_publishing(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import publish_event
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
path = _report(tmp_path)
|
|
rc = publish_event.main([
|
|
"--registry-dir", reg, "--job", job_id, "--event", "completed",
|
|
"--finalize-report", str(path), "--side-effect", "malformed",
|
|
])
|
|
assert rc == 1
|
|
|
|
|
|
# ==============================================================================
|
|
# 3. autocomplete — the gate, and everything that must stop it
|
|
# ==============================================================================
|
|
|
|
def _ready_job(tmp_path, mam_sandbox, *, backend="claude-code", opt_in=True,
|
|
artifacts=None, body="report\n"):
|
|
"""A job in exactly the state that should auto-complete, plus its evidence."""
|
|
from lib_py.delegation import finalize
|
|
from lib_py.loop import store
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg, agent=backend, expected_artifacts=artifacts or [])
|
|
from lib_py.delegation import engine
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
if opt_in:
|
|
engine.transition(job_id, "progress", registry_dir=reg,
|
|
fields={"auto_complete": True})
|
|
|
|
scenario = _scenario()
|
|
dispatch = scenario["dispatch_context"]
|
|
dispatch["job_id"] = job_id
|
|
for item in scenario["timeline"]:
|
|
env = item.get("collector_attribution")
|
|
if env:
|
|
env["job_id"] = job_id
|
|
# A non-herdr payload is a legitimate identifier source, so it has to
|
|
# agree too. Leaving the fixture's original id here is a contradiction,
|
|
# and the policy correctly refuses it.
|
|
payload = item.get("payload") or {}
|
|
if item.get("source_kind") != "herdr" and "job_id" in payload:
|
|
payload["job_id"] = job_id
|
|
# The dispatch record carries the record's own token for this attempt, which
|
|
# is what ties the evidence to the attempt that is currently running.
|
|
from lib_py.delegation import engine as _engine
|
|
token = _load(reg, job_id).get(_engine.DISPATCH_TOKEN_FIELD)
|
|
store.record_dispatch(job_id, dispatch["attempt_id"], dispatch["agent_session_uuid"],
|
|
dispatch["pane_process_generation"], workspace=str(mam_sandbox),
|
|
dispatch_token=token)
|
|
store.mark_execution_anchor(job_id, workspace=str(mam_sandbox))
|
|
|
|
report = _report(tmp_path, body)
|
|
finalize.finalize_report(report, job_id, dispatch["attempt_id"],
|
|
side_effects=finalize.parse_side_effects(
|
|
["{}=ref".format(a) for a in (artifacts or [])]))
|
|
return job_id, reg, scenario, report.parent
|
|
|
|
|
|
def test_gate_commits_when_every_condition_holds(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed, result.reason
|
|
record = _load(reg, job_id)
|
|
assert record["status"] == "completed"
|
|
assert record["completed_by"] == "auto_complete", "an inferred terminal must be marked"
|
|
assert record["pending_notification"]["event"] == "completed"
|
|
|
|
|
|
def test_g4_default_off_commits_nothing_and_leaves_the_registry_untouched(
|
|
mam_sandbox, tmp_path, monkeypatch
|
|
):
|
|
"""G4: with the flag unset, the registry must not change at all."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
before = _fingerprint(Path(reg))
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False
|
|
assert "disabled" in result.reason
|
|
assert _fingerprint(Path(reg)) == before, "the default must write nothing"
|
|
|
|
|
|
@pytest.mark.parametrize("backend", ["agy", "hermes", "opencode", "unknown-agent"])
|
|
def test_g9_a_backend_without_quiescence_evidence_never_auto_completes(
|
|
mam_sandbox, tmp_path, monkeypatch, backend
|
|
):
|
|
"""Inferring 'finished' from stillness needs stillness to have been observed."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox, backend=backend)
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "quiescent-state evidence" in result.reason
|
|
assert _load(reg, job_id)["status"] == "running"
|
|
|
|
|
|
def test_the_allowlist_can_be_narrowed_but_never_widened(monkeypatch):
|
|
from lib_py.loop import autocomplete
|
|
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE_BACKENDS", "claude")
|
|
assert autocomplete.allowed_backends() == ("claude",)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE_BACKENDS", "claude,hermes,opencode")
|
|
assert autocomplete.allowed_backends() == ("claude",), "unobserved backends are dropped"
|
|
|
|
|
|
def test_a_job_that_did_not_opt_in_is_not_auto_completed(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox, opt_in=False)
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "opt in" in result.reason
|
|
|
|
|
|
def test_g5_a_rewritten_report_stops_the_gate(mam_sandbox, tmp_path, monkeypatch):
|
|
"""G5: the manifest is checked against the file, so a late edit disqualifies."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
(report_dir / "report-final.md").write_text("rewritten after finalization\n", encoding="utf-8")
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "changed after finalization" in result.reason
|
|
assert _load(reg, job_id)["status"] == "running"
|
|
|
|
|
|
def test_g5_missing_side_effect_evidence_stops_the_gate(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
from lib_py.delegation import finalize
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox,
|
|
artifacts=["git_push"])
|
|
# re-declare without the evidence the job said it would produce
|
|
finalize.finalize_report(report_dir / "report-final.md", job_id, 1, side_effects=[])
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "git_push" in result.reason
|
|
|
|
|
|
def test_d7_a_policy_that_cannot_decide_holds(mam_sandbox, tmp_path, monkeypatch):
|
|
"""'Could not decide' is never 'done'."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete, policy
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
|
|
def explode(_scenario):
|
|
raise RuntimeError("evidence is contradictory")
|
|
|
|
monkeypatch.setattr(policy, "decide", explode)
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "policy error" in result.reason
|
|
assert _load(reg, job_id)["status"] == "running"
|
|
|
|
|
|
def test_a_late_explicit_terminal_beats_the_gate(mam_sandbox, tmp_path, monkeypatch):
|
|
"""If the agent already reported, the boundary refuses and that is correct."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.delegation import mqtt_common
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
mqtt_common.update_job_status(job_id, reg, _source="publish_event", status="error")
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False
|
|
record = _load(reg, job_id)
|
|
assert record["status"] == "error", "the explicit report wins"
|
|
assert "completed_by" not in record
|
|
|
|
|
|
# ==============================================================================
|
|
# 4. notifier — the intent is durable, and recovery re-sends it
|
|
# ==============================================================================
|
|
|
|
def test_the_notification_intent_is_committed_with_the_state(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine, notifier
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
engine.transition(job_id, "completed", registry_dir=reg, notify="completed")
|
|
record = _load(reg, job_id)
|
|
assert record["status"] == "completed"
|
|
intent = notifier.pending(job_id, reg)
|
|
assert intent and intent["event"] == "completed" and intent["seq"] == record["transition_seq"]
|
|
|
|
|
|
def test_flush_publishes_then_settles(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine, notifier
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
engine.transition(job_id, "completed", registry_dir=reg, notify="completed")
|
|
|
|
sent = []
|
|
assert notifier.flush(job_id, reg, publish=lambda j, i, r: sent.append(i) or 0)
|
|
assert len(sent) == 1 and sent[0]["event"] == "completed"
|
|
assert notifier.pending(job_id, reg) is None
|
|
# settling is idempotent: a second flush has nothing to do
|
|
assert notifier.flush(job_id, reg, publish=lambda *a: 1 / 0)
|
|
|
|
|
|
def test_a_failed_send_leaves_the_intent_pending(mam_sandbox, tmp_path, monkeypatch):
|
|
"""An unsent notification that looks sent is worse than one that is retried."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine, notifier
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
engine.transition(job_id, "completed", registry_dir=reg, notify="completed")
|
|
|
|
assert notifier.flush(job_id, reg, publish=lambda *a: 2) is False
|
|
assert notifier.pending(job_id, reg) is not None
|
|
assert notifier.flush(job_id, reg, publish=lambda *a: (_ for _ in ()).throw(OSError())) is False
|
|
assert notifier.pending(job_id, reg) is not None
|
|
assert _load(reg, job_id)["status"] == "completed", "the state stays committed regardless"
|
|
|
|
|
|
def test_g7_restart_recovery_resends_every_owed_notification(mam_sandbox, tmp_path, monkeypatch):
|
|
"""G7: the crash window between committing and publishing."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine, notifier
|
|
|
|
reg = _reg(tmp_path)
|
|
owed = []
|
|
for _ in range(3):
|
|
job_id = _make_job(reg)
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
engine.transition(job_id, "completed", registry_dir=reg, notify="completed")
|
|
owed.append(job_id)
|
|
settled = _make_job(reg)
|
|
engine.transition(settled, "claim", registry_dir=reg)
|
|
|
|
assert sorted(notifier.jobs_with_pending(reg)) == sorted(owed)
|
|
|
|
sent = []
|
|
results = notifier.recover(reg, publish=lambda j, i, r: sent.append(j) or 0)
|
|
assert all(results.values()) and sorted(sent) == sorted(owed)
|
|
assert notifier.jobs_with_pending(reg) == [], "nothing is owed after recovery"
|
|
|
|
|
|
# ==============================================================================
|
|
# 6. Cross-review regressions (job f63d8279)
|
|
#
|
|
# Three reviewers, seven blocking items, all reproduced before being accepted.
|
|
# The theme: a gate whose strictness depended on the caller, and a boundary that
|
|
# trusted its only current caller instead of enforcing its own rule.
|
|
# ==============================================================================
|
|
|
|
def test_f1_an_omitted_report_directory_is_a_refusal_not_a_shortcut(
|
|
mam_sandbox, tmp_path, monkeypatch
|
|
):
|
|
"""Independent verification cannot be optional: skipping it puts the
|
|
collector's own flags back in charge of the decision."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, _report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
before = _fingerprint(Path(reg))
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox)) # no report_dir
|
|
assert result.committed is False and "independently verify" in result.reason
|
|
assert _fingerprint(Path(reg)) == before, "a refusal must not write"
|
|
|
|
|
|
def test_f1_an_empty_report_directory_is_also_a_refusal(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, _rd = _ready_job(tmp_path, mam_sandbox)
|
|
empty = tmp_path / "empty-reports"
|
|
empty.mkdir()
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=empty)
|
|
assert result.committed is False and "no completion manifest" in result.reason
|
|
|
|
|
|
def test_r2_evidence_for_another_job_cannot_complete_this_one(
|
|
mam_sandbox, tmp_path, monkeypatch
|
|
):
|
|
"""The policy checks a scenario against the dispatch context inside that same
|
|
scenario, which is self-consistent by construction. The gate must bind the
|
|
evidence to the target."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
scenario["dispatch_context"]["job_id"] = "some-other-job"
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "not" in result.reason
|
|
assert _load(reg, job_id)["status"] == "running"
|
|
|
|
|
|
@pytest.mark.parametrize("field", ["attempt_id", "agent_session_uuid", "pane_process_generation"])
|
|
def test_r2_evidence_must_match_the_recorded_dispatch(mam_sandbox, tmp_path, monkeypatch, field):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
scenario["dispatch_context"][field] = "not-what-was-dispatched"
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and field in result.reason
|
|
|
|
|
|
def test_r2_a_job_with_no_recorded_dispatch_is_refused(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
(Path(mam_sandbox) / ".mam" / "shadow" / "attribution" / (job_id + ".json")).unlink()
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "no recorded dispatch" in result.reason
|
|
|
|
|
|
def test_the_cas_rollback_switch_also_disables_auto_completion(mam_sandbox, tmp_path, monkeypatch):
|
|
"""Inferred completion is built on the CAS boundary, so committing while
|
|
that boundary is rolled back would bypass it."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
monkeypatch.setenv("MAM_ENGINE_CAS", "0")
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "rolled back" in result.reason
|
|
assert _load(reg, job_id)["status"] == "running"
|
|
|
|
|
|
@pytest.mark.parametrize("agent", ["claude-experimental", "codexish", "grok2"])
|
|
def test_o4_a_backend_key_must_match_exactly_not_by_prefix(
|
|
mam_sandbox, tmp_path, monkeypatch, agent
|
|
):
|
|
"""A future name must not inherit evidence that was never collected for it."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox, backend=agent)
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "quiescent-state evidence" in result.reason
|
|
|
|
|
|
def test_a_reviewer_without_a_verdict_token_is_not_completed(mam_sandbox, tmp_path, monkeypatch):
|
|
"""Phase 0 safety contract: a reviewer that has not written its verdict has
|
|
not finished reviewing, and run_loop reads a missing token as NOT PASS."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.delegation import finalize
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(
|
|
tmp_path, mam_sandbox, body="a thorough review with no verdict line\n")
|
|
from lib_py.delegation import engine
|
|
engine.transition(job_id, "progress", registry_dir=reg, fields={"role": "Reviewer"})
|
|
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "verdict token" in result.reason
|
|
assert _load(reg, job_id)["status"] == "running"
|
|
|
|
# writing the verdict requires a fresh finalization, and then it passes
|
|
report = report_dir / "report-final.md"
|
|
report.write_text("a thorough review\n\n[VERDICT: PASS]\n", encoding="utf-8")
|
|
finalize.finalize_report(report, job_id, scenario["dispatch_context"]["attempt_id"])
|
|
ok = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert ok.committed, ok.reason
|
|
|
|
|
|
def test_a_worker_role_needs_no_verdict_token(mam_sandbox, tmp_path, monkeypatch):
|
|
"""The token requirement is a reviewer contract, not a general one."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox,
|
|
body="worker output, no verdict\n")
|
|
assert _load(reg, job_id)["role"] == "Worker"
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed, result.reason
|
|
|
|
|
|
def test_grok_f2_bookkeeping_cannot_merge_arbitrary_fields(mam_sandbox, tmp_path, monkeypatch):
|
|
"""A status-free request must not become an unrestricted merge on a record
|
|
nothing else is allowed to touch."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
engine.transition(job_id, "completed", registry_dir=reg, notify="completed")
|
|
before = _load(reg, job_id)
|
|
|
|
result = engine.transition(job_id, "notification_settled", registry_dir=reg,
|
|
fields={"pending_notification": None, "prompt": "HACKED",
|
|
"agent_session": "herdr:attacker"},
|
|
source="not-an-operator")
|
|
after = _load(reg, job_id)
|
|
assert result.committed is False and "may not merge" in result.reason
|
|
assert after["prompt"] == before["prompt"]
|
|
assert after["agent_session"] == before["agent_session"]
|
|
assert after["pending_notification"], "the intent survives a refused settle"
|
|
|
|
# the legitimate settle still works
|
|
from lib_py.delegation import notifier
|
|
assert notifier.settle(job_id, reg, sent_intent=after["pending_notification"])
|
|
assert notifier.pending(job_id, reg) is None
|
|
|
|
|
|
def test_r4_a_newer_intent_is_not_erased_by_a_slow_send(mam_sandbox, tmp_path, monkeypatch):
|
|
"""Publishing happens outside the lock, so a replacement can land mid-flight.
|
|
Settling unconditionally would drop a notification nobody sent."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine, notifier
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
engine.transition(job_id, "completed", registry_dir=reg, notify="completed")
|
|
|
|
def racing_publisher(job, intent, registry_dir):
|
|
engine.transition(job, "notification_settled", registry_dir=registry_dir,
|
|
fields={"pending_notification":
|
|
notifier.intent_for("error", 999, "later")})
|
|
return 0
|
|
|
|
notifier.flush(job_id, reg, publish=racing_publisher)
|
|
still_owed = notifier.pending(job_id, reg)
|
|
assert still_owed and still_owed["seq"] == 999, "the replacement must survive"
|
|
|
|
|
|
def test_r4_the_published_payload_carries_the_intent_identity(mam_sandbox, tmp_path, monkeypatch):
|
|
"""The transport sequence cannot identify a retry of the same intent, so the
|
|
intent's own identity travels in the payload for consumers to dedupe on."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine, notifier
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
record = _load(reg, job_id)
|
|
record["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
|
Path(reg, job_id + ".json").write_text(json.dumps(record), encoding="utf-8")
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
engine.transition(job_id, "completed", registry_dir=reg, notify="completed")
|
|
|
|
intent = notifier.pending(job_id, reg)
|
|
notifier.flush(job_id, reg, attempts=1) # real path; broker is unreachable
|
|
from lib_py.delegation import mqtt_common
|
|
events = list(mqtt_common.iter_logged_events(job_id, mqtt_common.get_logs_dir()))
|
|
published = [e for e in events if e.get("event") == "published"]
|
|
assert published, "the attempt is recorded even though the broker refused"
|
|
carried = published[-1]["payload"]["data"].get("notification_intent")
|
|
assert carried and carried["seq"] == intent["seq"]
|
|
|
|
|
|
@pytest.mark.parametrize("entry", [
|
|
{"kind": "deploy"},
|
|
{"kind": "deploy", "reference": " "},
|
|
{"reference": "abc"},
|
|
"not-an-object",
|
|
])
|
|
def test_r3_a_side_effect_entry_must_carry_real_evidence(tmp_path, entry):
|
|
"""A kind with no reference names something without evidencing it."""
|
|
from lib_py.delegation import finalize
|
|
|
|
path = _report(tmp_path)
|
|
with pytest.raises(ValueError):
|
|
finalize.finalize_report(path, "job1", 1, side_effects=[entry])
|
|
|
|
# and a hand-written manifest is rejected at verification too
|
|
finalize.finalize_report(path, "job1", 1,
|
|
side_effects=[{"kind": "deploy", "reference": "sha"}])
|
|
manifest_file = finalize.manifest_in(path.parent)
|
|
forged = json.loads(manifest_file.read_text(encoding="utf-8"))
|
|
forged["side_effects"] = [entry]
|
|
manifest_file.write_text(json.dumps(forged), encoding="utf-8")
|
|
verdict = finalize.verify_manifest(path.parent, "job1", expected_artifacts=["deploy"])
|
|
assert not verdict.ok
|
|
|
|
|
|
def test_o2_the_manifest_records_an_absolute_report_path(tmp_path, monkeypatch):
|
|
"""A relative path would resolve against whatever directory the reader is in."""
|
|
from lib_py.delegation import finalize
|
|
|
|
path = _report(tmp_path)
|
|
monkeypatch.chdir(tmp_path)
|
|
manifest = finalize.finalize_report(Path("claude-reports/report-final.md"), "job1", 1)
|
|
assert Path(manifest["report_path"]).is_absolute()
|
|
monkeypatch.chdir(REPO_ROOT)
|
|
assert finalize.verify_manifest(path.parent, "job1").ok, "verification survives a chdir"
|
|
|
|
|
|
# ==============================================================================
|
|
# 7. Cross-review round 2 regressions (job 4cfc1649)
|
|
#
|
|
# R2 (codex): attribution was checked before the terminal commit but nothing
|
|
# bound the two, so a job could complete, be re-dispatched and
|
|
# claimed again in between, and stale evidence finished the new
|
|
# attempt.
|
|
# R4 (codex): the intent identity was emitted but the real subscriber ignored
|
|
# it, so a replay with a fresh wire sequence was accepted twice.
|
|
# ==============================================================================
|
|
|
|
def test_r2_a_redispatch_between_the_check_and_the_commit_is_refused(
|
|
mam_sandbox, tmp_path, monkeypatch
|
|
):
|
|
"""The exact interleaving: complete, re-dispatch, claim again, then let the
|
|
original commit proceed. Evidence for attempt 1 must not finish attempt 2."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.delegation import engine
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
|
|
real_transition = engine.transition
|
|
fired = {}
|
|
|
|
def racing(job, request="merge", **kwargs):
|
|
if request == "completed" and kwargs.get("source") == "auto_complete" and not fired:
|
|
fired["yes"] = True
|
|
engine.transition = real_transition
|
|
real_transition(job, "completed", registry_dir=reg)
|
|
real_transition(job, "redispatch", registry_dir=reg,
|
|
source="registry_cli", fields={"iteration": 2})
|
|
real_transition(job, "claim", registry_dir=reg)
|
|
engine.transition = racing
|
|
return real_transition(job, request, **kwargs)
|
|
|
|
monkeypatch.setattr(engine, "transition", racing)
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
monkeypatch.setattr(engine, "transition", real_transition)
|
|
|
|
record = _load(reg, job_id)
|
|
assert result.committed is False and "dispatch token" in result.reason
|
|
assert record["status"] == "running", "the new attempt must still be running"
|
|
assert record["iteration"] == 2
|
|
assert "completed_by" not in record
|
|
|
|
|
|
def test_r2_evidence_from_a_superseded_dispatch_is_refused(mam_sandbox, tmp_path, monkeypatch):
|
|
"""Even without a race: evidence recorded against an older token is stale."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.delegation import engine
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
# a full re-dispatch mints a new token; the recorded evidence keeps the old
|
|
engine.transition(job_id, "completed", registry_dir=reg)
|
|
engine.transition(job_id, "redispatch", registry_dir=reg,
|
|
source="registry_cli", fields={"iteration": 2})
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "superseded dispatch" in result.reason
|
|
|
|
|
|
def test_r2_a_job_without_a_dispatch_token_is_refused(mam_sandbox, tmp_path, monkeypatch):
|
|
"""Nothing to bind the decision to is a refusal, not a pass."""
|
|
monkeypatch.chdir(mam_sandbox)
|
|
monkeypatch.setenv("MAM_AUTO_COMPLETE", "1")
|
|
from lib_py.delegation import engine, mqtt_common
|
|
from lib_py.loop import autocomplete
|
|
|
|
job_id, reg, scenario, report_dir = _ready_job(tmp_path, mam_sandbox)
|
|
record = _load(reg, job_id)
|
|
record.pop(engine.DISPATCH_TOKEN_FIELD, None)
|
|
mqtt_common._atomic_write_record(job_id, reg, record)
|
|
|
|
result = autocomplete.evaluate(job_id, scenario, registry_dir=reg,
|
|
workspace=str(mam_sandbox), report_dir=report_dir)
|
|
assert result.committed is False and "no dispatch token" in result.reason
|
|
|
|
|
|
def test_r2_a_new_attempt_mints_a_new_token(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
first = _load(reg, job_id)[engine.DISPATCH_TOKEN_FIELD]
|
|
engine.transition(job_id, "progress", registry_dir=reg, fields={"note": "x"})
|
|
assert _load(reg, job_id)[engine.DISPATCH_TOKEN_FIELD] == first, (
|
|
"an ordinary transition must not mint a new attempt"
|
|
)
|
|
engine.transition(job_id, "completed", registry_dir=reg)
|
|
engine.transition(job_id, "redispatch", registry_dir=reg,
|
|
source="registry_cli", fields={"iteration": 2})
|
|
assert _load(reg, job_id)[engine.DISPATCH_TOKEN_FIELD] != first
|
|
|
|
|
|
def test_r2_the_token_precondition_is_enforced_by_the_engine(mam_sandbox, tmp_path, monkeypatch):
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
before = _fingerprint(Path(reg))
|
|
|
|
stale = engine.transition(job_id, "completed", registry_dir=reg,
|
|
expected_dispatch_token="not-the-current-token")
|
|
assert stale.committed is False and "dispatch token changed" in stale.reason
|
|
assert _fingerprint(Path(reg)) == before, "a refused precondition must not write"
|
|
|
|
token = _load(reg, job_id)[engine.DISPATCH_TOKEN_FIELD]
|
|
assert engine.transition(job_id, "completed", registry_dir=reg,
|
|
expected_dispatch_token=token).committed
|
|
|
|
|
|
def test_r4_the_subscriber_drops_a_replayed_notification_intent():
|
|
"""A replay is re-published with a fresh wire sequence, so the monotonic
|
|
guard cannot see it; the stable identity is what must be deduplicated."""
|
|
import types
|
|
from lib_py.delegation import job_subscriber
|
|
|
|
watcher = job_subscriber._Watcher({"j1"}, {"j1": None}, {"j1": 0})
|
|
|
|
def deliver(wire_seq, intent_seq):
|
|
payload = {"schema_version": 1, "seq": wire_seq, "job_id": "j1",
|
|
"event": "completed", "timestamp": "t", "detail": "d",
|
|
"data": {"notification_intent": {"seq": intent_seq,
|
|
"event": "completed",
|
|
"timestamp": "T"}}}
|
|
watcher.on_message(None, None, types.SimpleNamespace(
|
|
topic="python/mqtt/jobs/j1/events",
|
|
payload=json.dumps(payload).encode()))
|
|
|
|
deliver(1, 7)
|
|
deliver(2, 7) # the same intent, retried
|
|
assert watcher.events.qsize() == 1, "a replayed intent must not be acted on twice"
|
|
deliver(3, 8) # a genuinely different notification
|
|
assert watcher.events.qsize() == 2
|
|
|
|
|
|
def test_r4_events_without_an_intent_are_unaffected():
|
|
"""The existing sequence guard remains the rule for ordinary events."""
|
|
import types
|
|
from lib_py.delegation import job_subscriber
|
|
|
|
watcher = job_subscriber._Watcher({"j1"}, {"j1": None}, {"j1": 0})
|
|
|
|
def deliver(wire_seq):
|
|
payload = {"schema_version": 1, "seq": wire_seq, "job_id": "j1",
|
|
"event": "progress", "timestamp": "t", "detail": "d", "data": {}}
|
|
watcher.on_message(None, None, types.SimpleNamespace(
|
|
topic="python/mqtt/jobs/j1/events",
|
|
payload=json.dumps(payload).encode()))
|
|
|
|
deliver(1)
|
|
deliver(2)
|
|
assert watcher.events.qsize() == 2
|
|
deliver(2) # non-increasing: still dropped by the sequence guard
|
|
assert watcher.events.qsize() == 2
|
|
|
|
|
|
_MISSING = object()
|
|
|
|
|
|
def _intent_watcher(last_seq=0):
|
|
from lib_py.delegation import job_subscriber
|
|
return job_subscriber._Watcher({"j1"}, {"j1": None}, {"j1": last_seq})
|
|
|
|
|
|
def _deliver_intent(watcher, wire_seq, intent, event="completed"):
|
|
"""Hand one schema-v1 event to the real callback. `intent` is placed in
|
|
data.notification_intent verbatim, so a malformed one can be modelled."""
|
|
import types
|
|
payload = {"schema_version": 1, "seq": wire_seq, "job_id": "j1",
|
|
"event": event, "timestamp": "t", "detail": "d", "data": {}}
|
|
if intent is not _MISSING:
|
|
payload["data"]["notification_intent"] = intent
|
|
watcher.on_message(None, None, types.SimpleNamespace(
|
|
topic="python/mqtt/jobs/j1/events",
|
|
payload=json.dumps(payload).encode()))
|
|
|
|
|
|
def test_r4b_a_stale_packet_does_not_burn_the_intent_for_its_own_retry():
|
|
"""The defect: the identity was recorded before the sequence guard ran, so a
|
|
late packet marked its intent as seen and the legitimate retry that followed
|
|
was mistaken for a replay of an event the subscriber never delivered."""
|
|
watcher = _intent_watcher(last_seq=10)
|
|
intent = {"seq": 7, "event": "completed", "timestamp": "T"}
|
|
|
|
_deliver_intent(watcher, 9, intent) # late/out-of-order: refused by the seq guard
|
|
assert watcher.events.qsize() == 0
|
|
assert watcher.seen_intents.get("j1", set()) == set(), (
|
|
"a refused event must not leave its intent marked as accepted"
|
|
)
|
|
|
|
_deliver_intent(watcher, 11, intent) # the genuine retry of that notification
|
|
assert watcher.events.qsize() == 1, "the retry of a never-delivered intent must be accepted"
|
|
|
|
_deliver_intent(watcher, 12, intent) # now it really is a replay
|
|
assert watcher.events.qsize() == 1
|
|
|
|
|
|
def test_r4b_a_duplicate_intent_does_not_advance_the_sequence_floor():
|
|
"""Dropping a replay must not consume sequence numbers: the wire seq of an
|
|
event that was never delivered has no claim on the monotonic floor."""
|
|
watcher = _intent_watcher()
|
|
intent = {"seq": 7, "event": "completed", "timestamp": "T"}
|
|
|
|
_deliver_intent(watcher, 1, intent)
|
|
assert watcher.events.qsize() == 1 and watcher.last_seq["j1"] == 1
|
|
_deliver_intent(watcher, 9, intent) # replay, refused
|
|
assert watcher.events.qsize() == 1
|
|
assert watcher.last_seq["j1"] == 1, "a refused replay must not raise the sequence floor"
|
|
|
|
_deliver_intent(watcher, 2, {"seq": 8, "event": "completed", "timestamp": "T"})
|
|
assert watcher.events.qsize() == 2, "an ordinary event after a refused replay still lands"
|
|
|
|
|
|
def test_r4b_an_unhashable_intent_is_refused_instead_of_raising():
|
|
"""A callback that raises takes down the network loop for every watched job,
|
|
so a malformed remote payload must be refused inside defensive parsing."""
|
|
watcher = _intent_watcher()
|
|
_deliver_intent(watcher, 1, {"seq": [1, 2], "event": "completed", "timestamp": "T"})
|
|
assert watcher.events.qsize() == 0
|
|
assert watcher.last_seq["j1"] == 0, "a refused event must not advance the floor"
|
|
|
|
# A well-formed intent on the same job is still accepted afterwards.
|
|
_deliver_intent(watcher, 2, {"seq": 7, "event": "completed", "timestamp": "T"})
|
|
assert watcher.events.qsize() == 1
|
|
|
|
|
|
def test_r4b_a_non_object_intent_is_refused():
|
|
"""Present but not an object: no stable identity can be derived, so the
|
|
replay guard could not protect it."""
|
|
watcher = _intent_watcher()
|
|
_deliver_intent(watcher, 1, "completed:7")
|
|
assert watcher.events.qsize() == 0
|
|
_deliver_intent(watcher, 2, ["completed", 7])
|
|
assert watcher.events.qsize() == 0
|
|
|
|
|
|
def test_r4b_an_absent_or_null_intent_is_treated_as_no_intent():
|
|
"""The ordinary path is unchanged: events that carry no intent are governed
|
|
by the sequence guard alone."""
|
|
watcher = _intent_watcher()
|
|
_deliver_intent(watcher, 1, _MISSING, event="progress")
|
|
_deliver_intent(watcher, 2, None, event="progress")
|
|
assert watcher.events.qsize() == 2
|
|
assert watcher.seen_intents.get("j1", set()) == set()
|
|
_deliver_intent(watcher, 2, None, event="progress") # non-increasing
|
|
assert watcher.events.qsize() == 2
|
|
|
|
|
|
def test_r4b_a_real_retry_is_deduplicated_by_the_real_consumer(mam_sandbox, tmp_path, monkeypatch):
|
|
"""End to end, no hand-built payload: the notifier's own retry of an unsent
|
|
notification is handed to the real subscriber callback and must land once."""
|
|
import types
|
|
monkeypatch.chdir(mam_sandbox)
|
|
from lib_py.delegation import engine, job_subscriber, mqtt_common, notifier
|
|
|
|
reg = _reg(tmp_path)
|
|
job_id = _make_job(reg)
|
|
record = _load(reg, job_id)
|
|
record["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
|
Path(reg, job_id + ".json").write_text(json.dumps(record), encoding="utf-8")
|
|
engine.transition(job_id, "claim", registry_dir=reg)
|
|
engine.transition(job_id, "completed", registry_dir=reg, notify="completed")
|
|
|
|
# The broker is unreachable, so the intent stays owed and flush retries it.
|
|
notifier.flush(job_id, reg, attempts=1)
|
|
assert notifier.pending(job_id, reg), "an unsent notification stays owed"
|
|
notifier.flush(job_id, reg, attempts=1)
|
|
|
|
payloads = [e["payload"] for e
|
|
in mqtt_common.iter_logged_events(job_id, mqtt_common.get_logs_dir())
|
|
if e.get("event") == "published"]
|
|
assert len(payloads) == 2, "two send attempts for one notification"
|
|
assert payloads[0]["seq"] != payloads[1]["seq"], (
|
|
"the retry gets a fresh wire sequence: the monotonic guard cannot see it"
|
|
)
|
|
|
|
watcher = job_subscriber._Watcher({job_id}, {job_id: None},
|
|
{job_id: payloads[0]["seq"] - 1})
|
|
for payload in payloads:
|
|
watcher.on_message(None, None, types.SimpleNamespace(
|
|
topic="python/mqtt/jobs/%s/events" % job_id,
|
|
payload=json.dumps(payload).encode()))
|
|
assert watcher.events.qsize() == 1, "the same notification must be acted on once"
|