multi-agent-mux/tests/test_issue5_phase2a_loop.py

#!/usr/bin/env python3
"""Issue #5 Phase 2a — lib_py/loop core and shadow watcher.

Written in the order the plan fixes: characterize the current writers BEFORE
routing them through a CAS boundary, so the refactor has an oracle rather than a
promise. Phase 1 proved the value of that ordering; it also proved that a fake
which cannot express a failure hides it, so every double here models the real
contract (herdr's exit/stream split, the registry lock, the record on disk).

Sections
  1. X-01..X-04  current writer behaviour (must hold before and after P2A-01).
  2. engine      the single CAS transition boundary and its lock discipline.
  3. watcher     herdr agent wait wrapping, stream split, reconnect backoff.
  4. policy      decisions checked against the phase-0 scenario contracts.
  5. shadow      the runner that cannot write, proven by hashing the registry.
"""
from __future__ import annotations

import hashlib
import json
import os
import subprocess
import sys
import time
from pathlib import Path

import pytest

REPO_ROOT = Path(__file__).resolve().parent.parent
SKILLS = REPO_ROOT / ".agents" / "skills"
WATCHER_FIXTURES = REPO_ROOT / "tests" / "fixtures" / "issue_5" / "watcher"

CAS_MODES = ("0", "1")  # MAM_ENGINE_CAS: legacy unconditional merge / CAS engine


# ==============================================================================
# Helpers
# ==============================================================================

def _reg(tmp_path) -> str:
    d = tmp_path / ".mam" / "jobs"
    d.mkdir(parents=True, exist_ok=True)
    return str(d)


def _make_job(registry_dir, **over):
    from lib_py.delegation import registry
    job_id = registry.register_job(over.pop("prompt", "p"), registry_dir=registry_dir, **over)
    return job_id


def _load(registry_dir, job_id):
    from lib_py.delegation import mqtt_common
    return mqtt_common.load_job(job_id, registry_dir)


def _dir_fingerprint(path: Path) -> dict:
    """File name -> sha256 for every file under path. Detects any byte change."""
    out = {}
    for p in sorted(Path(path).rglob("*")):
        if p.is_file():
            out[str(p.relative_to(path))] = hashlib.sha256(p.read_bytes()).hexdigest()
    return out


@pytest.fixture(autouse=True)
def _cas_on(monkeypatch):
    """Default every test to the CAS engine unless it says otherwise."""
    monkeypatch.setenv("MAM_ENGINE_CAS", "1")


# ==============================================================================
# 1. X-01..X-04 — the writers as they behave today
# ==============================================================================

@pytest.mark.parametrize("mode", CAS_MODES)
def test_x01_update_job_status_merges_arbitrary_fields(mam_sandbox, tmp_path, monkeypatch, mode):
    """X-01: update_job_status merges whatever it is given, in both modes.

    Its signature, return value and side effects are load-bearing: phase 1's
    G-1..G-4 assert through it, so routing it through the engine must not change
    what a caller observes."""
    monkeypatch.setenv("MAM_ENGINE_CAS", mode)
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import mqtt_common

    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    out = mqtt_common.update_job_status(job_id, reg, status="running", iteration=4)
    assert out["status"] == "running"
    assert out["iteration"] == 4
    assert out["updated_at"], "updated_at must always be refreshed"
    assert _load(reg, job_id)["status"] == "running"


@pytest.mark.parametrize("mode", CAS_MODES)
def test_x02_pick_pending_claims_only_pending_and_only_once(mam_sandbox, tmp_path, monkeypatch, mode):
    """X-02: the claim path's real contract.

    pick_pending is not broken today: it filters on pending under the lock, so
    pending->running is already a correct swap. It is a *duplicate* of the
    chokepoint, not a hole in it. Routing it through the engine must preserve
    exactly this behaviour."""
    monkeypatch.setenv("MAM_ENGINE_CAS", mode)
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import registry

    reg = _reg(tmp_path)
    job_id = _make_job(reg, agent_session="herdr:s1")
    assert registry.pick_pending("herdr:s1", reg) == job_id
    assert _load(reg, job_id)["status"] == "running"
    # already claimed -> no second winner
    assert registry.pick_pending("herdr:s1", reg) is None
    # a different session never sees another session's work
    other = _make_job(reg, agent_session="herdr:s2")
    assert registry.pick_pending("herdr:s1", reg) is None
    assert registry.pick_pending("herdr:s2", reg) == other


@pytest.mark.parametrize("mode", CAS_MODES)
def test_x03_status_write_mirrors_into_the_audit_log(mam_sandbox, tmp_path, monkeypatch, mode):
    """X-03: a status change is mirrored to the audit log, in both modes.

    P2A-01 moves that mirror out of the lock; it must not lose it."""
    monkeypatch.setenv("MAM_ENGINE_CAS", mode)
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import mqtt_common

    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    mqtt_common.update_job_status(job_id, reg, status="completed")

    logged = mqtt_common.read_logged_status(job_id, mqtt_common.get_logs_dir())
    assert logged and logged.get("status") == "completed"
    events = list(mqtt_common.iter_logged_events(job_id, mqtt_common.get_logs_dir()))
    changes = [e for e in events if e.get("event") == "status_changed"]
    assert changes, "a status change must appear in the audit log"
    assert changes[-1]["to"] == "completed"


@pytest.mark.parametrize("mode", CAS_MODES)
def test_x04_next_seq_and_register_are_unaffected(mam_sandbox, tmp_path, monkeypatch, mode):
    """X-04: the non-status writers keep their behaviour under either mode."""
    monkeypatch.setenv("MAM_ENGINE_CAS", mode)
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import mqtt_common

    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    assert _load(reg, job_id)["status"] == "pending"
    assert mqtt_common.next_seq(job_id, reg) == 1
    assert mqtt_common.next_seq(job_id, reg) == 2
    assert _load(reg, job_id)["last_seq"] == 2
    assert _load(reg, job_id)["status"] == "pending", "next_seq must not touch status"


# ==============================================================================
# 2. engine — the CAS table (state-transitions.md §3) and lock discipline
# ==============================================================================

def test_engine_pending_to_running_on_claim(mam_sandbox, tmp_path, monkeypatch):
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import engine
    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    res = engine.transition(job_id, "claim", registry_dir=reg, expected_status="pending")
    assert res.committed and res.from_status == "pending" and res.to_status == "running"


@pytest.mark.parametrize("request_", ["progress", "permission_required"])
def test_engine_non_terminal_requests_keep_running(mam_sandbox, tmp_path, monkeypatch, request_):
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import engine
    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    engine.transition(job_id, "claim", registry_dir=reg)
    res = engine.transition(job_id, request_, registry_dir=reg,
                            fields={"blocked_reason": "tool"})
    assert res.committed
    assert _load(reg, job_id)["status"] == "running", "must stay non-terminal"
    assert _load(reg, job_id)["blocked_reason"] == "tool"


@pytest.mark.parametrize("terminal", ["completed", "error", "cancelled"])
def test_engine_explicit_terminal_commits(mam_sandbox, tmp_path, monkeypatch, terminal):
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import engine
    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    engine.transition(job_id, "claim", registry_dir=reg)
    res = engine.transition(job_id, terminal, registry_dir=reg)
    assert res.committed and res.to_status == terminal


def test_engine_repeated_terminal_is_an_idempotent_noop(mam_sandbox, tmp_path, monkeypatch):
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import engine
    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    engine.transition(job_id, "claim", registry_dir=reg)
    engine.transition(job_id, "completed", registry_dir=reg)
    before = _load(reg, job_id)
    res = engine.transition(job_id, "completed", registry_dir=reg)
    assert res.committed is False
    assert "idempotent" in res.reason
    assert _load(reg, job_id)["transition_seq"] == before["transition_seq"], "no second write"


@pytest.mark.parametrize("late", ["error", "cancelled", "progress"])
def test_engine_refuses_to_reverse_a_terminal_and_audits_the_conflict(
    mam_sandbox, tmp_path, monkeypatch, late
):
    """A late or conflicting signal must never walk a terminal record backwards."""
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import mqtt_common
    from lib_py.delegation import engine
    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    engine.transition(job_id, "claim", registry_dir=reg)
    engine.transition(job_id, "completed", registry_dir=reg)

    res = engine.transition(job_id, late, registry_dir=reg, source="late-arrival")
    assert res.committed is False
    assert _load(reg, job_id)["status"] == "completed", "terminal must be preserved"
    events = list(mqtt_common.iter_logged_events(job_id, mqtt_common.get_logs_dir()))
    refusals = [e for e in events if e.get("event") == "conflict_refused"]
    assert refusals, "a refused conflict must be auditable"
    assert refusals[-1]["current"] == "completed"


def test_engine_rejects_a_stale_attempt(mam_sandbox, tmp_path, monkeypatch):
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import engine
    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    engine.transition(job_id, "claim", registry_dir=reg, fields={"attempt_id": 2})
    res = engine.transition(job_id, "completed", registry_dir=reg, expected_attempt=1)
    assert res.committed is False
    assert "stale attempt" in res.reason
    assert _load(reg, job_id)["status"] == "running"


def test_engine_deadline_timeout_becomes_error(mam_sandbox, tmp_path, monkeypatch):
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import engine
    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    engine.transition(job_id, "claim", registry_dir=reg)
    res = engine.transition(job_id, "deadline_timeout", registry_dir=reg)
    assert res.committed and res.to_status == "error"


def test_engine_precondition_failure_does_not_write(mam_sandbox, tmp_path, monkeypatch):
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import engine
    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    engine.transition(job_id, "claim", registry_dir=reg)          # now running
    res = engine.transition(job_id, "claim", registry_dir=reg, expected_status="pending")
    assert res.committed is False and "precondition failed" in res.reason


def test_engine_transition_seq_is_monotonic(mam_sandbox, tmp_path, monkeypatch):
    """The audit mirror runs outside the lock, so ordering rides on this counter."""
    monkeypatch.chdir(mam_sandbox)
    from lib_py.delegation import engine
    reg = _reg(tmp_path)
    job_id = _make_job(reg)
    seqs = []
    for req in ("claim", "progress", "progress", "completed"):
        engine.transition(job_id, req, registry_dir=reg)
        seqs.append(_load(reg, job_id)["transition_seq"])
    assert seqs == sorted(seqs) and len(set(seqs)) == len(seqs), seqs


def test_g5_no_blocking_io_while_the_registry_lock_is_held(mam_sandbox, tmp_path, monkeypatch):
    """G5: the lock may cover a decision and one record write, nothing else.

    Watches the calls that can block on another process or the network and
    asserts none of them happen between acquiring and releasing the lock."""
    monkeypatch.chdir(mam_sandbox)
    import socket
    from lib_py.delegation import mqtt_common
    from lib_py.delegation import engine

    reg = _reg(tmp_path)
    job_id = _make_job(reg)

    held = {"depth": 0}
    violations = []
    real_lock = mqtt_common.registry_lock

    import contextlib

    @contextlib.contextmanager
    def watched_lock(registry_dir):
        held["depth"] += 1
        try:
            with real_lock(registry_dir):
                yield
        finally:
            held["depth"] -= 1

    def guard(name):
        def spy(*args, **kwargs):
            if held["depth"] > 0:
                violations.append(name)
            raise AssertionError("{} must not run under the registry lock".format(name))
        return spy

    monkeypatch.setattr(mqtt_common, "registry_lock", watched_lock)
    monkeypatch.setattr(engine.mqtt_common, "registry_lock", watched_lock, raising=False)
    monkeypatch.setattr(subprocess, "run", guard("subprocess.run"))
    monkeypatch.setattr(subprocess, "Popen", guard("subprocess.Popen"))
    monkeypatch.setattr(socket, "socket", guard("socket.socket"))
    real_sleep = time.sleep

    def sleep_spy(seconds):
        if held["depth"] > 0:
            violations.append("time.sleep")
        return real_sleep(0)

    monkeypatch.setattr(time, "sleep", sleep_spy)

    for req in ("claim", "progress", "completed", "completed", "error"):
        engine.transition(job_id, req, registry_dir=reg)

    assert not violations, "blocking calls under the lock: {}".format(violations)


def test_g6_every_record_writer_is_the_engine_or_a_declared_exception():
    """G6: no new bypass of the transition boundary can be added unnoticed."""
    import ast

    allowed = {
        # creation, and the sequence counter, which never touches status
        ("registry.py", "register_job"),
        ("mqtt_common.py", "next_seq"),
        ("mqtt_common.py", "_update_job_status_legacy"),
        ("registry.py", "_pick_pending_legacy"),
        ("engine.py", "transition"),
    }
    found = set()
    for path in list((SKILLS / "lib_py" / "delegation").glob("*.py")) + \
                list((SKILLS / "lib_py" / "loop").glob("*.py")):
        tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
        for node in ast.walk(tree):
            if not isinstance(node, ast.FunctionDef):
                continue
            for inner in ast.walk(node):
                if isinstance(inner, ast.Call):
                    fn = inner.func
                    name = getattr(fn, "attr", getattr(fn, "id", None))
                    if name == "_atomic_write_record":
                        found.add((path.name, node.name))
    assert found <= allowed, (
        "job records are written outside the transition boundary: {}".format(found - allowed)
    )