GodopuandClaude Opus 5.5
1673818bb5
- job_subscriber.py returns only named codes (RC_COMPLETED 0, RC_ERROR 1, RC_NO_EVENT 2, RC_BROKER_UNAVAILABLE 3, RC_WALL_CLOCK 4). The wrapper mirrors them as SUB_RC_* and compares subscriber codes only by name; a test keeps the mirror equal to the Python constants and forbids numeric comparisons of $sub_rc/$sub_exit and numeric `return` in the subscriber. - `delegate-job wait` passed the subscriber's code straight through, so a spent budget would have reached its callers as 4; it is 2 again (B7 fails without the mapping). - tier1 G-8 gave both clocks 0.5 s and raced; the wall clock now runs out first by construction (idle 30 s) and the test asserts RC_WALL_CLOCK. Other subscriber rc asserts use the names. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1255 lines
53 KiB
Python
1255 lines
53 KiB
Python
#!/usr/bin/env python3
|
|
"""Issue #5 Phase 1 — delegation extraction characterization & contract tests.
|
|
|
|
Written BEFORE the `lib_py/delegation/` extraction (P1-00) so the refactor has a
|
|
real regression oracle: the three shell helpers in `lib.sh` had zero direct test
|
|
coverage. Every test here must pass identically against the legacy inline bodies
|
|
and against the Python-backed ones, which is why the shell tests are
|
|
parametrized over `MAM_DELEGATION_PY` in {"0", "1"}.
|
|
|
|
Sections
|
|
1. C-01..C-09 shell helper golden behaviour (non-fatal contract, stdout shape,
|
|
daemon PID/dedup/logging).
|
|
2. CLI contract snapshots — argparse option names/choices/defaults, asserted
|
|
against the parser rather than --help text so cosmetic wording is free.
|
|
3. Auto-completion invariant guards — the three surfaces that decide job status
|
|
must not shift by a single line during Phase 1.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
SKILLS = REPO_ROOT / ".agents" / "skills"
|
|
FIXTURES = REPO_ROOT / "tests" / "fixtures" / "issue_5" / "phase1"
|
|
|
|
DELEGATION_MODES = ("0", "1") # MAM_DELEGATION_PY: legacy inline body / python body
|
|
|
|
|
|
# ==============================================================================
|
|
# Helpers
|
|
# ==============================================================================
|
|
|
|
def _sandbox_skills(mam_sandbox) -> Path:
|
|
return Path(mam_sandbox) / "skills"
|
|
|
|
|
|
def _run_helper(mam_sandbox, snippet: str, env=None, timeout=30):
|
|
"""Source the sandbox lib.sh and run a shell snippet against it."""
|
|
lib_sh = _sandbox_skills(mam_sandbox) / "lib.sh"
|
|
run_env = dict(os.environ)
|
|
run_env.pop("HERDR_SESSION_NAME", None)
|
|
run_env.pop("HERDR_SERVER_NAME", None)
|
|
if env:
|
|
run_env.update({k: str(v) for k, v in env.items()})
|
|
return subprocess.run(
|
|
["bash", "-c", 'source "{}" >/dev/null 2>&1; {}'.format(lib_sh, snippet)],
|
|
capture_output=True, text=True, env=run_env, timeout=timeout,
|
|
)
|
|
|
|
|
|
def _stub_env(tmp_path, *, exit_code=0, stdout=""):
|
|
"""Point _delegate_py_bin at the recording stub via AGENT_PYTHON_BIN."""
|
|
log = tmp_path / "stub_argv.log"
|
|
return {
|
|
"AGENT_PYTHON_BIN": str(FIXTURES / "fake_python.sh"),
|
|
"STUB_ARGV_LOG": str(log),
|
|
"STUB_EXIT": str(exit_code),
|
|
"STUB_STDOUT": stdout,
|
|
}, log
|
|
|
|
|
|
def _install_fake_monitor(mam_sandbox) -> Path:
|
|
"""Replace the sandbox reconcile.sh with a sleeping stub and return its path."""
|
|
target = (Path(mam_sandbox) / ".agents" / "skills"
|
|
/ "multi-agent-mux-monitor" / "scripts" / "reconcile.sh")
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text((FIXTURES / "fake_reconcile.sh").read_text(encoding="utf-8"),
|
|
encoding="utf-8")
|
|
target.chmod(0o755)
|
|
return target
|
|
|
|
|
|
def _kill(pids):
|
|
for pid in pids:
|
|
try:
|
|
os.kill(int(pid), signal.SIGKILL)
|
|
except (OSError, ValueError):
|
|
pass
|
|
|
|
|
|
def _alive(pid) -> bool:
|
|
try:
|
|
os.kill(int(pid), 0)
|
|
return True
|
|
except (OSError, ValueError):
|
|
return False
|
|
|
|
|
|
# ==============================================================================
|
|
# 1. Shell helper golden behaviour (C-01 .. C-09)
|
|
# ==============================================================================
|
|
|
|
@pytest.mark.parametrize("mode", DELEGATION_MODES)
|
|
def test_c01_publish_event_empty_job_is_noop(mam_sandbox, mode):
|
|
"""C-01: an empty job id returns 0 without ever starting an interpreter."""
|
|
env, log = _stub_env(mam_sandbox, exit_code=7)
|
|
env["MAM_DELEGATION_PY"] = mode
|
|
res = _run_helper(mam_sandbox, 'delegate_publish_event "" started', env=env)
|
|
assert res.returncode == 0, res.stderr
|
|
assert not log.exists(), "python must not be invoked for an empty job id"
|
|
|
|
|
|
@pytest.mark.parametrize("mode", DELEGATION_MODES)
|
|
def test_c02_publish_event_missing_script_is_non_fatal(mam_sandbox, mode):
|
|
"""C-02: a missing publish_event.py must not abort the caller."""
|
|
script = (_sandbox_skills(mam_sandbox) / "multi-agent-mux-delegate-job"
|
|
/ "scripts" / "publish_event.py")
|
|
script.unlink()
|
|
env, _ = _stub_env(mam_sandbox)
|
|
env["MAM_DELEGATION_PY"] = mode
|
|
res = _run_helper(mam_sandbox, 'delegate_publish_event j1 started', env=env)
|
|
assert res.returncode == 0, res.stderr
|
|
|
|
|
|
@pytest.mark.parametrize("mode", DELEGATION_MODES)
|
|
def test_c03_publish_event_broker_failure_is_non_fatal(mam_sandbox, mode):
|
|
"""C-03: a non-zero interpreter exit (broker down) still returns 0."""
|
|
env, log = _stub_env(mam_sandbox, exit_code=2)
|
|
env["MAM_DELEGATION_PY"] = mode
|
|
res = _run_helper(
|
|
mam_sandbox,
|
|
'delegate_publish_event j1 completed "done"; echo "rc=$?"',
|
|
env=env,
|
|
)
|
|
assert res.returncode == 0, res.stderr
|
|
assert "rc=0" in res.stdout, res.stdout
|
|
assert log.exists(), "the interpreter should have been invoked"
|
|
|
|
|
|
@pytest.mark.parametrize("mode", DELEGATION_MODES)
|
|
def test_c04_publish_event_forwards_job_event_detail(mam_sandbox, mode):
|
|
"""C-04: the job id, event and detail reach the interpreter argv intact."""
|
|
env, log = _stub_env(mam_sandbox)
|
|
env["MAM_DELEGATION_PY"] = mode
|
|
res = _run_helper(
|
|
mam_sandbox,
|
|
'delegate_publish_event abc123 progress "half way"',
|
|
env=env,
|
|
)
|
|
assert res.returncode == 0, res.stderr
|
|
argv = log.read_text(encoding="utf-8")
|
|
assert "--job abc123" in argv, argv
|
|
assert "--event progress" in argv, argv
|
|
assert "--detail half way" in argv, argv
|
|
|
|
|
|
@pytest.mark.parametrize("mode", DELEGATION_MODES)
|
|
def test_c05_submit_job_stdout_is_exactly_the_job_id(mam_sandbox, mode):
|
|
"""C-05: create_session.sh captures this with $( ) — stdout must be the id alone."""
|
|
env, _ = _stub_env(mam_sandbox, stdout="deadbeef")
|
|
env["MAM_DELEGATION_PY"] = mode
|
|
res = _run_helper(
|
|
mam_sandbox,
|
|
'delegate_submit_job "a prompt" claude-code herdr:claude',
|
|
env=env,
|
|
)
|
|
assert res.returncode == 0, res.stderr
|
|
assert res.stdout == "deadbeef\n", repr(res.stdout)
|
|
|
|
|
|
@pytest.mark.parametrize("mode", DELEGATION_MODES)
|
|
def test_c06_watchdog_missing_monitor_script_errors(mam_sandbox, mode):
|
|
"""C-06: exact stderr wording and rc=1 when reconcile.sh is absent."""
|
|
workdir = Path(mam_sandbox) / "empty_ws"
|
|
workdir.mkdir()
|
|
expected = (workdir / ".agents" / "skills" / "multi-agent-mux-monitor"
|
|
/ "scripts" / "reconcile.sh")
|
|
res = _run_helper(
|
|
mam_sandbox,
|
|
'start_watchdog job1 "{}"'.format(workdir),
|
|
env={"MAM_DELEGATION_PY": mode},
|
|
)
|
|
assert res.returncode == 1, res.stdout
|
|
assert res.stderr.strip() == "ERROR: monitor script not found: {}".format(expected)
|
|
|
|
|
|
@pytest.mark.parametrize("mode", DELEGATION_MODES)
|
|
def test_c07_watchdog_spawns_daemon_with_workdir_and_log(mam_sandbox, mode):
|
|
"""C-07: prints a live numeric PID, anchors cwd at workdir, appends to the log."""
|
|
_install_fake_monitor(mam_sandbox)
|
|
workdir = Path(mam_sandbox)
|
|
(workdir / ".mam").mkdir(exist_ok=True)
|
|
spawned = []
|
|
try:
|
|
res = _run_helper(
|
|
mam_sandbox,
|
|
'start_watchdog job1 "{}"'.format(workdir),
|
|
env={"MAM_DELEGATION_PY": mode, "MAM_NO_SUBSCRIBER_DAEMON": "0"},
|
|
)
|
|
assert res.returncode == 0, res.stderr
|
|
pid = res.stdout.strip()
|
|
assert re.fullmatch(r"\d+", pid), repr(res.stdout)
|
|
spawned.append(pid)
|
|
deadline = time.time() + 5
|
|
log_file = workdir / ".mam" / "multi-agent-mux-monitor.log"
|
|
while time.time() < deadline and not log_file.exists():
|
|
time.sleep(0.1)
|
|
assert log_file.exists(), "monitor log was not created"
|
|
assert _alive(pid), "spawned daemon is not running"
|
|
finally:
|
|
_kill(spawned)
|
|
|
|
|
|
@pytest.mark.parametrize("mode", DELEGATION_MODES)
|
|
def test_c08_watchdog_second_call_reuses_running_daemon(mam_sandbox, mode):
|
|
"""C-08: the pgrep pre-check must return the existing PID, not spawn a second."""
|
|
monitor = _install_fake_monitor(mam_sandbox)
|
|
workdir = Path(mam_sandbox)
|
|
(workdir / ".mam").mkdir(exist_ok=True)
|
|
spawned = []
|
|
try:
|
|
first = _run_helper(mam_sandbox, 'start_watchdog job1 "{}"'.format(workdir),
|
|
env={"MAM_DELEGATION_PY": mode, "MAM_NO_SUBSCRIBER_DAEMON": "0"})
|
|
pid1 = first.stdout.strip()
|
|
spawned.append(pid1)
|
|
assert _alive(pid1)
|
|
second = _run_helper(mam_sandbox, 'start_watchdog job1 "{}"'.format(workdir),
|
|
env={"MAM_DELEGATION_PY": mode, "MAM_NO_SUBSCRIBER_DAEMON": "0"})
|
|
assert second.returncode == 0, second.stderr
|
|
assert second.stdout.strip() == pid1, (first.stdout, second.stdout)
|
|
matches = subprocess.run(
|
|
["pgrep", "-f", "bash {} --subscribe".format(monitor)],
|
|
capture_output=True, text=True,
|
|
).stdout.split()
|
|
spawned.extend(matches)
|
|
assert len(matches) == 1, "expected exactly one daemon, got {}".format(matches)
|
|
finally:
|
|
_kill(spawned)
|
|
|
|
|
|
@pytest.mark.parametrize("mode", DELEGATION_MODES)
|
|
def test_c09_start_watchdog_and_ensure_subscriber_daemon_agree(mam_sandbox, mode):
|
|
"""C-09: `start_watchdog` survives as an alias once the python body lands.
|
|
|
|
Passes on the legacy tree (only start_watchdog defined) and on the refactored
|
|
tree (both defined and behaviourally identical)."""
|
|
_install_fake_monitor(mam_sandbox)
|
|
workdir = Path(mam_sandbox)
|
|
(workdir / ".mam").mkdir(exist_ok=True)
|
|
probe = _run_helper(
|
|
mam_sandbox,
|
|
'declare -F start_watchdog >/dev/null && echo HAS_SW; '
|
|
'declare -F ensure_subscriber_daemon >/dev/null && echo HAS_ESD',
|
|
env={"MAM_DELEGATION_PY": mode},
|
|
)
|
|
assert "HAS_SW" in probe.stdout, "start_watchdog must always remain defined"
|
|
if "HAS_ESD" not in probe.stdout:
|
|
return # legacy tree: alias not introduced yet (P1-02)
|
|
|
|
spawned = []
|
|
try:
|
|
a = _run_helper(mam_sandbox, 'start_watchdog job1 "{}"'.format(workdir),
|
|
env={"MAM_DELEGATION_PY": mode, "MAM_NO_SUBSCRIBER_DAEMON": "0"})
|
|
b = _run_helper(mam_sandbox, 'ensure_subscriber_daemon job1 "{}"'.format(workdir),
|
|
env={"MAM_DELEGATION_PY": mode, "MAM_NO_SUBSCRIBER_DAEMON": "0"})
|
|
spawned += [a.stdout.strip(), b.stdout.strip()]
|
|
assert a.returncode == b.returncode == 0, (a.stderr, b.stderr)
|
|
assert a.stdout.strip() == b.stdout.strip(), (a.stdout, b.stdout)
|
|
finally:
|
|
_kill(spawned)
|
|
|
|
|
|
# ==============================================================================
|
|
# 2. CLI contract snapshots
|
|
# ==============================================================================
|
|
|
|
def _parser_snapshot(parser):
|
|
"""{option string: (nargs-ish kind, choices, default)} for one parser level."""
|
|
snap = {}
|
|
for action in parser._actions:
|
|
if not action.option_strings:
|
|
snap["<positional:{}>".format(action.dest)] = (
|
|
action.type.__name__ if action.type else None,
|
|
tuple(action.choices) if action.choices else None,
|
|
action.default,
|
|
)
|
|
continue
|
|
key = ",".join(sorted(action.option_strings))
|
|
snap[key] = (
|
|
action.type.__name__ if action.type else None,
|
|
tuple(action.choices) if action.choices else None,
|
|
action.default,
|
|
)
|
|
return snap
|
|
|
|
|
|
def _import_delegation(mam_sandbox, name):
|
|
"""Import a delegation module the way production callers do (top-level name)."""
|
|
scripts = str(_sandbox_skills(mam_sandbox) / "multi-agent-mux-delegate-job" / "scripts")
|
|
if scripts not in sys.path:
|
|
sys.path.insert(0, scripts)
|
|
return __import__(name)
|
|
|
|
|
|
def test_publish_event_cli_contract_snapshot(mam_sandbox):
|
|
publish_event = _import_delegation(mam_sandbox, "publish_event")
|
|
assert publish_event.VALID_EVENTS == (
|
|
"started", "permission_required", "progress", "completed", "error", "cancelled",
|
|
)
|
|
assert publish_event.TERMINAL_EVENTS == ("completed", "error", "cancelled")
|
|
# main() builds its parser inline, so --help is the only way to exercise it.
|
|
with pytest.raises(SystemExit) as exc:
|
|
publish_event.main(["--help"])
|
|
assert exc.value.code == 0
|
|
# Defaults that shell and wrapper callers depend on.
|
|
src = Path(publish_event.__file__).read_text(encoding="utf-8")
|
|
assert '"--attempts", type=int, default=3' in src
|
|
# Issue #15: resolved per process (MAM_JOBS_DIR, then WORKSPACE_ROOT, then
|
|
# the cwd-relative DEFAULT_REGISTRY_DIR) instead of the bare constant.
|
|
assert '"--registry-dir", default=default_registry_dir()' in src
|
|
assert '"--event", default="progress", choices=VALID_EVENTS' in src
|
|
|
|
|
|
def test_registry_cli_contract_snapshot(mam_sandbox):
|
|
registry = _import_delegation(mam_sandbox, "registry")
|
|
parser = registry._build_parser()
|
|
top = _parser_snapshot(parser)
|
|
assert "--registry-dir" in top
|
|
subparsers = [a for a in parser._actions if hasattr(a, "choices") and isinstance(a.choices, dict)]
|
|
assert subparsers, "registry must expose subcommands"
|
|
commands = set(subparsers[0].choices)
|
|
assert commands == {
|
|
"register", "list", "get", "status", "update", "close", "get-feedback", "pick", "logs",
|
|
"scrub-secrets", # Issue #12 migration
|
|
}, commands
|
|
reg = _parser_snapshot(subparsers[0].choices["register"])
|
|
assert reg["--prompt"][2] is None
|
|
assert reg["--agent"][2] == "claude-code"
|
|
assert reg["--agent-session"][2] == "herdr:claude"
|
|
assert reg["--role"][2] == "Worker"
|
|
assert reg["--timeout"][2] == 3600
|
|
# Issue #18: the idle window derives from the liveness contract (K x N =
|
|
# 4 x 30 = 120 s, the historical default) unless given explicitly.
|
|
assert reg["--idle-timeout"][2] is None
|
|
assert reg["--heartbeat-sec"][2] == 30
|
|
assert reg["--heartbeat-misses"][2] == 4
|
|
assert reg["--ack-timeout"][2] == 180
|
|
assert reg["--transport"][1] == ("herdr", "remote")
|
|
assert registry.liveness_window(reg["--heartbeat-sec"][2], reg["--heartbeat-misses"][2]) == 120
|
|
assert reg["--bits"][2] == 32
|
|
assert reg["--job-type"][1] == ("direct", "loop", "discuss")
|
|
assert reg["--max-iterations"][2] == 5
|
|
assert registry.VALID_STATUSES == (
|
|
"pending", "running", "completed", "error", "cancelled",
|
|
)
|
|
|
|
|
|
def test_job_subscriber_cli_contract_snapshot(mam_sandbox):
|
|
job_subscriber = _import_delegation(mam_sandbox, "job_subscriber")
|
|
assert job_subscriber.TERMINAL_EVENTS == ("completed", "error")
|
|
src = Path(job_subscriber.__file__).read_text(encoding="utf-8")
|
|
for doc in (" 0 all watched jobs reached", " 1 any watched job reached",
|
|
" 2 no event for the idle window", " 4 the wall-clock budget"):
|
|
assert doc in src, doc
|
|
# Issue #33: codes are named; their values are the frozen contract.
|
|
assert (job_subscriber.RC_COMPLETED, job_subscriber.RC_ERROR, job_subscriber.RC_NO_EVENT,
|
|
job_subscriber.RC_BROKER_UNAVAILABLE, job_subscriber.RC_WALL_CLOCK) == (0, 1, 2, 3, 4)
|
|
assert 'return RC_BROKER_UNAVAILABLE' in src, "fatal-error exit code 3 must survive"
|
|
assert not re.search(r"^\s+return [0-9]+\s*(#.*)?$", src, re.M), "exit codes are returned by name"
|
|
|
|
|
|
# ==============================================================================
|
|
# 3. Auto-completion invariant guards (Phase 1 must not move these)
|
|
# ==============================================================================
|
|
|
|
def test_event_to_status_mapping_is_frozen(mam_sandbox):
|
|
publish_event = _import_delegation(mam_sandbox, "publish_event")
|
|
assert publish_event.EVENT_TO_STATUS == {
|
|
"started": "running",
|
|
"completed": "completed",
|
|
"error": "error",
|
|
"cancelled": "cancelled",
|
|
}
|
|
|
|
|
|
def test_wrapper_exit_code_mapping_literals_are_frozen(mam_sandbox):
|
|
wrapper = SKILLS / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
|
|
text = wrapper.read_text(encoding="utf-8")
|
|
for literal in (
|
|
'if [[ $sub_rc -eq $SUB_RC_COMPLETED ]]; then',
|
|
'elif [[ $sub_rc -eq $SUB_RC_ERROR ]]; then',
|
|
'elif [[ $sub_rc -eq $SUB_RC_BROKER_UNAVAILABLE ]]; then',
|
|
'[ "$sub_rc" = "$SUB_RC_WALL_CLOCK" ] && clock="wall"',
|
|
'job_status="completed"',
|
|
'job_status="error"',
|
|
):
|
|
assert literal in text, literal
|
|
# Issue #33: the wrapper's SUB_RC_* mirror job_subscriber's RC_* exactly,
|
|
# and the wrapper compares subscriber codes only by those names.
|
|
job_subscriber = _import_delegation(mam_sandbox, "job_subscriber")
|
|
mirror = dict(re.findall(r"\bSUB_RC_([A-Z_]+)=([0-9]+)", text))
|
|
assert mirror == {name[3:]: str(getattr(job_subscriber, name))
|
|
for name in dir(job_subscriber) if name.startswith("RC_")}, mirror
|
|
assert not re.search(r"\$(sub_rc|sub_exit)\"? +(-eq|-ne|=) +\"?[0-9]", text), "subscriber codes compared by value"
|
|
|
|
|
|
def test_reconcile_terminal_mutation_literals_are_frozen():
|
|
reconcile = SKILLS / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
|
|
text = reconcile.read_text(encoding="utf-8")
|
|
for literal in (
|
|
"s['delegate_job_id'] = None",
|
|
"s['status'] = 'terminated'",
|
|
"kill-session",
|
|
"if _event in ('completed', 'cancelled')",
|
|
):
|
|
assert literal in text, literal
|
|
|
|
|
|
# ==============================================================================
|
|
# 4. P1-01 — package binding, provenance and context matrix (PV-1, PV-4, PV-5)
|
|
# ==============================================================================
|
|
|
|
DELEGATION_MODULES = ("mqtt_common", "registry", "publish_event", "job_subscriber")
|
|
PKG_DIR = SKILLS / "lib_py" / "delegation"
|
|
SHIM_DIR = SKILLS / "multi-agent-mux-delegate-job" / "scripts"
|
|
|
|
|
|
def test_pv5_all_four_shims_share_one_bootstrap_block():
|
|
"""PV-5: the four shims differ only by the module name they bind."""
|
|
normalised = {}
|
|
for mod in DELEGATION_MODULES:
|
|
text = (SHIM_DIR / (mod + ".py")).read_text(encoding="utf-8")
|
|
normalised[mod] = text.replace(mod, "{mod}")
|
|
base = normalised["mqtt_common"]
|
|
for mod, text in normalised.items():
|
|
assert text == base, "shim {} diverged from the shared bootstrap".format(mod)
|
|
assert "sys.path.insert" not in base, "the shim must never take sys.path precedence"
|
|
assert "sys.path.append(_SKILLS)" in base, "standalone fallback must remain"
|
|
|
|
|
|
def test_pv1_identity_and_provenance_of_every_alias(mam_sandbox, tmp_path_factory):
|
|
"""PV-1: each alias IS the package module, and all four share one tree."""
|
|
import lib_py.delegation
|
|
|
|
paths = set()
|
|
for mod in DELEGATION_MODULES:
|
|
alias = _import_delegation(mam_sandbox, mod)
|
|
pkg_mod = getattr(lib_py.delegation, mod, None)
|
|
if pkg_mod is None:
|
|
import importlib
|
|
pkg_mod = importlib.import_module("lib_py.delegation." + mod)
|
|
assert alias is pkg_mod, "{}: alias is not the package module".format(mod)
|
|
paths.add(str(Path(alias.__file__).resolve().parent))
|
|
|
|
assert len(paths) == 1, "delegation modules span multiple trees: {}".format(paths)
|
|
bound = Path(paths.pop())
|
|
real = PKG_DIR.resolve()
|
|
# Provenance, not just identity: the bound tree is either the real package or a
|
|
# copy under pytest's own basetemp. Checked against the actual basetemp rather
|
|
# than sniffing for "pytest-" in the path, so an unrelated directory that merely
|
|
# looks temporary cannot satisfy it.
|
|
basetemp = Path(tmp_path_factory.getbasetemp()).resolve()
|
|
try:
|
|
bound.relative_to(basetemp)
|
|
under_basetemp = True
|
|
except ValueError:
|
|
under_basetemp = False
|
|
assert bound == real or under_basetemp, (
|
|
"delegation bound to an unexpected tree: {} (real={}, basetemp={})".format(
|
|
bound, real, basetemp)
|
|
)
|
|
# the parent package must never be dragged along with it
|
|
assert Path(lib_py.__file__).resolve().parent == (SKILLS / "lib_py").resolve()
|
|
|
|
|
|
def test_delegation_package_imports_no_sibling_lib_py_modules():
|
|
"""D1(b): a frozen delegation package must not reach into the parent tree."""
|
|
import ast
|
|
|
|
offenders = []
|
|
for path in sorted(PKG_DIR.glob("*.py")):
|
|
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
|
for node in ast.walk(tree):
|
|
names = []
|
|
if isinstance(node, ast.Import):
|
|
names = [a.name for a in node.names]
|
|
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
names = [node.module]
|
|
for name in names:
|
|
if name in DELEGATION_MODULES:
|
|
offenders.append("{}:{} top-level `{}`".format(path.name, node.lineno, name))
|
|
if name == "lib_py" or (
|
|
name.startswith("lib_py.") and not name.startswith("lib_py.delegation")
|
|
):
|
|
offenders.append("{}:{} sibling `{}`".format(path.name, node.lineno, name))
|
|
assert not offenders, "forbidden imports inside lib_py.delegation:\n " + "\n ".join(offenders)
|
|
|
|
|
|
def test_importing_delegation_does_not_mutate_environment(mam_sandbox):
|
|
"""D6: importing the library must leave os.environ alone; env loads at entry points."""
|
|
import importlib
|
|
mqtt_common = _import_delegation(mam_sandbox, "mqtt_common")
|
|
before = set(os.environ)
|
|
importlib.reload(mqtt_common)
|
|
assert set(os.environ) == before, "library import injected environment variables"
|
|
# idempotent: the second call is a no-op
|
|
mqtt_common.ensure_env_loaded()
|
|
keys = set(os.environ)
|
|
mqtt_common.ensure_env_loaded()
|
|
assert set(os.environ) == keys
|
|
|
|
|
|
def _normalize_help(text: str) -> str:
|
|
text = text.replace("optional arguments:", "options:")
|
|
return re.sub(r"\s+", " ", text).strip()
|
|
|
|
|
|
def test_shim_help_output_matches_pre_move_capture():
|
|
"""The shim must be indistinguishable from the implementation at the CLI.
|
|
|
|
The captures were taken before the phase 1 move to prove the move changed
|
|
nothing. They are refreshed only when the CLI is *intentionally* extended
|
|
(phase 2b added the finalization options to publish_event); the guard's job
|
|
is to catch drift nobody meant, which it still does.
|
|
"""
|
|
for mod in ("publish_event", "registry", "job_subscriber"):
|
|
golden = (FIXTURES / "help-{}.txt".format(mod)).read_text(encoding="utf-8")
|
|
res = subprocess.run(
|
|
[sys.executable, str(SHIM_DIR / (mod + ".py")), "--help"],
|
|
capture_output=True, text=True, timeout=60,
|
|
)
|
|
assert res.returncode == 0, res.stderr
|
|
# argparse re-wraps usage/help text and renamed the "optional arguments"
|
|
# group across Python versions (Issue #28: the captures were taken on
|
|
# 3.9, the Linux host runs 3.14). Compare the content, not the layout:
|
|
# whitespace runs collapsed, the group label unified.
|
|
assert _normalize_help(res.stdout + res.stderr) == _normalize_help(golden), \
|
|
"--help drifted for {}".format(mod)
|
|
|
|
|
|
# --- PV-4: the context matrix, each scenario in a fresh interpreter ------------
|
|
|
|
@pytest.fixture(scope="module")
|
|
def shim_trees(tmp_path_factory):
|
|
"""Four copies of the live skills tree: real / sandbox / sandbox2 / freeze."""
|
|
import shutil
|
|
|
|
root = tmp_path_factory.mktemp("shim_trees")
|
|
for name in ("real", "sandbox", "sandbox2", "freeze"):
|
|
dest = root / name
|
|
(dest / "multi-agent-mux-delegate-job").mkdir(parents=True)
|
|
shutil.copytree(SKILLS / "lib_py", dest / "lib_py")
|
|
shutil.copytree(SHIM_DIR, dest / "multi-agent-mux-delegate-job" / "scripts")
|
|
return root
|
|
|
|
|
|
def _run_scenario(shim_trees, scenario, pythonpath=None):
|
|
env = dict(os.environ)
|
|
env.pop("PYTHONPATH", None)
|
|
if pythonpath:
|
|
env["PYTHONPATH"] = str(shim_trees / pythonpath)
|
|
res = subprocess.run(
|
|
[sys.executable, str(FIXTURES / "shim_contexts.py"), scenario]
|
|
+ [str(shim_trees / t) for t in ("real", "sandbox", "sandbox2", "freeze")],
|
|
capture_output=True, text=True, env=env, timeout=120,
|
|
)
|
|
assert res.returncode == 0, "scenario {} failed:\n{}".format(scenario, res.stderr)
|
|
return json.loads(res.stdout)
|
|
|
|
|
|
def _delegation_trees(report):
|
|
keys = ("alias_mqtt_common", "alias_publish_event", "pkg_delegation",
|
|
"pkg_mqtt_common", "pkg_publish_event")
|
|
return {report[k] for k in keys if report[k] is not None}
|
|
|
|
|
|
@pytest.mark.parametrize("scenario,expected_delegation", [
|
|
("A", "sandbox"), # sandbox shim binds first
|
|
("A_pe", "sandbox"), # non-leaf module alone: dependencies must follow it
|
|
("B", "sandbox"), # real lib_py.agents already imported: delegation still binds to shim tree
|
|
("B2", "real"), # real delegation already bound: bind-once keeps it
|
|
("D", "sandbox"), # a second sandbox must not rebind
|
|
("E", "sandbox"), # direct package import after a shim resolves to the bound tree
|
|
])
|
|
def test_pv4_pytest_family_binds_one_tree_and_spares_the_parent(
|
|
shim_trees, scenario, expected_delegation
|
|
):
|
|
rep = _run_scenario(shim_trees, scenario)
|
|
assert _delegation_trees(rep) == {expected_delegation}, rep
|
|
assert rep["lib_py"] == "real", "parent lib_py must stay on the real tree: {}".format(rep)
|
|
if rep["lib_py_agents"] is not None:
|
|
assert rep["lib_py_agents"] == "real", rep
|
|
for name, same in rep["identity"].items():
|
|
if same is not None:
|
|
assert same, "identity broken for {} in {}".format(name, scenario)
|
|
assert "sandbox" not in rep["lib_py_path_entries"], rep
|
|
assert "sandbox2" not in rep["lib_py_path_entries"], rep
|
|
|
|
|
|
@pytest.mark.parametrize("scenario", ["C1", "C1_pe"])
|
|
def test_pv4_standalone_bootstraps_from_its_own_tree(shim_trees, scenario):
|
|
"""K4: no ambient lib_py anywhere, so the shim may append its own tree."""
|
|
rep = _run_scenario(shim_trees, scenario)
|
|
assert _delegation_trees(rep) == {"freeze"}, rep
|
|
assert rep["lib_py"] == "freeze", rep
|
|
|
|
|
|
@pytest.mark.parametrize("scenario", ["C2", "C2_pe"])
|
|
def test_pv4_frozen_copy_wins_over_a_live_pythonpath(shim_trees, scenario):
|
|
"""K2: the shape every delegated job runs -- frozen script, live PYTHONPATH.
|
|
|
|
B-13 requires the frozen code to execute; the parent lib_py may stay live
|
|
because D1(b) forbids the delegation package from importing siblings."""
|
|
rep = _run_scenario(shim_trees, scenario, pythonpath="real")
|
|
assert _delegation_trees(rep) == {"freeze"}, rep
|
|
assert rep["lib_py"] == "real", rep
|
|
for name, same in rep["identity"].items():
|
|
if same is not None:
|
|
assert same, name
|
|
|
|
|
|
def test_pv4_k3_live_helpers_resolve_to_the_live_tree(shim_trees):
|
|
rep = _run_scenario(shim_trees, "K3")
|
|
assert _delegation_trees(rep) == {"real"}, rep
|
|
assert rep["lib_py"] == "real", rep
|
|
|
|
|
|
def test_k1_module_execution_follows_pythonpath(shim_trees):
|
|
"""K1: `python -m lib_py.delegation.<mod>` is resolved by PYTHONPATH, not by a shim."""
|
|
for tree in ("real", "freeze"):
|
|
env = dict(os.environ)
|
|
env["PYTHONPATH"] = str(shim_trees / tree)
|
|
res = subprocess.run(
|
|
[sys.executable, "-c",
|
|
"import lib_py.delegation as d; print(list(d.__path__)[0])"],
|
|
capture_output=True, text=True, env=env, timeout=60,
|
|
)
|
|
assert res.returncode == 0, res.stderr
|
|
assert res.stdout.strip() == str(shim_trees / tree / "lib_py" / "delegation"), res.stdout
|
|
|
|
|
|
def test_k6_subprocess_executes_the_copy_it_was_given(shim_trees):
|
|
"""K6: e2e-style subprocess execution runs the tree the path points at."""
|
|
env = dict(os.environ)
|
|
env.pop("PYTHONPATH", None)
|
|
shim = shim_trees / "sandbox" / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"
|
|
res = subprocess.run([sys.executable, str(shim), "--help"],
|
|
capture_output=True, text=True, env=env, timeout=60)
|
|
assert res.returncode == 0, res.stderr
|
|
probe = subprocess.run(
|
|
[sys.executable, "-c",
|
|
"import sys; sys.path.insert(0, {!r}); import publish_event;"
|
|
" print(publish_event.__file__)".format(str(shim.parent))],
|
|
capture_output=True, text=True, env=env, timeout=60,
|
|
)
|
|
assert probe.returncode == 0, probe.stderr
|
|
assert str(shim_trees / "sandbox") in probe.stdout, probe.stdout
|
|
|
|
|
|
# ==============================================================================
|
|
# 5. P1-02 — daemon module, cli router, shell-to-python equivalence
|
|
# ==============================================================================
|
|
|
|
def _delegation_env(mam_sandbox):
|
|
"""Env for running the package out of the sandbox copy."""
|
|
env = dict(os.environ)
|
|
env["PYTHONPATH"] = str(_sandbox_skills(mam_sandbox))
|
|
env["MAM_ENV_FILE"] = str(Path(mam_sandbox) / ".mam.env.absent")
|
|
return env
|
|
|
|
|
|
def test_daemon_missing_monitor_raises_with_the_legacy_path(mam_sandbox):
|
|
from lib_py.delegation import daemon
|
|
|
|
workdir = Path(mam_sandbox) / "no_monitor"
|
|
workdir.mkdir()
|
|
with pytest.raises(FileNotFoundError) as exc:
|
|
daemon.ensure_subscriber_daemon("job1", str(workdir))
|
|
assert exc.value.args[0] == str(daemon.monitor_script_path(str(workdir)))
|
|
|
|
|
|
def test_daemon_spawn_dedup_and_survives_parent_exit(mam_sandbox, monkeypatch):
|
|
"""The daemon must outlive the shell that started it and never double-spawn."""
|
|
from lib_py.delegation import daemon
|
|
|
|
monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False) # issue #19 sandbox default
|
|
monitor = _install_fake_monitor(mam_sandbox)
|
|
workdir = str(mam_sandbox)
|
|
spawned = []
|
|
try:
|
|
first = daemon.ensure_subscriber_daemon("job1", workdir)
|
|
spawned += first
|
|
assert len(first) == 1 and _alive(first[0])
|
|
assert (Path(workdir) / ".mam" / "multi-agent-mux-monitor.log").exists()
|
|
second = daemon.ensure_subscriber_daemon("job1", workdir)
|
|
assert second == first, "a running daemon must be reused, not replaced"
|
|
# detached into its own session, so a parent's process-group signal misses it
|
|
pgid = os.getpgid(int(first[0]))
|
|
assert pgid != os.getpgid(os.getpid())
|
|
assert daemon.find_running(str(monitor)) == first
|
|
finally:
|
|
_kill(spawned)
|
|
|
|
|
|
def test_cli_ensure_daemon_reproduces_the_legacy_failure_line(mam_sandbox):
|
|
workdir = Path(mam_sandbox) / "no_monitor_cli"
|
|
workdir.mkdir()
|
|
res = subprocess.run(
|
|
[sys.executable, "-m", "lib_py.delegation.cli",
|
|
"ensure-daemon", "--job", "j1", "--workdir", str(workdir)],
|
|
capture_output=True, text=True, env=_delegation_env(mam_sandbox), timeout=60,
|
|
)
|
|
assert res.returncode == 1
|
|
expected = str(workdir / ".agents" / "skills" / "multi-agent-mux-monitor"
|
|
/ "scripts" / "reconcile.sh")
|
|
assert res.stderr.strip() == "ERROR: monitor script not found: {}".format(expected)
|
|
assert res.stdout == ""
|
|
|
|
|
|
def test_cli_submit_prints_only_the_job_id(mam_sandbox):
|
|
reg_dir = Path(mam_sandbox) / ".mam" / "jobs"
|
|
reg_dir.mkdir(parents=True, exist_ok=True)
|
|
res = subprocess.run(
|
|
[sys.executable, "-m", "lib_py.delegation.cli", "submit",
|
|
"--prompt", "hello", "--agent", "claude-code",
|
|
"--agent-session", "herdr:claude", "--registry-dir", str(reg_dir)],
|
|
capture_output=True, text=True, env=_delegation_env(mam_sandbox), timeout=60,
|
|
)
|
|
assert res.returncode == 0, res.stderr
|
|
job_id = res.stdout.strip()
|
|
assert re.fullmatch(r"[0-9a-f]+", job_id), repr(res.stdout)
|
|
assert res.stdout == job_id + "\n", "stdout must carry the id and nothing else"
|
|
assert (reg_dir / (job_id + ".json")).exists()
|
|
|
|
|
|
@pytest.mark.parametrize("entry", ["script", "cli"])
|
|
def test_publish_failure_semantics_match_across_entry_points(mam_sandbox, entry):
|
|
"""G-1/G-2/G-4 semantics must be identical through the shim and through the cli.
|
|
|
|
Broker points at a non-routable port, so nothing touches the network."""
|
|
reg_dir = Path(mam_sandbox) / ".mam" / "jobs"
|
|
reg_dir.mkdir(parents=True, exist_ok=True)
|
|
env = _delegation_env(mam_sandbox)
|
|
job_id = subprocess.run(
|
|
[sys.executable, "-m", "lib_py.delegation.cli", "submit", "--prompt", "p",
|
|
"--registry-dir", str(reg_dir)],
|
|
capture_output=True, text=True, env=env, timeout=60,
|
|
).stdout.strip()
|
|
|
|
record = json.loads((reg_dir / (job_id + ".json")).read_text(encoding="utf-8"))
|
|
record["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
|
(reg_dir / (job_id + ".json")).write_text(json.dumps(record), encoding="utf-8")
|
|
|
|
if entry == "cli":
|
|
cmd = [sys.executable, "-m", "lib_py.delegation.cli", "publish"]
|
|
else:
|
|
cmd = [sys.executable, str(SHIM_DIR / "publish_event.py")]
|
|
res = subprocess.run(
|
|
cmd + ["--job", job_id, "--event", "completed", "--detail", "done",
|
|
"--attempts", "1", "--registry-dir", str(reg_dir)],
|
|
capture_output=True, text=True, env=env, cwd=str(mam_sandbox), timeout=120,
|
|
)
|
|
assert res.returncode == 2, (res.stdout, res.stderr)
|
|
assert res.stdout == "", "publish must stay silent on stdout"
|
|
after = json.loads((reg_dir / (job_id + ".json")).read_text(encoding="utf-8"))
|
|
assert after["status"] == "completed", "status syncs even when the publish fails"
|
|
assert after.get("last_seq") == 1, "seq is consumed on failure by design"
|
|
audit = (Path(mam_sandbox) / ".mam" / "delegate_job_logs" / job_id / "events.ndjson")
|
|
events = [json.loads(l) for l in audit.read_text(encoding="utf-8").splitlines() if l.strip()]
|
|
published = [e for e in events if e.get("event") == "published"]
|
|
assert published and published[-1]["published"] is False
|
|
assert published[-1]["publish_error"]
|
|
|
|
|
|
# ==============================================================================
|
|
# 6. P1-03 — transports, parity and the benchmark harness (no network at all)
|
|
# ==============================================================================
|
|
|
|
def _prepared(topic="python/mqtt/jobs/j/events", host="broker.example", port=1883):
|
|
from lib_py.delegation import mqtt_common, publish_event
|
|
|
|
config = mqtt_common.broker_config_from_env({"host": host, "port": port})
|
|
return publish_event.PreparedEvent(
|
|
job_id="j", event="progress", detail="d", seq=1, topic=topic, retain=False,
|
|
payload={"seq": 1}, body=b'{"seq": 1}', config=config,
|
|
)
|
|
|
|
|
|
def _fake_factory(counters, publishes_ok=True, connect_rc=0):
|
|
from lib_py.delegation.bench import FakeClient, FakeInfo
|
|
|
|
def factory(_role, _config):
|
|
client = FakeClient(connect_latency=0.0, counters=counters)
|
|
if not publishes_ok:
|
|
def unacked(*_a, **_k):
|
|
counters["publish"] = counters.get("publish", 0) + 1
|
|
return FakeInfo(published=False)
|
|
client.publish = unacked
|
|
if connect_rc:
|
|
original = client.connect
|
|
|
|
def connect(host, port, keepalive=60):
|
|
original(host, port, keepalive)
|
|
client.on_connect(client, None, None, connect_rc, None)
|
|
client.connect = connect
|
|
return client
|
|
return factory
|
|
|
|
|
|
def test_persistent_publisher_reuses_one_connection():
|
|
"""The whole point of the prototype: N publishes, 1 connection."""
|
|
from lib_py.delegation.publisher import OneShotPublisher, PersistentPublisher
|
|
|
|
prepared = _prepared()
|
|
one_counters, persist_counters = {}, {}
|
|
one = OneShotPublisher(client_factory=_fake_factory(one_counters))
|
|
persist = PersistentPublisher(client_factory=_fake_factory(persist_counters))
|
|
try:
|
|
for _ in range(5):
|
|
assert one.publish_with_retry(prepared, attempts=1) == (True, None)
|
|
assert persist.publish_with_retry(prepared, attempts=1) == (True, None)
|
|
finally:
|
|
persist.close()
|
|
assert one_counters["connect"] == 5
|
|
assert persist_counters["connect"] == 1
|
|
assert one_counters["publish"] == persist_counters["publish"] == 5
|
|
|
|
|
|
def test_transport_parity_of_registry_and_audit_records(mam_sandbox, monkeypatch):
|
|
"""Same input, two transports: registry and audit records must match field for field."""
|
|
from lib_py.delegation import mqtt_common, publish_event, registry
|
|
from lib_py.delegation.publisher import OneShotPublisher, PersistentPublisher
|
|
|
|
monkeypatch.chdir(mam_sandbox)
|
|
reg_dir = str(Path(mam_sandbox) / ".mam" / "jobs")
|
|
os.makedirs(reg_dir, exist_ok=True)
|
|
|
|
results = {}
|
|
for label, publisher in (
|
|
("one_shot", OneShotPublisher(client_factory=_fake_factory({}))),
|
|
("persistent", PersistentPublisher(client_factory=_fake_factory({}))),
|
|
):
|
|
job_id = registry.register_job("parity " + label, registry_dir=reg_dir)
|
|
prepared = publish_event.prepare_event(
|
|
job_id=job_id, event="completed", detail="done", registry_dir=reg_dir,
|
|
)
|
|
ok, err = publisher.publish_with_retry(prepared, attempts=1)
|
|
publish_event.record_outcome(prepared, reg_dir, ok, err)
|
|
if hasattr(publisher, "close"):
|
|
publisher.close()
|
|
record = mqtt_common.load_job(job_id, reg_dir)
|
|
events = list(mqtt_common.iter_logged_events(job_id, mqtt_common.get_logs_dir()))
|
|
audit = [e for e in events if e.get("event") == "published"][-1]
|
|
results[label] = {
|
|
"ok": ok,
|
|
"status": record["status"],
|
|
"last_seq": record.get("last_seq"),
|
|
"retain": audit["retain"],
|
|
"published": audit["published"],
|
|
"seq": audit["seq"],
|
|
"has_sig": "hmac_sig" in audit["payload"].get("data", {}),
|
|
"source_event": audit["source_event"],
|
|
}
|
|
|
|
assert results["one_shot"] == results["persistent"], results
|
|
assert results["one_shot"]["retain"] is True, "terminal events must retain"
|
|
assert results["one_shot"]["status"] == "completed"
|
|
|
|
|
|
def test_persistent_publisher_unacked_publish_retries_then_records_failure(mam_sandbox, monkeypatch):
|
|
"""No PUBACK -> exception -> with_retry -> record_outcome(published=False)."""
|
|
from lib_py.delegation import mqtt_common, publish_event, registry
|
|
from lib_py.delegation.publisher import PersistentPublisher
|
|
|
|
monkeypatch.chdir(mam_sandbox)
|
|
reg_dir = str(Path(mam_sandbox) / ".mam" / "jobs")
|
|
os.makedirs(reg_dir, exist_ok=True)
|
|
job_id = registry.register_job("unacked", registry_dir=reg_dir)
|
|
prepared = publish_event.prepare_event(
|
|
job_id=job_id, event="completed", detail="d", registry_dir=reg_dir,
|
|
)
|
|
counters = {}
|
|
publisher = PersistentPublisher(client_factory=_fake_factory(counters, publishes_ok=False))
|
|
try:
|
|
ok, err = publisher.publish_with_retry(prepared, attempts=3)
|
|
finally:
|
|
publisher.close()
|
|
assert ok is False and "not acknowledged" in err
|
|
assert counters["publish"] == 3, "every attempt must reach the transport"
|
|
assert counters["connect"] == 1, "retries reuse the connection"
|
|
|
|
publish_event.record_outcome(prepared, reg_dir, ok, err)
|
|
record = mqtt_common.load_job(job_id, reg_dir)
|
|
assert record["status"] == "completed", "status syncs even when the publish failed"
|
|
assert record.get("last_seq") == prepared.seq
|
|
audit = [e for e in mqtt_common.iter_logged_events(job_id, mqtt_common.get_logs_dir())
|
|
if e.get("event") == "published"][-1]
|
|
assert audit["published"] is False and audit["publish_error"]
|
|
|
|
|
|
def test_persistent_publisher_refuses_a_second_broker():
|
|
from lib_py.delegation.publisher import PersistentPublisher
|
|
|
|
publisher = PersistentPublisher(client_factory=_fake_factory({}))
|
|
try:
|
|
assert publisher.publish_with_retry(_prepared(host="a.example"), attempts=1)[0] is True
|
|
ok, err = publisher.publish_with_retry(_prepared(host="b.example"), attempts=1)
|
|
assert ok is False
|
|
assert "bound to a.example" in err, err
|
|
finally:
|
|
publisher.close()
|
|
|
|
|
|
def test_bench_fake_transport_schema_and_counts():
|
|
from lib_py.delegation import bench
|
|
|
|
result = bench.run_benchmark(n=7, transport="fake", connect_latency_ms=1.0)
|
|
assert result["transport"] == "fake"
|
|
for arm in ("one_shot", "persistent"):
|
|
assert result[arm]["publishes"] == 7
|
|
assert result[arm]["failures"] == 0
|
|
assert set(result[arm]["latency_ms"]) == {"p50", "p95", "max"}
|
|
assert result["one_shot"]["client_connects"] == 7
|
|
assert result["persistent"]["client_connects"] == 1
|
|
assert result["persistent"]["latency_ms"]["p50"] < result["one_shot"]["latency_ms"]["p50"]
|
|
|
|
|
|
def test_bench_broker_transport_requires_a_job():
|
|
from lib_py.delegation import bench
|
|
|
|
with pytest.raises(ValueError):
|
|
bench.run_benchmark(n=1, transport="broker")
|
|
|
|
|
|
# ==============================================================================
|
|
# 7. Self-review regressions — the connect handshake is per-connection
|
|
#
|
|
# Found reviewing phase 1: PersistentPublisher held the CONNACK result on the
|
|
# instance, so a refused attempt left its reason code behind for the next one.
|
|
# The retry then either rejected a healthy connection or, worse, accepted one
|
|
# whose CONNACK had never arrived. Both need an asynchronous callback to
|
|
# reproduce, which is how real paho delivers it.
|
|
# ==============================================================================
|
|
|
|
class _AsyncAckClient(object):
|
|
"""paho-shaped stub: connect() returns at once, CONNACK lands on a timer."""
|
|
|
|
def __init__(self, rc_sequence, calls, ack_delay=0.05):
|
|
self._rc_sequence = rc_sequence
|
|
self._calls = calls
|
|
self._ack_delay = ack_delay
|
|
self.on_connect = None
|
|
|
|
def connect(self, host, port, keepalive=60):
|
|
import threading
|
|
idx = self._calls["n"]
|
|
self._calls["n"] += 1
|
|
rc = self._rc_sequence[min(idx, len(self._rc_sequence) - 1)]
|
|
if rc is None:
|
|
return # models a broker that never acknowledges
|
|
timer = threading.Timer(self._ack_delay,
|
|
lambda: self.on_connect(self, None, None, rc, None))
|
|
timer.daemon = True
|
|
timer.start()
|
|
|
|
def loop_start(self):
|
|
return None
|
|
|
|
def loop_stop(self):
|
|
self._calls["loop_stop"] = self._calls.get("loop_stop", 0) + 1
|
|
|
|
def reconnect_delay_set(self, **_kw):
|
|
return None
|
|
|
|
def publish(self, *_a, **_k):
|
|
from lib_py.delegation.bench import FakeInfo
|
|
self._calls["publish"] = self._calls.get("publish", 0) + 1
|
|
return FakeInfo()
|
|
|
|
def disconnect(self):
|
|
self._calls["disconnect"] = self._calls.get("disconnect", 0) + 1
|
|
|
|
|
|
def _async_factory(rc_sequence, calls):
|
|
def factory(_role, _config):
|
|
return _AsyncAckClient(rc_sequence, calls)
|
|
return factory
|
|
|
|
|
|
def test_persistent_publisher_recovers_on_the_attempt_after_a_refusal():
|
|
"""A refused connection must not decide the next attempt's outcome."""
|
|
from lib_py.delegation.publisher import PersistentPublisher
|
|
|
|
calls = {"n": 0}
|
|
publisher = PersistentPublisher(client_factory=_async_factory([5, 0], calls))
|
|
try:
|
|
ok, err = publisher.publish_with_retry(_prepared(), attempts=3)
|
|
finally:
|
|
publisher.close()
|
|
assert ok is True, err
|
|
assert calls["n"] == 2, (
|
|
"attempt 2 had a healthy connection and should have been the last one; "
|
|
"got {} connect() calls".format(calls["n"])
|
|
)
|
|
assert calls.get("disconnect", 0) >= 1, "the refused client must be torn down, not leaked"
|
|
|
|
|
|
def test_persistent_publisher_never_accepts_a_connection_that_did_not_ack():
|
|
"""A silent broker must time out even when an earlier attempt acked."""
|
|
from lib_py.delegation.publisher import PersistentPublisher
|
|
import lib_py.delegation.publisher as publisher_mod
|
|
|
|
calls = {"n": 0}
|
|
# first connection is refused (leaves a reason code behind), second never acks
|
|
publisher = PersistentPublisher(client_factory=_async_factory([5, None], calls))
|
|
original = publisher_mod.CONNECT_ACK_TIMEOUT
|
|
publisher_mod.CONNECT_ACK_TIMEOUT = 0.3 # keep the test quick
|
|
try:
|
|
ok, err = publisher.publish_with_retry(_prepared(), attempts=2)
|
|
finally:
|
|
publisher_mod.CONNECT_ACK_TIMEOUT = original
|
|
publisher.close()
|
|
assert ok is False
|
|
assert "no CONNACK" in err, (
|
|
"the silent second connection must time out on its own evidence, not "
|
|
"inherit the first attempt's reason code; got: {}".format(err)
|
|
)
|
|
|
|
|
|
def test_persistent_publisher_reuses_a_healthy_connection_across_events():
|
|
"""The successful path still connects exactly once."""
|
|
from lib_py.delegation.publisher import PersistentPublisher
|
|
|
|
calls = {"n": 0}
|
|
publisher = PersistentPublisher(client_factory=_async_factory([0], calls))
|
|
try:
|
|
for _ in range(4):
|
|
assert publisher.publish_with_retry(_prepared(), attempts=1) == (True, None)
|
|
finally:
|
|
publisher.close()
|
|
assert calls["n"] == 1, "a healthy connection must be reused"
|
|
assert calls["publish"] == 4
|
|
|
|
|
|
# ==============================================================================
|
|
# 8. Cross-review regressions (job 60c980ac)
|
|
#
|
|
# R1 (codex): `python -m` puts the working directory ahead of PYTHONPATH, so a
|
|
# frozen loop whose cwd contained a lib_py package silently ran live
|
|
# code. The shell helpers now use an explicit bootstrap instead.
|
|
# R2 (codex): the persistent connection was reused on host/port alone, ignoring
|
|
# credentials and TLS.
|
|
# F-1 (claude): paho queues a publish issued while disconnected and then raises
|
|
# RuntimeError, which sat outside the retry tuple -- so the one state
|
|
# a persistent connection exists to survive failed on first attempt.
|
|
# ==============================================================================
|
|
|
|
def _decoy_tree(root: Path) -> Path:
|
|
"""A directory that looks like a skills tree and screams if it is imported."""
|
|
pkg = root / "lib_py" / "delegation"
|
|
pkg.mkdir(parents=True)
|
|
(root / "lib_py" / "__init__.py").write_text("", encoding="utf-8")
|
|
(pkg / "__init__.py").write_text("", encoding="utf-8")
|
|
(pkg / "cli.py").write_text(
|
|
"def main(argv=None):\n"
|
|
" print('DECOY-TREE-EXECUTED')\n"
|
|
" return 0\n",
|
|
encoding="utf-8",
|
|
)
|
|
return root
|
|
|
|
|
|
def test_r1_shell_helper_ignores_a_lib_py_in_the_working_directory(mam_sandbox, tmp_path):
|
|
"""R1: the helper must run its own tree even when cwd provides lib_py.
|
|
|
|
This is the frozen-loop shape: a decoy package sitting in the working
|
|
directory must not win over the tree the sourced lib.sh belongs to.
|
|
"""
|
|
decoy = _decoy_tree(tmp_path / "decoy_ws")
|
|
(decoy / ".mam" / "jobs").mkdir(parents=True)
|
|
|
|
lib_sh = _sandbox_skills(mam_sandbox) / "lib.sh"
|
|
env = dict(os.environ)
|
|
env.pop("HERDR_SESSION_NAME", None)
|
|
env.pop("PYTHONPATH", None)
|
|
env["MAM_ENV_FILE"] = str(Path(mam_sandbox) / ".mam.env.absent")
|
|
|
|
res = subprocess.run(
|
|
["bash", "-c",
|
|
'cd "{}" && source "{}" >/dev/null 2>&1 && '
|
|
'delegate_submit_job "r1 probe" claude-code herdr:claude'.format(decoy, lib_sh)],
|
|
capture_output=True, text=True, env=env, timeout=120,
|
|
)
|
|
assert "DECOY-TREE-EXECUTED" not in res.stdout, (
|
|
"the working directory's lib_py shadowed the helper's own tree:\n" + res.stdout
|
|
)
|
|
assert res.returncode == 0, res.stderr
|
|
job_id = res.stdout.strip()
|
|
assert re.fullmatch(r"[0-9a-f]+", job_id), repr(res.stdout)
|
|
# Issue #15: the registry follows MAM_JOBS_DIR / WORKSPACE_ROOT (both point
|
|
# at the sandbox here), no longer the cwd — the decoy workspace's own
|
|
# .mam/jobs must stay empty.
|
|
registry = Path(env["MAM_JOBS_DIR"])
|
|
assert (registry / (job_id + ".json")).exists(), (
|
|
"the real cli should have registered the job in the sandbox registry"
|
|
)
|
|
assert not (decoy / ".mam" / "jobs" / (job_id + ".json")).exists(), (
|
|
"the job must not be registered relative to the cwd"
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("field,first,second", [
|
|
("username", "account-a", "account-b"),
|
|
("password", "secret-a", "secret-b"),
|
|
("tls", False, True),
|
|
("ca_certs", None, "/tmp/ca.pem"),
|
|
])
|
|
def test_r2_persistent_publisher_refuses_a_different_connection_identity(field, first, second):
|
|
"""R2: same endpoint is not the same connection when auth or TLS differ."""
|
|
from lib_py.delegation import mqtt_common, publish_event
|
|
from lib_py.delegation.publisher import PersistentPublisher
|
|
|
|
built = []
|
|
|
|
def factory(_role, config):
|
|
built.append(getattr(config, field))
|
|
from lib_py.delegation.bench import FakeClient
|
|
client = FakeClient(connect_latency=0.0, counters={})
|
|
return client
|
|
|
|
def prepared(value):
|
|
config = mqtt_common.broker_config_from_env(
|
|
{"host": "same.example", "port": 1883, field: value})
|
|
return publish_event.PreparedEvent(
|
|
"j", "progress", "d", 1, "t", False, {}, b"{}", config)
|
|
|
|
publisher = PersistentPublisher(client_factory=factory)
|
|
try:
|
|
ok, _err = publisher.publish_with_retry(prepared(first), attempts=1)
|
|
assert ok is True
|
|
ok2, err2 = publisher.publish_with_retry(prepared(second), attempts=1)
|
|
finally:
|
|
publisher.close()
|
|
|
|
assert ok2 is False, "an event with a different {} must not reuse the connection".format(field)
|
|
assert field in err2, err2
|
|
assert str(second) not in err2, "the error must name the field, never its value"
|
|
assert built == [first], "the second event must not have been published on the first client"
|
|
|
|
|
|
def test_r2_identical_configuration_still_reuses_the_connection():
|
|
"""The guard must not become so strict that reuse never happens."""
|
|
from lib_py.delegation.publisher import PersistentPublisher
|
|
|
|
counters = {"n": 0}
|
|
|
|
def factory(_role, _config):
|
|
counters["n"] += 1
|
|
from lib_py.delegation.bench import FakeClient
|
|
return FakeClient(connect_latency=0.0, counters={})
|
|
|
|
publisher = PersistentPublisher(client_factory=factory)
|
|
try:
|
|
for _ in range(3):
|
|
assert publisher.publish_with_retry(_prepared(), attempts=1) == (True, None)
|
|
finally:
|
|
publisher.close()
|
|
assert counters["n"] == 1
|
|
|
|
|
|
class _FlakyConnectionClient(object):
|
|
"""paho-shaped: the first N publishes are issued while disconnected."""
|
|
|
|
def __init__(self, no_conn_times, calls):
|
|
import paho.mqtt.client as mqtt
|
|
self._mqtt = mqtt
|
|
self._no_conn_times = no_conn_times
|
|
self._calls = calls
|
|
self.on_connect = None
|
|
|
|
def connect(self, host, port, keepalive=60):
|
|
self._calls["connect"] = self._calls.get("connect", 0) + 1
|
|
self.on_connect(self, None, None, 0, None)
|
|
|
|
def loop_start(self):
|
|
return None
|
|
|
|
def loop_stop(self):
|
|
return None
|
|
|
|
def reconnect_delay_set(self, **_kw):
|
|
return None
|
|
|
|
def publish(self, *_a, **_k):
|
|
from lib_py.delegation.bench import FakeInfo
|
|
n = self._calls.get("publish", 0)
|
|
self._calls["publish"] = n + 1
|
|
if n < self._no_conn_times:
|
|
return FakeInfo(published=False, rc=int(self._mqtt.MQTT_ERR_NO_CONN))
|
|
return FakeInfo(published=True, rc=0)
|
|
|
|
def disconnect(self):
|
|
return None
|
|
|
|
|
|
def test_f1_publish_issued_while_disconnected_is_retried_until_reconnect():
|
|
"""F-1: the gap between a drop and paho's reconnect must be survivable.
|
|
|
|
paho queues such a publish and raises RuntimeError from wait_for_publish,
|
|
which is not in the retry tuple; it used to end the attempt immediately."""
|
|
from lib_py.delegation.publisher import PersistentPublisher
|
|
|
|
calls = {}
|
|
publisher = PersistentPublisher(
|
|
client_factory=lambda _r, _c: _FlakyConnectionClient(2, calls))
|
|
try:
|
|
ok, err = publisher.publish_with_retry(_prepared(), attempts=3)
|
|
finally:
|
|
publisher.close()
|
|
assert ok is True, "should have recovered on the third attempt, got: {}".format(err)
|
|
assert calls["publish"] == 3, "every attempt must reach the transport"
|
|
assert calls["connect"] == 1, "recovery must reuse the connection, not rebuild it"
|
|
|
|
|
|
def test_f1_disconnected_publish_still_fails_when_it_never_recovers():
|
|
"""Retrying must not mask a genuinely dead connection."""
|
|
from lib_py.delegation.publisher import PersistentPublisher
|
|
|
|
calls = {}
|
|
publisher = PersistentPublisher(
|
|
client_factory=lambda _r, _c: _FlakyConnectionClient(99, calls))
|
|
try:
|
|
ok, err = publisher.publish_with_retry(_prepared(), attempts=2)
|
|
finally:
|
|
publisher.close()
|
|
assert ok is False
|
|
assert "not connected" in err, err
|
|
assert calls["publish"] == 2
|
|
|
|
|
|
def test_one_shot_publish_has_a_single_implementation():
|
|
"""O-1: the CLI path and the service path must not drift apart again."""
|
|
from lib_py.delegation import publish_event
|
|
from lib_py.delegation.publisher import OneShotPublisher
|
|
|
|
source = Path(publish_event.__file__).read_text(encoding="utf-8")
|
|
assert "OneShotPublisher().publish_raw(" in source, (
|
|
"_publish_once must delegate to the transport module"
|
|
)
|
|
for leaked in ("client.loop_start()", "no CONNACK from broker", "wait_for_publish("):
|
|
assert leaked not in source, (
|
|
"publish_event still carries its own copy of the transport logic: {}".format(leaked)
|
|
)
|
|
assert publish_event.CONNECT_ACK_TIMEOUT is not None
|
|
assert hasattr(OneShotPublisher, "publish_raw")
|