tmpl/multi-agent-mux
1
0 Fork 0
Code Issues 9 Pull Requests 1 Actions Packages Projects Releases Wiki Activity
Files
ae98cf18c4db573af9cc630ab2e2fe22ede3f31a
multi-agent-mux/tests/test_issue57_logs_follow_registry.py
T
GodopuandClaude Fable 5.1 a7cb90e63a fix(delegation): the sibling rule pairs with the registry's real path (Issue #57, Mac R2) 
A registry directory that is itself a symlink got two audit logs: submit
resolved --registry-dir with pwd -P and wrote next to the target, while logs
derived the sibling from the link path. sibling_logs_dir() now takes the
registry's realpath; an explicit DELEGATE_JOB_LOGS_DIR still wins.
Also: forwarded_keys() normalises both sides of its comparison, and the
design doc's status row lists the audit log next to the registry.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-15 00:06:30 +00:00

634 lines
35 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""Issue #57 — the audit log follows the registry.
`--registry-dir` / `MAM_JOBS_DIR` moved the registry, but the audit log kept
its own root (`DELEGATE_JOB_LOGS_DIR`, else the workspace's), so an isolated
experiment left audit-only directories in the real control plane. The rule is
now one function, `mqtt_common.get_logs_dir(registry_dir)`: an explicit
`DELEGATE_JOB_LOGS_DIR` wins, otherwise the audit log is the
`delegate_job_logs` next to the registry in use. Every test ends by asserting
that the workspace's own `.mam/delegate_job_logs` was never created.
Comment 5 of the issue: `job_subscriber._check_disk_fallback` compared the
dict `read_logged_status()` returns with status strings, so an audit-only
terminal status was never recognised.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
import pytest
import _mam_state_guard # noqa: F401
REPO_ROOT = Path(__file__).resolve().parent.parent
SKILLS = REPO_ROOT / ".agents" / "skills"
WRAPPER = SKILLS / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
REGISTRY_PY = SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "registry.py"
PUBLISH_PY = SKILLS / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"
if str(SKILLS) not in sys.path:
sys.path.insert(0, str(SKILLS))
from lib_py import control_plane # noqa: E402
from lib_py.delegation import engine, job_subscriber, mqtt_common, registry # noqa: E402
from test_issue21_phase2_loop_a import _mutate, _prepare, _rows, _skill, _wt # noqa: E402
from test_issue21_phase2_loop_a import ctl # noqa: E402,F401
from test_issue21_worktree_prereqs import _env_flags, _sent_texts # noqa: E402
UNREACHABLE = {"MQTT_BROKER": "127.0.0.1", "MQTT_PORT": "9", "MQTT_TLS": "0"}
DROP = ("MAM_JOBS_DIR", "DELEGATE_JOB_LOGS_DIR")
@pytest.fixture
def moved(tmp_path, monkeypatch):
"""A workspace whose registry lives elsewhere, with neither override set:
conftest's autouse defaults would otherwise hide the fallback (comment 7)."""
for k in DROP:
monkeypatch.delenv(k, raising=False)
ws = tmp_path / "ws"
(ws / ".mam").mkdir(parents=True)
reg = tmp_path / "elsewhere" / "jobs"
monkeypatch.setenv("WORKSPACE_ROOT", str(ws))
monkeypatch.chdir(ws)
yield ws, reg, tmp_path / "elsewhere" / "delegate_job_logs"
assert not (ws / ".mam" / "delegate_job_logs").exists(), "the workspace's audit-log root was created"
def _env(ws: Path, **extra) -> dict:
env = {k: v for k, v in os.environ.items() if k not in DROP}
env.update(WORKSPACE_ROOT=str(ws), PYTHONPATH=str(SKILLS), DELEGATE_JOB_PYTHON=sys.executable,
AGENT_PYTHON_BIN=sys.executable, MAM_ENV_FILE=str(ws / ".mam.env.absent"), **UNREACHABLE)
env.update({k: str(v) for k, v in extra.items()})
return env
# ---------------------------------------------------------------------------
# comment 5: the disk fallback reads status.json's status, not the dict
# ---------------------------------------------------------------------------
def test_disk_fallback_reads_the_audit_only_terminal_status(moved):
ws, reg, logs = moved
reg.mkdir(parents=True)
# no registry record at all: only the audit log says the job ended
jid = "a57fb001"
mqtt_common.update_logged_status(jid, "completed", logs_dir=mqtt_common.get_logs_dir(str(reg)))
pending, terminal = {jid}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, None, None)
assert terminal == {jid: "completed"}
assert pending == set()
# a pending record with a dispatch token, an audit-only `error`
tok = "0123456789abcdef0123456789abcdef"
jid2 = registry.register_job("fb2", registry_dir=str(reg), agent_session="herdr:t57", dispatch_token=tok)
mqtt_common.update_logged_status(jid2, "error", logs_dir=mqtt_common.get_logs_dir(str(reg)))
pending, terminal = {jid2}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, {jid2: tok}, None)
assert terminal == {jid2: "error"} and pending == set()
# a different expected token still ignores it (#14/#18 rule, unchanged)
pending, terminal = {jid2}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, {jid2: "f" * 32}, None)
assert terminal == {} and pending == {jid2}
# ---------------------------------------------------------------------------
# L1 — the resolver
# ---------------------------------------------------------------------------
def test_resolver_pairs_the_logs_with_the_registry(moved, monkeypatch):
ws, reg, logs = moved
# the usual roots are unchanged
assert mqtt_common.get_logs_dir() == str(ws / ".mam" / "delegate_job_logs")
assert mqtt_common.get_logs_dir(str(ws / ".mam" / "jobs")) == str(ws / ".mam" / "delegate_job_logs")
# a registry the caller names
assert mqtt_common.get_logs_dir(str(reg)) == str(logs)
# a registry moved by MAM_JOBS_DIR, and the compat alias
monkeypatch.setenv("MAM_JOBS_DIR", str(reg))
assert mqtt_common.get_logs_dir() == str(logs)
assert mqtt_common.LOGS_DIR == str(logs)
# an explicit root still wins
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(ws / "x"))
assert mqtt_common.get_logs_dir() == str(ws / "x")
assert mqtt_common.get_logs_dir(str(reg)) == str(ws / "x")
monkeypatch.delenv("DELEGATE_JOB_LOGS_DIR")
# a relative registry resolves against the cwd, as registry.py opens it
assert mqtt_common.get_logs_dir("r/jobs") == os.path.join(os.getcwd(), "r", "delegate_job_logs")
# the sibling rule itself reads no environment
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(ws / "x"))
assert mqtt_common.sibling_logs_dir(str(reg)) == str(logs)
# ---------------------------------------------------------------------------
# L2 — registry.py writes and reads next to --registry-dir
# ---------------------------------------------------------------------------
def _registry(ws: Path, reg: Path, *args) -> subprocess.CompletedProcess:
return subprocess.run([sys.executable, str(REGISTRY_PY), "--registry-dir", str(reg), *args],
capture_output=True, text=True, env=_env(ws), cwd=str(ws), timeout=60)
def test_registry_cli_writes_and_reads_next_to_the_registry(moved):
ws, reg, logs = moved
res = _registry(ws, reg, "register", "--prompt", "p57", "--agent", "human", "--agent-session", "herdr:t57")
assert res.returncode == 0, res.stderr
jid = res.stdout.strip()
assert (logs / jid / "meta.json").is_file() and (logs / jid / "status.json").is_file()
assert _registry(ws, reg, "pick", "--agent-session", "herdr:t57").stdout.strip() == jid
assert _registry(ws, reg, "update", "--job", jid, "--status", "completed").returncode == 0
assert json.loads((logs / jid / "status.json").read_text())["status"] == "completed"
assert _registry(ws, reg, "get-feedback", "--job", jid).returncode == 0
res = _registry(ws, reg, "logs", jid)
assert res.returncode == 0, res.stderr
assert "registered" in res.stdout and "status_changed" in res.stdout
res = _registry(ws, reg, "logs", "--list")
assert res.returncode == 0 and jid in res.stdout, res.stdout + res.stderr
events = [json.loads(l) for l in (logs / jid / "events.ndjson").read_text().splitlines() if l.strip()]
assert [e["event"] for e in events][:2] == ["registered", "status_changed"]
# ---------------------------------------------------------------------------
# L3 — the wrapper's last line names the directory it wrote
# ---------------------------------------------------------------------------
def _wrapper(ws: Path, *args, cwd: Path = None, **extra) -> subprocess.CompletedProcess:
return subprocess.run(["bash", str(WRAPPER), *args], capture_output=True, text=True,
env=_env(ws, **extra), cwd=str(cwd or ws), timeout=120)
def _only_job_id(reg: Path) -> str:
ids = [p.stem for p in reg.glob("*.json")]
assert len(ids) == 1, ids
return ids[0]
def test_wrapper_output_names_the_dir_it_wrote(moved):
ws, reg, logs = moved
real_logs = Path(os.path.realpath(logs.parent)) / "delegate_job_logs"
# (a) dry-run: registered and seeded, both next to the registry
res = _wrapper(ws, "submit", "--dry-run", "--agent", "human", "--registry-dir", str(reg),
"--workdir", str(ws), "--prompt", "p57")
assert res.returncode == 0, res.stdout[-800:] + res.stderr[-800:]
jid = _only_job_id(reg)
assert res.stdout.strip().splitlines()[-1] == str(real_logs / jid)
assert (logs / jid / "meta.json").is_file()
# (b) a real human job against an unreachable broker: rc 3, same rule
res = _wrapper(ws, "submit", "--agent", "human", "--registry-dir", str(reg), "--workdir", str(ws),
"--prompt", "p57b", "--timeout", "3", "--idle-timeout", "2")
assert res.returncode == 3, res.stdout[-800:] + res.stderr[-800:]
jid2 = [p.stem for p in reg.glob("*.json") if p.stem != jid]
assert len(jid2) == 1, jid2
jid2 = jid2[0]
assert res.stdout.strip().splitlines()[-1] == str(real_logs / jid2)
events = [json.loads(l) for l in (logs / jid2 / "events.ndjson").read_text().splitlines() if l.strip()]
assert "published" in {e.get("event") for e in events}, events
# (c) wait and (d) logs find the same registry and its audit log
res = _wrapper(ws, "wait", "--registry-dir", str(reg), "--job", str(jid2), "--timeout", "2")
assert res.returncode in (0, 1, 2, 3), res.stdout[-400:] + res.stderr[-400:]
res = _wrapper(ws, "logs", "--registry-dir", str(reg), jid2)
assert res.returncode == 0 and "registered" in res.stdout, res.stdout + res.stderr
res = _wrapper(ws, "logs", "--registry-dir", str(reg), "--list")
assert res.returncode == 0 and jid2 in res.stdout, res.stdout + res.stderr
# ---------------------------------------------------------------------------
# L4 — publish_event.py records next to --registry-dir
# ---------------------------------------------------------------------------
def test_publish_event_records_next_to_the_registry(moved):
ws, reg, logs = moved
jid = registry.register_job("p4", registry_dir=str(reg), agent_session="herdr:t57")
(logs / jid / "events.ndjson").unlink() # only what publish_event writes counts
def publish(*args):
return subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(reg), "--job", jid,
"--attempts", "1", *args],
capture_output=True, text=True, env=_env(ws), cwd=str(ws), timeout=60)
publish("--event", "progress", "--detail", "x")
events = [json.loads(l) for l in (logs / jid / "events.ndjson").read_text().splitlines() if l.strip()]
assert [e for e in events if e.get("event") == "published" and e.get("source_event") == "progress"], events
publish("--event", "completed", "--detail", "done")
assert json.loads((logs / jid / "status.json").read_text())["status"] == "completed"
# ---------------------------------------------------------------------------
# the session binding derives the audit-log root from the registry
# ---------------------------------------------------------------------------
def test_from_env_derives_the_logs_dir_from_a_moved_registry(moved):
ws, reg, logs = moved
b = control_plane.from_env({"WORKSPACE_ROOT": str(ws), "MAM_JOBS_DIR": str(reg)})
assert b["mam_jobs_dir"] == os.path.realpath(reg)
assert b["delegate_job_logs_dir"] == os.path.realpath(logs)
# an explicit root is kept as it is
b = control_plane.from_env({"WORKSPACE_ROOT": str(ws), "MAM_JOBS_DIR": str(reg),
"DELEGATE_JOB_LOGS_DIR": str(ws / "custom-logs")})
assert b["delegate_job_logs_dir"] == os.path.realpath(ws / "custom-logs")
# the usual binding is unchanged
assert control_plane.from_env({"WORKSPACE_ROOT": str(ws)}) == control_plane.defaults(str(ws))
# and the CLI lib.sh calls agrees
res = subprocess.run([sys.executable, "-m", "lib_py.control_plane", "json", str(ws / "wt")],
capture_output=True, text=True, env=_env(ws, MAM_JOBS_DIR=str(reg)), cwd=str(ws), timeout=60)
assert res.returncode == 0, res.stderr
assert json.loads(res.stdout)["delegate_job_logs_dir"] == os.path.realpath(logs)
def test_an_explicit_logs_dir_still_wins_over_a_moved_registry(tmp_path, monkeypatch):
"""Comment 1: DELEGATE_JOB_LOGS_DIR means the audit log was moved to a place
that cannot be derived; it wins over --registry-dir as before."""
for k in DROP:
monkeypatch.delenv(k, raising=False)
main, custom, exp = tmp_path / "main", tmp_path / "custom-logs", tmp_path / "exp" / "jobs"
(main / ".mam").mkdir(parents=True)
monkeypatch.setenv("WORKSPACE_ROOT", str(main))
monkeypatch.chdir(main)
env = dict(MAM_JOBS_DIR=str(main / ".mam" / "jobs"), DELEGATE_JOB_LOGS_DIR=str(custom))
res = _wrapper(main, "submit", "--dry-run", "--agent", "human", "--registry-dir", str(exp),
"--workdir", str(main), "--prompt", "p57", **env)
assert res.returncode == 0, res.stdout[-800:] + res.stderr[-800:]
jid = _only_job_id(exp)
assert res.stdout.strip().splitlines()[-1] == str(custom / jid)
assert (custom / jid / "meta.json").is_file()
assert not (tmp_path / "exp" / "delegate_job_logs").exists()
res = subprocess.run([sys.executable, str(PUBLISH_PY), "--registry-dir", str(exp), "--job", jid,
"--attempts", "1", "--event", "progress", "--detail", "x"],
capture_output=True, text=True, env=_env(main, **env), cwd=str(main), timeout=60)
events = [json.loads(l) for l in (custom / jid / "events.ndjson").read_text().splitlines() if l.strip()]
assert [e for e in events if e.get("event") == "published"], events
assert not (tmp_path / "exp" / "delegate_job_logs").exists()
assert not (main / ".mam" / "delegate_job_logs").exists()
# ---------------------------------------------------------------------------
# worktree_prepare keeps the creator's audit log next to the control root's registry
# ---------------------------------------------------------------------------
def test_worktree_prepare_keeps_the_logs_next_to_the_control_root_registry(ctl, mock_herdr, mock_agents, monkeypatch):
main = ctl
reg = main / "elsewhere" / "jobs"
monkeypatch.setenv("MAM_JOBS_DIR", str(reg))
monkeypatch.delenv("DELEGATE_JOB_LOGS_DIR", raising=False)
wt = _wt(main)
res = _prepare(main, wt, "--creator-agent", "claude", "--onboard")
assert res.returncode == 0, res.stdout + res.stderr
(row,) = _rows(main)
assert row["control_plane"]["mam_jobs_dir"] == os.path.realpath(reg)
assert row["control_plane"]["delegate_job_logs_dir"] == os.path.realpath(main / "elsewhere" / "delegate_job_logs")
jid = row["delegate_job_id"]
assert (main / "elsewhere" / "delegate_job_logs" / jid / "status.json").is_file()
assert not (main / ".mam" / "delegate_job_logs").exists()
# the pane carries the registry, not the audit-log root derived from it
flags = _env_flags(mock_herdr)
assert any(f == f"MAM_JOBS_DIR={os.path.realpath(reg)}" for f in flags), flags
assert not [f for f in flags if f.startswith("DELEGATE_JOB_LOGS_DIR=")], flags
# ---------------------------------------------------------------------------
# status.sh reads the audit log next to a moved registry
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("mode", ["json", "text"])
def test_status_reads_the_audit_log_next_to_a_moved_registry(mam_sandbox, monkeypatch, mode):
tmp = mam_sandbox
reg, logs = tmp / "elsewhere" / "jobs", tmp / "elsewhere" / "delegate_job_logs"
jid = "s57aa001"
(logs / jid).mkdir(parents=True)
(logs / jid / "status.json").write_text('{"job_id": "s57aa001", "status": "completed"}')
reg.mkdir(parents=True) # the registry exists, but the record was cleaned up
_mutate(tmp, f"""
d['herdr_sessions'] = [{{'name': 's57-creator-claude', 'status': 'running', 'delegate_job_id': {jid!r},
'pane': {{'cwd': {str(tmp)!r}}}}}]
""")
monkeypatch.setenv("MAM_JOBS_DIR", str(reg))
monkeypatch.delenv("DELEGATE_JOB_LOGS_DIR", raising=False)
args = ["--json"] if mode == "json" else []
res = subprocess.run(["bash", _skill(tmp, "multi-agent-mux-status", "scripts", "status.sh"), *args],
capture_output=True, text=True, timeout=120)
assert res.returncode == 0, res.stderr
if mode == "json":
assert json.loads(res.stdout)["sessions_detail"][0]["job_status"] == "completed"
else:
(line,) = [l for l in res.stdout.splitlines() if l.startswith("s57-creator-claude")]
assert jid in line and "completed" in line, line
assert not (tmp / ".mam" / "delegate_job_logs").exists()
# ---------------------------------------------------------------------------
# review (codex P2): conflict refusals and legacy claims follow the registry too
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("path", ["conflict", "legacy-claim"])
def test_conflict_refusals_and_legacy_claims_follow_the_registry(moved, monkeypatch, path):
ws, reg, logs = moved
jid = registry.register_job("p2", registry_dir=str(reg), agent_session="herdr:t57")
if path == "conflict":
assert engine.transition(jid, "completed", registry_dir=str(reg), source="operator").committed
assert not engine.transition(jid, "error", registry_dir=str(reg), source="late").committed
expected = "conflict_refused"
else:
monkeypatch.setenv("MAM_ENGINE_CAS", "0")
assert registry.pick_pending("herdr:t57", registry_dir=str(reg)) == jid
expected = "status_changed"
assert expected in [e.get("event") for e in mqtt_common.iter_logged_events(jid, str(logs))]
# ---------------------------------------------------------------------------
# review (codex P1): an old audit terminal must not close a redispatched attempt
# ---------------------------------------------------------------------------
def test_old_audit_terminal_does_not_complete_the_redispatched_attempt(moved, monkeypatch):
ws, reg, logs = moved
jid = registry.register_job("p1", registry_dir=str(reg), agent_session="herdr:t57")
assert engine.transition(jid, "completed", registry_dir=str(reg), source="operator").committed
old_audit = mqtt_common.read_logged_status(jid, str(logs))
flush, observed = engine._flush, {}
def before_the_mirror(notes, registry_dir):
# the real gap: attempt 2 is committed, the audit still says attempt 1 completed
rec = mqtt_common.load_job(jid, registry_dir)
assert rec["status"] == "pending" and rec["attempt_id"] == 2
assert old_audit["transition_seq"] < rec["transition_seq"]
pending, terminal = {jid}, {}
job_subscriber._check_disk_fallback(pending, registry_dir, terminal, {jid: rec["dispatch_token"]}, {jid: 2})
observed.update(pending=pending, terminal=terminal)
flush(notes, registry_dir)
monkeypatch.setattr(engine, "_flush", before_the_mirror)
assert engine.transition(jid, "redispatch", registry_dir=str(reg), source="operator",
fields={"attempt_id": 2}).committed
assert observed == {"pending": {jid}, "terminal": {}}, "the old completion was attributed to attempt 2"
rec = mqtt_common.load_job(jid, str(reg))
# a failed mirror leaves the old status.json behind (update_logged_status
# itself refuses to go backwards, so the file is written directly): still ignored
mqtt_common.job_log_path(jid, mqtt_common.STATUS_FILENAME, str(logs)).write_text(json.dumps(old_audit))
assert mqtt_common.read_logged_status(jid, str(logs))["status"] == "completed"
pending, terminal = {jid}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, {jid: rec["dispatch_token"]}, {jid: 2})
assert pending == {jid} and terminal == {}
# an audit status without a sequence cannot prove it is about this attempt
mqtt_common.update_logged_status(jid, "completed", logs_dir=str(logs))
pending, terminal = {jid}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, {jid: rec["dispatch_token"]}, {jid: 2})
assert pending == {jid} and terminal == {}
# a damaged sequence is ignored too, and does not take the subscriber down
mqtt_common.job_log_path(jid, mqtt_common.STATUS_FILENAME, str(logs)).write_text(
json.dumps({"job_id": jid, "status": "completed", "transition_seq": "n/a"}))
pending, terminal = {jid}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, {jid: rec["dispatch_token"]}, {jid: 2})
assert pending == {jid} and terminal == {}
# a newer audit-only terminal is still taken
mqtt_common.update_logged_status(jid, "completed", logs_dir=str(logs), transition_seq=rec["transition_seq"] + 1)
pending, terminal = {jid}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, {jid: rec["dispatch_token"]}, {jid: 2})
assert pending == set() and terminal == {jid: "completed"}
# ---------------------------------------------------------------------------
# plan §2-B (user decision): the derivable audit-log root is recorded, not forwarded
# ---------------------------------------------------------------------------
def _cp_cli(ws: Path, *args, **extra) -> subprocess.CompletedProcess:
return subprocess.run([sys.executable, "-m", "lib_py.control_plane", *args], capture_output=True, text=True,
env=_env(ws, **extra), cwd=str(ws), timeout=60)
def test_prefix_forwards_only_what_cannot_be_derived(moved, monkeypatch, tmp_path):
ws, reg, logs = moved
monkeypatch.setenv("MAM_JOBS_DIR", str(reg))
d = control_plane.describe(str(ws / "wt"))
assert d["forward"] is True
assert d["delegate_job_logs_dir"] == os.path.realpath(logs), "the row still records the sibling"
p = control_plane.prefix(d)
assert "WORKSPACE_ROOT=" in p and "AGENT_SESSIONS_YAML=" in p and f"MAM_JOBS_DIR={os.path.realpath(reg)} " in p, p
assert "DELEGATE_JOB_LOGS_DIR=" not in p, p
# an audit-log root that cannot be derived is still forwarded
custom = tmp_path / "custom-logs"
monkeypatch.setenv("DELEGATE_JOB_LOGS_DIR", str(custom))
d2 = control_plane.describe(str(ws / "wt"))
assert d2["delegate_job_logs_dir"] == os.path.realpath(custom)
assert f"DELEGATE_JOB_LOGS_DIR={os.path.realpath(custom)} " in control_plane.prefix(d2)
# the CLI lib.sh calls agrees on both
monkeypatch.delenv("DELEGATE_JOB_LOGS_DIR")
res = _cp_cli(ws, "json", str(ws / "wt"), MAM_JOBS_DIR=str(reg))
assert res.returncode == 0, res.stderr
cli = _cp_cli(ws, "prefix", res.stdout.strip(), MAM_JOBS_DIR=str(reg))
assert cli.returncode == 0 and "MAM_JOBS_DIR=" in cli.stdout and "DELEGATE_JOB_LOGS_DIR=" not in cli.stdout, cli.stdout
res = _cp_cli(ws, "json", str(ws / "wt"), MAM_JOBS_DIR=str(reg), DELEGATE_JOB_LOGS_DIR=str(custom))
cli = _cp_cli(ws, "prefix", res.stdout.strip(), MAM_JOBS_DIR=str(reg), DELEGATE_JOB_LOGS_DIR=str(custom))
assert f"DELEGATE_JOB_LOGS_DIR={os.path.realpath(custom)} " in cli.stdout, cli.stdout
def test_create_binds_the_sibling_and_does_not_forward_it(mam_sandbox, mock_herdr, mock_agents, monkeypatch):
tmp = mam_sandbox
reg, sibling = tmp / "elsewhere" / "jobs", tmp / "elsewhere" / "delegate_job_logs"
monkeypatch.setenv("MAM_JOBS_DIR", str(reg))
monkeypatch.delenv("DELEGATE_JOB_LOGS_DIR", raising=False)
wt = tmp / "wt"
wt.mkdir()
res = subprocess.run(["bash", _skill(tmp, "multi-agent-mux-create", "scripts", "create_session.sh"),
"--workspace", str(wt), "--agent", "claude", "--role", "creator", "--onboard"],
capture_output=True, text=True, cwd=str(tmp), timeout=180)
assert res.returncode == 0, res.stdout + res.stderr
(row,) = _rows(tmp)
assert row["control_plane"]["mam_jobs_dir"] == os.path.realpath(reg)
assert row["control_plane"]["delegate_job_logs_dir"] == os.path.realpath(sibling)
flags = _env_flags(mock_herdr)
assert any(f == f"MAM_JOBS_DIR={os.path.realpath(reg)}" for f in flags), flags
assert not [f for f in flags if f.startswith("DELEGATE_JOB_LOGS_DIR=")], flags
# the injected publish commands still pin the root per command
texts = [t for t in _sent_texts(json.loads(Path(mock_herdr).read_text())) if "publish_event.py" in t]
assert texts and f"DELEGATE_JOB_LOGS_DIR={os.path.realpath(sibling)} " in "".join(texts), texts
jid = row["delegate_job_id"]
assert (sibling / jid / "status.json").is_file()
assert not (tmp / ".mam" / "delegate_job_logs").exists()
def test_a_forwarded_pane_moves_its_logs_with_the_registry(tmp_path, monkeypatch):
"""The issue's scene: a worktree creator pane after §2-B carries the control
root's WORKSPACE_ROOT and MAM_JOBS_DIR but no DELEGATE_JOB_LOGS_DIR."""
for k in DROP:
monkeypatch.delenv(k, raising=False)
main, wt, exp = tmp_path / "main", tmp_path / "wt", tmp_path / "exp" / "jobs"
real_logs = main / ".mam" / "delegate_job_logs"
real_logs.mkdir(parents=True)
wt.mkdir()
pane = dict(WORKSPACE_ROOT=str(main), MAM_JOBS_DIR=str(main / ".mam" / "jobs"))
res = _wrapper(main, "submit", "--dry-run", "--agent", "human", "--registry-dir", str(exp),
"--workdir", str(wt), "--prompt", "t", cwd=wt, **pane)
assert res.returncode == 0, res.stdout[-800:] + res.stderr[-800:]
jid = _only_job_id(exp)
assert res.stdout.strip().splitlines()[-1] == str(Path(os.path.realpath(tmp_path / "exp")) / "delegate_job_logs" / jid)
assert (tmp_path / "exp" / "delegate_job_logs" / jid / "meta.json").is_file()
assert os.listdir(real_logs) == [], "the experiment leaked into the control root"
# the usual run in that pane is unchanged: the default registry's sibling
res = _wrapper(main, "submit", "--dry-run", "--agent", "human", "--workdir", str(wt), "--prompt", "t",
cwd=wt, **pane)
assert res.returncode == 0, res.stdout[-800:] + res.stderr[-800:]
jid2 = _only_job_id(main / ".mam" / "jobs")
assert res.stdout.strip().splitlines()[-1] == str(Path(os.path.realpath(main)) / ".mam" / "delegate_job_logs" / jid2)
assert os.listdir(real_logs) == [jid2]
assert not (wt / ".mam" / "delegate_job_logs").exists()
# ---------------------------------------------------------------------------
# Mac review F1: the path helpers must not need paho
# ---------------------------------------------------------------------------
@pytest.fixture
def nopaho(tmp_path):
"""A `paho` package that fails to import, first on PYTHONPATH: the real
import path then behaves like an interpreter without paho-mqtt."""
d = tmp_path / "nopaho"
(d / "paho").mkdir(parents=True)
(d / "paho" / "__init__.py").write_text("raise ModuleNotFoundError(\"No module named 'paho'\")\n")
probe = subprocess.run([sys.executable, "-c", "import paho.mqtt.client"], capture_output=True, text=True,
env={**os.environ, "PYTHONPATH": f"{d}:{SKILLS}"}, timeout=60)
assert probe.returncode != 0 and "No module named 'paho'" in probe.stderr, probe.stderr
return d
def test_control_plane_binding_works_without_paho(moved, nopaho):
"""_delegate_py_bin's last fallback is the PATH's python3, which may lack
paho: create/resume's binding and lib.sh's publish-command prefix must still
resolve the audit-log root."""
ws, reg, logs = moved
env = _env(ws, MAM_JOBS_DIR=str(reg), PYTHONPATH=f"{nopaho}:{SKILLS}", AGENT_PYTHON_BIN=sys.executable)
res = subprocess.run([sys.executable, "-m", "lib_py.control_plane", "json", str(ws / "wt")],
capture_output=True, text=True, env=env, cwd=str(ws), timeout=60)
assert res.returncode == 0, res.stderr
desc = json.loads(res.stdout)
assert desc["delegate_job_logs_dir"] == os.path.realpath(logs)
res = subprocess.run([sys.executable, "-m", "lib_py.control_plane", "prefix", json.dumps(desc)],
capture_output=True, text=True, env=env, cwd=str(ws), timeout=60)
assert res.returncode == 0 and "MAM_JOBS_DIR=" in res.stdout and "DELEGATE_JOB_LOGS_DIR=" not in res.stdout, res.stdout
# lib.sh's injected publish command still pins the root (it resolved it with the same interpreter)
res = subprocess.run(["bash", "-c", f"source '{SKILLS}/lib.sh' && delegate_publish_cmd j57"],
capture_output=True, text=True, env=env, cwd=str(ws), timeout=60)
assert res.returncode == 0, res.stderr
assert res.stdout.startswith(f"DELEGATE_JOB_LOGS_DIR={os.path.realpath(logs)} "), res.stdout + res.stderr
@pytest.mark.parametrize("mode", ["json", "text"])
def test_status_works_without_paho(mam_sandbox, nopaho, monkeypatch, mode):
tmp = mam_sandbox
reg = tmp / ".mam" / "jobs"
jid = registry.register_job("s57 nopaho", registry_dir=str(reg), agent_session="herdr:s57-nopaho")
registry.update_status(jid, str(reg), "completed")
_mutate(tmp, f"""
d['herdr_sessions'] = [{{'name': 's57-creator-claude', 'status': 'running', 'delegate_job_id': {jid!r},
'pane': {{'cwd': {str(tmp)!r}}}}}]
""")
monkeypatch.setenv("PYTHONPATH", str(nopaho))
monkeypatch.setenv("AGENT_PYTHON_BIN", sys.executable)
args = ["--json"] if mode == "json" else []
res = subprocess.run(["bash", _skill(tmp, "multi-agent-mux-status", "scripts", "status.sh"), *args],
capture_output=True, text=True, timeout=120)
assert res.returncode == 0, res.stdout[-500:] + res.stderr[-800:]
if mode == "json":
assert json.loads(res.stdout)["sessions_detail"][0]["job_status"] == "completed"
else:
(line,) = [l for l in res.stdout.splitlines() if l.startswith("s57-creator-claude")]
assert "completed" in line, line
# ---------------------------------------------------------------------------
# Mac review F2: `logs` without --registry-dir follows the workspace, not the cwd
# ---------------------------------------------------------------------------
def test_logs_without_registry_dir_follows_the_workspace_from_any_cwd(tmp_path, monkeypatch):
for k in DROP:
monkeypatch.delenv(k, raising=False)
ws, elsewhere = tmp_path / "ws", tmp_path / "elsewhere"
(ws / ".mam").mkdir(parents=True)
elsewhere.mkdir()
monkeypatch.setenv("WORKSPACE_ROOT", str(ws))
res = subprocess.run([sys.executable, str(REGISTRY_PY), "register", "--prompt", "f2", "--agent", "human",
"--agent-session", "herdr:f2"], capture_output=True, text=True, env=_env(ws),
cwd=str(elsewhere), timeout=60)
assert res.returncode == 0, res.stderr
jid = res.stdout.strip()
assert (ws / ".mam" / "jobs" / f"{jid}.json").is_file() and (ws / ".mam" / "delegate_job_logs" / jid).is_dir()
res = _wrapper(ws, "logs", "--list", cwd=elsewhere)
assert res.returncode == 0 and jid in res.stdout, res.stdout + res.stderr
res = _wrapper(ws, "logs", jid, cwd=elsewhere)
assert res.returncode == 0 and "registered" in res.stdout, res.stdout + res.stderr
assert not (elsewhere / ".mam").exists()
# ---------------------------------------------------------------------------
# Mac review R1: an audit terminal of a later dispatch is not the earlier one's success
# ---------------------------------------------------------------------------
def test_disk_fallback_rejects_a_terminal_that_belongs_to_a_later_dispatch(moved, monkeypatch):
"""Between the registry read and the audit read: A error -> redispatch
(attempt 2) -> B completed. The caller waiting on A must not be told
"completed"; the caller of B must."""
ws, reg, logs = moved
jid = registry.register_job("r1", registry_dir=str(reg), agent_session="herdr:t57")
assert registry.pick_pending("herdr:t57", registry_dir=str(reg)) == jid # claim: running, token A
rec = mqtt_common.load_job(jid, str(reg))
tok_a = rec["dispatch_token"]
assert rec["status"] == "running" and rec["attempt_id"] == 1
read, fired = mqtt_common.read_logged_status, []
def interleave(job, logs_dir=None):
if not fired: # once, right between the two reads, with real transitions
fired.append(1)
assert engine.transition(jid, "error", registry_dir=str(reg), source="operator").committed
assert engine.transition(jid, "redispatch", registry_dir=str(reg), source="operator",
fields={"attempt_id": 2}).committed
assert engine.transition(jid, "completed", registry_dir=str(reg), source="operator").committed
return read(job, logs_dir)
monkeypatch.setattr(mqtt_common, "read_logged_status", interleave)
pending, terminal = {jid}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, {jid: tok_a}, {jid: 1})
assert fired, "the interleaving did not run"
assert pending == {jid} and terminal == {}, "B's completion was attributed to A"
rec = mqtt_common.load_job(jid, str(reg))
tok_b = rec["dispatch_token"]
assert tok_b != tok_a and rec["attempt_id"] == 2 and rec["status"] == "completed"
# the next round, and the caller of B, see the truth
pending, terminal = {jid}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, {jid: tok_b}, {jid: 2})
assert pending == set() and terminal == {jid: "completed"}
pending, terminal = {jid}, {}
job_subscriber._check_disk_fallback(pending, str(reg), terminal, {jid: tok_a}, {jid: 1})
assert pending == {jid} and terminal == {}
# ---------------------------------------------------------------------------
# Mac review R2: a registry directory that is itself a symlink has one audit log
# ---------------------------------------------------------------------------
def test_a_symlinked_registry_dir_resolves_to_one_audit_log(tmp_path, monkeypatch):
"""submit resolves --registry-dir with pwd -P, so its audit log lands next
to the link's target; the resolver must pair the same place for logs."""
from lib_py.delegation import logroot
for k in DROP:
monkeypatch.delenv(k, raising=False)
ws = tmp_path / "ws"
(ws / ".mam").mkdir(parents=True)
monkeypatch.setenv("WORKSPACE_ROOT", str(ws))
actual = tmp_path / "actual" / "jobs"
actual.mkdir(parents=True)
alias = tmp_path / "alias"
alias.mkdir()
(alias / "jobs").symlink_to(actual)
real_actual = Path(os.path.realpath(tmp_path / "actual"))
assert logroot.sibling_logs_dir(str(alias / "jobs")) == str(real_actual / "delegate_job_logs")
res = _wrapper(ws, "submit", "--dry-run", "--agent", "human", "--registry-dir", str(alias / "jobs"),
"--workdir", str(ws), "--prompt", "t")
assert res.returncode == 0, res.stdout[-800:] + res.stderr[-800:]
jid = _only_job_id(actual)
assert res.stdout.strip().splitlines()[-1] == str(real_actual / "delegate_job_logs" / jid)
assert (real_actual / "delegate_job_logs" / jid / "meta.json").is_file()
assert not (tmp_path / "alias" / "delegate_job_logs").exists()
res = _wrapper(ws, "logs", "--registry-dir", str(alias / "jobs"), jid)
assert res.returncode == 0 and "registered" in res.stdout, res.stdout + res.stderr
assert not (ws / ".mam" / "delegate_job_logs").exists()
Reference in new issue View Git Blame Copy Permalink