356 lines
11 KiB
Python
356 lines
11 KiB
Python
import os
|
|
import sys
|
|
import subprocess
|
|
import time
|
|
import pytest
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).parent.parent
|
|
LOCK_SCRIPT = REPO_ROOT / ".agents" / "skills" / "multi-agent-mux-loop" / "scripts" / "loop_lock.sh"
|
|
RUN_LOOP = REPO_ROOT / ".agents" / "skills" / "multi-agent-mux-loop" / "scripts" / "run_loop.sh"
|
|
|
|
|
|
def run_lock_cmd(action, marker_path, env=None):
|
|
cmd = ["bash", str(LOCK_SCRIPT), action, str(marker_path)]
|
|
run_env = dict(os.environ)
|
|
if env:
|
|
run_env.update(env)
|
|
res = subprocess.run(cmd, capture_output=True, text=True, env=run_env)
|
|
return res
|
|
|
|
|
|
def acquire_bg(marker_path, sleep_secs=10):
|
|
cmd = f"source '{LOCK_SCRIPT}' && mam_acquire_loop_lock '{marker_path}' && sleep {sleep_secs}"
|
|
proc = subprocess.Popen(["bash", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
time.sleep(0.3)
|
|
return proc
|
|
|
|
|
|
def get_lstart(pid=None):
|
|
if pid is None:
|
|
pid = os.getpid()
|
|
res = subprocess.run(["ps", "-p", str(pid), "-o", "lstart="], capture_output=True, text=True)
|
|
return res.stdout.strip()
|
|
|
|
|
|
def test_o2_1_acquire_empty_marker(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
assert marker.is_file()
|
|
content = marker.read_text()
|
|
assert f"pid={proc.pid}" in content
|
|
assert "lstart=" in content
|
|
assert "started=" in content
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_2_live_holder_rejected(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
res = run_lock_cmd("acquire", marker)
|
|
assert res.returncode == 3
|
|
assert f"pid={proc.pid}" in marker.read_text()
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_3_dead_holder_reclaimed(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
marker.write_text("pid=999999\nlstart=Sat Jan 1 00:00:00 2000\nstarted=2000-01-01T00:00:00Z\n")
|
|
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
assert marker.is_file()
|
|
assert f"pid={proc.pid}" in marker.read_text()
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_4_pid_rollover_reclaimed(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
proc_dummy = subprocess.Popen(["sleep", "10"])
|
|
try:
|
|
# Same PID as proc_dummy, but lstart differs -> stale!
|
|
marker.write_text(f"pid={proc_dummy.pid}\nlstart=Sat Jan 1 00:00:00 2000\nstarted=2000-01-01T00:00:00Z\n")
|
|
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
assert marker.is_file()
|
|
assert f"pid={proc.pid}" in marker.read_text()
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
finally:
|
|
proc_dummy.kill()
|
|
proc_dummy.wait()
|
|
|
|
|
|
def test_o2_5_release_other_marker_fails(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
res = run_lock_cmd("release", marker)
|
|
assert res.returncode == 1
|
|
assert marker.is_file()
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_6_release_own_marker_succeeds(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
cmd = f"source '{LOCK_SCRIPT}' && mam_acquire_loop_lock '{marker}' && mam_release_loop_lock '{marker}'"
|
|
res = subprocess.run(["bash", "-c", cmd], capture_output=True, text=True)
|
|
assert res.returncode == 0
|
|
assert not marker.exists()
|
|
|
|
|
|
def test_o2_7_no_leftover_temp_files(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
cmd = f"source '{LOCK_SCRIPT}' && mam_acquire_loop_lock '{marker}' && mam_release_loop_lock '{marker}'"
|
|
subprocess.run(["bash", "-c", cmd], capture_output=True, text=True)
|
|
|
|
files = list(tmp_path.glob("*"))
|
|
assert files == []
|
|
|
|
|
|
def test_o2_8_concurrent_acquisition_single_winner(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
script = f"""
|
|
source '{LOCK_SCRIPT}'
|
|
if mam_acquire_loop_lock '{marker}'; then
|
|
sleep 2
|
|
exit 0
|
|
else
|
|
exit 3
|
|
fi
|
|
"""
|
|
procs = []
|
|
for _ in range(8):
|
|
p = subprocess.Popen(["bash", "-c", script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
procs.append(p)
|
|
|
|
rcs = [p.wait() for p in procs]
|
|
winners = [rc for rc in rcs if rc == 0]
|
|
losers = [rc for rc in rcs if rc == 3]
|
|
|
|
assert len(winners) == 1
|
|
assert len(losers) == 7
|
|
|
|
|
|
def test_o2_9_marker_never_observed_partial(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
content = marker.read_text()
|
|
assert content.startswith("pid=")
|
|
assert "lstart=" in content
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_10_ps_unavailable_fails_safe(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
cmd = f"export PATH=/bin/nonexistent:$PATH && source '{LOCK_SCRIPT}' && mam_acquire_loop_lock '{marker}'"
|
|
res = subprocess.run(["bash", "-c", cmd], capture_output=True, text=True)
|
|
assert res.returncode in (3, 4)
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_11_legacy_marker_respected(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
proc_dummy = subprocess.Popen(["sleep", "10"])
|
|
try:
|
|
marker.write_text(f"pid={proc_dummy.pid}\n")
|
|
|
|
cmd = f"source '{LOCK_SCRIPT}' && mam_acquire_loop_lock '{marker}'"
|
|
res = subprocess.run(["bash", "-c", cmd], capture_output=True, text=True)
|
|
assert res.returncode == 3
|
|
finally:
|
|
proc_dummy.kill()
|
|
proc_dummy.wait()
|
|
|
|
|
|
def test_o2_12_run_loop_exits_on_lock_failure(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
cmd = ["bash", str(RUN_LOOP), "--target-agent", "dummy-agent", "--task", "test"]
|
|
run_env = dict(os.environ)
|
|
run_env["MAM_LOOP_MARKER"] = str(marker)
|
|
res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(tmp_path), env=run_env)
|
|
assert res.returncode != 0
|
|
assert "Another multi-agent-mux-loop is already running" in res.stdout or "Another multi-agent-mux-loop is already running" in res.stderr
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_13_o3_guard_compatibility(mam_sandbox):
|
|
marker = mam_sandbox / ".mam" / "loop-guard-active"
|
|
marker.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
guard_script = REPO_ROOT / ".agents" / "hooks" / "loop_delegation_guard.sh"
|
|
payload = f'{{"toolCall":{{"name":"file_change"}},"workspacePaths":["{mam_sandbox}"]}}'
|
|
res = subprocess.run(["bash", str(guard_script)], input=payload, capture_output=True, text=True, cwd=str(mam_sandbox))
|
|
assert res.returncode == 0
|
|
assert '"decision":"deny"' in res.stdout.replace(" ", "")
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_14_bash_syntax_clean():
|
|
res1 = subprocess.run(["bash", "-n", str(LOCK_SCRIPT)], capture_output=True)
|
|
res2 = subprocess.run(["bash", "-n", str(RUN_LOOP)], capture_output=True)
|
|
assert res1.returncode == 0, res1.stderr
|
|
assert res2.returncode == 0, res2.stderr
|
|
|
|
|
|
def test_o2_15_rejected_loop_preserves_holder_marker(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
holder_content = marker.read_text()
|
|
|
|
# Subshell attempt that gets rejected
|
|
cmd = f"source '{LOCK_SCRIPT}' && (mam_acquire_loop_lock '{marker}' || true) && mam_release_loop_lock '{marker}'"
|
|
subprocess.run(["bash", "-c", cmd], capture_output=True, text=True)
|
|
|
|
assert marker.is_file()
|
|
assert marker.read_text() == holder_content
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_16_concurrent_stale_recovery(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
marker.write_text("pid=999999\nlstart=Sat Jan 1 00:00:00 2000\n")
|
|
|
|
script = f"""
|
|
source '{LOCK_SCRIPT}'
|
|
if mam_acquire_loop_lock '{marker}'; then
|
|
sleep 2
|
|
exit 0
|
|
else
|
|
exit 3
|
|
fi
|
|
"""
|
|
procs = []
|
|
for _ in range(6):
|
|
p = subprocess.Popen(["bash", "-c", script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
procs.append(p)
|
|
|
|
rcs = [p.wait() for p in procs]
|
|
winners = [rc for rc in rcs if rc == 0]
|
|
assert len(winners) == 1
|
|
|
|
|
|
def test_o2_17_staggered_stale_recovery(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
marker.write_text("pid=999999\nlstart=Sat Jan 1 00:00:00 2000\n")
|
|
|
|
procs = []
|
|
for i in range(5):
|
|
script = f"""
|
|
sleep 0.05
|
|
source '{LOCK_SCRIPT}'
|
|
if mam_acquire_loop_lock '{marker}'; then
|
|
sleep 2
|
|
exit 0
|
|
else
|
|
exit 3
|
|
fi
|
|
"""
|
|
p = subprocess.Popen(["bash", "-c", script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
procs.append(p)
|
|
|
|
rcs = [p.wait() for p in procs]
|
|
winners = [rc for rc in rcs if rc == 0]
|
|
assert len(winners) == 1
|
|
|
|
|
|
def test_o2_18_orphan_steal_lock_recovered(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
marker.write_text("pid=999999\nlstart=Sat Jan 1 00:00:00 2000\n")
|
|
|
|
steal_lock = tmp_path / "loop-guard-active.steal"
|
|
steal_lock.write_text("pid=999998\nlstart=Sat Jan 1 00:00:00 2000\n")
|
|
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
assert marker.is_file()
|
|
assert f"pid={proc.pid}" in marker.read_text()
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_19_active_steal_in_progress_not_displaced(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
marker.write_text("pid=999999\nlstart=Sat Jan 1 00:00:00 2000\n")
|
|
steal_lock = tmp_path / "loop-guard-active.steal"
|
|
|
|
proc_steal = acquire_bg(steal_lock)
|
|
try:
|
|
cmd = f"source '{LOCK_SCRIPT}' && mam_acquire_loop_lock '{marker}'"
|
|
res = subprocess.run(["bash", "-c", cmd], capture_output=True, text=True)
|
|
assert res.returncode == 3
|
|
finally:
|
|
proc_steal.kill()
|
|
proc_steal.wait()
|
|
|
|
|
|
def test_o2_20_contentless_steal_lock_reclaimed(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
marker.write_text("pid=999999\nlstart=Sat Jan 1 00:00:00 2000\n")
|
|
|
|
steal_lock = tmp_path / "loop-guard-active.steal"
|
|
steal_lock.write_text("")
|
|
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
assert marker.is_file()
|
|
assert f"pid={proc.pid}" in marker.read_text()
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_21_legacy_steal_directory_cleaned(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
steal_dir = tmp_path / "loop-guard-active.steal"
|
|
steal_dir.mkdir()
|
|
|
|
proc = acquire_bg(marker)
|
|
try:
|
|
assert marker.is_file()
|
|
assert not steal_dir.exists()
|
|
finally:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def test_o2_22_repeated_invocations_converge(tmp_path):
|
|
marker = tmp_path / "loop-guard-active"
|
|
for i in range(5):
|
|
cmd = f"source '{LOCK_SCRIPT}' && mam_acquire_loop_lock '{marker}' && mam_release_loop_lock '{marker}'"
|
|
res = subprocess.run(["bash", "-c", cmd], capture_output=True, text=True)
|
|
assert res.returncode == 0
|
|
assert not marker.exists()
|