feat: multi-agent-mux-update skill and in-place install (Issue #72)
The update is a skill (decision 4); deploy/update.sh and the installed .mam_deploy/update.sh are gone. - Phase 1 (installed code, update.sh): publish the update marker under the exclusive admission lock; refuse before any change while a running row, a live herdr agent in the workspace on any running server, an open job (unreadable JSON included, dry_run not), a live loop marker or a monitor.lock held by a monitor the skills did not start (U3) is there - the calling agent (server + pane, its row and one running job) excepted (lib_py/update.py precheck, decisions 1 and 7). Stop the monitor (TERM, then KILL and restart it if it does not end; decision 2), handle a leftover backup by its state (U4), back up .agents, .mam_deploy, root files, .venv, the installer's .mam files and the registry (SQLite backup API under the registry lock) into .mam/update-backup/, then exec a copy of install_phase.sh outside the tree (decision 5). - The copy (no lib.sh/lib_py, bash 3.2) runs the installer in place; on failure it restores the backup and restarts the monitor from the restored tree; on success it runs phase 2, post_update.sh of the new tree, which converts the registry with lib_py.registry_format and restarts the monitor if it ran. The backup and the marker are removed at the end. - install.sh updates in place (decisions 3, 4, 9): install_state keys are set, one line each; no updater is installed and an old one (.mam_deploy/update.sh, or a root update.sh the manifest lists) is removed; framework files the new version no longer ships are deleted when unmodified, kept and listed when modified, and dropped from base/, the manifest and asset_hashes.txt; a local source's .mam is not staged. remove.sh knows the new skill; CI shellchecks its scripts; docs point at the skill. - test_issue8_update_state and test_deploy_layout td6/td27 run the skill. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
GodopuandClaude Opus 5.5 committed
1 parent
6359d65999
commit
47fe24e07c
19 files changed
+1820
-314
No files matched your search
@@ -463,6 +463,20 @@
|
||||
"evidence": [],
|
||||
"mismatch": null,
|
||||
"note": "post-stop checks; no script offers them as a command - stage 3"
|
||||
},
|
||||
{
|
||||
"skill": "multi-agent-mux-update",
|
||||
"start": 17,
|
||||
"end": 19,
|
||||
"lang": "bash",
|
||||
"sha": "63f73ade58d3",
|
||||
"class": "c",
|
||||
"kind": "invocation",
|
||||
"evidence": [
|
||||
"multi-agent-mux-update/scripts/update.sh:3"
|
||||
],
|
||||
"mismatch": null,
|
||||
"note": "entry point call (Issue #72)"
|
||||
}
|
||||
],
|
||||
"removed": [
|
||||
|
||||
@@ -48,7 +48,7 @@ class TestDeployLayout(unittest.TestCase):
|
||||
self.assertTrue(os.path.exists(os.path.join(self.test_dir, ".agents", "skills", "lib.sh")))
|
||||
|
||||
def test_td6_td7_td8_mam_deploy_layout_and_removal(self):
|
||||
"""T-D6~D8: Verify remove.sh/update.sh installed in .mam_deploy and work properly."""
|
||||
"""T-D6~D8: Verify remove.sh installed in .mam_deploy (update.sh not, Issue #72) and works properly."""
|
||||
subprocess.run(
|
||||
["bash", os.path.join(self.repo_root, "deploy/install.sh"), self.test_dir],
|
||||
env=self.env,
|
||||
@@ -59,7 +59,7 @@ class TestDeployLayout(unittest.TestCase):
|
||||
remover = os.path.join(self.test_dir, ".mam_deploy", "remove.sh")
|
||||
updater = os.path.join(self.test_dir, ".mam_deploy", "update.sh")
|
||||
self.assertTrue(os.path.exists(remover))
|
||||
self.assertTrue(os.path.exists(updater))
|
||||
self.assertFalse(os.path.exists(updater))
|
||||
self.assertTrue(os.access(remover, os.X_OK))
|
||||
|
||||
# T-D8: Execution from inside .mam_deploy directory targets parent workspace
|
||||
@@ -129,17 +129,29 @@ class TestDeployLayout(unittest.TestCase):
|
||||
self.assertTrue(os.path.exists(backups_dir))
|
||||
|
||||
def test_td27_td28_update_preserves_mam_state(self):
|
||||
"""T-D27~D28: Verify update.sh preserves .mam state files across cycles (B-5 gate)."""
|
||||
"""T-D27~D28: Verify the update skill preserves .mam state files across cycles (B-5 gate)."""
|
||||
subprocess.run(
|
||||
["bash", os.path.join(self.repo_root, "deploy/install.sh"), self.test_dir],
|
||||
env=self.env,
|
||||
check=True
|
||||
)
|
||||
|
||||
# Run update.sh
|
||||
updater = os.path.join(self.test_dir, ".mam_deploy", "update.sh")
|
||||
res = subprocess.run(["bash", updater, "--force"], cwd=self.test_dir, env=self.env, capture_output=True, text=True)
|
||||
self.assertEqual(res.returncode, 0, f"update.sh failed: {res.stderr}")
|
||||
# Run the update skill (Issue #72: .mam_deploy/update.sh is gone)
|
||||
updater = os.path.join(self.test_dir, ".agents", "skills", "multi-agent-mux-update", "scripts", "update.sh")
|
||||
env = {k: v for k, v in self.env.items() if not k.startswith("HERDR_")
|
||||
and k not in ("WORKSPACE_ROOT", "AGENT_SESSIONS_YAML", "MAM_JOBS_DIR")}
|
||||
env["MAM_NO_SUBSCRIBER_DAEMON"] = "1"
|
||||
# The update lists the live herdr agents; never ask a real server here.
|
||||
fake = os.path.join(self.temp_dir, "fake-herdr-bin")
|
||||
os.makedirs(fake)
|
||||
with open(os.path.join(fake, "herdr"), "w") as f:
|
||||
f.write('#!/bin/sh\necho \'{"sessions": []}\'\n')
|
||||
os.chmod(os.path.join(fake, "herdr"), 0o755)
|
||||
env["PATH"] = fake + ":" + env["PATH"]
|
||||
res = subprocess.run(["bash", updater, "--yes"], cwd=self.test_dir, env=env, capture_output=True, text=True,
|
||||
stdin=subprocess.DEVNULL)
|
||||
self.assertEqual(res.returncode, 0, f"the update skill failed: {res.stdout}{res.stderr}")
|
||||
self.assertFalse(os.path.exists(os.path.join(self.test_dir, ".mam_deploy", "update.sh")))
|
||||
|
||||
# Verify asset_hashes.txt and version.txt still exist
|
||||
self.assertTrue(os.path.exists(os.path.join(self.test_dir, ".mam", "asset_hashes.txt")))
|
||||
|
||||
@@ -115,8 +115,8 @@ def test_inventory_matches_skill_md(skill):
|
||||
f"actual (start, end, lang, sha): {json.dumps(actual)}")
|
||||
|
||||
|
||||
def test_inventory_covers_eight_skills():
|
||||
assert len(SKILL_NAMES) == 8
|
||||
def test_inventory_covers_nine_skills():
|
||||
assert len(SKILL_NAMES) == 9 # Issue #72: + multi-agent-mux-update
|
||||
assert {b["skill"] for b in DATA["blocks"]} == set(SKILL_NAMES)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Issue #72 — the installer side of the multi-agent-mux-update skill.
|
||||
|
||||
The update skill runs the new install.sh in place over an existing install
|
||||
(no remove.sh, .mam kept). These pin what that in-place run must do:
|
||||
install_state keeps one line per key, no updater is shipped any more and an
|
||||
old one is cleaned up, and framework files the new version no longer ships
|
||||
are deleted unless the user modified them.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
# Extra framework-owned files only the "old" source ships.
|
||||
RETIRED_SKILL = ".agents/skills/multi-agent-mux-retired72"
|
||||
RETIRED_UNMODIFIED = [RETIRED_SKILL + "/SKILL.md", RETIRED_SKILL + "/scripts/old.sh"]
|
||||
RETIRED_SHARED_DIR = ".agents/skills/multi-agent-mux-retired72b"
|
||||
RETIRED_SHARED = RETIRED_SHARED_DIR + "/SKILL.md"
|
||||
RETIRED_MODIFIED = ".agents/hooks/retired72_hook.sh"
|
||||
|
||||
|
||||
def _copy_tree(dest):
|
||||
shutil.copytree(REPO_ROOT, dest,
|
||||
ignore=shutil.ignore_patterns(".git", ".venv", ".mam", ".mam_deploy",
|
||||
"__pycache__", ".pytest_cache"),
|
||||
symlinks=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def sources():
|
||||
"""(new, old): the tree as is, and the same tree plus files it no longer ships."""
|
||||
tmp = tempfile.mkdtemp(prefix="mam_issue72_src_")
|
||||
new, old = os.path.join(tmp, "new"), os.path.join(tmp, "old")
|
||||
_copy_tree(new)
|
||||
_copy_tree(old)
|
||||
for rel in RETIRED_UNMODIFIED + [RETIRED_SHARED, RETIRED_MODIFIED]:
|
||||
path = os.path.join(old, rel)
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
f.write("# shipped by the old version only: %s\n" % rel)
|
||||
try:
|
||||
yield new, old
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ws():
|
||||
tmp = tempfile.mkdtemp(prefix="mam_issue72_ws_")
|
||||
try:
|
||||
yield os.path.join(tmp, "workspace")
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def _install(src, ws):
|
||||
env = dict(os.environ)
|
||||
env.update({"MAM_REPO_URL": src, "MAM_SKIP_VENV": "1"})
|
||||
res = subprocess.run(["bash", os.path.join(src, "deploy", "install.sh"), ws],
|
||||
env=env, capture_output=True, text=True, timeout=600)
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
return res
|
||||
|
||||
|
||||
def _remove(ws):
|
||||
res = subprocess.run(["bash", os.path.join(ws, ".mam_deploy", "remove.sh"), "--force"],
|
||||
cwd=ws, capture_output=True, text=True, timeout=600)
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
return res
|
||||
|
||||
|
||||
def _read(ws, rel):
|
||||
with open(os.path.join(ws, rel)) as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def _manifest(ws):
|
||||
return _read(ws, ".mam/install_manifest.txt").splitlines()
|
||||
|
||||
|
||||
def _hashed_paths(ws):
|
||||
return [line.split(None, 1)[1] for line in _read(ws, ".mam/asset_hashes.txt").splitlines()
|
||||
if line.strip()]
|
||||
|
||||
|
||||
def test_in_place_reinstall_keeps_one_line_per_install_state_key(sources, ws):
|
||||
new, _ = sources
|
||||
_install(new, ws)
|
||||
with open(os.path.join(ws, ".mam", "install_state"), "w") as f:
|
||||
f.write("first=a\ngitignore_created=1\nlast=b\n")
|
||||
|
||||
_install(new, ws)
|
||||
|
||||
assert _read(ws, ".mam/install_state").splitlines() == [
|
||||
"first=a", "gitignore_created=1", "last=b"]
|
||||
# remove.sh must read the key as 1: the .gitignore it created goes away.
|
||||
_remove(ws)
|
||||
assert not os.path.exists(os.path.join(ws, ".gitignore"))
|
||||
|
||||
|
||||
def test_fresh_install_ships_no_updater(sources, ws):
|
||||
new, _ = sources
|
||||
res = _install(new, ws)
|
||||
assert not os.path.exists(os.path.join(ws, ".mam_deploy", "update.sh"))
|
||||
manifest = _manifest(ws)
|
||||
assert ".mam_deploy/update.sh" not in manifest and "update.sh" not in manifest
|
||||
assert "bash .mam_deploy/update.sh" not in res.stdout
|
||||
assert "multi-agent-mux-update/scripts/update.sh" in res.stdout
|
||||
|
||||
|
||||
def test_in_place_reinstall_deletes_an_old_mam_deploy_updater(sources, ws):
|
||||
new, _ = sources
|
||||
_install(new, ws)
|
||||
with open(os.path.join(ws, ".mam_deploy", "update.sh"), "w") as f:
|
||||
f.write("#!/usr/bin/env bash\n# old updater\n")
|
||||
with open(os.path.join(ws, ".mam", "install_manifest.txt"), "a") as f:
|
||||
f.write(".mam_deploy/update.sh\n")
|
||||
|
||||
_install(new, ws)
|
||||
|
||||
assert not os.path.exists(os.path.join(ws, ".mam_deploy", "update.sh"))
|
||||
assert ".mam_deploy/update.sh" not in _manifest(ws)
|
||||
|
||||
|
||||
def test_legacy_root_updater_goes_only_when_the_manifest_lists_it(sources, ws):
|
||||
new, _ = sources
|
||||
_install(new, ws)
|
||||
root_updater = os.path.join(ws, "update.sh")
|
||||
|
||||
# Listed: the installer put it there (very old layout).
|
||||
with open(root_updater, "w") as f:
|
||||
f.write("# legacy updater\n")
|
||||
with open(os.path.join(ws, ".mam", "install_manifest.txt"), "a") as f:
|
||||
f.write("update.sh\n")
|
||||
_install(new, ws)
|
||||
assert not os.path.exists(root_updater)
|
||||
assert not os.path.exists(os.path.join(ws, ".mam_deploy", "update.sh"))
|
||||
manifest = _manifest(ws)
|
||||
assert "update.sh" not in manifest and ".mam_deploy/update.sh" not in manifest
|
||||
|
||||
# Not listed: the user's own file stays untouched.
|
||||
with open(root_updater, "w") as f:
|
||||
f.write("# user's own script\n")
|
||||
_install(new, ws)
|
||||
assert _read(ws, "update.sh") == "# user's own script\n"
|
||||
|
||||
|
||||
def test_files_the_new_version_no_longer_ships(sources, ws):
|
||||
new, old = sources
|
||||
_install(old, ws)
|
||||
for rel in RETIRED_UNMODIFIED + [RETIRED_SHARED, RETIRED_MODIFIED]:
|
||||
assert rel in _manifest(ws) and rel in _hashed_paths(ws), rel
|
||||
# The user edits one retired file and adds their own file next to another.
|
||||
with open(os.path.join(ws, RETIRED_MODIFIED), "a") as f:
|
||||
f.write("# my change\n")
|
||||
with open(os.path.join(ws, RETIRED_SHARED_DIR, "notes.txt"), "w") as f:
|
||||
f.write("mine\n")
|
||||
# A base/ copy of a retired file must go with it.
|
||||
base_copy = os.path.join(ws, ".mam", "base", RETIRED_UNMODIFIED[0])
|
||||
os.makedirs(os.path.dirname(base_copy))
|
||||
shutil.copy(os.path.join(ws, RETIRED_UNMODIFIED[0]), base_copy)
|
||||
|
||||
res = _install(new, ws)
|
||||
|
||||
for rel in RETIRED_UNMODIFIED + [RETIRED_SHARED]:
|
||||
assert not os.path.exists(os.path.join(ws, rel)), rel
|
||||
assert not os.path.exists(os.path.join(ws, RETIRED_SKILL)), "emptied directory left behind"
|
||||
assert _read(ws, RETIRED_SHARED_DIR + "/notes.txt") == "mine\n"
|
||||
assert "# my change" in _read(ws, RETIRED_MODIFIED)
|
||||
assert "Local modification detected" in res.stderr and RETIRED_MODIFIED in res.stderr
|
||||
for rel in RETIRED_UNMODIFIED:
|
||||
assert rel not in res.stderr, rel
|
||||
assert not os.path.exists(base_copy)
|
||||
manifest, hashed = _manifest(ws), _hashed_paths(ws)
|
||||
for rel in RETIRED_UNMODIFIED + [RETIRED_SHARED, RETIRED_MODIFIED]:
|
||||
assert rel not in manifest and rel not in hashed, rel
|
||||
# Still-shipped framework files are untouched by the cleanup.
|
||||
assert ".agents/skills/lib.sh" in manifest and ".agents/skills/lib.sh" in hashed
|
||||
|
||||
# The kept file is the user's now: uninstalling leaves it in place.
|
||||
_remove(ws)
|
||||
assert "# my change" in _read(ws, RETIRED_MODIFIED)
|
||||
|
||||
|
||||
@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0,
|
||||
reason="root reads a mode-000 file")
|
||||
def test_local_source_update_backup_is_not_staged(sources, ws):
|
||||
new, _ = sources
|
||||
backup = os.path.join(new, ".mam", "update-backup")
|
||||
os.makedirs(backup)
|
||||
unreadable = os.path.join(backup, "unreadable")
|
||||
with open(unreadable, "w") as f:
|
||||
f.write("x")
|
||||
os.chmod(unreadable, 0)
|
||||
try:
|
||||
_install(new, ws)
|
||||
finally:
|
||||
os.chmod(unreadable, 0o600)
|
||||
shutil.rmtree(os.path.join(new, ".mam"))
|
||||
@@ -0,0 +1,710 @@
|
||||
"""Issue #72 - the multi-agent-mux-update skill.
|
||||
|
||||
Every test updates a copy of one workspace installed from this checkout
|
||||
(MAM_REPO_URL, MAM_SKIP_VENV=1). herdr is a fake that answers `session list
|
||||
--json` and `--session S agent list` from a JSON spec, so the pre-check's view
|
||||
of the live agents is set per test; the monitor daemon is a stub reconcile.sh
|
||||
that holds .mam/monitor.lock like the real one. Nothing here touches a real
|
||||
herdr server or the control root's monitor.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
UPDATE_REL = ".agents/skills/multi-agent-mux-update/scripts/update.sh"
|
||||
MONITOR_REL = ".agents/skills/multi-agent-mux-monitor/scripts/reconcile.sh"
|
||||
|
||||
FAKE_HERDR = r'''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
spec = json.load(open(os.environ["FAKE_HERDR_SPEC"]))
|
||||
a = sys.argv[1:]
|
||||
server = "default"
|
||||
if a[:1] == ["--session"]:
|
||||
server, a = a[1], a[2:]
|
||||
with open(os.environ["FAKE_HERDR_SPEC"] + ".calls", "a") as f:
|
||||
f.write(json.dumps([server] + a) + "\n")
|
||||
if a[:2] == ["session", "list"]:
|
||||
if spec.get("fail") == "session-list":
|
||||
sys.exit(1)
|
||||
print(json.dumps({"sessions": [{"name": n, "running": r} for n, r in spec.get("sessions", {"default": True}).items()]}))
|
||||
elif a[:2] == ["agent", "list"]:
|
||||
if spec.get("fail") == "agent-list:" + server:
|
||||
sys.exit(1)
|
||||
print(json.dumps({"result": {"agents": spec.get("agents", {}).get(server, [])}}))
|
||||
'''
|
||||
|
||||
STUB_MONITOR = r'''#!/usr/bin/env bash
|
||||
# Issue #72 test stub of reconcile.sh: a wrapper whose python child holds monitor.lock.
|
||||
d="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd -P)"
|
||||
echo "start $$ $*" >> "$d/.mam/stub-monitor.log"
|
||||
python3 -c '
|
||||
import fcntl, os, signal, sys, time
|
||||
if os.environ.get("STUB_MONITOR_IGNORE_TERM") == "1":
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
f = open(sys.argv[1], "w")
|
||||
fcntl.flock(f, fcntl.LOCK_EX)
|
||||
time.sleep(600)
|
||||
' "$d/.mam/monitor.lock" &
|
||||
child=$!
|
||||
if [ "${STUB_MONITOR_IGNORE_TERM:-0}" = "1" ]; then
|
||||
trap '' TERM
|
||||
else
|
||||
trap 'kill $child 2>/dev/null; exit 143' TERM
|
||||
fi
|
||||
while kill -0 "$child" 2>/dev/null; do wait "$child"; done
|
||||
'''
|
||||
|
||||
|
||||
def _clean_env():
|
||||
env = {k: v for k, v in os.environ.items()
|
||||
if not k.startswith(("HERDR_", "XDG_", "MAM_", "STUB_"))
|
||||
and k not in ("WORKSPACE_ROOT", "AGENT_SESSIONS_YAML", "DELEGATE_JOB_LOGS_DIR", "PYTHONPATH")}
|
||||
env.update(MAM_REPO_URL=str(REPO_ROOT), MAM_SKIP_VENV="1",
|
||||
MAM_INSTALLER_URL="file://" + str(REPO_ROOT / "deploy" / "install.sh"),
|
||||
MAM_ENV_FILE="/nonexistent/.mam.env", PYTHONDONTWRITEBYTECODE="1")
|
||||
return env
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def template(tmp_path_factory):
|
||||
ws = tmp_path_factory.mktemp("tpl72") / "ws"
|
||||
ws.mkdir()
|
||||
res = subprocess.run(["bash", str(REPO_ROOT / "deploy" / "install.sh"), str(ws)], env=_clean_env(),
|
||||
capture_output=True, text=True, timeout=600)
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
return ws
|
||||
|
||||
|
||||
class Box:
|
||||
def __init__(self, root: Path, ws: Path):
|
||||
self.root, self.ws = root, ws
|
||||
self.mam = ws / ".mam"
|
||||
self.bin = root / "bin"
|
||||
self.spec = root / "herdr.json"
|
||||
self.env = _clean_env()
|
||||
self.env["PATH"] = "%s:%s" % (self.bin, os.environ["PATH"])
|
||||
self.env["FAKE_HERDR_SPEC"] = str(self.spec)
|
||||
self.helpers = []
|
||||
self.herdr()
|
||||
|
||||
def herdr(self, sessions=None, agents=None, fail=None):
|
||||
self.spec.write_text(json.dumps({"sessions": sessions or {"default": True},
|
||||
"agents": agents or {}, "fail": fail}))
|
||||
|
||||
def update(self, *args, env=None, timeout=300):
|
||||
e = dict(self.env, **(env or {}))
|
||||
return subprocess.run(["bash", str(self.ws / UPDATE_REL), "--yes", *args], cwd=str(self.ws), env=e,
|
||||
stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
def lib(self, script, env=None, timeout=120):
|
||||
e = dict(self.env, WORKSPACE_ROOT=str(self.ws), **(env or {}))
|
||||
return subprocess.run(["bash", "-c", 'source "$1/.agents/skills/lib.sh"\n' + script, "_", str(self.ws)],
|
||||
env=e, capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
def installer(self, body):
|
||||
p = self.root / "installer.sh"
|
||||
p.write_text("#!/usr/bin/env bash\nws=\"$1\"\n" + body)
|
||||
self.env["MAM_INSTALLER_URL"] = "file://" + str(p)
|
||||
|
||||
def helper(self, *argv, **kw):
|
||||
p = subprocess.Popen(list(argv), start_new_session=True, **kw)
|
||||
self.helpers.append(p)
|
||||
return p
|
||||
|
||||
def stub_monitor(self):
|
||||
p = self.ws / MONITOR_REL
|
||||
p.write_text(STUB_MONITOR)
|
||||
|
||||
def start_monitor(self, env=None):
|
||||
"""The way create and resume start it: lib.sh start_watchdog (detached, so
|
||||
not a child of this test that would linger as a zombie)."""
|
||||
res = self.lib('start_watchdog test "$WORKSPACE_ROOT"', env=dict(env or {}))
|
||||
assert res.returncode == 0 and res.stdout.strip(), res.stdout + res.stderr
|
||||
_until(lambda: _lock_held(self.mam / "monitor.lock"))
|
||||
|
||||
def monitor_pids(self):
|
||||
r = subprocess.run(["pgrep", "-f", "bash %s --subscribe" % (self.ws / MONITOR_REL)],
|
||||
capture_output=True, text=True)
|
||||
return r.stdout.split()
|
||||
|
||||
def monitor_starts(self):
|
||||
p = self.mam / "stub-monitor.log"
|
||||
return p.read_text().count("start ") if p.exists() else 0
|
||||
|
||||
|
||||
def _until(cond, timeout=15.0):
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
if cond():
|
||||
return True
|
||||
time.sleep(0.1)
|
||||
raise AssertionError("condition not met within %ss" % timeout)
|
||||
|
||||
|
||||
def _lock_held(path: Path) -> bool:
|
||||
if not path.exists():
|
||||
return False
|
||||
with open(path, "a") as fh:
|
||||
try:
|
||||
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
fcntl.flock(fh, fcntl.LOCK_UN)
|
||||
return False
|
||||
except BlockingIOError:
|
||||
return True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def box(template, tmp_path):
|
||||
root = Path(os.path.realpath(tmp_path))
|
||||
ws = root / "ws"
|
||||
shutil.copytree(template, ws, symlinks=True)
|
||||
(root / "bin").mkdir()
|
||||
herdr = root / "bin" / "herdr"
|
||||
herdr.write_text(FAKE_HERDR)
|
||||
herdr.chmod(0o755)
|
||||
b = Box(root, ws)
|
||||
try:
|
||||
yield b
|
||||
finally:
|
||||
for p in b.helpers:
|
||||
try:
|
||||
os.killpg(p.pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
subprocess.run(["pkill", "-KILL", "-f", str(ws)], capture_output=True)
|
||||
|
||||
|
||||
# --- helpers ---------------------------------------------------------------------
|
||||
|
||||
SKIP_SNAPSHOT = ("shim/", "update-admission.lock", "agent-sessions.db.lock", "monitor.lock", "stub-monitor.log")
|
||||
|
||||
|
||||
def snapshot(ws: Path) -> dict:
|
||||
out = {}
|
||||
for dp, dns, fns in os.walk(ws):
|
||||
dns[:] = [d for d in dns if d != "__pycache__"]
|
||||
for f in fns:
|
||||
p = Path(dp) / f
|
||||
rel = str(p.relative_to(ws))
|
||||
if rel.startswith(".mam/") and any(s in rel for s in SKIP_SNAPSHOT):
|
||||
continue
|
||||
out[rel] = "L:" + os.readlink(p) if p.is_symlink() else hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
return out
|
||||
|
||||
|
||||
def write_rows(ws: Path, rows, **top):
|
||||
d = dict(top, herdr_sessions=rows)
|
||||
(ws / ".mam" / "agent-sessions.yaml").write_text(yaml.safe_dump(d, sort_keys=False))
|
||||
|
||||
|
||||
def row(name, status="running", server="default", cwd=None, **extra):
|
||||
r = {"name": name, "status": status, "role": "creator", "herdr_session": server, "herdr_server": server,
|
||||
"pane": {"cwd": cwd or "/elsewhere", "cmd": "claude"}}
|
||||
r.update(extra)
|
||||
return r
|
||||
|
||||
|
||||
def write_job(d: Path, jid, status="running", session="herdr:x", **extra):
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
rec = dict({"job_id": jid, "status": status, "agent_session": session}, **extra)
|
||||
(d / (jid + ".json")).write_text(json.dumps(rec))
|
||||
|
||||
|
||||
def live_marker(path: Path, box: Box):
|
||||
"""A marker owned by a live process (pid + lstart as loop_lock.sh writes them)."""
|
||||
p = box.helper("sleep", "120")
|
||||
ls = subprocess.run(["ps", "-p", str(p.pid), "-o", "lstart="], capture_output=True, text=True).stdout
|
||||
ls = " ".join(ls.split())
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("pid=%d\nlstart=%s\nstarted=x\n" % (p.pid, ls))
|
||||
return p
|
||||
|
||||
|
||||
def dead_marker(path: Path):
|
||||
p = subprocess.Popen(["true"])
|
||||
p.wait()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("pid=%d\nlstart=Thu Jan 1 00:00:00 1970\nstarted=x\n" % p.pid)
|
||||
|
||||
|
||||
def assert_refused_unchanged(box: Box, before: dict, res, *needles):
|
||||
assert res.returncode == 1, res.stdout + res.stderr
|
||||
assert "update refused; nothing was changed" in res.stderr, res.stderr
|
||||
for n in needles:
|
||||
assert n in res.stderr, (n, res.stderr)
|
||||
assert snapshot(box.ws) == before
|
||||
assert not (box.mam / "update-in-progress").exists(), "the refusing update releases its marker"
|
||||
assert not (box.mam / "update-backup").exists()
|
||||
|
||||
|
||||
# --- success and the #8 state rules ------------------------------------------------
|
||||
|
||||
def test_update_in_place_keeps_mam_and_merges_install_state(box):
|
||||
mam = box.mam
|
||||
(mam / "version.txt").write_text("source=old\ncommit=stale0000\nfetched_at=then\nmethod=git\n")
|
||||
(mam / "herdr_workspace_id").write_text("w9\n")
|
||||
for d in ("briefs", "diagnostics", "resumelogs", "stoplogs", "jobs", "delegate_job_logs/j1"):
|
||||
(mam / d).mkdir(parents=True, exist_ok=True)
|
||||
(mam / d / "keep.txt").write_text(d)
|
||||
write_job(mam / "jobs", "done1", status="completed")
|
||||
(mam / "multi-agent-mux-monitor.log").write_text("log\n")
|
||||
(box.ws / ".mam.env").write_text("MQTT_BROKER=example\n")
|
||||
with open(mam / "install_state", "a") as fh:
|
||||
fh.write("custom_marker=kept\n")
|
||||
write_rows(box.ws, [row("a", status="stopped")], orchestrator_uuids=["u1"])
|
||||
keep = {k: v for k, v in snapshot(box.ws).items()
|
||||
if k.startswith((".mam/herdr_workspace_id", ".mam/briefs", ".mam/diagnostics", ".mam/resumelogs",
|
||||
".mam/stoplogs", ".mam/jobs", ".mam/delegate_job_logs", ".mam/multi-agent-mux-monitor.log"))
|
||||
or k == ".mam.env"}
|
||||
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
after = snapshot(box.ws)
|
||||
assert {k: after.get(k) for k in keep} == keep, "everything that is not the installer's is kept"
|
||||
assert "stale0000" not in (mam / "version.txt").read_text()
|
||||
state = (mam / "install_state").read_text().splitlines()
|
||||
assert "custom_marker=kept" in state and state.count("gitignore_created=1") == 1, state
|
||||
assert not (box.ws / ".mam_deploy" / "update.sh").exists()
|
||||
assert not (mam / "update-backup").exists() and not (mam / "update-in-progress").exists()
|
||||
assert "Update complete." in res.stdout
|
||||
|
||||
|
||||
def test_old_format_registry_is_converted_by_the_new_code(box):
|
||||
old = row("old-one", status="stopped", start_command="HERDR_SESSION_NAME=x herdr new-session",
|
||||
kill_command="HERDR_SESSION_NAME=x herdr kill-session", attach_command="HERDR_SESSION_NAME=x herdr agent attach old-one")
|
||||
write_rows(box.ws, [old])
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert "registry-format: 1 -> 2, rows changed: 1" in res.stdout
|
||||
y = yaml.safe_load((box.mam / "agent-sessions.yaml").read_text())
|
||||
assert y["registry_format"] == 2
|
||||
conn = sqlite3.connect(str(box.mam / "agent-sessions.db"))
|
||||
try:
|
||||
db = {r["name"]: r for r in (json.loads(x[0]) for x in conn.execute("SELECT data FROM sessions"))}
|
||||
st = json.loads(conn.execute("SELECT data FROM state WHERE id=1").fetchone()[0])
|
||||
finally:
|
||||
conn.close()
|
||||
assert {r["name"]: r for r in y["herdr_sessions"]} == db and st["registry_format"] == 2
|
||||
assert "kill_command" not in db["old-one"] and "start_command" not in db["old-one"]
|
||||
|
||||
|
||||
def test_no_registry_is_nothing_to_convert(box):
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert "registry: nothing to convert (no registry)" in res.stdout
|
||||
assert not (box.mam / "agent-sessions.yaml").exists()
|
||||
|
||||
|
||||
def test_a_failed_conversion_keeps_the_install_and_exits_nonzero(box, tmp_path):
|
||||
src = tmp_path / "src"
|
||||
shutil.copytree(REPO_ROOT, src, symlinks=True,
|
||||
ignore=shutil.ignore_patterns(".git", ".venv", ".mam", "__pycache__", "tests", "nats-docker"))
|
||||
fmt = src / ".agents" / "skills" / "lib_py" / "registry_format.py"
|
||||
fmt.write_text("import sys\nif __name__ == '__main__':\n sys.exit(3)\n" + fmt.read_text())
|
||||
box.env["MAM_REPO_URL"] = str(src)
|
||||
box.stub_monitor()
|
||||
box.start_monitor()
|
||||
write_rows(box.ws, [row("a", status="stopped")])
|
||||
res = box.update()
|
||||
assert res.returncode == 3, res.stdout + res.stderr
|
||||
assert "the registry was not converted" in res.stderr
|
||||
assert (box.ws / ".agents/skills/lib_py/registry_format.py").read_text() == fmt.read_text(), "install kept"
|
||||
assert box.monitor_pids() and box.monitor_starts() == 2, "the monitor is started again anyway"
|
||||
assert not (box.mam / "update-backup").exists() and not (box.mam / "update-in-progress").exists()
|
||||
|
||||
|
||||
def test_refuses_outside_an_install_and_without_yes(box):
|
||||
(box.ws / ".mam_deploy" / "remove.sh").unlink()
|
||||
res = box.update()
|
||||
assert res.returncode == 1 and "not an installed MAM workspace" in res.stderr
|
||||
shutil.copy(REPO_ROOT / "deploy" / "remove.sh", box.ws / ".mam_deploy" / "remove.sh")
|
||||
res = subprocess.run(["bash", str(box.ws / UPDATE_REL)], env=box.env, stdin=subprocess.DEVNULL,
|
||||
capture_output=True, text=True, timeout=60)
|
||||
assert res.returncode == 1 and "-y/--yes" in res.stderr
|
||||
res = box.update("--force")
|
||||
assert res.returncode == 2, "--force is not an alias of --yes"
|
||||
|
||||
|
||||
# --- refusals (decision 1) ---------------------------------------------------------
|
||||
|
||||
def test_refuses_a_running_row(box):
|
||||
write_rows(box.ws, [row("busy-creator", cwd="/somewhere/else")])
|
||||
before = snapshot(box.ws)
|
||||
res = box.update()
|
||||
assert_refused_unchanged(box, before, res, "session busy-creator is running", "stop_session.sh --session busy-creator")
|
||||
|
||||
|
||||
def test_refuses_an_unreadable_registry(box):
|
||||
(box.mam / "agent-sessions.db").write_text("not a database")
|
||||
before = snapshot(box.ws)
|
||||
assert_refused_unchanged(box, before, box.update(), "cannot be read")
|
||||
|
||||
|
||||
def test_refuses_a_rowless_agent_on_any_running_server(box):
|
||||
box.herdr(sessions={"default": True, "picked-by-hand": True, "down": False},
|
||||
agents={"picked-by-hand": [{"name": "stray", "pane_id": "w3:p1", "cwd": str(box.ws / "sub")}],
|
||||
"down": [{"name": "ghost", "pane_id": "w4:p1", "cwd": str(box.ws)}]})
|
||||
before = snapshot(box.ws)
|
||||
res = box.update()
|
||||
assert_refused_unchanged(box, before, res, "herdr agent stray (pane w3:p1, server picked-by-hand)")
|
||||
assert "ghost" not in res.stderr, "a server that is not running is not asked"
|
||||
calls = (box.root / "herdr.json.calls").read_text()
|
||||
assert '"down"' not in calls
|
||||
|
||||
|
||||
def test_an_agent_outside_the_workspace_does_not_refuse(box):
|
||||
box.herdr(agents={"default": [{"name": "other", "pane_id": "w1:p2", "cwd": str(box.root / "elsewhere")}]})
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail", ["session-list", "agent-list:default"])
|
||||
def test_refuses_when_herdr_cannot_answer(box, fail):
|
||||
box.herdr(fail=fail)
|
||||
before = snapshot(box.ws)
|
||||
assert_refused_unchanged(box, before, box.update(), "cannot list the live agents")
|
||||
|
||||
|
||||
def test_open_jobs_refuse_and_terminal_or_dry_run_do_not(box, tmp_path):
|
||||
jobs = box.mam / "jobs"
|
||||
write_job(jobs, "done1", status="completed")
|
||||
write_job(jobs, "err1", status="error")
|
||||
write_job(jobs, "can1", status="cancelled")
|
||||
write_job(jobs, "dry1", status="pending", dry_run=True)
|
||||
assert box.update().returncode == 0
|
||||
|
||||
write_job(jobs, "open1", status="pending", session="remote:x")
|
||||
before = snapshot(box.ws)
|
||||
res = box.update()
|
||||
assert_refused_unchanged(box, before, res, "job open1", "close --job open1 --request cancelled")
|
||||
(jobs / "open1.json").unlink()
|
||||
|
||||
(jobs / "broken.json").write_text("{not json")
|
||||
before = snapshot(box.ws)
|
||||
assert_refused_unchanged(box, before, box.update(), "job record %s cannot be read" % (jobs / "broken.json"))
|
||||
(jobs / "broken.json").unlink()
|
||||
|
||||
extra = tmp_path / "mam-jobs-dir"
|
||||
write_job(extra, "env1", status="running")
|
||||
box.env["MAM_JOBS_DIR"] = str(extra)
|
||||
before = snapshot(box.ws)
|
||||
assert_refused_unchanged(box, before, box.update(), "job env1")
|
||||
del box.env["MAM_JOBS_DIR"]
|
||||
|
||||
|
||||
def test_other_control_planes_count_only_jobs_addressed_to_this_registry(box, tmp_path):
|
||||
other = tmp_path / "other-plane" / "jobs"
|
||||
write_job(other, "theirs", status="running", session="herdr:not-ours")
|
||||
write_rows(box.ws, [row("wt-creator", status="stopped", control_plane={"mam_jobs_dir": str(other)})])
|
||||
assert box.update().returncode == 0, "a job of another registry's session is not ours"
|
||||
write_job(other, "ours", status="running", session="herdr:wt-creator")
|
||||
before = snapshot(box.ws)
|
||||
assert_refused_unchanged(box, before, box.update(), "job ours")
|
||||
|
||||
|
||||
def test_loop_marker_live_refuses_stale_does_not(box):
|
||||
live_marker(box.mam / "loop-guard-active", box)
|
||||
before = snapshot(box.ws)
|
||||
assert_refused_unchanged(box, before, box.update(), "a review loop is running")
|
||||
dead_marker(box.mam / "loop-guard-active")
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert "stale loop marker" in res.stdout
|
||||
|
||||
|
||||
# --- the calling agent (decision 7, N2) ----------------------------------------------
|
||||
|
||||
def _me_env(server="mam-x", pane="w5:p1"):
|
||||
return {"HERDR_SESSION": server, "HERDR_PANE_ID": pane}
|
||||
|
||||
|
||||
def test_the_calling_agent_its_row_and_its_running_job_are_excluded(box):
|
||||
me = {"name": "me-creator", "pane_id": "w5:p1", "cwd": str(box.ws)}
|
||||
box.herdr(sessions={"default": True, "mam-x": True}, agents={"mam-x": [me]})
|
||||
write_rows(box.ws, [row("me-creator", server="mam-x", cwd=str(box.ws))])
|
||||
write_job(box.mam / "jobs", "myjob", status="running", session="herdr:me-creator")
|
||||
res = box.update(env=_me_env())
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
|
||||
write_job(box.mam / "jobs", "myjob2", status="pending", session="herdr:me-creator")
|
||||
before = snapshot(box.ws)
|
||||
assert_refused_unchanged(box, before, box.update(env=_me_env()), "job myjob2")
|
||||
|
||||
|
||||
def test_same_pane_id_on_another_server_is_not_excluded(box):
|
||||
box.herdr(sessions={"default": True, "mam-x": True, "mam-y": True},
|
||||
agents={"mam-x": [{"name": "me", "pane_id": "w5:p1", "cwd": str(box.ws)}],
|
||||
"mam-y": [{"name": "twin", "pane_id": "w5:p1", "cwd": str(box.ws)}]})
|
||||
before = snapshot(box.ws)
|
||||
res = box.update(env=_me_env())
|
||||
assert_refused_unchanged(box, before, res, "herdr agent twin (pane w5:p1, server mam-y)")
|
||||
assert "agent me " not in res.stderr
|
||||
|
||||
|
||||
def test_a_nameless_caller_pane_is_excluded_by_pane_only(box):
|
||||
box.herdr(sessions={"default": True, "mam-x": True},
|
||||
agents={"mam-x": [{"name": None, "pane_id": "w5:p1", "cwd": str(box.ws)}]})
|
||||
assert box.update(env=_me_env()).returncode == 0
|
||||
write_job(box.mam / "jobs", "nameless", status="running", session="herdr:")
|
||||
write_job(box.mam / "jobs", "nonename", status="running", session="herdr:None")
|
||||
before = snapshot(box.ws)
|
||||
assert_refused_unchanged(box, before, box.update(env=_me_env()), "job nameless", "job nonename")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("env", [{}, {"HERDR_PANE_ID": "w5:p1"}, {"HERDR_SESSION": "mam-x"}])
|
||||
def test_nothing_is_excluded_when_the_caller_is_not_identified(box, env):
|
||||
box.herdr(sessions={"default": True, "mam-x": True},
|
||||
agents={"mam-x": [{"name": "me", "pane_id": "w5:p1", "cwd": str(box.ws)}]})
|
||||
before = snapshot(box.ws)
|
||||
res = box.update(env=env)
|
||||
assert_refused_unchanged(box, before, res, "herdr agent me", "nothing was excluded as the calling agent")
|
||||
|
||||
|
||||
# --- the monitor (decision 2, U3) ----------------------------------------------------
|
||||
|
||||
def test_a_running_monitor_is_stopped_and_started_again(box):
|
||||
box.stub_monitor()
|
||||
box.start_monitor()
|
||||
first = set(box.monitor_pids())
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert "monitor: stopping the daemon" in res.stdout and "monitor: restarted" in res.stdout
|
||||
now = set(box.monitor_pids())
|
||||
assert now and not (now & first) and box.monitor_starts() == 2
|
||||
|
||||
|
||||
def test_no_monitor_before_means_none_after(box):
|
||||
box.stub_monitor()
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert not box.monitor_pids() and box.monitor_starts() == 0
|
||||
|
||||
|
||||
def test_no_subscriber_daemon_flag_wins(box):
|
||||
box.stub_monitor()
|
||||
box.start_monitor()
|
||||
res = box.update(env={"MAM_NO_SUBSCRIBER_DAEMON": "1"})
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert "monitor: not restarted (MAM_NO_SUBSCRIBER_DAEMON=1)" in res.stdout
|
||||
assert not box.monitor_pids()
|
||||
|
||||
|
||||
def test_a_monitor_that_ignores_term_is_killed_restarted_and_nothing_installed(box):
|
||||
box.stub_monitor()
|
||||
box.start_monitor(env={"STUB_MONITOR_IGNORE_TERM": "1"})
|
||||
box.installer("touch \"$ws/INSTALLER-RAN\"\n")
|
||||
before = snapshot(box.ws)
|
||||
res = box.update(env={"MAM_UPDATE_MONITOR_STOP_S": "1"})
|
||||
assert res.returncode == 1, res.stdout + res.stderr
|
||||
assert "it was killed and is started again" in res.stderr
|
||||
assert not (box.ws / "INSTALLER-RAN").exists()
|
||||
assert snapshot(box.ws) == before
|
||||
assert box.monitor_pids() and box.monitor_starts() == 2
|
||||
assert not (box.mam / "update-in-progress").exists()
|
||||
|
||||
|
||||
def test_a_monitor_lock_held_by_an_unknown_process_refuses(box):
|
||||
lock = box.mam / "monitor.lock"
|
||||
with open(lock, "w") as fh:
|
||||
fcntl.flock(fh, fcntl.LOCK_EX)
|
||||
before = snapshot(box.ws)
|
||||
res = box.update()
|
||||
assert_refused_unchanged(box, before, res, "monitor.lock is held by a monitor the skills did not start")
|
||||
|
||||
|
||||
# --- rollback (decisions 3, 10; U4) --------------------------------------------------
|
||||
|
||||
FAILING_INSTALL = """
|
||||
echo broken >> "$ws/.agents/skills/lib.sh"
|
||||
rm -f "$ws/.mam_deploy/remove.sh"
|
||||
echo new > "$ws/MESSAGING.md"
|
||||
mkdir -p "$ws/.venv/lib" && echo junk > "$ws/.venv/lib/junk"
|
||||
echo junk > "$ws/.mam/version.txt"
|
||||
echo junk > "$ws/.mam/asset_hashes.txt"
|
||||
mkdir -p "$ws/.agents/skills/half-installed" && echo x > "$ws/.agents/skills/half-installed/f"
|
||||
python3 -c 'import yaml,sys; p=sys.argv[1]; d=yaml.safe_load(open(p)); d["herdr_sessions"]=[]; yaml.safe_dump(d, open(p, "w"))' "$ws/.mam/agent-sessions.yaml"
|
||||
exit 7
|
||||
"""
|
||||
|
||||
|
||||
def test_a_failed_install_rolls_everything_back(box):
|
||||
write_rows(box.ws, [row("a", status="stopped")])
|
||||
db = box.mam / "agent-sessions.db"
|
||||
conn = sqlite3.connect(str(db))
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA wal_autocheckpoint=0")
|
||||
conn.execute("CREATE TABLE state (id INTEGER PRIMARY KEY, data TEXT)")
|
||||
conn.execute("CREATE TABLE sessions (name TEXT PRIMARY KEY, status TEXT, pane_cwd TEXT, data JSON)")
|
||||
conn.execute("INSERT INTO state VALUES (1, ?)", (json.dumps({"orchestrator_uuids": ["in-wal"]}),))
|
||||
conn.execute("INSERT INTO sessions VALUES ('a', 'stopped', '/x', ?)", (json.dumps(row("a", status="stopped")),))
|
||||
conn.commit()
|
||||
assert (box.mam / "agent-sessions.db-wal").stat().st_size > 0, "the commit is still in the WAL"
|
||||
box.stub_monitor()
|
||||
box.start_monitor()
|
||||
box.installer(FAILING_INSTALL)
|
||||
before = snapshot(box.ws)
|
||||
try:
|
||||
res = box.update()
|
||||
finally:
|
||||
conn.close()
|
||||
assert res.returncode == 1, res.stdout + res.stderr
|
||||
assert "the install failed (rc 7); rolling back" in res.stderr and "rolled back" in res.stdout
|
||||
after = snapshot(box.ws)
|
||||
after.pop(".mam/agent-sessions.db", None)
|
||||
before.pop(".mam/agent-sessions.db", None)
|
||||
before.pop(".mam/agent-sessions.db-wal", None)
|
||||
before.pop(".mam/agent-sessions.db-shm", None)
|
||||
assert after == before, sorted(set(after.items()) ^ set(before.items()))[:10]
|
||||
assert not (box.ws / "MESSAGING.md").exists() and not (box.ws / ".venv").exists()
|
||||
check = sqlite3.connect(str(db))
|
||||
try:
|
||||
st = json.loads(check.execute("SELECT data FROM state WHERE id=1").fetchone()[0])
|
||||
finally:
|
||||
check.close()
|
||||
assert st == {"orchestrator_uuids": ["in-wal"]}, "the backup carried the WAL's commit"
|
||||
assert not (box.mam / "update-backup").exists() and not (box.mam / "update-in-progress").exists()
|
||||
assert box.monitor_pids() and box.monitor_starts() == 2, "the restored tree starts the monitor again"
|
||||
|
||||
box.env["MAM_INSTALLER_URL"] = "file://" + str(REPO_ROOT / "deploy" / "install.sh")
|
||||
res = box.update()
|
||||
assert res.returncode == 0, "the skill runs again after a rollback: " + res.stdout + res.stderr
|
||||
|
||||
|
||||
# --- leftovers (U4) ------------------------------------------------------------------
|
||||
|
||||
def _leftover(box: Box, state):
|
||||
bk = box.mam / "update-backup"
|
||||
(bk / "files" / ".agents" / "skills").mkdir(parents=True)
|
||||
shutil.copy2(box.ws / ".agents/skills/lib.sh", bk / "files/.agents/skills/lib.sh")
|
||||
(bk / "items").write_text(".agents/skills/lib.sh\nMESSAGING.md\n")
|
||||
(bk / "absent").write_text("MESSAGING.md\n")
|
||||
(bk / "registry").mkdir()
|
||||
(bk / "registry" / "agent-sessions.yaml").write_text("herdr_sessions: []\n")
|
||||
if state is not None:
|
||||
(bk / "state").write_text(state + "\n")
|
||||
return bk
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", [None, "backing-up", "installed"])
|
||||
def test_leftover_that_changed_nothing_or_finished_is_removed(box, state):
|
||||
_leftover(box, state)
|
||||
(box.ws / "MESSAGING.md").write_text("must survive: nothing is restored\n")
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert "removing the backup a previous update left" in res.stdout
|
||||
assert (box.ws / "MESSAGING.md").exists()
|
||||
assert not (box.mam / "update-backup").exists()
|
||||
|
||||
|
||||
def test_leftover_installing_restores_files_but_not_the_registry(box):
|
||||
_leftover(box, "installing")
|
||||
pristine = (box.ws / ".agents/skills/lib.sh").read_bytes()
|
||||
with open(box.ws / ".agents/skills/lib.sh", "a") as fh:
|
||||
fh.write("# half installed\n")
|
||||
(box.ws / "MESSAGING.md").write_text("made by the dead install\n")
|
||||
write_rows(box.ws, [row("written-after-crash", status="stopped")])
|
||||
box.installer("grep -q 'half installed' \"$ws/.agents/skills/lib.sh\" && exit 9\n"
|
||||
"test -e \"$ws/MESSAGING.md\" && exit 8\n"
|
||||
"cp \"$ws/.mam/agent-sessions.yaml\" \"$ws/../seen-registry.yaml\"\nexit 0\n")
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert "the previous update was rolled back" in res.stdout
|
||||
assert (box.ws / ".agents/skills/lib.sh").read_bytes() == pristine
|
||||
assert "written-after-crash" in (box.root / "seen-registry.yaml").read_text(), "registry left as it was"
|
||||
assert not (box.mam / "update-backup").exists()
|
||||
|
||||
|
||||
def test_leftover_whose_restore_fails_is_kept_and_nothing_more_changes(box):
|
||||
bk = _leftover(box, "installing")
|
||||
(bk / "items").write_text(".agents/skills/lib.sh\n.agents/skills/missing-from-backup\n")
|
||||
box.stub_monitor()
|
||||
box.start_monitor()
|
||||
box.installer("touch \"$ws/INSTALLER-RAN\"\n")
|
||||
res = box.update()
|
||||
assert res.returncode == 1, res.stdout + res.stderr
|
||||
assert "restoring the previous update's backup failed" in res.stderr
|
||||
assert bk.exists() and (bk / "state").read_text().strip() == "installing"
|
||||
assert not (box.ws / "INSTALLER-RAN").exists()
|
||||
assert not box.monitor_pids(), "the monitor stays stopped"
|
||||
assert not (box.mam / "update-in-progress").exists()
|
||||
|
||||
|
||||
def test_the_backup_lives_in_the_workspace_and_is_gone_after_success(box):
|
||||
box.installer("test -f \"$ws/.mam/update-backup/state\" && grep -qx installing \"$ws/.mam/update-backup/state\" "
|
||||
"&& test -f \"$ws/.mam/update-backup/files/.agents/skills/lib.sh\" && touch \"$ws/../saw-backup\"\n")
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert (box.root / "saw-backup").exists()
|
||||
assert not (box.mam / "update-backup").exists()
|
||||
|
||||
|
||||
# --- the marker and the admission lock (decision 8) -----------------------------------
|
||||
|
||||
def test_a_live_update_marker_refuses_a_second_update_and_is_left_alone(box):
|
||||
live_marker(box.mam / "update-in-progress", box)
|
||||
text = (box.mam / "update-in-progress").read_text()
|
||||
before = snapshot(box.ws)
|
||||
res = box.update()
|
||||
assert res.returncode == 1 and "another update is in progress" in res.stderr, res.stderr
|
||||
assert (box.mam / "update-in-progress").read_text() == text
|
||||
assert snapshot(box.ws) == before
|
||||
|
||||
|
||||
def test_a_stale_update_marker_is_taken_over(box):
|
||||
dead_marker(box.mam / "update-in-progress")
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert "stale update marker" in res.stdout
|
||||
assert not (box.mam / "update-in-progress").exists()
|
||||
|
||||
|
||||
def test_two_updates_at_once_one_proceeds(box):
|
||||
box.installer("touch \"$ws/../first-installing\"; sleep 4; exit 0\n")
|
||||
first = box.helper("bash", str(box.ws / UPDATE_REL), "--yes", cwd=str(box.ws), env=box.env,
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
_until(lambda: (box.root / "first-installing").exists())
|
||||
marker = (box.mam / "update-in-progress").read_text()
|
||||
second = box.update()
|
||||
assert second.returncode == 1 and "another update is in progress" in second.stderr, second.stderr
|
||||
assert (box.mam / "update-in-progress").read_text() == marker, "the running update's marker is untouched"
|
||||
out, err = first.communicate(timeout=120)
|
||||
assert first.returncode == 0, out + err
|
||||
|
||||
|
||||
def test_entries_are_refused_while_the_copy_installs(box):
|
||||
box.installer('if bash -c \'source "$1/.agents/skills/lib.sh" && mam_update_admit\' _ "$ws" 2> "$ws/../admit.err"; '
|
||||
'then echo admitted > "$ws/../admit"; else echo refused > "$ws/../admit"; fi\n')
|
||||
res = box.update()
|
||||
assert res.returncode == 0, res.stdout + res.stderr
|
||||
assert (box.root / "admit").read_text().strip() == "refused"
|
||||
assert "an update of %s is in progress" % box.ws in (box.root / "admit.err").read_text()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wait,needle", [("30", "session late-row is running"), ("1", "admission lock past")])
|
||||
def test_an_entry_holding_the_admission_lock_is_waited_for_or_refuses(box, wait, needle):
|
||||
script = ('mam_update_admit || exit 1\necho admitted\nsleep 3\n'
|
||||
'printf "herdr_sessions:\\n- name: late-row\\n status: running\\n" > "$WORKSPACE_ROOT/.mam/agent-sessions.yaml"\n'
|
||||
'mam_update_admit_release\nsleep 30\n')
|
||||
entry = box.helper("bash", "-c", 'source "$1/.agents/skills/lib.sh"\n' + script, "_", str(box.ws),
|
||||
env=dict(box.env, WORKSPACE_ROOT=str(box.ws)), stdout=subprocess.PIPE, text=True)
|
||||
assert entry.stdout.readline().strip() == "admitted"
|
||||
box.installer("touch \"$ws/INSTALLER-RAN\"\n")
|
||||
res = box.update(env={"MAM_UPDATE_ADMISSION_WAIT_S": wait})
|
||||
assert res.returncode == 1, res.stdout + res.stderr
|
||||
assert needle in res.stderr, res.stderr
|
||||
assert not (box.ws / "INSTALLER-RAN").exists()
|
||||
assert not (box.mam / "update-in-progress").exists()
|
||||
@@ -1,6 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Issue #8 — update.sh must keep the installer's fresh version.txt and
|
||||
asset_hashes.txt instead of restoring the pre-update copies.
|
||||
"""Issue #8 — an update must keep the installer's fresh version.txt and
|
||||
asset_hashes.txt instead of restoring the pre-update copies. Since Issue #72
|
||||
the update is the multi-agent-mux-update skill (in place, no remove.sh) and
|
||||
no .mam_deploy/update.sh is installed.
|
||||
|
||||
Restoring them left version.txt naming the previous commit and made remove.sh
|
||||
compare against stale hashes, so every updated-but-pristine file was backed up
|
||||
@@ -51,8 +53,19 @@ def test_update_keeps_fresh_version_and_hashes_and_merges_install_state(installe
|
||||
with open(mam / "install_state", "a") as fh:
|
||||
fh.write("custom_marker=kept\n")
|
||||
|
||||
res = subprocess.run(["bash", str(ws / ".mam_deploy" / "update.sh"), "--force"],
|
||||
cwd=str(ws), env=env, capture_output=True, text=True, timeout=600)
|
||||
assert not (ws / ".mam_deploy" / "update.sh").exists()
|
||||
upd_env = {k: v for k, v in env.items() if not k.startswith("HERDR_")
|
||||
and k not in ("WORKSPACE_ROOT", "AGENT_SESSIONS_YAML", "MAM_JOBS_DIR")}
|
||||
upd_env["MAM_NO_SUBSCRIBER_DAEMON"] = "1"
|
||||
# The update lists the live herdr agents; never ask a real server here.
|
||||
fake = ws.parent / "fake-herdr-bin"
|
||||
fake.mkdir()
|
||||
(fake / "herdr").write_text('#!/bin/sh\necho \'{"sessions": []}\'\n')
|
||||
(fake / "herdr").chmod(0o755)
|
||||
upd_env["PATH"] = "%s:%s" % (fake, upd_env["PATH"])
|
||||
res = subprocess.run(["bash", str(ws / ".agents/skills/multi-agent-mux-update/scripts/update.sh"), "--yes"],
|
||||
cwd=str(ws), env=upd_env, capture_output=True, text=True, timeout=600,
|
||||
stdin=subprocess.DEVNULL)
|
||||
assert res.returncode == 0, res.stderr + res.stdout
|
||||
|
||||
version = (mam / "version.txt").read_text()
|
||||
@@ -64,3 +77,4 @@ def test_update_keeps_fresh_version_and_hashes_and_merges_install_state(installe
|
||||
state = (mam / "install_state").read_text()
|
||||
assert "custom_marker=kept" in state, state
|
||||
assert state.count("gitignore_created=1") == 1, "merged keys must not be duplicated:\n" + state
|
||||
assert not (ws / ".mam_deploy" / "update.sh").exists()
|
||||
Reference in new issue
Block a user