fix(issue-19): bound the subscriber daemon to its workspace, forward SIGTERM, keep sandboxes from leaking it
create_session.sh detaches `reconcile.sh --subscribe --idle-timeout 0` per workspace; test sandboxes never reaped it and the daemon neither noticed its workspace vanish nor honoured SIGTERM (bash deferred the signal behind the foreground python child). 202 orphans were found holding public-broker wildcard subscriptions for deleted pytest tmpdirs. - daemon.py / lib.sh: MAM_NO_SUBSCRIBER_DAEMON=1 (literal) skips the spawn but still reports a running daemon; cli prints a stderr note, stdout stays empty. - reconcile.sh: python subscriber runs as a waited background child with a TERM/INT trap (exit 143); the MQTT loop exits when the workspace dir is gone or .mam/monitor.lock is no longer the held inode; SIGTERM/SIGINT handled via the stop event so loop_stop/disconnect run; the polling fallback checks the same and sleeps in a waited child. - conftest: sandbox default MAM_NO_SUBSCRIBER_DAEMON=1, per-sandbox reaper finalizer, session-scope leak guard over the pytest basetemp. - tests: 11 lifecycle tests (tests/test_issue19_subscriber_daemon_lifecycle.py); phase-1 watchdog/daemon characterization tests opt back in. - docs: monitor SKILL.md lifecycle contract + pitfalls, create SKILL.md note. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff
This commit is contained in:
1 parent
8e7282c948
commit
ce315d8c5e
9 files changed
+536
-12
No files matched your search
+98
-1
@@ -11,13 +11,89 @@ SKILLS_DIR = str(Path(__file__).resolve().parent.parent / ".agents" / "skills")
|
||||
if SKILLS_DIR not in sys.path:
|
||||
sys.path.insert(0, SKILLS_DIR)
|
||||
|
||||
def _subscriber_daemon_pids(root):
|
||||
"""PIDs of `reconcile.sh --subscribe` daemons whose command line lives under
|
||||
`root`, plus their children (the python subscriber the bash wrapper waits on).
|
||||
|
||||
Issue #19: `create_session.sh` starts one such daemon per workspace,
|
||||
detached into its own session with `--idle-timeout 0`. A sandbox that ran
|
||||
it must reap it, or every suite run leaves ~30 daemons holding wildcard
|
||||
subscriptions on the default public broker behind."""
|
||||
import subprocess as _sp
|
||||
root = str(root)
|
||||
try:
|
||||
out = _sp.run(["pgrep", "-f", "reconcile.sh --subscribe"],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
except OSError:
|
||||
return []
|
||||
found = []
|
||||
for pid in out:
|
||||
try:
|
||||
args = _sp.run(["ps", "-o", "args=", "-p", pid],
|
||||
capture_output=True, text=True).stdout
|
||||
except OSError:
|
||||
continue
|
||||
if root in args:
|
||||
found.append(int(pid))
|
||||
kids = []
|
||||
for pid in found:
|
||||
try:
|
||||
kids += [int(k) for k in _sp.run(["pgrep", "-P", str(pid)],
|
||||
capture_output=True, text=True).stdout.split()]
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return found + kids
|
||||
|
||||
|
||||
def _reap_subscriber_daemons(root):
|
||||
"""Terminate (then kill) every subscriber daemon anchored under `root`.
|
||||
Returns the PIDs that were still alive before the sweep."""
|
||||
import signal as _sig
|
||||
import time as _t
|
||||
pids = _subscriber_daemon_pids(root)
|
||||
for sig in (_sig.SIGTERM, _sig.SIGKILL):
|
||||
for pid in pids:
|
||||
try:
|
||||
os.kill(pid, sig)
|
||||
except OSError:
|
||||
pass
|
||||
deadline = _t.time() + 3
|
||||
while _t.time() < deadline and any(_pid_alive(p) for p in pids):
|
||||
_t.sleep(0.1)
|
||||
if not any(_pid_alive(p) for p in pids):
|
||||
break
|
||||
return pids
|
||||
|
||||
|
||||
def _pid_alive(pid):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
# a zombie still answers kill(0); ps state Z means it is gone for our purposes
|
||||
import subprocess as _sp
|
||||
try:
|
||||
state = _sp.run(["ps", "-o", "stat=", "-p", str(pid)],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
except OSError:
|
||||
return True
|
||||
return bool(state) and not state.startswith("Z")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mam_sandbox(tmp_path, monkeypatch):
|
||||
def mam_sandbox(tmp_path, monkeypatch, request):
|
||||
"""
|
||||
Manages a temporary sandboxed directory for .mam/ configuration files
|
||||
and copies the scripts from .agents/skills to prevent contaminating the real codebase.
|
||||
Also overrides environments (HOME, PATH, AGENT_SESSIONS_YAML, etc.) for isolation.
|
||||
"""
|
||||
# Issue #19: reap any subscriber daemon a test spawned under this sandbox,
|
||||
# whatever path it came from (create_session.sh, start_watchdog, a direct
|
||||
# `reconcile.sh --subscribe`). Registered first so it runs after every
|
||||
# other finalizer of the test.
|
||||
request.addfinalizer(lambda: _reap_subscriber_daemons(tmp_path))
|
||||
# 1. Copy the skill scripts to sandboxed tmp_path/skills and tmp_path/.agents/skills
|
||||
src_skills = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".agents", "skills"))
|
||||
shutil.copytree(src_skills, tmp_path / "skills")
|
||||
@@ -52,6 +128,10 @@ def mam_sandbox(tmp_path, monkeypatch):
|
||||
# exist, so importing or running delegation code inside a sandbox never injects the
|
||||
# real .mam.env into this process's os.environ.
|
||||
monkeypatch.setenv("MAM_ENV_FILE", str(tmp_path / ".mam.env.absent"))
|
||||
# Issue #19: never let code under test detach a real subscriber daemon
|
||||
# (`reconcile.sh --subscribe --idle-timeout 0`) from a sandbox. Tests that
|
||||
# exercise the daemon itself override this with "0" or delenv.
|
||||
monkeypatch.setenv("MAM_NO_SUBSCRIBER_DAEMON", "1")
|
||||
monkeypatch.delenv("HERDR_SESSION_NAME", raising=False)
|
||||
monkeypatch.delenv("HERDR_SERVER_NAME", raising=False)
|
||||
monkeypatch.delenv("HERDR_WORKSPACE_ID", raising=False)
|
||||
@@ -1274,6 +1354,23 @@ def _drop_ambient_mam_version():
|
||||
os.environ["MAM_VERSION"] = previous
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def subscriber_daemon_leak_guard(tmp_path_factory):
|
||||
"""Issue #19: no `reconcile.sh --subscribe` daemon anchored in this session's
|
||||
pytest basetemp may survive the suite. Mirrors the #15 registry guard: the
|
||||
per-sandbox finalizer is the fix, this is the regression alarm. Leftovers
|
||||
are reaped after being reported so a failing run still cleans up."""
|
||||
yield
|
||||
basetemp = str(Path(tmp_path_factory.getbasetemp()).resolve())
|
||||
leaked = _subscriber_daemon_pids(basetemp)
|
||||
if leaked:
|
||||
_reap_subscriber_daemons(basetemp)
|
||||
assert not leaked, (
|
||||
"Issue #19: {} subscriber daemon process(es) anchored under {} survived the "
|
||||
"suite (reaped now): {}".format(len(leaked), basetemp, leaked)
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def delegation_tree_provenance_guard(tmp_path_factory):
|
||||
yield
|
||||
|
||||
Reference in new issue
Block a user