feat(resume): resume ensures the monitor daemon (Issue #72, decision 2)
Only create started the per-workspace monitor, and only on its job path, so the stop -> update -> resume flow the update skill requires left no monitor. resume now calls start_watchdog with the control root as workdir (as create does since #21 A8, also for a worktree session) after the row is written, on both the already-running and the spawn path. MAM_NO_SUBSCRIBER_DAEMON=1 starts none; a failure warns and does not fail the resume. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ec378e82c9
commit
179473e5d5
3 files changed
+202
-5
No files matched your search
@@ -154,6 +154,18 @@ if [ -z "$UUID" ]; then
|
|||||||
esac
|
esac
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Issue #72: a resumed session gets the monitor daemon create starts - the
|
||||||
|
# control root's (Issue #21 A8), also for a worktree --workspace. Called after
|
||||||
|
# the row is written; MAM_NO_SUBSCRIBER_DAEMON=1 starts none. Never fails the resume.
|
||||||
|
_ensure_monitor_daemon() {
|
||||||
|
local wd_pid
|
||||||
|
if wd_pid=$(start_watchdog "resume" "$WORKSPACE_ROOT"); then
|
||||||
|
echo "watchdog PID: $wd_pid"
|
||||||
|
else
|
||||||
|
echo "WARNING: could not start the monitor daemon of '$WORKSPACE_ROOT'; the session itself was resumed" >&2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# 2. If herdr is alive, print warning or attach.
|
# 2. If herdr is alive, print warning or attach.
|
||||||
if [ "$HERDR_ALIVE" = "1" ]; then
|
if [ "$HERDR_ALIVE" = "1" ]; then
|
||||||
if [ "${DRY_RUN:-0}" = "1" ]; then
|
if [ "${DRY_RUN:-0}" = "1" ]; then
|
||||||
@@ -166,6 +178,7 @@ if [ "$HERDR_ALIVE" = "1" ]; then
|
|||||||
--session "$SESSION_NAME" --uuid "$UUID" --agent "$AGENT" --workspace "$WORKSPACE" \
|
--session "$SESSION_NAME" --uuid "$UUID" --agent "$AGENT" --workspace "$WORKSPACE" \
|
||||||
--herdr-session "$HERDR_SESSION_NAME" --already-running \
|
--herdr-session "$HERDR_SESSION_NAME" --already-running \
|
||||||
${HERDR_WORKSPACE_OPT:+--herdr-workspace "$HERDR_WORKSPACE_OPT"}
|
${HERDR_WORKSPACE_OPT:+--herdr-workspace "$HERDR_WORKSPACE_OPT"}
|
||||||
|
_ensure_monitor_daemon
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -274,5 +287,6 @@ CONTROL_PLANE_JSON="$CP_JSON" bash "$(dirname "${BASH_SOURCE[0]}")/update_yaml_r
|
|||||||
--herdr-session "$HERDR_SESSION_NAME" --cmd-full "$CMD_FULL" \
|
--herdr-session "$HERDR_SESSION_NAME" --cmd-full "$CMD_FULL" \
|
||||||
${DROP_FLAG:+$DROP_FLAG} \
|
${DROP_FLAG:+$DROP_FLAG} \
|
||||||
${HERDR_WORKSPACE_OPT:+--herdr-workspace "$HERDR_WORKSPACE_OPT"}
|
${HERDR_WORKSPACE_OPT:+--herdr-workspace "$HERDR_WORKSPACE_OPT"}
|
||||||
|
_ensure_monitor_daemon
|
||||||
|
|
||||||
echo "Successfully resumed $SESSION_NAME ($AGENT)"
|
echo "Successfully resumed $SESSION_NAME ($AGENT)"
|
||||||
@@ -476,11 +476,11 @@
|
|||||||
"evidence": [
|
"evidence": [
|
||||||
"multi-agent-mux-resume/scripts/resume_session.sh:111-130",
|
"multi-agent-mux-resume/scripts/resume_session.sh:111-130",
|
||||||
"multi-agent-mux-resume/scripts/resume_session.sh:133-144",
|
"multi-agent-mux-resume/scripts/resume_session.sh:133-144",
|
||||||
"multi-agent-mux-resume/scripts/resume_session.sh:147-158",
|
"multi-agent-mux-resume/scripts/resume_session.sh:147-170",
|
||||||
"multi-agent-mux-resume/scripts/resume_session.sh:172-197",
|
"multi-agent-mux-resume/scripts/resume_session.sh:185-210",
|
||||||
"multi-agent-mux-resume/scripts/resume_session.sh:245",
|
"multi-agent-mux-resume/scripts/resume_session.sh:258",
|
||||||
"multi-agent-mux-resume/scripts/resume_session.sh:249",
|
"multi-agent-mux-resume/scripts/resume_session.sh:262",
|
||||||
"multi-agent-mux-resume/scripts/resume_session.sh:255-258",
|
"multi-agent-mux-resume/scripts/resume_session.sh:268-271",
|
||||||
"lib_py/agents/adapters/claude.py:109"
|
"lib_py/agents/adapters/claude.py:109"
|
||||||
],
|
],
|
||||||
"mismatch": [
|
"mismatch": [
|
||||||
|
|||||||
@@ -0,0 +1,183 @@
|
|||||||
|
"""Issue #72 (decision 2): resume ensures the monitor daemon, as create does.
|
||||||
|
|
||||||
|
Only create_session.sh started the per-workspace subscriber daemon, so a
|
||||||
|
session brought back by resume_session.sh had no monitor unless one already
|
||||||
|
ran. Resume now calls the same `start_watchdog` with the control root as the
|
||||||
|
workdir (Issue #21 A8: never a worktree --workspace's copy), on the spawn path
|
||||||
|
and the already-running path, after the row is written.
|
||||||
|
|
||||||
|
What is real: the shipped create/stop/resume scripts and the real
|
||||||
|
start_watchdog -> lib_py ensure-daemon, against the mock herdr in a sandbox.
|
||||||
|
What is not: the sandbox's monitor script is a stub that records how it was
|
||||||
|
started and sleeps, so the daemon's `pgrep -f "bash <path> --subscribe"`
|
||||||
|
check sees it but nothing reconciles. Every stub is killed in teardown.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
CREATE = "multi-agent-mux-create/scripts/create_session.sh"
|
||||||
|
STOP = "multi-agent-mux-stop/scripts/stop_session.sh"
|
||||||
|
RESUME = "multi-agent-mux-resume/scripts/resume_session.sh"
|
||||||
|
MONITOR_REL = os.path.join(".agents", "skills", "multi-agent-mux-monitor", "scripts", "reconcile.sh")
|
||||||
|
|
||||||
|
|
||||||
|
def _script(root, rel):
|
||||||
|
return str(root / ".agents" / "skills" / rel)
|
||||||
|
|
||||||
|
|
||||||
|
def _install_stub_monitor(base, log):
|
||||||
|
"""Replace <base>'s monitor with a stub that writes `pid|$0|cwd|args`."""
|
||||||
|
path = base / MONITOR_REL
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text('#!/bin/bash\nprintf \'%%s|%%s|%%s|%%s\\n\' "$$" "$0" "$PWD" "$*" >> "%s"\n'
|
||||||
|
'sleep 60 &\nwait $!\n' % log)
|
||||||
|
path.chmod(0o755)
|
||||||
|
return str(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _starts(log):
|
||||||
|
return [line.split("|") for line in log.read_text().splitlines()] if log.exists() else []
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_starts(log, n, timeout=10.0):
|
||||||
|
deadline = time.time() + timeout
|
||||||
|
while time.time() < deadline and len(_starts(log)) < n:
|
||||||
|
time.sleep(0.1)
|
||||||
|
return _starts(log)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def stub(mam_sandbox):
|
||||||
|
"""The control root's stub monitor; kills every stub process it recorded
|
||||||
|
(the mam_sandbox reaper also sweeps anything under the sandbox)."""
|
||||||
|
logs = []
|
||||||
|
|
||||||
|
def install(base, name):
|
||||||
|
log = mam_sandbox / ("monitor-starts-%s.log" % name)
|
||||||
|
logs.append(log)
|
||||||
|
return _install_stub_monitor(base, log), log
|
||||||
|
|
||||||
|
yield install
|
||||||
|
for log in logs:
|
||||||
|
for row in _starts(log):
|
||||||
|
pid = int(row[0])
|
||||||
|
for target in subprocess.run(["pgrep", "-P", str(pid)], capture_output=True,
|
||||||
|
text=True).stdout.split() + [str(pid)]:
|
||||||
|
try:
|
||||||
|
os.kill(int(target), signal.SIGKILL)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _create(root, ws, name, stop):
|
||||||
|
subprocess.run(["bash", _script(root, CREATE), "--workspace", ws, "--agent", "claude",
|
||||||
|
"--role", "creator", "--session", name, "--no-onboard"],
|
||||||
|
check=True, cwd=str(root), capture_output=True, text=True)
|
||||||
|
if stop:
|
||||||
|
subprocess.run(["bash", _script(root, STOP), "--session", name, "--agent", "claude"],
|
||||||
|
check=True, cwd=str(root), capture_output=True, text=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _resume(root, ws, name):
|
||||||
|
return subprocess.run(["bash", _script(root, RESUME), "--workspace", ws, "--agent", "claude",
|
||||||
|
"--session", name], cwd=str(root), capture_output=True, text=True, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_root_monitor_started(root, monitor, log, res):
|
||||||
|
starts = _wait_starts(log, 1)
|
||||||
|
assert len(starts) == 1, starts
|
||||||
|
pid, script, cwd, args = starts[0]
|
||||||
|
assert os.path.realpath(script) == os.path.realpath(monitor)
|
||||||
|
assert os.path.realpath(cwd) == os.path.realpath(str(root))
|
||||||
|
assert args == "--subscribe --idle-timeout 0"
|
||||||
|
assert "watchdog PID: %s" % pid in res.stdout, res.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_spawn_path_ensures_the_control_root_monitor(mam_sandbox, mock_herdr, mock_agents, stub, monkeypatch):
|
||||||
|
root = mam_sandbox
|
||||||
|
_create(root, str(root), "issue72-spawn", stop=True)
|
||||||
|
monitor, log = stub(root, "root")
|
||||||
|
monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
|
||||||
|
res = _resume(root, str(root), "issue72-spawn")
|
||||||
|
assert res.returncode == 0, res.stdout + res.stderr
|
||||||
|
assert "Successfully resumed" in res.stdout
|
||||||
|
_assert_root_monitor_started(root, monitor, log, res)
|
||||||
|
|
||||||
|
|
||||||
|
def test_already_running_path_ensures_the_monitor_once(mam_sandbox, mock_herdr, mock_agents, stub, monkeypatch):
|
||||||
|
root = mam_sandbox
|
||||||
|
_create(root, str(root), "issue72-alive", stop=False)
|
||||||
|
monitor, log = stub(root, "root")
|
||||||
|
monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
|
||||||
|
res = _resume(root, str(root), "issue72-alive")
|
||||||
|
assert res.returncode == 0, res.stdout + res.stderr
|
||||||
|
assert "already running" in res.stdout
|
||||||
|
_assert_root_monitor_started(root, monitor, log, res)
|
||||||
|
# A second resume finds the running daemon and starts no other.
|
||||||
|
pid = _starts(log)[0][0]
|
||||||
|
again = _resume(root, str(root), "issue72-alive")
|
||||||
|
assert again.returncode == 0, again.stdout + again.stderr
|
||||||
|
assert "watchdog PID: %s" % pid in again.stdout
|
||||||
|
time.sleep(1)
|
||||||
|
assert len(_starts(log)) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("stop", [True, False], ids=["spawn", "already-running"])
|
||||||
|
def test_worktree_session_uses_the_control_root_monitor(mam_sandbox, mock_herdr, mock_agents, stub, monkeypatch, stop):
|
||||||
|
"""A session whose pane cwd (--workspace) is not the control root still
|
||||||
|
gets the control root's monitor, never the copy in its own tree."""
|
||||||
|
root = mam_sandbox
|
||||||
|
wt = root / "wt-issue72"
|
||||||
|
wt.mkdir()
|
||||||
|
name = "issue72-wt-" + ("spawn" if stop else "alive")
|
||||||
|
_create(root, str(wt), name, stop=stop)
|
||||||
|
monitor, log = stub(root, "root")
|
||||||
|
_, wt_log = stub(wt, "worktree")
|
||||||
|
monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
|
||||||
|
res = _resume(root, str(wt), name)
|
||||||
|
assert res.returncode == 0, res.stdout + res.stderr
|
||||||
|
_assert_root_monitor_started(root, monitor, log, res)
|
||||||
|
assert _starts(wt_log) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_opt_out_starts_nothing_and_reports_a_running_daemon(mam_sandbox, mock_herdr, mock_agents, stub):
|
||||||
|
"""MAM_NO_SUBSCRIBER_DAEMON=1 (the sandbox default): resume starts no
|
||||||
|
daemon; one that already runs is still found and reported."""
|
||||||
|
root = mam_sandbox
|
||||||
|
assert os.environ.get("MAM_NO_SUBSCRIBER_DAEMON") == "1"
|
||||||
|
_create(root, str(root), "issue72-optout", stop=True)
|
||||||
|
monitor, log = stub(root, "root")
|
||||||
|
res = _resume(root, str(root), "issue72-optout") # spawn path
|
||||||
|
assert res.returncode == 0, res.stdout + res.stderr
|
||||||
|
time.sleep(1)
|
||||||
|
assert _starts(log) == []
|
||||||
|
|
||||||
|
pre = subprocess.Popen(["bash", monitor, "--subscribe", "--idle-timeout", "0"], cwd=str(root),
|
||||||
|
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||||
|
start_new_session=True)
|
||||||
|
try:
|
||||||
|
assert len(_wait_starts(log, 1)) == 1
|
||||||
|
res = _resume(root, str(root), "issue72-optout") # already-running path
|
||||||
|
assert res.returncode == 0, res.stdout + res.stderr
|
||||||
|
assert "watchdog PID: %d" % pre.pid in res.stdout, res.stdout
|
||||||
|
time.sleep(1)
|
||||||
|
assert len(_starts(log)) == 1
|
||||||
|
finally:
|
||||||
|
pre.kill()
|
||||||
|
pre.wait()
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_missing_monitor_warns_and_the_resume_still_succeeds(mam_sandbox, mock_herdr, mock_agents, monkeypatch):
|
||||||
|
root = mam_sandbox
|
||||||
|
_create(root, str(root), "issue72-nomon", stop=True)
|
||||||
|
os.remove(str(root / MONITOR_REL))
|
||||||
|
monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
|
||||||
|
res = _resume(root, str(root), "issue72-nomon")
|
||||||
|
assert res.returncode == 0, res.stdout + res.stderr
|
||||||
|
assert "Successfully resumed" in res.stdout
|
||||||
|
assert "WARNING: could not start the monitor daemon" in res.stderr
|
||||||
Reference in new issue
Block a user