multi-agent-mux/tests/test_issue19_subscriber_daemon_lifecycle.py

#!/usr/bin/env python3
"""Issue #19 — the per-workspace subscriber daemon must not outlive its workspace.

`create_session.sh` detaches `reconcile.sh --subscribe --idle-timeout 0` into
its own session for every workspace. Three gaps let ~30 of them pile up per
suite run, each holding a wildcard subscription on the default public broker:

  * nothing in the test sandbox stopped the spawn or reaped the daemon;
  * the daemon never noticed its workspace (a deleted pytest tmpdir) was gone;
  * SIGTERM did not work - bash kept the python child in the foreground and
    deferred the signal until the child exited, which with idle-timeout 0 was
    never.

This file pins the fix from all three sides: the MAM_NO_SUBSCRIBER_DAEMON
opt-out that sandboxes set by default, the daemon's self-termination when its
workspace or `.mam/monitor.lock` disappears, and prompt SIGTERM handling on
both the MQTT loop and the broker-down polling fallback.
"""
from __future__ import annotations

import os
import re
import signal
import subprocess
import sys
import time
from pathlib import Path

import pytest

from _local_mqtt import local_broker_env

FIXTURES = Path(__file__).parent / "fixtures" / "issue_5" / "phase1"
DELEGATION_MODES = ("0", "1")


def _skills(sandbox: Path) -> Path:
    return sandbox / ".agents" / "skills"


def _reconcile(sandbox: Path) -> Path:
    return _skills(sandbox) / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"


def _install_fake_monitor(sandbox: Path) -> Path:
    target = _reconcile(sandbox)
    target.write_text((FIXTURES / "fake_reconcile.sh").read_text(encoding="utf-8"), encoding="utf-8")
    target.chmod(0o755)
    return target


def _run_helper(sandbox: Path, snippet: str, env=None) -> subprocess.CompletedProcess:
    run_env = dict(os.environ)
    if env:
        run_env.update(env)
    return subprocess.run(
        ["bash", "-c", 'source "{}" >/dev/null 2>&1; {}'.format(_skills(sandbox) / "lib.sh", snippet)],
        capture_output=True, text=True, env=run_env, timeout=60,
    )


def _daemons_for(sandbox: Path) -> list:
    """Same argv-shape selection as the conftest reaper (review R-2, P2)."""
    from conftest import _is_subscriber_daemon_argv

    out = subprocess.run(["pgrep", "-f", "reconcile.sh --subscribe"], capture_output=True, text=True).stdout.split()
    hits = []
    for pid in out:
        args = subprocess.run(["ps", "-o", "args=", "-p", pid], capture_output=True, text=True).stdout
        if _is_subscriber_daemon_argv(args, str(sandbox)):
            hits.append(int(pid))
    return hits


def _alive(pid: int) -> bool:
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return False
    state = subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True).stdout.strip()
    return bool(state) and not state.startswith("Z")


def _kill(pids) -> None:
    for pid in pids:
        try:
            os.kill(int(pid), signal.SIGKILL)
        except (OSError, ValueError):
            pass


def _start_subscriber(sandbox: Path, log: Path, extra_env: dict) -> subprocess.Popen:
    env = dict(os.environ)
    env.update(extra_env)
    handle = open(log, "ab")
    try:
        return subprocess.Popen(
            ["bash", str(_reconcile(sandbox)), "--subscribe", "--idle-timeout", "0"],
            cwd=str(sandbox), env=env, stdin=subprocess.DEVNULL, stdout=handle, stderr=subprocess.STDOUT,
            start_new_session=True,
        )
    finally:
        handle.close()


def _wait_for(log: Path, pattern: str, timeout: float) -> bool:
    deadline = time.time() + timeout
    while time.time() < deadline:
        if log.exists() and re.search(pattern, log.read_text(errors="replace")):
            return True
        time.sleep(0.1)
    return False


def _wait_exit(proc: subprocess.Popen, timeout: float):
    try:
        return proc.wait(timeout=timeout)
    except subprocess.TimeoutExpired:
        return None


def _unreachable_broker_env() -> dict:
    # Port 9 (discard) is closed on a stock macOS/Linux host: connect() is
    # refused at once, so the daemon drops into its polling fallback quickly.
    return {"MQTT_BROKER": "127.0.0.1", "MQTT_PORT": "9", "MQTT_TLS": "0", "RECONCILE_POLL_INTERVAL": "1"}


# ---------------------------------------------------------------------------
# 1. the sandbox opt-out
# ---------------------------------------------------------------------------

@pytest.mark.parametrize("mode", DELEGATION_MODES)
def test_sandbox_default_never_spawns_a_daemon(mam_sandbox, mode):
    """With the conftest default MAM_NO_SUBSCRIBER_DAEMON=1, start_watchdog
    prints no PID and leaves no process behind, in both helper bodies."""
    assert os.environ.get("MAM_NO_SUBSCRIBER_DAEMON") == "1"
    _install_fake_monitor(mam_sandbox)
    res = _run_helper(mam_sandbox, 'start_watchdog job1 "{}"'.format(mam_sandbox), env={"MAM_DELEGATION_PY": mode})
    assert res.returncode == 0, res.stderr
    assert res.stdout.strip() == "", res.stdout
    assert "MAM_NO_SUBSCRIBER_DAEMON=1" in res.stderr
    assert _daemons_for(mam_sandbox) == []


def test_opt_out_still_reports_a_daemon_that_already_runs(mam_sandbox, monkeypatch):
    """Opting out means "do not start", not "pretend nothing runs": an existing
    daemon's PID is still returned so callers keep their dedup contract."""
    from lib_py.delegation import daemon

    monitor = _install_fake_monitor(mam_sandbox)
    spawned = []
    try:
        monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
        first = daemon.ensure_subscriber_daemon("job1", str(mam_sandbox))
        spawned += first
        assert len(first) == 1 and _alive(int(first[0]))

        monkeypatch.setenv("MAM_NO_SUBSCRIBER_DAEMON", "1")
        assert daemon.daemon_disabled()
        assert daemon.ensure_subscriber_daemon("job1", str(mam_sandbox)) == first
        assert daemon.find_running(str(monitor)) == first
    finally:
        _kill(spawned)


def test_only_the_literal_one_disables_the_spawn(mam_sandbox, monkeypatch):
    from lib_py.delegation import daemon

    for value in ("0", "", "yes", "true"):
        monkeypatch.setenv("MAM_NO_SUBSCRIBER_DAEMON", value)
        assert not daemon.daemon_disabled(), value
    monkeypatch.setenv("MAM_NO_SUBSCRIBER_DAEMON", "1")
    assert daemon.daemon_disabled()


def _create_session(sandbox: Path, name: str) -> subprocess.CompletedProcess:
    script = _skills(sandbox) / "multi-agent-mux-create" / "scripts" / "create_session.sh"
    return subprocess.run(
        ["bash", str(script), "--workspace", str(sandbox), "--agent", "claude", "--role", "Creator", "--session", name],
        capture_output=True, text=True, cwd=str(sandbox), timeout=180,
    )


def test_create_session_under_sandbox_leaves_no_daemon(mam_sandbox, mock_herdr, mock_agents):
    """The leaking path itself: a full create_session.sh run in a sandbox."""
    res = _create_session(mam_sandbox, "issue19-creator-claude")
    assert res.returncode == 0, res.stderr + res.stdout
    assert "watchdog PID:" in res.stdout
    assert _daemons_for(mam_sandbox) == []


def test_create_session_without_opt_out_spawns_a_daemon_the_reaper_removes(mam_sandbox, mock_herdr, mock_agents, monkeypatch):
    """Belt and braces: with the opt-out lifted the historic leak is reproduced
    (create_session.sh does detach a daemon), and the sandbox reaper takes it
    down - so a test that forgets the opt-out still cannot leak."""
    from conftest import _reap_subscriber_daemons

    monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
    res = _create_session(mam_sandbox, "issue19-leaky-creator-claude")
    assert res.returncode == 0, res.stderr + res.stdout
    pids = _daemons_for(mam_sandbox)
    try:
        assert pids, "expected create_session.sh to detach a subscriber daemon"
        reaped = _reap_subscriber_daemons(mam_sandbox)
        assert set(pids) <= set(reaped)
        deadline = time.time() + 5
        while time.time() < deadline and any(_alive(p) for p in pids):
            time.sleep(0.1)
        assert not any(_alive(p) for p in pids)
    finally:
        _kill(pids)


# ---------------------------------------------------------------------------
# 2. the sandbox reaper
# ---------------------------------------------------------------------------

def test_reaper_selects_only_the_daemon_argv_shape(tmp_path):
    """Review R-2: a process that quotes the words in a brief, or lives under
    a sibling root that is a string prefix, must never be selected."""
    from conftest import _is_subscriber_daemon_argv

    root = str(tmp_path / "sbx1")
    daemon = "bash {}/.agents/skills/multi-agent-mux-monitor/scripts/reconcile.sh --subscribe --idle-timeout 0".format(root)
    assert _is_subscriber_daemon_argv(daemon, root)
    assert _is_subscriber_daemon_argv("/bin/sh " + daemon.split(" ", 1)[1], root)
    # sibling root that is a prefix of this one
    assert not _is_subscriber_daemon_argv(daemon, root[:-1])
    assert not _is_subscriber_daemon_argv(daemon, root + "0")
    # brief-carrying wrapper that merely quotes the words and the path
    quoting = "python3 -m lib_py.delegation.cli submit --prompt 'note: reconcile.sh --subscribe under {}/.agents/skills/x'".format(root)
    assert not _is_subscriber_daemon_argv(quoting, root)
    # ...and one run by bash from under the root, its prompt ending in the daemon's words
    wrapper = "bash {0}/.agents/skills/multi-agent-mux-delegate-job/multi-agent-mux-delegate-job submit --prompt restart {1}".format(root, daemon)
    assert not _is_subscriber_daemon_argv(wrapper, root)
    # the real reconcile.sh in a different mode
    assert not _is_subscriber_daemon_argv(daemon.replace("--subscribe", "--once"), root)
    assert not _is_subscriber_daemon_argv("", root)


def _spaced_monitor(tmp_path: Path):
    """A monitor script under a sandbox path with a space in it, as a pytest
    `--basetemp` or temp dir may have (codex review P2)."""
    root = tmp_path / "pytest review" / "case"
    script = root / ".agents" / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
    script.parent.mkdir(parents=True)
    script.write_text("#!/usr/bin/env bash\nwhile :; do sleep 1; done\n", encoding="utf-8")
    script.chmod(0o755)
    return root, script


def test_reaper_selects_daemons_under_whitespace_paths(tmp_path):
    """`ps -o args=` does not quote: the script path must be matched whole, both
    at the sandbox root (fixture reaper) and at an ancestor (session guard)."""
    from conftest import _is_subscriber_daemon_argv

    root, script = _spaced_monitor(tmp_path)
    daemon = "bash {} --subscribe --idle-timeout 0".format(script)
    assert _is_subscriber_daemon_argv(daemon + "\n", str(root))
    assert _is_subscriber_daemon_argv(daemon, str(tmp_path))
    assert _is_subscriber_daemon_argv("/bin/sh " + daemon.split(" ", 1)[1], str(root))
    # the text before the space names another root, not an ancestor
    assert not _is_subscriber_daemon_argv(daemon, str(tmp_path / "pytest"))
    # sibling root that this one is a string prefix of
    assert not _is_subscriber_daemon_argv(daemon.replace("/case/", "/case2/"), str(root))
    assert not _is_subscriber_daemon_argv(daemon.replace("--subscribe", "--once"), str(root))
    # wrappers carrying the words: python, and bash from under the spaced root
    quoting = "python3 -m lib_py.delegation.cli submit --prompt note: {}".format(daemon)
    assert not _is_subscriber_daemon_argv(quoting, str(root))
    wrapper = "bash {0}/.agents/skills/multi-agent-mux-delegate-job/multi-agent-mux-delegate-job submit --prompt restart {1}".format(root, daemon)
    assert not _is_subscriber_daemon_argv(wrapper, str(root))


def test_reaper_removes_a_live_daemon_under_a_whitespace_path(tmp_path):
    """The owned live-process half of P2: the reaper and the leak guard find a
    running `bash <spaced path>/reconcile.sh --subscribe` and take it down."""
    from conftest import _reap_subscriber_daemons, _subscriber_daemon_pids

    root, script = _spaced_monitor(tmp_path)
    proc = subprocess.Popen(
        ["bash", str(script), "--subscribe", "--idle-timeout", "0"],
        stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
        start_new_session=True,
    )
    try:
        deadline = time.time() + 5
        while time.time() < deadline and proc.pid not in _subscriber_daemon_pids(root):
            time.sleep(0.1)
        assert proc.pid in _subscriber_daemon_pids(root), "fixture reaper does not see the daemon"
        assert proc.pid in _subscriber_daemon_pids(tmp_path), "session leak guard does not see the daemon"
        assert proc.pid in _reap_subscriber_daemons(root)
        assert _wait_exit(proc, 5) is not None, "reaper left the daemon running"
        assert _subscriber_daemon_pids(root) == []
    finally:
        if proc.poll() is None:
            try:
                os.killpg(proc.pid, signal.SIGKILL)
            except OSError:
                pass
            proc.wait(timeout=5)


def test_sandbox_reaper_terminates_daemons_anchored_in_the_sandbox(mam_sandbox, monkeypatch):
    from conftest import _reap_subscriber_daemons, _subscriber_daemon_pids
    from lib_py.delegation import daemon

    _install_fake_monitor(mam_sandbox)
    monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
    pid = int(daemon.ensure_subscriber_daemon("job1", str(mam_sandbox))[0])
    try:
        assert pid in _subscriber_daemon_pids(mam_sandbox)
        reaped = _reap_subscriber_daemons(mam_sandbox)
        assert pid in reaped
        deadline = time.time() + 5
        while time.time() < deadline and _alive(pid):
            time.sleep(0.1)
        assert not _alive(pid), "reaper left the daemon running"
        assert _subscriber_daemon_pids(mam_sandbox) == []
    finally:
        _kill([pid])


# ---------------------------------------------------------------------------
# 3. self-termination and signals - polling fallback (no broker needed)
# ---------------------------------------------------------------------------

def test_poll_fallback_exits_when_monitor_lock_disappears(mam_sandbox, mock_herdr, monkeypatch):
    monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
    log = mam_sandbox / "sub.log"
    proc = _start_subscriber(mam_sandbox, log, _unreachable_broker_env())
    try:
        assert _wait_for(log, r"falling back to polling", 30), log.read_text()
        lock = mam_sandbox / ".mam" / "monitor.lock"
        assert lock.exists()
        lock.unlink()
        rc = _wait_exit(proc, 15)
        assert rc == 0, (rc, log.read_text())
        assert "monitor.lock is gone" in log.read_text()
    finally:
        if proc.poll() is None:
            proc.kill()


def test_poll_fallback_sigterm_exits_promptly(mam_sandbox, mock_herdr, monkeypatch):
    monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
    monkeypatch.setenv("RECONCILE_POLL_INTERVAL", "60")  # a foreground sleep would defer the signal for a minute
    log = mam_sandbox / "sub.log"
    proc = _start_subscriber(mam_sandbox, log, _unreachable_broker_env() | {"RECONCILE_POLL_INTERVAL": "60"})
    try:
        assert _wait_for(log, r"falling back to polling", 30), log.read_text()
        time.sleep(1.0)  # let the loop reach its sleep
        t0 = time.time()
        proc.send_signal(signal.SIGTERM)
        rc = _wait_exit(proc, 10)
        assert rc is not None, "SIGTERM was deferred: daemon still running after 10s"
        assert time.time() - t0 < 5, "SIGTERM took too long to act"
        assert rc == 143, rc
    finally:
        if proc.poll() is None:
            proc.kill()


def test_sigterm_during_broker_handshake_takes_the_clean_path(mam_sandbox, mock_herdr, monkeypatch):
    """Review N1: the python handlers are installed before connect/loop_start,
    so a SIGTERM while the subscriber still waits for CONNACK (a local listener
    that never answers) exits through the same clean path as a later one - the
    "signal 15 received" line, rc 143 from the wrapper - and never drops into
    the polling fallback."""
    import socket

    monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
    silent = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    silent.bind(("127.0.0.1", 0))
    silent.listen(1)  # the kernel completes the TCP handshake; no CONNACK ever comes
    log = mam_sandbox / "sub.log"
    proc = _start_subscriber(mam_sandbox, log, {
        "MQTT_BROKER": "127.0.0.1", "MQTT_PORT": str(silent.getsockname()[1]), "MQTT_TLS": "0",
    })
    try:
        assert _wait_for(log, r"MQTT Monitor: connecting to 127\.0\.0\.1", 30), log.read_text()
        time.sleep(1.0)  # well inside the 5 s CONNACK wait
        t0 = time.time()
        proc.send_signal(signal.SIGTERM)
        rc = _wait_exit(proc, 10)
        assert rc is not None, "SIGTERM was deferred: daemon still running after 10s"
        assert time.time() - t0 < 5
        assert rc == 143, (rc, log.read_text())
        text = log.read_text()
        assert "signal 15 received, exiting" in text, text
        assert "falling back to polling" not in text, text
    finally:
        if proc.poll() is None:
            proc.kill()
        silent.close()


def test_sub_stop_trap_exits_143_before_the_first_background_child():
    """Review B1: the wrapper's `_sub_stop` trap must end with rc 143 even when
    the signal lands before the first `&` - bash 3.2 treats `$!` as unbound
    under `set -u` there, and the trap died with rc 1. The function is taken
    verbatim from reconcile.sh and run without and with a child to stop."""
    src = (Path(__file__).resolve().parent.parent / ".agents" / "skills" / "multi-agent-mux-monitor"
           / "scripts" / "reconcile.sh").read_text(encoding="utf-8")
    m = re.search(r"^  _sub_stop\(\) \{\n.*?^  \}\n", src, re.S | re.M)
    assert m, "_sub_stop not found in reconcile.sh"
    # With a child, let it exec first: a TERM forwarded in the instant between
    # its fork and its exec is swallowed by the child's copy of the trap - a
    # separate window from B1, which is about reading `$!` at all.
    for child in ("", "sleep 30 &\nsleep 0.5\n"):
        script = "set -euo pipefail\n_sub_pid=''\n{}trap _sub_stop TERM INT\n{}kill -TERM $$\nsleep 5\n".format(m.group(0), child)
        res = subprocess.run(["bash", "-c", script], capture_output=True, text=True, timeout=30)
        assert res.returncode == 143, (child, res.returncode, res.stderr)


# ---------------------------------------------------------------------------
# 4. self-termination and signals - MQTT loop (needs a reachable broker)
# ---------------------------------------------------------------------------

def _start_connected(mam_sandbox, log):
    # Issue #26: the session's in-process broker is always up - not reaching it fails.
    proc = _start_subscriber(mam_sandbox, log, local_broker_env())
    if not _wait_for(log, r"MQTT Monitor: subscribed to mam/[0-9a-f]{12}/jobs/\+/events", 45):
        proc.kill()
        proc.wait(timeout=10)
        pytest.fail("the monitor daemon never subscribed on the local broker: " + log.read_text()[-300:])
    return proc


def test_mqtt_loop_exits_when_workspace_lock_is_replaced(mam_sandbox, monkeypatch):
    monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
    log = mam_sandbox / "sub.log"
    proc = _start_connected(mam_sandbox, log)
    try:
        lock = mam_sandbox / ".mam" / "monitor.lock"
        lock.unlink()
        lock.write_text("")  # a re-created .mam/: a fresh daemon could now take this lock
        rc = _wait_exit(proc, 15)
        assert rc == 0, (rc, log.read_text())
        assert re.search(r"monitor\.lock (was replaced|no longer exists); exiting", log.read_text()), log.read_text()
    finally:
        if proc.poll() is None:
            proc.kill()


def test_mqtt_loop_sigterm_exits_cleanly(mam_sandbox, monkeypatch):
    monkeypatch.delenv("MAM_NO_SUBSCRIBER_DAEMON", raising=False)
    log = mam_sandbox / "sub.log"
    proc = _start_connected(mam_sandbox, log)
    try:
        t0 = time.time()
        proc.send_signal(signal.SIGTERM)
        rc = _wait_exit(proc, 10)
        assert rc is not None, "SIGTERM was deferred: daemon still running after 10s"
        assert time.time() - t0 < 6
        assert rc == 143, (rc, log.read_text())
        assert "signal 15 received, exiting" in log.read_text(), log.read_text()
    finally:
        if proc.poll() is None:
            proc.kill()