Implement full E2E, integration, component, and unit tests, and resolve all leftover tmux-to-herdr issues in UI and core scripts

This commit is contained in:
2026-07-19 20:05:48 +09:00
parent 30e606b0fa
commit ff7a2873f9
29 changed files with 3099 additions and 95 deletions
+121 -21
View File
@@ -28,7 +28,7 @@ AGENT_SESSIONS_YAML="${AGENT_SESSIONS_YAML:-$WORKSPACE_ROOT/.mam/agent-sessions.
# Add common Homebrew and local binary paths to PATH to ensure they are available in non-interactive shells # Add common Homebrew and local binary paths to PATH to ensure they are available in non-interactive shells
for dir in /home/linuxbrew/.linuxbrew/bin /home/linuxbrew/.linuxbrew/sbin "$HOME/.local/bin" "$HOME/.npm-global/bin"; do for dir in /home/linuxbrew/.linuxbrew/bin /home/linuxbrew/.linuxbrew/sbin "$HOME/.local/bin" "$HOME/.npm-global/bin"; do
if [ -d "$dir" ] && [[ ":$PATH:" != *":$dir:"* ]]; then if [ -d "$dir" ] && [[ ":$PATH:" != *":$dir:"* ]]; then
export PATH="$dir:$PATH" export PATH="$PATH:$dir"
fi fi
done done
@@ -58,7 +58,16 @@ _resolve_real_herdr_path() {
_init_herdr_isolation() { _init_herdr_isolation() {
local wrapper_dir="$WORKSPACE_ROOT/.mam/shim" local wrapper_dir="$WORKSPACE_ROOT/.mam/shim"
mkdir -p "$wrapper_dir" mkdir -p "$wrapper_dir"
cat <<'EOF' > "$wrapper_dir/herdr" if [ -x "$wrapper_dir/herdr" ]; then
if [[ ":$PATH:" != *":$wrapper_dir:"* ]]; then
export PATH="$wrapper_dir:$PATH"
fi
return 0
fi
local tmp_file
tmp_file=$(mktemp "$wrapper_dir/herdr.XXXXXX")
cat <<'EOF' > "$tmp_file"
#!/usr/bin/env bash #!/usr/bin/env bash
# Herdr-to-Herdr translation shim wrapper # Herdr-to-Herdr translation shim wrapper
set -euo pipefail set -euo pipefail
@@ -81,6 +90,16 @@ _resolve_real_herdr() {
} }
REAL_HERDR=$(_resolve_real_herdr) REAL_HERDR=$(_resolve_real_herdr)
# Support parsing -L <server> before the subcommand
while [ "${1:-}" = "-L" ]; do
if [ $# -lt 2 ]; then
echo "herdr shim: -L requires an argument" >&2
exit 1
fi
export HERDR_SERVER_NAME="$2"
shift 2
done
wrapper_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) wrapper_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
cmd="${1:-}" cmd="${1:-}"
if [ -z "$cmd" ]; then if [ -z "$cmd" ]; then
@@ -89,12 +108,20 @@ if [ -z "$cmd" ]; then
fi fi
shift shift
case "$cmd" in case "$cmd" in
has-session) has-session)
sess="" sess=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
-t) sess="$2"; shift 2 ;; -t)
if [ $# -lt 2 ]; then
echo "Error: -t requires a value" >&2
exit 1
fi
sess="$2"
shift 2
;;
*) shift ;; *) shift ;;
esac esac
done done
@@ -104,12 +131,29 @@ case "$cmd" in
name="" ws="" run_cmd="" name="" ws="" run_cmd=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
-s) name="$2"; shift 2 ;; -s)
-c) ws="$2"; shift 2 ;; if [ $# -lt 2 ]; then
echo "Error: -s requires a value" >&2
exit 1
fi
name="$2"
shift 2
;;
-c)
if [ $# -lt 2 ]; then
echo "Error: -c requires a value" >&2
exit 1
fi
ws="$2"
shift 2
;;
-d|-x|-y) shift ;; -d|-x|-y) shift ;;
*) run_cmd="$1"; shift ;; *) run_cmd="$1"; shift ;;
esac esac
done done
if [ -z "$run_cmd" ]; then
run_cmd="${SHELL:-/bin/bash}"
fi
"$REAL_HERDR" workspace create --label "${HERDR_SERVER_NAME:-default}" --cwd "${ws:-.}" >/dev/null 2>&1 || true "$REAL_HERDR" workspace create --label "${HERDR_SERVER_NAME:-default}" --cwd "${ws:-.}" >/dev/null 2>&1 || true
parsed=$(python3 -c " parsed=$(python3 -c "
@@ -136,13 +180,14 @@ print('\t'.join(env_flags) + '\n' + ' '.join(binary_tokens))
ws_label="${HERDR_SERVER_NAME:-default}" ws_label="${HERDR_SERVER_NAME:-default}"
ws_id="w1" ws_id="w1"
if [ "$ws_label" != "default" ]; then if [ "$ws_label" != "default" ]; then
ws_id=$("$REAL_HERDR" workspace list 2>/dev/null | python3 -c " ws_id=$("$REAL_HERDR" workspace list 2>/dev/null | WS_LABEL="$ws_label" python3 -c "
import sys, json import sys, os, json
try: try:
ws_label = os.environ.get('WS_LABEL', '')
data = json.loads(sys.stdin.read()) data = json.loads(sys.stdin.read())
res = data.get('result', data) res = data.get('result', data)
for ws in res.get('workspaces', []): for ws in res.get('workspaces', []):
if ws.get('label') == '$ws_label' or ws.get('workspace_id') == '$ws_label': if ws.get('label') == ws_label or ws.get('workspace_id') == ws_label:
print(ws.get('workspace_id')) print(ws.get('workspace_id'))
sys.exit(0) sys.exit(0)
except Exception: except Exception:
@@ -157,7 +202,14 @@ except Exception:
sess="" sess=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
-t) sess="$2"; shift 2 ;; -t)
if [ $# -lt 2 ]; then
echo "Error: -t requires a value" >&2
exit 1
fi
sess="$2"
shift 2
;;
*) shift ;; *) shift ;;
esac esac
done done
@@ -168,8 +220,22 @@ except Exception:
sess="" format="" sess="" format=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
-t) sess="$2"; shift 2 ;; -t)
-F) format="$2"; shift 2 ;; if [ $# -lt 2 ]; then
echo "Error: -t requires a value" >&2
exit 1
fi
sess="$2"
shift 2
;;
-F)
if [ $# -lt 2 ]; then
echo "Error: -F requires a value" >&2
exit 1
fi
format="$2"
shift 2
;;
*) shift ;; *) shift ;;
esac esac
done done
@@ -198,7 +264,14 @@ except Exception:
sess="" sess=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
-t) sess="$2"; shift 2 ;; -t)
if [ $# -lt 2 ]; then
echo "Error: -t requires a value" >&2
exit 1
fi
sess="$2"
shift 2
;;
*) shift ;; *) shift ;;
esac esac
done done
@@ -208,7 +281,14 @@ except Exception:
sess="" key="" sess="" key=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
-t) sess="$2"; shift 2 ;; -t)
if [ $# -lt 2 ]; then
echo "Error: -t requires a value" >&2
exit 1
fi
sess="$2"
shift 2
;;
*) key="$1"; shift ;; *) key="$1"; shift ;;
esac esac
done done
@@ -221,7 +301,14 @@ except Exception:
text="" text=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
-b) buf="$2"; shift 2 ;; -b)
if [ $# -lt 2 ]; then
echo "Error: -b requires a value" >&2
exit 1
fi
buf="$2"
shift 2
;;
*) text="$1"; shift ;; *) text="$1"; shift ;;
esac esac
done done
@@ -231,7 +318,14 @@ except Exception:
sess="" sess=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
-t) sess="$2"; shift 2 ;; -t)
if [ $# -lt 2 ]; then
echo "Error: -t requires a value" >&2
exit 1
fi
sess="$2"
shift 2
;;
*) shift ;; *) shift ;;
esac esac
done done
@@ -249,7 +343,11 @@ try:
data = json.load(sys.stdin) data = json.load(sys.stdin)
res = data.get('result', data) res = data.get('result', data)
for a in res.get('agents', []): for a in res.get('agents', []):
print(f\"{a['name']}|0\") try:
name = a.get('name') or a.get('agent') or 'unknown'
print(f\"{name}|0\")
except Exception:
pass
except Exception: except Exception:
pass pass
" || true " || true
@@ -259,7 +357,8 @@ except Exception:
;; ;;
esac esac
EOF EOF
chmod +x "$wrapper_dir/herdr" chmod +x "$tmp_file"
mv -f "$tmp_file" "$wrapper_dir/herdr"
if [[ ":$PATH:" != *":$wrapper_dir:"* ]]; then if [[ ":$PATH:" != *":$wrapper_dir:"* ]]; then
export PATH="$wrapper_dir:$PATH" export PATH="$wrapper_dir:$PATH"
fi fi
@@ -334,7 +433,7 @@ name = os.environ['SESSION_NAME']
d = json.loads(os.environ.get('MAM_STATE_JSON', '{}')) d = json.loads(os.environ.get('MAM_STATE_JSON', '{}'))
for s in d.get('herdr_sessions', []): for s in d.get('herdr_sessions', []):
if s.get('name') == name: if s.get('name') == name:
print(s.get('herdr_workspace', 'default')) print(s.get('herdr_workspace') or s.get('herdr_server') or 'default')
sys.exit(0) sys.exit(0)
print(os.environ.get('HERDR_SERVER_NAME', 'default')) print(os.environ.get('HERDR_SERVER_NAME', 'default'))
") ")
@@ -342,13 +441,14 @@ print(os.environ.get('HERDR_SERVER_NAME', 'default'))
echo "w1" echo "w1"
else else
local ws_id local ws_id
ws_id=$(herdr workspace list 2>/dev/null | python3 -c " ws_id=$(herdr workspace list 2>/dev/null | RAW_LABEL="$raw_label" python3 -c "
import sys, json import sys, os, json
try: try:
raw_label = os.environ.get('RAW_LABEL', '')
data = json.loads(sys.stdin.read()) data = json.loads(sys.stdin.read())
res = data.get('result', data) res = data.get('result', data)
for ws in res.get('workspaces', []): for ws in res.get('workspaces', []):
if ws.get('label') == '$raw_label' or ws.get('workspace_id') == '$raw_label': if ws.get('label') == raw_label or ws.get('workspace_id') == raw_label:
print(ws.get('workspace_id')) print(ws.get('workspace_id'))
sys.exit(0) sys.exit(0)
except Exception: except Exception:
@@ -11,7 +11,7 @@
# #
# This is a reference wrapper: it shells out to the python scripts that live # This is a reference wrapper: it shells out to the python scripts that live
# next to it. Copy it into your project and customise as needed. It never hard # next to it. Copy it into your project and customise as needed. It never hard
# fails if `claude`/`codex`/`tmux` are missing — it prints what it would run. # fails if `claude`/`codex`/`herdr` are missing — it prints what it would run.
set -euo pipefail set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -28,6 +28,8 @@ pick_python() {
local py_bin local py_bin
if [[ -n "${DELEGATE_JOB_PYTHON:-}" ]]; then if [[ -n "${DELEGATE_JOB_PYTHON:-}" ]]; then
py_bin="$DELEGATE_JOB_PYTHON" py_bin="$DELEGATE_JOB_PYTHON"
elif [[ -n "${AGENT_PYTHON_BIN:-}" ]] && [[ -x "$AGENT_PYTHON_BIN" ]]; then
py_bin="$AGENT_PYTHON_BIN"
elif [[ -x "${WORKDIR:-.}/.venv/bin/python" ]]; then elif [[ -x "${WORKDIR:-.}/.venv/bin/python" ]]; then
py_bin="${WORKDIR}/.venv/bin/python" py_bin="${WORKDIR}/.venv/bin/python"
elif [[ -x ".venv/bin/python" ]]; then elif [[ -x ".venv/bin/python" ]]; then
@@ -56,7 +58,7 @@ multi-agent-mux-delegate-job <command> [options]
[--type <direct|loop|discuss>] [--reviewer <reviewer_agent>] [--type <direct|loop|discuss>] [--reviewer <reviewer_agent>]
[--reviewer-session <reviewer_session>] [--max-iterations <count>] [--reviewer-session <reviewer_session>] [--max-iterations <count>]
[--counterpart-role <role_name>] [--strict-role-check] [--counterpart-role <role_name>] [--strict-role-check]
# The skill is tmux-interactive only; --mode print was removed. # The skill is herdr-interactive only; --mode print was removed.
status --job <id> [--registry-dir <dir>] status --job <id> [--registry-dir <dir>]
list [--registry-dir <dir>] list [--registry-dir <dir>]
verify --job <id> --validate <script> [--registry-dir <dir>] verify --job <id> --validate <script> [--registry-dir <dir>]
@@ -66,10 +68,10 @@ EOF
} }
# ---- arg parsing helpers -------------------------------------------------- # ---- arg parsing helpers --------------------------------------------------
AGENT="claude-code"; PROMPT=""; WORKDIR="$(pwd)"; AGENT_SESSION="tmux:claude" AGENT="claude-code"; PROMPT=""; WORKDIR="$(pwd)"; AGENT_SESSION="herdr:claude"
TIMEOUT=3600; IDLE_TIMEOUT=120; VALIDATE=""; DRY_RUN=0 TIMEOUT=3600; IDLE_TIMEOUT=120; VALIDATE=""; DRY_RUN=0
JOB_ID=""; REGISTRY_DIR="$REGISTRY_DIR_DEFAULT"; DELEGATE_ROLE="Worker" JOB_ID=""; REGISTRY_DIR="$REGISTRY_DIR_DEFAULT"; DELEGATE_ROLE="Worker"
TYPE="direct"; REVIEWER="hermes"; REVIEWER_SESSION="tmux:hermes"; MAX_ITERATIONS=5 TYPE="direct"; REVIEWER="hermes"; REVIEWER_SESSION="herdr:hermes"; MAX_ITERATIONS=5
DEFAULT_COUNTERPART_ROLE="Reviewer" DEFAULT_COUNTERPART_ROLE="Reviewer"
COUNTERPART_ROLE="$DEFAULT_COUNTERPART_ROLE" COUNTERPART_ROLE="$DEFAULT_COUNTERPART_ROLE"
STRICT_ROLE_CHECK=0 STRICT_ROLE_CHECK=0
@@ -153,8 +155,15 @@ EOF
break break
fi fi
else else
echo "ERROR: subscriber died early (pid=$sub_pid, check $logf)" >&2 wait "$sub_pid" 2>/dev/null
exit 1 local sub_exit=$?
if [ $sub_exit -eq 0 ]; then
sub_ready=1
break
else
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
exit 1
fi
fi fi
sleep 0.2 sleep 0.2
done done
@@ -224,7 +233,7 @@ EOF
# 1-1) Provision job directory and write iteration brief.md (MAM Job Restructuring) # 1-1) Provision job directory and write iteration brief.md (MAM Job Restructuring)
local job_dir="$REGISTRY_DIR/$JOB_ID" local job_dir="$REGISTRY_DIR/$JOB_ID"
local clean_session="${current_session#tmux:}" local clean_session="${current_session#herdr:}"
if [[ "$DRY_RUN" != "1" ]]; then if [[ "$DRY_RUN" != "1" ]]; then
mkdir -p "$job_dir" mkdir -p "$job_dir"
cat <<EOF > "$job_dir/brief.md" cat <<EOF > "$job_dir/brief.md"
@@ -267,8 +276,15 @@ EOF
break break
fi fi
else else
echo "ERROR: subscriber died early (pid=$sub_pid, check $logf)" >&2 wait "$sub_pid" 2>/dev/null
exit 1 local sub_exit=$?
if [ $sub_exit -eq 0 ]; then
sub_ready=1
break
else
echo "ERROR: subscriber died early (pid=$sub_pid, exit=$sub_exit, check $logf)" >&2
exit 1
fi
fi fi
sleep 0.2 sleep 0.2
done done
@@ -387,38 +403,38 @@ run_agent() {
# The skill is INTERACTIVE-ONLY. We never invoke `claude -p` or any other # The skill is INTERACTIVE-ONLY. We never invoke `claude -p` or any other
# one-shot print mode, because: # one-shot print mode, because:
# - claude -p exits the moment stdin is drained, so there's nothing to # - claude -p exits the moment stdin is drained, so there's nothing to
# `tmux attach` to afterwards. # `herdr session attach` to afterwards.
# - fire-and-forget via wrapper defeats the whole point of the audit log # - fire-and-forget via wrapper defeats the whole point of the audit log
# (you can't tell what happened if the agent crashes mid-turn). # (you can't tell what happened if the agent crashes mid-turn).
# - the job registry already gives us an authoritative completion signal, # - the job registry already gives us an authoritative completion signal,
# so we don't need a wrapper-side exit code to know "done". # so we don't need a wrapper-side exit code to know "done".
# The user attaches with `tmux attach -t <session>` and types follow-up # The user attaches with `herdr session attach <session>` and types follow-up
# prompts themselves. We pre-load the first prompt via stdin and `read` # prompts themselves. We pre-load the first prompt via stdin and `read`
# keeps the pane open after the agent exits so the user can review. # keeps the pane open after the agent exits so the user can review.
if [ "$AGENT" = "human" ]; then if [ "$AGENT" = "human" ]; then
echo "[human agent] complete the task, then run publish_event.py --event completed" echo "[human agent] complete the task, then run publish_event.py --event completed"
return return
fi fi
local sess="${target_session#tmux:}" local sess="${target_session#herdr:}"
if [[ "$DRY_RUN" == "1" ]]; then if [[ "$DRY_RUN" == "1" ]]; then
echo "[dry-run] would delegate task to running agent '$AGENT' in tmux session '$sess' with instructions:" echo "[dry-run] would delegate task to running agent '$AGENT' in herdr session '$sess' with instructions:"
echo "----"; echo "$instructions"; echo "----" echo "----"; echo "$instructions"; echo "----"
return return
fi fi
if ! command -v tmux >/dev/null 2>&1; then if ! command -v herdr >/dev/null 2>&1; then
echo "ERROR: this skill requires tmux (interactive agent sessions)." >&2 echo "ERROR: this skill requires herdr (interactive agent sessions)." >&2
echo " Install with: brew install tmux (or your package manager)" >&2 echo " Ensure herdr is installed and executable." >&2
return 1 return 1
fi fi
local _tmux="tmux" local _herdr="herdr"
if [ -n "${TMUX_SERVER_NAME:-}" ]; then if [ -n "${HERDR_SERVER_NAME:-}" ]; then
_tmux="tmux -L $TMUX_SERVER_NAME" _herdr="herdr -L $HERDR_SERVER_NAME"
fi fi
if ! $_tmux has-session -t "$sess" 2>/dev/null; then if ! $_herdr has-session -t "$sess" 2>/dev/null; then
echo "ERROR: 에이전트 세션 '$sess'이 존재하지 않습니다. 작업을 위임하기 전에 먼저 에이전트 세션을 기동해 주세요." >&2 echo "ERROR: 에이전트 세션 '$sess'이 존재하지 않습니다. 작업을 위임하기 전에 먼저 에이전트 세션을 기동해 주세요." >&2
echo " 팁: 'multi-agent-mux-resume' 또는 'multi-agent-mux-create'를 통해 에이전트를 먼저 생성할 수 있습니다." >&2 echo " 팁: 'multi-agent-mux-resume' 또는 'multi-agent-mux-create'를 통해 에이전트를 먼저 생성할 수 있습니다." >&2
return 1 return 1
@@ -431,7 +447,7 @@ run_agent() {
import os, json import os, json
d = json.loads(os.environ.get('MAM_STATE_JSON', '{}')) d = json.loads(os.environ.get('MAM_STATE_JSON', '{}'))
name = os.environ.get('SESS_NAME') name = os.environ.get('SESS_NAME')
for s in d.get('tmux_sessions', []): for s in d.get('herdr_sessions', []):
if s.get('name') == name: if s.get('name') == name:
print(s.get('role', '')) print(s.get('role', ''))
break break
@@ -483,7 +499,7 @@ else:
return 1 return 1
fi fi
echo "작업이 세션 '$sess'에 전송되었습니다. (연결하려면: $_tmux attach -t $sess)" echo "작업이 세션 '$sess'에 전송되었습니다. (연결하려면: $_herdr session attach $sess)"
trap - EXIT trap - EXIT
} }
@@ -153,7 +153,11 @@ def main(argv=None) -> int:
expected_ids: Set[str] = {j["job_id"] for j in jobs} expected_ids: Set[str] = {j["job_id"] for j in jobs}
tokens = {j["job_id"]: j.get("auth_token") for j in jobs} tokens = {j["job_id"]: j.get("auth_token") for j in jobs}
seqs = {j["job_id"]: int(j.get("last_seq", 0)) for j in jobs} seqs = {}
for j in jobs:
jid = j["job_id"]
last_seq = int(j.get("last_seq", 0))
seqs[jid] = max(0, last_seq - 1)
watcher = _Watcher(expected_ids, tokens, seqs) watcher = _Watcher(expected_ids, tokens, seqs)
# Resolve timeouts from CLI, falling back to the (first) job's settings. # Resolve timeouts from CLI, falling back to the (first) job's settings.
@@ -159,7 +159,7 @@ import os, json
d = json.loads(os.environ.get('MAM_STATE_JSON', '{}')) d = json.loads(os.environ.get('MAM_STATE_JSON', '{}'))
target_agent = os.environ.get('TARGET_AGENT') target_agent = os.environ.get('TARGET_AGENT')
reviewers = [s.get('name') for s in d.get('herdr_sessions', []) reviewers = [s.get('name') for s in d.get('herdr_sessions', [])
if 'reviewer' in s.get('role', '') and s.get('name') != target_agent] if 'reviewer' in s.get('role', '').lower() and s.get('name') != target_agent]
print(','.join(reviewers)) print(','.join(reviewers))
" "
} }
@@ -199,7 +199,7 @@ import os, json
d = json.loads(os.environ.get('MAM_STATE_JSON', '{}')) d = json.loads(os.environ.get('MAM_STATE_JSON', '{}'))
planner = '' planner = ''
for s in d.get('herdr_sessions', []): for s in d.get('herdr_sessions', []):
if 'planner' in s.get('role', ''): if 'planner' in s.get('role', '').lower():
planner = s.get('name') planner = s.get('name')
break break
print(planner) print(planner)
@@ -14,7 +14,9 @@
# Exit codes: 0 = ok | 1 = YAML not found | 2 = error # Exit codes: 0 = ok | 1 = YAML not found | 2 = error
set -euo pipefail set -euo pipefail
source "$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/lib.sh" SKILLS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
LIB_SH="$SKILLS_DIR/lib.sh"
source "$LIB_SH"
export WORKSPACE_ROOT export WORKSPACE_ROOT
STATE_DIR="${AGENT_SESSIONS_STATE_DIR:-$WORKSPACE_ROOT/.cache/multi-agent-mux-monitor}" STATE_DIR="${AGENT_SESSIONS_STATE_DIR:-$WORKSPACE_ROOT/.cache/multi-agent-mux-monitor}"
@@ -119,7 +121,7 @@ _changed = False
for s in d.get('herdr_sessions', []): for s in d.get('herdr_sessions', []):
if s.get('delegate_job_id') == _jid and s.get('status') == 'running': if s.get('delegate_job_id') == _jid and s.get('status') == 'running':
_name = s.get('name') _name = s.get('name')
_srv = s.get('herdr_server') or 'default' _srv = s.get('herdr_workspace') or s.get('herdr_server') or 'default'
if _event == 'completed': if _event == 'completed':
s['delegate_job_id'] = None s['delegate_job_id'] = None
print('MQTT Monitor: job completed on ' + str(_name) + ' — session kept alive', flush=True) print('MQTT Monitor: job completed on ' + str(_name) + ' — session kept alive', flush=True)
@@ -316,10 +318,13 @@ except NameError:
import subprocess import subprocess
d = {} d = {}
try: try:
ws_root = os.environ.get('WORKSPACE_ROOT') lib_sh = os.environ.get('LIB_SH')
if not ws_root: if not lib_sh:
ws_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..')) ws_root = os.environ.get('WORKSPACE_ROOT')
script = f"source '{ws_root}/.agents/skills/lib.sh' && load_state_json" if not ws_root:
ws_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))
lib_sh = os.path.join(ws_root, '.agents/skills/lib.sh')
script = f"source '{lib_sh}' && load_state_json"
out = subprocess.check_output(['bash', '-c', script], stderr=subprocess.DEVNULL) out = subprocess.check_output(['bash', '-c', script], stderr=subprocess.DEVNULL)
d = json.loads(out.decode('utf-8')) d = json.loads(out.decode('utf-8'))
except Exception: except Exception:
@@ -337,7 +342,7 @@ unique_servers = {'default'}
if 'HERDR_SERVER_NAME' in os.environ: if 'HERDR_SERVER_NAME' in os.environ:
unique_servers.add(os.environ['HERDR_SERVER_NAME']) unique_servers.add(os.environ['HERDR_SERVER_NAME'])
for s in d.get('herdr_sessions', []): for s in d.get('herdr_sessions', []):
srv = s.get('herdr_server') or 'default' srv = s.get('herdr_workspace') or s.get('herdr_server') or 'default'
unique_servers.add(srv) unique_servers.add(srv)
try: try:
@@ -391,7 +396,7 @@ if herdr_confirmed:
# (없으면 herdr-dead stopped 세션을 'terminated' 로 덮어써 resumable 플래그가 소실됨) # (없으면 herdr-dead stopped 세션을 'terminated' 로 덮어써 resumable 플래그가 소실됨)
if s.get('status') in ('terminated', 'archived', 'stopped'): if s.get('status') in ('terminated', 'archived', 'stopped'):
continue continue
srv = s.get('herdr_server') or 'default' srv = s.get('herdr_workspace') or s.get('herdr_server') or 'default'
if (name, srv) not in alive_set: if (name, srv) not in alive_set:
s['status'] = 'terminated' s['status'] = 'terminated'
s['terminated_at'] = now_iso s['terminated_at'] = now_iso
@@ -643,7 +648,7 @@ if not actions:
PYEOF PYEOF
if [ "$DRY_RUN" = "1" ]; then if [ "$DRY_RUN" = "1" ]; then
printf '%s' "$RECON_SRC" | env_python "$AGENT_SESSIONS_YAML" printf '%s' "$RECON_SRC" | LIB_SH="$LIB_SH" env_python "$AGENT_SESSIONS_YAML"
else else
printf '%s' "$RECON_SRC" | atomic_dump_yaml "$AGENT_SESSIONS_YAML" printf '%s' "$RECON_SRC" | LIB_SH="$LIB_SH" atomic_dump_yaml "$AGENT_SESSIONS_YAML"
fi fi
@@ -37,7 +37,8 @@ if [ -z "$UUID" ]; then
exit 1 exit 1
fi fi
export HERDR_SERVER_NAME="$(resolve_herdr_workspace "$SESSION_NAME")" HERDR_SERVER_NAME="$(resolve_herdr_workspace "$SESSION_NAME")"
export HERDR_SERVER_NAME
# 2. If herdr is alive, print warning or attach. # 2. If herdr is alive, print warning or attach.
if herdr has-session -t "$SESSION_NAME" 2>/dev/null; then if herdr has-session -t "$SESSION_NAME" 2>/dev/null; then
@@ -33,7 +33,8 @@ done
[ -n "$UUID" ] || { echo "ERROR: --uuid required" >&2; exit 2; } [ -n "$UUID" ] || { echo "ERROR: --uuid required" >&2; exit 2; }
[ -f "$AGENT_SESSIONS_YAML" ] || { echo "ERROR: $AGENT_SESSIONS_YAML not found" >&2; exit 1; } [ -f "$AGENT_SESSIONS_YAML" ] || { echo "ERROR: $AGENT_SESSIONS_YAML not found" >&2; exit 1; }
export HERDR_SERVER_NAME="$(resolve_herdr_workspace "$SESSION_NAME")" HERDR_SERVER_NAME="$(resolve_herdr_workspace "$SESSION_NAME")"
export HERDR_SERVER_NAME
# --agent 미지정 시 이름 suffix 로 fallback (P1-F: 가능하면 --agent 명시) # --agent 미지정 시 이름 suffix 로 fallback (P1-F: 가능하면 --agent 명시)
if [ -z "$AGENT" ]; then if [ -z "$AGENT" ]; then
@@ -93,7 +93,7 @@ def get_job_status(s):
sessions_detail = [] sessions_detail = []
for s in d.get('herdr_sessions', []): for s in d.get('herdr_sessions', []):
name = s.get('name', '?') name = s.get('name', '?')
server = s.get('herdr_server') or 'default' server = s.get('herdr_workspace') or s.get('herdr_server') or 'default'
jid, jstatus = get_job_status(s) jid, jstatus = get_job_status(s)
pane = s.get('pane') or {} pane = s.get('pane') or {}
sessions_detail.append({ sessions_detail.append({
@@ -197,7 +197,7 @@ if not sessions:
print("(no sessions registered)") print("(no sessions registered)")
for s in sessions: for s in sessions:
name = s.get('name', '?') name = s.get('name', '?')
server = s.get('herdr_server') or 'default' server = s.get('herdr_workspace') or s.get('herdr_server') or 'default'
status = s.get('status', '?') status = s.get('status', '?')
herdr = 'alive' if f"{name}|{server}" in alive else 'dead' herdr = 'alive' if f"{name}|{server}" in alive else 'dead'
cmd = (s.get('pane') or {}).get('cmd', '?') cmd = (s.get('pane') or {}).get('cmd', '?')
@@ -168,7 +168,7 @@ class _Header extends StatelessWidget {
runSpacing: 8, runSpacing: 8,
children: [ children: [
_Pill(text: s.status.toUpperCase()), _Pill(text: s.status.toUpperCase()),
_Pill(text: s.tmuxAlive ? 'TMUX ALIVE' : 'TMUX DEAD'), _Pill(text: s.herdrAlive ? 'HERDR ALIVE' : 'HERDR DEAD'),
if (s.role != null) _Pill(text: s.role!.toUpperCase()), if (s.role != null) _Pill(text: s.role!.toUpperCase()),
_Pill(text: 'SERVER: ${s.server}'), _Pill(text: 'SERVER: ${s.server}'),
], ],
@@ -47,7 +47,7 @@ class SessionTable extends StatelessWidget {
DataColumn2(label: Text('NAME'), size: ColumnSize.L), DataColumn2(label: Text('NAME'), size: ColumnSize.L),
DataColumn2(label: Text('SERVER'), size: ColumnSize.S), DataColumn2(label: Text('SERVER'), size: ColumnSize.S),
DataColumn2(label: Text('YAML'), size: ColumnSize.S), DataColumn2(label: Text('YAML'), size: ColumnSize.S),
DataColumn2(label: Text('TMUX'), size: ColumnSize.S), DataColumn2(label: Text('HERDR'), size: ColumnSize.S),
DataColumn2(label: Text('CMD'), size: ColumnSize.S), DataColumn2(label: Text('CMD'), size: ColumnSize.S),
DataColumn2(label: Text('RESUME'), size: ColumnSize.S), DataColumn2(label: Text('RESUME'), size: ColumnSize.S),
DataColumn2(label: Text('JOB_ID'), size: ColumnSize.S), DataColumn2(label: Text('JOB_ID'), size: ColumnSize.S),
@@ -79,7 +79,7 @@ class SessionTable extends StatelessWidget {
), ),
), ),
DataCell(_StatusChip(status: s.status)), DataCell(_StatusChip(status: s.status)),
DataCell(_TmuxChip(alive: s.tmuxAlive)), DataCell(_HerdrChip(alive: s.herdrAlive)),
DataCell( DataCell(
Text( Text(
s.pane.cmd ?? '?', s.pane.cmd ?? '?',
@@ -150,9 +150,9 @@ class _StatusChip extends StatelessWidget {
} }
} }
class _TmuxChip extends StatelessWidget { class _HerdrChip extends StatelessWidget {
final bool alive; final bool alive;
const _TmuxChip({required this.alive}); const _HerdrChip({required this.alive});
@override @override
Widget build(BuildContext context) { Widget build(BuildContext context) {
@@ -40,8 +40,8 @@ class _TerminalPaneState extends State<TerminalPane> {
Future<void> _startPty() async { Future<void> _startPty() async {
try { try {
final pty = await PtySession.start( final pty = await PtySession.start(
'tmux', 'herdr',
['-L', widget.serverName, 'attach', '-t', widget.sessionName], ['-L', widget.serverName, 'session', 'attach', widget.sessionName],
); );
if (!mounted) { if (!mounted) {
pty.close(); pty.close();
@@ -15,7 +15,7 @@ void main() {
name: 'demo-session', name: 'demo-session',
server: 'default', server: 'default',
status: 'running', status: 'running',
tmuxAlive: true, herdrAlive: true,
pane: const Pane( pane: const Pane(
pid: 123, pid: 123,
cwd: '/x', cwd: '/x',
@@ -26,7 +26,7 @@ void main() {
resumeState: 'yes', resumeState: 'yes',
jobId: '-', jobId: '-',
jobStatus: '-', jobStatus: '-',
attachCommand: 'tmux attach -t demo-session', attachCommand: 'herdr session attach demo-session',
driftClasses: const ['B'], driftClasses: const ['B'],
), ),
]; ];
@@ -1,6 +1,6 @@
import 'package:meta/meta.dart'; import 'package:meta/meta.dart';
/// The tmux pane backing a session row, as reported by `sessions_detail`. /// The herdr pane backing a session row, as reported by `sessions_detail`.
@immutable @immutable
class Pane { class Pane {
final int? pid; final int? pid;
@@ -13,7 +13,7 @@ class SessionRow {
final String name; final String name;
final String server; final String server;
final String status; final String status;
final bool tmuxAlive; final bool herdrAlive;
final Pane pane; final Pane pane;
final String? role; final String? role;
final String resumeState; final String resumeState;
@@ -28,7 +28,7 @@ class SessionRow {
required this.name, required this.name,
required this.server, required this.server,
required this.status, required this.status,
required this.tmuxAlive, required this.herdrAlive,
required this.pane, required this.pane,
this.role, this.role,
required this.resumeState, required this.resumeState,
@@ -49,7 +49,7 @@ class SessionRow {
name: json['name'] as String? ?? '?', name: json['name'] as String? ?? '?',
server: json['server'] as String? ?? 'default', server: json['server'] as String? ?? 'default',
status: json['status'] as String? ?? '?', status: json['status'] as String? ?? '?',
tmuxAlive: json['tmux_alive'] as bool? ?? false, herdrAlive: json['herdr_alive'] as bool? ?? false,
pane: Pane.fromSessionJson(json), pane: Pane.fromSessionJson(json),
role: json['role'] as String?, role: json['role'] as String?,
resumeState: json['resume_state'] as String? ?? '?', resumeState: json['resume_state'] as String? ?? '?',
@@ -64,3 +64,4 @@ class SessionRow {
); );
} }
} }
@@ -4,14 +4,14 @@ import 'drift_entry.dart';
import 'session_row.dart'; import 'session_row.dart';
/// The full parsed result of one `status.sh --json` call: the untouched /// The full parsed result of one `status.sh --json` call: the untouched
/// pre-D8 keys (timestamp/yaml_path/tmux_sessions_alive/tmux_confirmed/ /// pre-D8 keys (timestamp/yaml_path/herdr_sessions_alive/herdr_confirmed/
/// drifts/actions) plus the additive `sessions_detail` -> [SessionRow]. /// drifts/actions) plus the additive `sessions_detail` -> [SessionRow].
@immutable @immutable
class SessionsSnapshot { class SessionsSnapshot {
final DateTime timestamp; final DateTime timestamp;
final String yamlPath; final String yamlPath;
final List<String> tmuxSessionsAlive; final List<String> herdrSessionsAlive;
final bool tmuxConfirmed; final bool herdrConfirmed;
final List<DriftEntry> drifts; final List<DriftEntry> drifts;
final List<String> actions; final List<String> actions;
final List<SessionRow> sessions; final List<SessionRow> sessions;
@@ -19,8 +19,8 @@ class SessionsSnapshot {
const SessionsSnapshot({ const SessionsSnapshot({
required this.timestamp, required this.timestamp,
required this.yamlPath, required this.yamlPath,
required this.tmuxSessionsAlive, required this.herdrSessionsAlive,
required this.tmuxConfirmed, required this.herdrConfirmed,
required this.drifts, required this.drifts,
required this.actions, required this.actions,
required this.sessions, required this.sessions,
@@ -32,11 +32,11 @@ class SessionsSnapshot {
DateTime.tryParse(json['timestamp'] as String? ?? '')?.toUtc() ?? DateTime.tryParse(json['timestamp'] as String? ?? '')?.toUtc() ??
DateTime.now().toUtc(), DateTime.now().toUtc(),
yamlPath: json['yaml_path'] as String? ?? '', yamlPath: json['yaml_path'] as String? ?? '',
tmuxSessionsAlive: herdrSessionsAlive:
(json['tmux_sessions_alive'] as List<dynamic>? ?? const []) (json['herdr_sessions_alive'] as List<dynamic>? ?? const [])
.map((e) => e.toString()) .map((e) => e.toString())
.toList(growable: false), .toList(growable: false),
tmuxConfirmed: json['tmux_confirmed'] as bool? ?? false, herdrConfirmed: json['herdr_confirmed'] as bool? ?? false,
drifts: (json['drifts'] as List<dynamic>? ?? const []) drifts: (json['drifts'] as List<dynamic>? ?? const [])
.map((e) => DriftEntry.fromJson(e as Map<String, dynamic>)) .map((e) => DriftEntry.fromJson(e as Map<String, dynamic>))
.toList(growable: false), .toList(growable: false),
@@ -49,3 +49,4 @@ class SessionsSnapshot {
); );
} }
} }
@@ -37,9 +37,9 @@ class SessionService {
throw StatusFetchException('preflight failed: bash is missing or not executable'); throw StatusFetchException('preflight failed: bash is missing or not executable');
} }
final tmuxResult = await runCommand(['tmux', '-V'], timeout: timeout); final herdrResult = await runCommand(['herdr', '-V'], timeout: timeout);
if (tmuxResult.rc != 0) { if (herdrResult.rc != 0) {
throw StatusFetchException('preflight failed: tmux is missing or not executable'); throw StatusFetchException('preflight failed: herdr is missing or not executable');
} }
_preflightChecked = true; _preflightChecked = true;
@@ -24,18 +24,18 @@ void main() {
{ {
"timestamp": "2026-07-16T11:56:54Z", "timestamp": "2026-07-16T11:56:54Z",
"yaml_path": "/x/.mam/agent-sessions.yaml", "yaml_path": "/x/.mam/agent-sessions.yaml",
"tmux_sessions_alive": ["a|default"], "herdr_sessions_alive": ["a|default"],
"tmux_confirmed": true, "herdr_confirmed": true,
"drifts": [{"class": "B", "name": "a", "msg": "registered: a"}], "drifts": [{"class": "B", "name": "a", "msg": "registered: a"}],
"actions": ["registered: a"], "actions": ["registered: a"],
"sessions_detail": [ "sessions_detail": [
{ {
"name": "a", "server": "default", "status": "running", "name": "a", "server": "default", "status": "running",
"tmux_alive": true, "cmd": "claude", "role": "creator", "herdr_alive": true, "cmd": "claude", "role": "creator",
"resume_state": "yes", "job_id": "-", "job_status": "-", "resume_state": "yes", "job_id": "-", "job_status": "-",
"pane_cwd": "/x", "attach_command": "tmux attach -t a", "pane_cwd": "/x", "attach_command": "herdr session attach a",
"drift_classes": ["B"], "pane_pid": 123, "cmd_full": "claude --foo", "drift_classes": ["B"], "pane_pid": 123, "cmd_full": "claude --foo",
"start_command": "tmux new-session ...", "last_visible_status": "running" "start_command": "herdr agent start ...", "last_visible_status": "running"
} }
] ]
} }
@@ -43,7 +43,7 @@ void main() {
final snapshot = SessionsSnapshot.fromJson(json); final snapshot = SessionsSnapshot.fromJson(json);
expect(snapshot.tmuxConfirmed, isTrue); expect(snapshot.herdrConfirmed, isTrue);
expect(snapshot.drifts, hasLength(1)); expect(snapshot.drifts, hasLength(1));
expect(snapshot.drifts.single.driftClass, 'B'); expect(snapshot.drifts.single.driftClass, 'B');
expect(snapshot.sessions, hasLength(1)); expect(snapshot.sessions, hasLength(1));
@@ -56,7 +56,7 @@ void main() {
expect(row.isTerminal, isFalse); expect(row.isTerminal, isFalse);
expect(row.pane.pid, 123); expect(row.pane.pid, 123);
expect(row.pane.cmdFull, 'claude --foo'); expect(row.pane.cmdFull, 'claude --foo');
expect(row.attachCommand, 'tmux attach -t a'); expect(row.attachCommand, 'herdr session attach a');
}); });
test('SessionsSnapshot.fromJson tolerates missing optional fields', () { test('SessionsSnapshot.fromJson tolerates missing optional fields', () {
@@ -64,7 +64,7 @@ void main() {
as Map<String, dynamic>; as Map<String, dynamic>;
final snapshot = SessionsSnapshot.fromJson(json); final snapshot = SessionsSnapshot.fromJson(json);
expect(snapshot.tmuxConfirmed, isFalse); expect(snapshot.herdrConfirmed, isFalse);
expect(snapshot.sessions.single.name, 'bare'); expect(snapshot.sessions.single.name, 'bare');
expect(snapshot.sessions.single.resumeState, '?'); expect(snapshot.sessions.single.resumeState, '?');
expect(snapshot.sessions.single.pane.pid, isNull); expect(snapshot.sessions.single.pane.pid, isNull);
@@ -85,8 +85,8 @@ void main() {
final snapshot = await service.fetchSnapshot(); final snapshot = await service.fetchSnapshot();
expect(snapshot.yamlPath, isNotEmpty); expect(snapshot.yamlPath, isNotEmpty);
// sessions_detail row count must match tmux_sessions_alive's distinct names. // sessions_detail row count must match herdr_sessions_alive's distinct names.
final aliveNames = snapshot.tmuxSessionsAlive final aliveNames = snapshot.herdrSessionsAlive
.map((entry) => entry.split('|').first) .map((entry) => entry.split('|').first)
.toSet(); .toSet();
final detailNames = snapshot.sessions.map((s) => s.name).toSet(); final detailNames = snapshot.sessions.map((s) => s.name).toSet();
@@ -95,3 +95,4 @@ void main() {
timeout: const Timeout(Duration(seconds: 15)), timeout: const Timeout(Duration(seconds: 15)),
); );
} }
@@ -182,7 +182,9 @@ class PtySession {
// A. Disassociate controlling terminal // A. Disassociate controlling terminal
setsid(); setsid();
// B. Isolate from nested TMUX environments (§6.7) // B. Isolate from nested TMUX/Herdr environments (§6.7)
// We must unset TMUX and TMUX_PANE to ensure the child shell is properly isolated
// from any parent terminal multiplexer session, preventing unexpected nested behavior.
unsetenv(tmuxNamePtr.cast<ffi.Char>()); unsetenv(tmuxNamePtr.cast<ffi.Char>());
unsetenv(tmuxPaneNamePtr.cast<ffi.Char>()); unsetenv(tmuxPaneNamePtr.cast<ffi.Char>());
+49
View File
@@ -0,0 +1,49 @@
# Test Infrastructure Specification
## Test Philosophy
We adopt an **opaque-box, requirement-driven** testing philosophy for the tmux-to-herdr migration scripts.
This approach ensures that the test suite validates external behaviors, input/output contracts, and side-effects rather than asserting internal code structure or layout. The scripts are treated as black boxes that:
- Accept CLI arguments and environment variables.
- Query/interact with the `herdr` daemon through the `_herdr` shim (using mock executable interception).
- Perform state mutations inside `.mam/agent-sessions.yaml` and `.mam/agent-sessions.db`.
- Interact with background agent runners (`claude`, `agy`, `hermes`, `cline`).
This guarantees that our test assertions remain stable even if the script implementation details are refactored, as long as the functional requirements are met.
## Feature Inventory
The test suite is structured around five core features, mapping out verification checks across Tiers 1, 2, and 3:
| Feature | Tier 1 (Unit Checks) | Tier 2 (Component Checks) | Tier 3 (Integration Checks) |
|---|---|---|---|
| **Create Session** | - `derive_session_name` slug generation checks<br>- Workspace-to-slug character translation<br>- Invalid workspace path filtering<br>- Role parameter sanity validations<br>- Session override string generation | - Verification of state serialization to YAML schema<br>- Isolation home directory structure validation<br>- SQLite DB connection verification<br>- Concurrency check for database registration lock<br>- Database schema validation on write | - Spawn session execution with mock `herdr` and mock agent<br>- TUI readiness wait check<br>- Cleanup trap execution on crash<br>- Argument validation logic verification<br>- Isolation directory creation checks |
| **Resume Session** | - Workspace UUID resolution order unit tests<br>- CLI session ID parser validations<br>- Check prioritization (yaml file -> disk scan -> cache)<br>- Workspace path boundary check<br>- Empty UUID handling logic | - Configuration restore verification<br>- Environment overrides assertion<br>- Integrity check on retrieved SQLite metadata<br>- Validation of session ownership verification<br>- Config parsing for resume options | - Run `resume_session.sh` with mock agents<br>- Intercept agent command structure inside mock `herdr`<br>- Verify agent receives correct conversation UUID flag<br>- Invalid/missing UUID recovery path test<br>- Workspace resume CLI args verification |
| **Stop Session** | - Session name verification check<br>- Purge verification confirmations logic<br>- Command derivation format validation<br>- Timeout calculation helper tests<br>- Reason logging serializer test | - Safe folder path validation (shutil protection)<br>- Database status field mutation serialization<br>- Isolation folder cleanup check<br>- Lock file release checks on stop<br>- Concurrency handling of stop mutations | - Execute `stop_session.sh` with graceful key delivery (`/exit`)<br>- Fallback to forcible termination (`herdr kill-session`) check<br>- Fallback to PID termination (`kill -9`) verify<br>- Purge files verification on disk (`--purge-conversation`)<br>- CLI flag verification with yes/no confirmation |
| **Status Query** | - JSON converter unit tests<br>- Diff formatter text generators<br>- Output alignment tests<br>- Table grid column math verify<br>- CLI status argument parse tests | - Status read locks verification<br>- Parsing of drift status classifications<br>- Concurrency read protection test<br>- Registry YAML-to-JSON structural translation<br>- Verification of database read access checks | - Running `status.sh` with `--json`<br>- Verify console output match formatting rules<br>- Verify exit status codes on different states<br>- Integration test with `reconcile.sh` read-only diff emission<br>- Verify status command doesn't trigger side effects |
| **Monitor/Reconcile** | - Drift state classification unit tests<br>- Signature verification checks<br>- Subscription topic parsing tests<br>- MQTT message structure validator<br>- HMAC validation logic tests | - Concurrency lock checks (`.mam/monitor.lock`)<br>- Verify YAML and SQLite database reconciliation logic<br>- DB validation on drift updates<br>- HMAC signature signature verification<br>- SQLite journal mode fallback check (WAL vs DELETE) | - Execute `reconcile.sh` in single-pass mode (`--once`)<br>- MQTT subscription execution with mock messages<br>- Verify auto-termination of orphaned herdr sessions<br>- Verify auto-registration of untracked herdr sessions<br>- Lock contention handling testing |
## Test Architecture
The E2E testing framework is built using **pytest** and relies on two main pillars to ensure hermetic and reproducible test runs:
1. **Environment Sandboxing**:
All tests run inside a temporary, isolated directory structure provided by the pytest `tmp_path` fixture. The workspace environment is sandboxed by:
- Creating a temporary `.mam/` directory.
- Using the `monkeypatch` fixture to override `AGENT_SESSIONS_YAML` pointing to the sandboxed path.
- Overriding relevant environment variables (like `HOME`, `WORKSPACE_ROOT`, etc.) to prevent tests from modifying the developer's system state.
2. **Mock Binaries Interception**:
To prevent tests from interacting with external systems or relying on running daemons:
- A mock `herdr` script is dynamically generated and placed in a temporary bin folder, which is prepended to the system `PATH`. This mock binary reads/writes to a JSON file (`mock_herdr_state.json`) which acts as the control pane for tests to assert that `herdr` was called with correct arguments and return mocked outputs (session list, capture-pane output, exit codes).
- Mock agent binaries (`claude`, `agy`, `hermes`, `cline`) are also generated and prepended to `PATH`. They emulate successful login verification commands (e.g. `claude auth status`) and mock conversation UUID generation on disk.
## Real-World Application Scenarios (Tier 4)
We define five key E2E scenarios representing end-to-end user workflows:
1. **Standard Agent Session Lifecycle**: Spawning a new worker agent session via `create_session.sh`, verifying it is registered correctly in the YAML database, checking its status via `status.sh`, and then gracefully stopping it via `stop_session.sh`.
2. **Session Disconnect and Resume**: Creating a session, simulating a network disconnect/agent pane termination (updating herdr state), calling `resume_session.sh` to restore it using the workspace-scoped UUID, and asserting that the session returns to the active state in both herdr and the registry.
3. **Drift Detection and Auto-Reconciliation**: Artificially introducing drift (e.g. terminating a herdr session manually from the backend while keeping it registered in the YAML registry, or starting a herdr session outside the scripts), running `reconcile.sh --once`, and verifying that orphaned sessions are terminated and registry state is updated.
4. **Parallel Session Operations with flock Locking**: Simulating concurrent creation/stop script invocations to verify that SQLite flock transactions block lost update races, and that the registry data remains consistent.
5. **Multi-Agent Orchestrator Review Loop**: Running the orchestrator loop (`run_loop.sh`) where a worker agent and a reviewer agent are spawned, reviewer verdicts (`PASS` and `NOT PASS`) are processed, loops are iterated, and planner escalation is triggered on failure.
## Coverage Thresholds
To ensure the test suite is comprehensive, we define the following coverage thresholds:
- **Tier 1 (Unit Tests)**: Minimum >=5 unit tests per feature (total >=25 unit tests).
- **Tier 2 (Component Tests)**: Minimum >=5 component tests per feature (total >=25 component tests).
- **Tier 3 (Integration Tests)**: Pairwise combination testing covering CLI options and environment overrides for all features.
- **Tier 4 (E2E Scenarios)**: At least 5 full real-world scenario tests implemented and passing.
+129
View File
@@ -0,0 +1,129 @@
# Test Ready Report
## Test Runner
- **Command**: `.venv/bin/pytest tests/`
- **Expected**: All tests pass with exit code 0
## Coverage Summary
- **1. Feature Coverage (Tier 1)**: 29 tests
- **2. Boundary & Corner (Tier 2)**: 26 tests
- **3. Cross-Feature (Tier 3)**: 5 tests
- **4. Real-World Application (Tier 4)**: 5 tests
- **Sanity Checks**: 2 tests
- **Challenger/M2 Unit**: 7 tests
- **Total**: 74 tests
## Feature Checklist
### 1. Create Session
- **Tier 1 (Unit Checks)**
- [x] `derive_session_name` slug generation checks
- [x] Workspace-to-slug character translation
- [x] Invalid workspace path filtering
- [x] Role parameter sanity validations
- [x] Session override string generation
- **Tier 2 (Component Checks)**
- [x] Verification of state serialization to YAML schema
- [x] Isolation home directory structure validation
- [x] SQLite DB connection verification
- [x] Concurrency check for database registration lock
- [x] Database schema validation on write
- **Tier 3 (Integration Checks)**
- [x] Spawn session execution with mock `herdr` and mock agent
- [x] TUI readiness wait check
- [x] Cleanup trap execution on crash
- [x] Argument validation logic verification
- [x] Isolation directory creation checks
- **Tier 4 (Real-World Application Scenarios)**
- [x] Standard Agent Session Lifecycle E2E test (Scenario 1)
- [x] Parallel Session Operations with flock Locking E2E test (Scenario 4)
### 2. Resume Session
- **Tier 1 (Unit Checks)**
- [x] Workspace UUID resolution order unit tests
- [x] CLI session ID parser validations
- [x] Check prioritization (yaml file -> disk scan -> cache)
- [x] Workspace path boundary check
- [x] Empty UUID handling logic
- **Tier 2 (Component Checks)**
- [x] Configuration restore verification
- [x] Environment overrides assertion
- [x] Integrity check on retrieved SQLite metadata
- [x] Validation of session ownership verification
- [x] Config parsing for resume options
- **Tier 3 (Integration Checks)**
- [x] Run `resume_session.sh` with mock agents
- [x] Intercept agent command structure inside mock `herdr`
- [x] Verify agent receives correct conversation UUID flag
- [x] Invalid/missing UUID recovery path test
- [x] Workspace resume CLI args verification
- **Tier 4 (Real-World Application Scenarios)**
- [x] Session Disconnect and Resume E2E test (Scenario 2)
### 3. Stop Session
- **Tier 1 (Unit Checks)**
- [x] Session name verification check
- [x] Purge verification confirmations logic
- [x] Command derivation format validation
- [x] Timeout calculation helper tests
- [x] Reason logging serializer test
- **Tier 2 (Component Checks)**
- [x] Safe folder path validation (shutil protection)
- [x] Database status field mutation serialization
- [x] Isolation folder cleanup check
- [x] Lock file release checks on stop
- [x] Concurrency handling of stop mutations
- **Tier 3 (Integration Checks)**
- [x] Execute `stop_session.sh` with graceful key delivery (`/exit`)
- [x] Fallback to forcible termination (`herdr kill-session`) check
- [x] Fallback to PID termination (`kill -9`) verify
- [x] Purge files verification on disk (`--purge-conversation`)
- [x] CLI flag verification with yes/no confirmation
- **Tier 4 (Real-World Application Scenarios)**
- [x] Standard Agent Session Lifecycle E2E test (Scenario 1)
- [x] Parallel Session Operations with flock Locking E2E test (Scenario 4)
### 4. Status Query
- **Tier 1 (Unit Checks)**
- [x] JSON converter unit tests
- [x] Diff formatter text generators
- [x] Output alignment tests
- [x] Table grid column math verify
- [x] CLI status argument parse tests
- **Tier 2 (Component Checks)**
- [x] Status read locks verification
- [x] Parsing of drift status classifications
- [x] Concurrency read protection test
- [x] Registry YAML-to-JSON structural translation
- [x] Verification of database read access checks
- **Tier 3 (Integration Checks)**
- [x] Running `status.sh` with `--json`
- [x] Verify console output match formatting rules
- [x] Verify exit status codes on different states
- [x] Integration test with `reconcile.sh` read-only diff emission
- [x] Verify status command doesn't trigger side effects
- **Tier 4 (Real-World Application Scenarios)**
- [x] Standard Agent Session Lifecycle E2E test (Scenario 1)
- [x] Drift Detection and Auto-Reconciliation E2E test (Scenario 3)
### 5. Monitor/Reconcile
- **Tier 1 (Unit Checks)**
- [x] Drift state classification unit tests
- [x] Signature verification checks
- [x] Subscription topic parsing tests
- [x] MQTT message structure validator
- [x] HMAC validation logic tests
- **Tier 2 (Component Checks)**
- [x] Concurrency lock checks (`.mam/monitor.lock`)
- [x] Verify YAML and SQLite database reconciliation logic
- [x] DB validation on drift updates
- [x] HMAC signature signature verification
- [x] SQLite journal mode fallback check (WAL vs DELETE)
- **Tier 3 (Integration Checks)**
- [x] Execute `reconcile.sh` in single-pass mode (`--once`)
- [x] MQTT subscription execution with mock messages
- [x] Verify auto-termination of orphaned herdr sessions
- [x] Verify auto-registration of untracked herdr sessions
- [x] Lock contention handling testing
- **Tier 4 (Real-World Application Scenarios)**
- [x] Drift Detection and Auto-Reconciliation E2E test (Scenario 3)
+516
View File
@@ -0,0 +1,516 @@
import os
import shutil
import json
import uuid
import sqlite3
import pytest
@pytest.fixture
def mam_sandbox(tmp_path, monkeypatch):
"""
Manages a temporary sandboxed directory for .mam/ configuration files
and copies the scripts from .agents/skills to prevent contaminating the real codebase.
Also overrides environments (HOME, PATH, AGENT_SESSIONS_YAML, etc.) for isolation.
"""
# 1. Copy the skill scripts to sandboxed tmp_path/skills and tmp_path/.agents/skills
src_skills = "/home/godopu16/PuKi/laa/canary_projects/multi-agent-mux/.agents/skills"
shutil.copytree(src_skills, tmp_path / "skills", ignore=shutil.ignore_patterns('multi-agent-mux-ui'))
shutil.copytree(src_skills, tmp_path / ".agents" / "skills", ignore=shutil.ignore_patterns('multi-agent-mux-ui'))
# 2. Create sandboxed directory structure
mam_dir = tmp_path / ".mam"
mam_dir.mkdir(parents=True, exist_ok=True)
yaml_path = mam_dir / "agent-sessions.yaml"
# Create empty registry structure
yaml_path.write_text("herdr_sessions: []\ndelegation_jobs: []\n")
# 3. Setup mock home directory contents
(tmp_path / ".claude" / "projects").mkdir(parents=True, exist_ok=True)
(tmp_path / ".local" / "bin").mkdir(parents=True, exist_ok=True)
# 4. Monkeypatch environment variables to point inside the sandbox
monkeypatch.setenv("HOME", str(tmp_path))
monkeypatch.setenv("HOME_DIR", str(tmp_path))
monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path / ".claude" / "projects"))
monkeypatch.setenv("LOCAL_BIN", str(tmp_path / ".local" / "bin"))
monkeypatch.setenv("AGENT_SESSIONS_YAML", str(yaml_path))
monkeypatch.setenv("WORKSPACE_ROOT", str(tmp_path))
import sys
monkeypatch.setenv("AGENT_PYTHON_BIN", sys.executable)
return tmp_path
@pytest.fixture
def mock_herdr(mam_sandbox, monkeypatch):
"""
Generates a mock 'herdr' executable and prepends it to PATH.
Allows tests to control and verify state via mock_herdr_state.json.
Automatically generates mock session database files upon 'agent start'.
"""
tmp_path = mam_sandbox
state_file = tmp_path / "mock_herdr_state.json"
# Initial mock state
initial_state = {
"workspaces": [
{
"workspace_id": "w1",
"label": "default",
"cwd": str(tmp_path)
}
],
"agents": {},
"calls": []
}
state_file.write_text(json.dumps(initial_state, indent=2))
bin_dir = tmp_path / "bin"
bin_dir.mkdir(parents=True, exist_ok=True)
herdr_bin = bin_dir / "herdr"
# Write Python implementation of mock herdr (using string replacement to avoid f-string escaping issues)
# Double escape backslashes for newline (\\\\n) and write correctly
code = """#!/usr/bin/env python3
import sys
import os
import json
import uuid
import sqlite3
import fcntl
state_file = "STATE_FILE_PLACEHOLDER"
if not os.path.exists(state_file):
sys.exit(0)
# Lock the state file exclusively to prevent concurrent race conditions
lock_f = open(state_file + ".lock", "w")
fcntl.flock(lock_f, fcntl.LOCK_EX)
with open(state_file, 'r') as f:
state = json.load(f)
# Record the command call
state["calls"].append(sys.argv[1:])
def save_state():
with open(state_file, 'w') as f:
json.dump(state, f, indent=2)
# Save calls immediately so they persist even if we exit early or error out
save_state()
args = sys.argv[1:]
if not args:
sys.exit(0)
cmd1 = args[0]
if cmd1 == "workspace":
if len(args) < 2:
sys.exit(0)
cmd2 = args[1]
if cmd2 == "list":
res = {"workspaces": state.get("workspaces", [])}
print(json.dumps({"result": res}))
sys.exit(0)
elif cmd2 == "create":
label = "default"
cwd = "."
i = 2
while i < len(args):
if args[i] == "--label":
label = args[i+1]
i += 2
elif args[i] == "--cwd":
cwd = args[i+1]
i += 2
else:
i += 1
workspaces = state.get("workspaces", [])
if not any(w["label"] == label for w in workspaces):
ws_id = f"w{len(workspaces) + 1}"
workspaces.append({"workspace_id": ws_id, "label": label, "cwd": cwd})
state["workspaces"] = workspaces
save_state()
sys.exit(0)
elif cmd1 == "agent":
if len(args) < 2:
sys.exit(0)
cmd2 = args[1]
if cmd2 == "list":
agents_list = []
for name, data in state.get("agents", {}).items():
agents_list.append({
"name": name,
"agent": data.get("agent", "claude"),
"agent_status": data.get("status", "running"),
"cwd": data.get("cwd", ""),
"pane_id": data.get("pane_id", "w1:p1"),
"workspace_id": data.get("workspace_id", "w1")
})
print(json.dumps({"result": {"agents": agents_list}}))
sys.exit(0)
elif cmd2 == "start":
if len(args) < 3:
sys.exit(1)
name = args[2]
ws = ""
cwd = ""
# Find where -- is
try:
double_dash_idx = args.index("--")
agent_cmd = args[double_dash_idx+1:]
opts = args[3:double_dash_idx]
except ValueError:
agent_cmd = []
opts = args[3:]
i = 0
while i < len(opts):
if opts[i] == "--workspace":
ws = opts[i+1]
i += 2
elif opts[i] == "--cwd":
cwd = opts[i+1]
i += 2
elif opts[i] == "--env":
env_val = opts[i+1]
if "=" in env_val:
k, v = env_val.split("=", 1)
os.environ[k] = v
i += 2
else:
i += 1
# Determine the agent type (claude, agy, hermes, cline)
agent_type = "claude"
if agent_cmd:
if "claude" in agent_cmd[0]:
agent_type = "claude"
elif "agy" in agent_cmd[0]:
agent_type = "agy"
elif "hermes" in agent_cmd[0]:
agent_type = "hermes"
elif "cline" in agent_cmd[0]:
agent_type = "cline"
else:
# guess from name
if "claude" in name:
agent_type = "claude"
elif "agy" in name:
agent_type = "agy"
elif "hermes" in name:
agent_type = "hermes"
elif "cline" in name:
agent_type = "cline"
# TUI Welcome Tokens definition to prevent TUI readiness check timeout
buffer_content = {
"claude": "Anthropic Claude Ready",
"agy": "Antigravity Ready",
"hermes": "Hermes Ready",
"cline": "Cline Chat Ready"
}.get(agent_type, "Ready")
agents = state.get("agents", {})
agents[name] = {
"agent": agent_type,
"status": "running",
"cwd": cwd or "TMP_PATH_PLACEHOLDER",
"workspace_id": ws or "w1",
"pid": 9999,
"pane_id": "w1:p1",
"command": " ".join(agent_cmd),
"buffer": buffer_content
}
# Generate session/conversation UUID and files to simulate agent startup
session_uuid = str(uuid.uuid4())
own_key_map = {
"claude": "claude_session_id_own",
"agy": "agy_conversation_id_own",
"hermes": "hermes_conversation_id_own",
"cline": "cline_conversation_id_own"
}
own_key = own_key_map.get(agent_type)
if own_key:
agents[name][own_key] = session_uuid
# Find isolation directory (e.g. if HOME has agent_homes/<uuid>)
home_dir = os.environ.get("HOME", "TMP_PATH_PLACEHOLDER")
ws_abs = os.path.abspath(cwd or "TMP_PATH_PLACEHOLDER")
ws_key = ws_abs.replace('/', '-').replace('_', '-')
if agent_type == "claude":
ccd = os.environ.get("CLAUDE_CONFIG_DIR")
if ccd:
cp_dir = os.path.join(ccd, "projects")
else:
cp_dir = os.environ.get("CLAUDE_PROJECT_DIR", f"{home_dir}/.claude/projects")
proj_dir = os.path.join(cp_dir, ws_key)
os.makedirs(proj_dir, exist_ok=True)
jsonl_file = os.path.join(proj_dir, f"{session_uuid}.jsonl")
with open(jsonl_file, 'w') as jf:
jf.write(json.dumps({"sessionId": session_uuid}) + "\\\\n")
elif agent_type == "agy":
db_dir = os.path.join(home_dir, ".gemini", "antigravity-cli", "conversations")
os.makedirs(db_dir, exist_ok=True)
db_file = os.path.join(db_dir, f"{session_uuid}.db")
with open(db_file, 'w') as df:
df.write("")
lc_dir = os.path.join(home_dir, ".gemini", "antigravity-cli", "cache")
os.makedirs(lc_dir, exist_ok=True)
lc_file = os.path.join(lc_dir, "last_conversations.json")
lc_data = {}
if os.path.exists(lc_file):
try:
with open(lc_file, 'r') as lcf:
lc_data = json.load(lcf)
except Exception:
pass
lc_data[ws_abs] = session_uuid
with open(lc_file, 'w') as lcf:
json.dump(lc_data, lcf)
elif agent_type == "hermes":
hermes_dir = os.path.join(home_dir, ".hermes")
os.makedirs(hermes_dir, exist_ok=True)
hdb = os.path.join(hermes_dir, "state.db")
conn = sqlite3.connect(hdb)
conn.execute("CREATE TABLE IF NOT EXISTS sessions (id TEXT PRIMARY KEY, cwd TEXT, started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)")
conn.execute("INSERT OR REPLACE INTO sessions (id, cwd) VALUES (?, ?)", (session_uuid, ws_abs))
conn.commit()
conn.close()
elif agent_type == "cline":
clin_base = os.path.join(home_dir, ".cline", "data", "sessions")
if "agent_homes" in home_dir:
clin_base = os.path.join(home_dir, "sessions")
session_dir = os.path.join(clin_base, session_uuid)
os.makedirs(session_dir, exist_ok=True)
json_file = os.path.join(session_dir, f"{session_uuid}.json")
with open(json_file, 'w') as jf:
jf.write(json.dumps({"id": session_uuid}))
state["agents"] = agents
save_state()
sys.exit(0)
elif cmd2 == "get":
if len(args) < 3:
sys.exit(1)
name = args[2]
agents = state.get("agents", {})
if name in agents:
agent_data = agents[name]
pane_info = {
"pid": agent_data.get("pid", 9999),
"cwd": agent_data.get("cwd", ""),
"command": agent_data.get("command", ""),
"argv": agent_data.get("command", "")
}
print(json.dumps({"pane": pane_info}))
sys.exit(0)
else:
sys.stderr.write("Agent " + name + " not found\\\\n")
sys.exit(1)
elif cmd2 == "read":
if len(args) < 3:
sys.exit(1)
name = args[2]
agents = state.get("agents", {})
if name in agents:
buffer_content = agents[name].get("buffer", "Ready")
print(buffer_content)
sys.exit(0)
else:
sys.stderr.write("Agent " + name + " not found\\\\n")
sys.exit(1)
elif cmd2 == "send":
if len(args) < 4:
sys.exit(1)
name = args[2]
text = args[3]
agents = state.get("agents", {})
if name in agents:
agents[name]["sent_text"] = agents[name].get("sent_text", "") + text
if text == "C-m":
agents[name]["buffer"] = agents[name].get("buffer", "") + "\\nesc to interrupt"
else:
agents[name]["buffer"] = agents[name].get("buffer", "") + "\\n" + text
if "/exit" in text or "exit" in text or "Exit" in text:
agents[name]["status"] = "stopped"
state["agents"] = agents
save_state()
sys.exit(0)
else:
sys.stderr.write("Agent " + name + " not found\\\\n")
sys.exit(1)
elif cmd1 == "session":
if len(args) < 3:
sys.exit(1)
cmd2 = args[1]
name = args[2]
agents = state.get("agents", {})
if cmd2 == "stop":
if name in agents:
agents[name]["status"] = "stopped"
state["agents"] = agents
save_state()
sys.exit(0)
else:
sys.exit(1)
elif cmd2 == "delete":
if name in agents:
del agents[name]
state["agents"] = agents
save_state()
sys.exit(0)
else:
sys.exit(1)
elif cmd1 == "pane":
if len(args) < 4:
sys.exit(1)
cmd2 = args[1]
if cmd2 == "send-keys":
name = args[2]
key = args[3]
agents = state.get("agents", {})
if name in agents:
agents[name]["sent_keys"] = agents[name].get("sent_keys", []) + [key]
if key in ("Enter", "C-m"):
agents[name]["buffer"] = agents[name].get("buffer", "") + "\\nesc to interrupt"
state["agents"] = agents
save_state()
sys.exit(0)
else:
sys.exit(1)
elif cmd1 == "ls":
if "-F" in args:
for name, data in state.get("agents", {}).items():
print(f"{name}|999999")
sys.exit(0)
with open("/tmp/debug_mock_herdr.log", "a") as f_debug:
f_debug.write(f"ARGS: {sys.argv[1:]} | AGENTS: {list(state.get('agents', {}).keys())} | PATH: {os.path.exists(state_file)}\\n")
agents_list = []
for name, data in state.get("agents", {}).items():
agents_list.append({
"name": name,
"agent": data.get("agent", "claude"),
"agent_status": data.get("status", "running")
})
print(json.dumps({"result": {"agents": agents_list}}))
sys.exit(0)
sys.exit(0)
""".replace("STATE_FILE_PLACEHOLDER", str(state_file)).replace("TMP_PATH_PLACEHOLDER", str(tmp_path))
herdr_bin.write_text(code)
herdr_bin.chmod(0o755)
# Update PATH using monkeypatch.
# Prepend bin_dir, and append standard Homebrew/system paths so lib.sh won't prepend them
old_path = os.environ.get("PATH", "")
extra_dirs = [
"/home/linuxbrew/.linuxbrew/bin",
"/home/linuxbrew/.linuxbrew/sbin",
f"{tmp_path}/.local/bin",
f"{tmp_path}/.npm-global/bin"
]
new_path = old_path
for d in extra_dirs:
if d not in new_path:
new_path = f"{new_path}:{d}"
monkeypatch.setenv("PATH", f"{bin_dir}:{new_path}")
return state_file
@pytest.fixture
def mock_agents(mam_sandbox, monkeypatch):
"""
Generates mock agent binaries (claude, agy, hermes, cline)
and places them in the sandboxed bin folder to satisfy preflight checks.
"""
tmp_path = mam_sandbox
bin_dir = tmp_path / "bin"
bin_dir.mkdir(parents=True, exist_ok=True)
# 1. claude mock
claude_bin = bin_dir / "claude"
claude_bin.write_text("""#!/usr/bin/env python3
import sys
import json
args = sys.argv[1:]
if len(args) >= 2 and args[0] == "auth" and args[1] == "status":
print(json.dumps({"loggedIn": True}))
sys.exit(0)
sys.exit(0)
""")
claude_bin.chmod(0o755)
# 2. agy mock
agy_bin = bin_dir / "agy"
agy_bin.write_text("""#!/usr/bin/env python3
import sys
args = sys.argv[1:]
if len(args) >= 1 and args[0] == "models":
print("gemini-1.5-pro\\ngemini-1.5-flash")
sys.exit(0)
sys.exit(0)
""")
agy_bin.chmod(0o755)
# 3. hermes mock
hermes_bin = bin_dir / "hermes"
hermes_bin.write_text("""#!/usr/bin/env python3
import sys
args = sys.argv[1:]
if len(args) >= 1 and args[0] == "status":
print("Hermes functional")
sys.exit(0)
sys.exit(0)
""")
hermes_bin.chmod(0o755)
# 4. cline mock
cline_bin = bin_dir / "cline"
cline_bin.write_text("""#!/usr/bin/env python3
import sys
import json
args = sys.argv[1:]
if len(args) >= 1 and args[0] == "history":
print(json.dumps([]))
sys.exit(0)
sys.exit(0)
""")
cline_bin.chmod(0o755)
# 5. uuidgen mock to guarantee isolated creation UUIDs
uuidgen_bin = bin_dir / "uuidgen"
uuidgen_bin.write_text("""#!/usr/bin/env python3
import uuid
print(str(uuid.uuid4()))
""")
uuidgen_bin.chmod(0o755)
# Prepend bin directory to PATH
old_path = os.environ.get("PATH", "")
extra_dirs = [
"/home/linuxbrew/.linuxbrew/bin",
"/home/linuxbrew/.linuxbrew/sbin",
f"{tmp_path}/.local/bin",
f"{tmp_path}/.npm-global/bin"
]
new_path = old_path
for d in extra_dirs:
if d not in new_path:
new_path = f"{new_path}:{d}"
monkeypatch.setenv("PATH", f"{bin_dir}:{new_path}")
return bin_dir
+182
View File
@@ -0,0 +1,182 @@
import os
import subprocess
import json
import pytest
import sys
from pathlib import Path
# Helper to run mutation on agent-sessions.yaml using atomic_dump_yaml in bash
def run_mutation(mam_sandbox, mutation_str, env=None):
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
cmd_str = f"source {lib_path} && atomic_dump_yaml {yaml_path}"
run_env = dict(os.environ)
if env:
run_env.update(env)
res = subprocess.run(["bash", "-c", cmd_str], input=mutation_str, capture_output=True, text=True, env=run_env)
return res
def test_unbound_key_handling(mam_sandbox, mock_herdr):
"""
Verify unbound key handling: herdr has-session/kill-session with trailing parameters
handles it gracefully (returns status 1 with error) instead of crashing with unbound var or shift errors.
"""
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
subprocess.run(["bash", "-c", f"source {lib_path}"], cwd=str(mam_sandbox), env=dict(os.environ, HOME=str(mam_sandbox), WORKSPACE_ROOT=str(mam_sandbox)))
shim_path = mam_sandbox / ".mam" / "shim" / "herdr"
assert shim_path.exists(), "Shim herdr was not created"
# Run has-session with trailing -t
res = subprocess.run([str(shim_path), "has-session", "-t"], capture_output=True, text=True, env=dict(os.environ, HOME=str(mam_sandbox), WORKSPACE_ROOT=str(mam_sandbox), REAL_HERDR=str(mock_herdr)))
assert res.returncode == 1
assert "Error: -t requires a value" in res.stderr
# Run kill-session with trailing -t
res = subprocess.run([str(shim_path), "kill-session", "-t"], capture_output=True, text=True, env=dict(os.environ, HOME=str(mam_sandbox), WORKSPACE_ROOT=str(mam_sandbox), REAL_HERDR=str(mock_herdr)))
assert res.returncode == 1
assert "Error: -t requires a value" in res.stderr
def test_new_session_fallback_shell(mam_sandbox, mock_herdr):
"""
Verify new-session: launches shell when no command is provided.
"""
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
subprocess.run(["bash", "-c", f"source {lib_path}"], cwd=str(mam_sandbox), env=dict(os.environ, HOME=str(mam_sandbox), WORKSPACE_ROOT=str(mam_sandbox)))
shim_path = mam_sandbox / ".mam" / "shim" / "herdr"
test_shell = "/bin/custom_sh"
run_env = dict(os.environ, HOME=str(mam_sandbox), WORKSPACE_ROOT=str(mam_sandbox), REAL_HERDR=str(mock_herdr), SHELL=test_shell)
res = subprocess.run([str(shim_path), "new-session", "-s", "fallback-session"], capture_output=True, text=True, env=run_env)
assert res.returncode == 0, f"Stderr: {res.stderr}"
with open(mock_herdr, 'r') as f:
state = json.load(f)
calls = state.get("calls", [])
start_call = None
for call in calls:
if "agent" in call and "start" in call and "fallback-session" in call:
start_call = call
break
assert start_call is not None, f"No agent start call found in: {calls}"
assert start_call[-1] == test_shell, f"Expected last arg to be {test_shell}, got {start_call[-1]}"
def test_ls_key_error(mam_sandbox):
"""
Verify ls key error: parses agent list without 'name' correctly using fallback keys.
"""
py_code = """
import sys, json
try:
data = json.load(sys.stdin)
res = data.get('result', data)
for a in res.get('agents', []):
try:
name = a.get('name') or a.get('agent') or 'unknown'
print(f"{name}|0")
except Exception:
pass
except Exception:
pass
"""
input_json = json.dumps({
"result": {
"agents": [
{"agent": "claude", "status": "running"},
{"name": "agent-with-name", "agent": "agy"}
]
}
})
res = subprocess.run([sys.executable, "-c", py_code], input=input_json, capture_output=True, text=True)
assert res.returncode == 0
lines = res.stdout.strip().split('\n')
assert "claude|0" in lines
assert "agent-with-name|0" in lines
def test_reconcile_relocatability(mam_sandbox, mock_herdr):
"""
Verify relocatability: reconcile.sh runs when relocated or inside different paths.
"""
reconcile_script = mam_sandbox / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
run_dir = mam_sandbox / "some_other_dir"
run_dir.mkdir()
res = subprocess.run(["bash", str(reconcile_script), "--once", "--emit-diff", "--dry-run"], cwd=str(run_dir), capture_output=True, text=True, env=dict(os.environ, HOME=str(mam_sandbox), WORKSPACE_ROOT=str(mam_sandbox), REAL_HERDR=str(mock_herdr)))
assert res.returncode == 0, f"Failed when run from different directory. Stderr: {res.stderr}"
def test_workspace_server_mapping(mam_sandbox, mock_herdr):
"""
Verify workspace/server mapping: status output workspace is correct.
"""
status_script = mam_sandbox / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
session_name = "test-mapping-sess-creator-claude"
mutation = f"""
d['herdr_sessions'] = [{{
'name': '{session_name}',
'status': 'running',
'role': 'Creator',
'herdr_workspace': 'my-custom-workspace',
'pane': {{
'cwd': 'WS_PLACEHOLDER',
'pid': 7777,
'cmd': 'claude',
'cmd_full': 'claude'
}}
}}]
""".replace("WS_PLACEHOLDER", str(mam_sandbox))
res_mut = run_mutation(mam_sandbox, mutation)
assert res_mut.returncode == 0
res = subprocess.run(["bash", str(status_script), "--json"], capture_output=True, text=True, cwd=str(mam_sandbox), env=dict(os.environ, HOME=str(mam_sandbox), WORKSPACE_ROOT=str(mam_sandbox), REAL_HERDR=str(mock_herdr)))
assert res.returncode == 0, f"Stderr: {res.stderr}"
data = json.loads(res.stdout)
sess_detail = data["sessions_detail"]
target_sess = [s for s in sess_detail if s["name"] == session_name][0]
assert target_sess["server"] == "my-custom-workspace"
def test_variable_splicing_injection_safety(mam_sandbox, mock_herdr):
"""
Verify variable splicing injection: no vulnerabilities or syntax errors remain.
"""
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
subprocess.run(["bash", "-c", f"source {lib_path}"], cwd=str(mam_sandbox), env=dict(os.environ, HOME=str(mam_sandbox), WORKSPACE_ROOT=str(mam_sandbox)))
shim_path = mam_sandbox / ".mam" / "shim" / "herdr"
adversarial_name = 'my"server; import os; os.system("echo INJECTED")'
run_env = dict(os.environ, HOME=str(mam_sandbox), WORKSPACE_ROOT=str(mam_sandbox), REAL_HERDR=str(mock_herdr), HERDR_SERVER_NAME=adversarial_name)
res = subprocess.run([str(shim_path), "new-session", "-s", "injection-test-session"], capture_output=True, text=True, env=run_env)
assert res.returncode == 0, f"Failed with adversarial HERDR_SERVER_NAME. Stderr: {res.stderr}"
with open(mock_herdr, 'r') as f:
state = json.load(f)
assert any("injection-test-session" in call for call in state["calls"]), "Expected injection-test-session call to succeed"
def test_export_masking_exit_code_preservation():
"""
Verify export masking: exit codes are preserved when assigning and exporting.
"""
# 1. Export masked version exits 0 (silent fail)
cmd_masked = ["bash", "-c", "set -e; export TEST_VAR=$(false); echo 'survived'"]
res_masked = subprocess.run(cmd_masked, capture_output=True, text=True)
assert res_masked.returncode == 0
assert res_masked.stdout.strip() == "survived"
# 2. Fixed split version exits non-zero (preserves failure exit code)
cmd_fixed = ["bash", "-c", "set -e; TEST_VAR=$(false); export TEST_VAR; echo 'survived'"]
res_fixed = subprocess.run(cmd_fixed, capture_output=True, text=True)
assert res_fixed.returncode != 0
assert res_fixed.stdout.strip() != "survived"
+74
View File
@@ -0,0 +1,74 @@
import subprocess
import json
import yaml
def test_create_session_dry_run(mam_sandbox, mock_herdr, mock_agents):
"""
Sanity test verifying that create_session.sh --dry-run parses inputs,
invokes herdr's preflight checks, and exits successfully without side effects.
"""
tmp_path = mam_sandbox
script_path = tmp_path / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"
cmd = [
"bash", str(script_path),
"--workspace", str(tmp_path),
"--agent", "claude",
"--role", "Creator",
"--dry-run"
]
res = subprocess.run(cmd, capture_output=True, text=True)
assert res.returncode == 0, f"Stdout: {res.stdout}\nStderr: {res.stderr}"
assert "[dry-run] would provision isolation" in res.stdout
assert "[dry-run] would spawn" in res.stdout
# Verify that mock herdr was called for checking session existence (which gets translated to agent get)
with open(mock_herdr, 'r') as f:
state = json.load(f)
calls = state.get("calls", [])
assert any("agent" in call and "get" in call for call in calls), f"Calls: {calls}"
def test_create_session_full(mam_sandbox, mock_herdr, mock_agents):
"""
Sanity test verifying that a full create_session.sh run intercepts herdr calls,
completes the TUI readiness check using mock responses, and serializes state
correctly to the sandboxed agent-sessions.yaml registry.
"""
tmp_path = mam_sandbox
script_path = tmp_path / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"
cmd = [
"bash", str(script_path),
"--workspace", str(tmp_path),
"--agent", "claude",
"--role", "Creator"
]
res = subprocess.run(cmd, capture_output=True, text=True)
assert res.returncode == 0, f"Stdout: {res.stdout}\nStderr: {res.stderr}"
# 1. Verify mock herdr state file registers the agent session
with open(mock_herdr, 'r') as f:
state = json.load(f)
agents = state.get("agents", {})
assert len(agents) == 1, "There should be exactly one registered agent in the mock herdr state."
session_name = list(agents.keys())[0]
assert session_name.endswith("-creator-claude")
assert agents[session_name]["status"] == "running"
assert agents[session_name]["agent"] == "claude"
# 2. Verify that agent-sessions.yaml is successfully written
yaml_path = tmp_path / ".mam" / "agent-sessions.yaml"
assert yaml_path.exists(), "The agent-sessions.yaml configuration file was not created."
with open(yaml_path, 'r') as f:
reg = yaml.safe_load(f)
sessions = reg.get("herdr_sessions", [])
assert len(sessions) == 1
assert sessions[0]["name"] == session_name
assert sessions[0]["status"] == "running"
assert sessions[0]["role"] == "Creator"
assert sessions[0]["isolation"]["uuid"] is not None
+363
View File
@@ -0,0 +1,363 @@
import os
import subprocess
import json
import hmac
import hashlib
import shlex
import sys
import pytest
# Helper to run bash snippets sourcing lib.sh
def run_lib_func(mam_sandbox, func_name, *args, env=None):
lib_path = mam_sandbox / "skills" / "lib.sh"
cmd_str = f"source {lib_path} && {func_name} " + " ".join(shlex.quote(str(a)) for a in args)
run_env = dict(os.environ)
if env:
run_env.update(env)
res = subprocess.run(["bash", "-c", cmd_str], capture_output=True, text=True, env=run_env)
return res
def get_mqtt_common(mam_sandbox):
script_path = str(mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts")
if script_path not in sys.path:
sys.path.insert(0, script_path)
import mqtt_common
return mqtt_common
# ==============================================================================
# FEATURE 1: Create Session (7 Test Cases)
# ==============================================================================
def test_create_derive_session_name_standard(mam_sandbox):
"""Test standard derive_session_name slug generation."""
res = run_lib_func(mam_sandbox, "derive_session_name", "/home/user/project", "claude")
assert res.returncode == 0
assert res.stdout.strip() == "user-project-creator-claude"
def test_create_derive_session_name_nested(mam_sandbox):
"""Test derive_session_name with nested paths, upper casing, and underscores."""
res = run_lib_func(mam_sandbox, "derive_session_name", "/home/User_Name/My_New_Project", "agy")
assert res.returncode == 0
assert res.stdout.strip() == "user-name-my-new-project-creator-agy"
def test_create_derive_session_name_weird_characters(mam_sandbox):
"""Test derive_session_name with spaces and punctuation in the path."""
res = run_lib_func(mam_sandbox, "derive_session_name", "/a/b c/d-e!f", "hermes")
assert res.returncode == 0
assert res.stdout.strip() == "bc-d-ef-creator-hermes"
def test_create_isolation_lever(mam_sandbox):
"""Test isolation_lever outputs for each supported agent."""
agents = {
"claude": "claude_config_dir",
"cline": "cline_data_dir",
"agy": "home",
"hermes": "home",
"unknown": ""
}
for agent, expected in agents.items():
res = run_lib_func(mam_sandbox, "isolation_lever", agent)
assert res.returncode == 0
assert res.stdout.strip() == expected
def test_create_isolation_env_prefix(mam_sandbox):
"""Test isolation_env_prefix format outputs."""
res = run_lib_func(mam_sandbox, "isolation_env_prefix", "claude", "/tmp/iso")
assert res.returncode == 0
assert res.stdout == "CLAUDE_CONFIG_DIR=/tmp/iso "
res2 = run_lib_func(mam_sandbox, "isolation_env_prefix", "agy", "/tmp/iso")
assert res2.returncode == 0
assert res2.stdout == "HOME=/tmp/iso "
res3 = run_lib_func(mam_sandbox, "isolation_env_prefix", "cline", "/tmp/iso")
assert res3.returncode == 0
assert res3.stdout == ""
def test_create_isolation_cmd_args(mam_sandbox):
"""Test isolation_cmd_args format outputs."""
res = run_lib_func(mam_sandbox, "isolation_cmd_args", "cline", "/tmp/iso")
assert res.returncode == 0
assert res.stdout == "--data-dir /tmp/iso"
res2 = run_lib_func(mam_sandbox, "isolation_cmd_args", "claude", "/tmp/iso")
assert res2.returncode == 0
assert res2.stdout == ""
def test_create_validate_env_key(mam_sandbox):
"""Test _validate_env_key function with valid and blocked environment keys."""
# Valid key
res = run_lib_func(mam_sandbox, "_validate_env_key", "MY_VALID_KEY")
assert res.returncode == 0
# Malformed key (starts with number)
res2 = run_lib_func(mam_sandbox, "_validate_env_key", "123BAD")
assert res2.returncode != 0
# Blocked key (LD_PRELOAD)
res3 = run_lib_func(mam_sandbox, "_validate_env_key", "LD_PRELOAD")
assert res3.returncode != 0
# ==============================================================================
# FEATURE 2: Resume Session (6 Test Cases)
# ==============================================================================
def test_resume_resolve_herdr_workspace_default(mam_sandbox):
"""Test resolve_herdr_workspace fallback behavior when session is not in YAML."""
res = run_lib_func(mam_sandbox, "resolve_herdr_workspace", "non-existent-session")
assert res.returncode == 0
assert res.stdout.strip() == "w1"
def test_resume_resolve_herdr_workspace_env(mam_sandbox):
"""Test resolve_herdr_workspace fallback to HERDR_SERVER_NAME env var."""
res = run_lib_func(mam_sandbox, "resolve_herdr_workspace", "non-existent-session", env={"HERDR_SERVER_NAME": "custom_server"})
assert res.returncode == 0
assert res.stdout.strip() == "w1" # default mapping to w1 unless resolved from workspace list
def test_resume_find_workspace_uuid_empty(mam_sandbox):
"""Test find_workspace_uuid returns empty string for non-existent workspace."""
res = run_lib_func(mam_sandbox, "find_workspace_uuid", "/non/existent/path", "claude")
assert res.returncode == 0
assert res.stdout.strip() == ""
def test_resume_find_workspace_uuid_target_non_existent(mam_sandbox):
"""Test target session query with target that does not exist in YAML."""
res = run_lib_func(mam_sandbox, "find_workspace_uuid", str(mam_sandbox), "claude", "non-existent-session")
assert res.returncode == 0
assert res.stdout.strip() == ""
def test_resume_find_workspace_uuid_invalid_agent(mam_sandbox):
"""Test find_workspace_uuid behavior with an unsupported agent name."""
res = run_lib_func(mam_sandbox, "find_workspace_uuid", str(mam_sandbox), "invalidagent")
assert res.returncode == 0
assert res.stdout.strip() == ""
def test_resume_script_invalid_args(mam_sandbox):
"""Test calling resolve_session_id.sh with missing arguments."""
script_path = mam_sandbox / "skills" / "multi-agent-mux-resume" / "scripts" / "resolve_session_id.sh"
res = subprocess.run(["bash", str(script_path), "--workspace", str(mam_sandbox)], capture_output=True, text=True)
assert res.returncode == 2
assert "ERROR: --agent required" in res.stderr
# ==============================================================================
# FEATURE 3: Stop Session (5 Test Cases)
# ==============================================================================
def test_stop_check_is_nfs_local(mam_sandbox):
"""Test that _check_is_nfs on local temp directory returns non-zero (not NFS)."""
res = run_lib_func(mam_sandbox, "_check_is_nfs", str(mam_sandbox))
# It will exit with 1 if it is not NFS
assert res.returncode == 1
def test_stop_is_already_stopped_not_found(mam_sandbox):
"""Test is_already_stopped exits with 1 when session is not in YAML."""
res = run_lib_func(mam_sandbox, "is_already_stopped", "non-existent-session")
assert res.returncode == 1
def test_stop_session_invalid_agent_suffix(mam_sandbox):
"""Test stop_session.sh fails when agent cannot be inferred from session name."""
script_path = mam_sandbox / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
res = subprocess.run(["bash", str(script_path), "--session", "bad-session-name"], capture_output=True, text=True)
assert res.returncode == 2
assert "ERROR: cannot infer agent" in res.stderr
def test_stop_session_missing_required_args(mam_sandbox):
"""Test stop_session.sh fails when session name is missing."""
script_path = mam_sandbox / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
res = subprocess.run(["bash", str(script_path)], capture_output=True, text=True)
assert res.returncode == 2
assert "ERROR: --session required" in res.stderr
def test_stop_session_purge_no_yes(mam_sandbox):
"""Test stop_session.sh exits with 3 when purge is requested without --yes."""
script_path = mam_sandbox / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
# Seed yaml with the session first to avoid "not in yaml" exit 1
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
yaml_path.write_text("""herdr_sessions:
- name: test-project-creator-claude
status: running
pane:
cwd: /tmp
""")
res = subprocess.run(["bash", str(script_path), "--session", "test-project-creator-claude", "--purge-conversation"], capture_output=True, text=True)
assert res.returncode == 3
assert "DANGER: --purge-conversation will DELETE" in res.stdout
# ==============================================================================
# FEATURE 4: Status Query (5 Test Cases)
# ==============================================================================
def test_status_json_schema_fields(mam_sandbox, mock_herdr, mock_agents):
"""Verify status.sh output JSON contains the expected structure."""
script_path = mam_sandbox / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
# Run status.sh with --json
res = subprocess.run(["bash", str(script_path), "--json"], capture_output=True, text=True)
assert res.returncode == 0
data = json.loads(res.stdout)
assert "timestamp" in data
assert "yaml_path" in data
assert "drifts" in data
assert "sessions_detail" in data
def test_status_text_headers_presence(mam_sandbox):
"""Verify status.sh output in text mode includes header columns."""
script_path = mam_sandbox / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
res = subprocess.run(["bash", str(script_path)], capture_output=True, text=True)
assert res.returncode == 0
assert "NAME" in res.stdout
assert "WORKSPACE" in res.stdout
assert "YAML" in res.stdout
assert "HERDR" in res.stdout
assert "DRIFT" in res.stdout
def test_status_resume_on_disk_helper_claude(mam_sandbox):
"""Verify resume_on_disk behavior for claude inside status.sh logic via mock YAML queries."""
# We can write a custom yaml and check status
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
yaml_path.write_text("""herdr_sessions:
- name: test-project-creator-claude
status: running
claude_session_id_own: some-uuid
pane:
cwd: /tmp/nonexistent-workspace
""")
script_path = mam_sandbox / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
res = subprocess.run(["bash", str(script_path), "--json"], capture_output=True, text=True)
assert res.returncode == 0
data = json.loads(res.stdout)
detail = data["sessions_detail"][0]
assert detail["name"] == "test-project-creator-claude"
assert detail["resume_state"] == "MISSING"
def test_status_resume_on_disk_helper_agy(mam_sandbox):
"""Verify resume_on_disk behavior for agy inside status.sh logic."""
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
yaml_path.write_text("""herdr_sessions:
- name: test-project-creator-agy
status: running
agy_conversation_id_own: some-uuid
pane:
cwd: /tmp/nonexistent-workspace
""")
script_path = mam_sandbox / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
res = subprocess.run(["bash", str(script_path), "--json"], capture_output=True, text=True)
assert res.returncode == 0
data = json.loads(res.stdout)
detail = data["sessions_detail"][0]
assert detail["name"] == "test-project-creator-agy"
assert detail["resume_state"] == "MISSING"
def test_status_get_job_status_helper(mam_sandbox):
"""Verify get_job_status parses non-existent jobs gracefully."""
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
yaml_path.write_text("""herdr_sessions:
- name: test-project-creator-claude
status: running
delegate_job_id: nonexistent-job-id
pane:
cwd: /tmp
""")
script_path = mam_sandbox / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
res = subprocess.run(["bash", str(script_path), "--json"], capture_output=True, text=True)
assert res.returncode == 0
data = json.loads(res.stdout)
detail = data["sessions_detail"][0]
assert detail["job_id"] == "nonexistent-job-id"
assert detail["job_status"] == "unknown"
# ==============================================================================
# FEATURE 5: Monitor/Reconcile (6 Test Cases)
# ==============================================================================
def test_mqtt_topic_prefix_for(mam_sandbox):
"""Test topic prefix helper methods in mqtt_common."""
mqtt_common = get_mqtt_common(mam_sandbox)
prefix = mqtt_common.topic_prefix_for("job123")
assert prefix == "python/mqtt/jobs/job123"
events_topic = mqtt_common.events_topic_for("job123")
assert events_topic == "python/mqtt/jobs/job123/events"
def test_mqtt_verify_hmac_no_token(mam_sandbox):
"""Verify verify_hmac returns True when no token is present."""
mqtt_common = get_mqtt_common(mam_sandbox)
payload = {"data": {"hmac_sig": "somesig"}}
assert mqtt_common.verify_hmac(payload, None) is True
assert mqtt_common.verify_hmac(payload, "") is True
def test_mqtt_verify_hmac_valid_invalid(mam_sandbox):
"""Verify verify_hmac signature validation matching logic."""
mqtt_common = get_mqtt_common(mam_sandbox)
token = "secret_key"
payload = {
"job_id": "job1",
"event": "started",
"seq": 1,
"timestamp": "2026-07-19T00:00:00Z",
"data": {
"some_key": "some_val"
}
}
# Calculate HMAC
msg = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
sig = hmac.new(token.encode(), msg, hashlib.sha256).hexdigest()
# Put HMAC signature inside data block
payload["data"]["hmac_sig"] = sig
assert mqtt_common.verify_hmac(payload, token) is True
# Modifying payload should cause verification to fail
payload["seq"] = 2
assert mqtt_common.verify_hmac(payload, token) is False
def test_mqtt_reason_code_value(mam_sandbox):
"""Verify reason_code_value correctly extracts values from paho reason codes."""
mqtt_common = get_mqtt_common(mam_sandbox)
# Test plain int
assert mqtt_common.reason_code_value(0) == 0
assert mqtt_common.reason_code_value(5) == 5
# Test object with .value attribute
class DummyReasonCode:
def __init__(self, val):
self.value = val
assert mqtt_common.reason_code_value(DummyReasonCode(0)) == 0
assert mqtt_common.reason_code_value(DummyReasonCode(16)) == 16
def test_mqtt_with_retry_success(mam_sandbox):
"""Verify with_retry decorator works on direct success."""
mqtt_common = get_mqtt_common(mam_sandbox)
calls = []
@mqtt_common.with_retry(attempts=3)
def dummy_func(x):
calls.append(x)
return x * 2
res = dummy_func(5)
assert res == 10
assert calls == [5]
def test_mqtt_with_retry_failure(mam_sandbox):
"""Verify with_retry decorator raises error after specified attempts."""
mqtt_common = get_mqtt_common(mam_sandbox)
calls = []
@mqtt_common.with_retry(attempts=3, base_delay=0.01)
def failing_func():
calls.append(1)
raise ValueError("failing")
with pytest.raises(ValueError, match="failing"):
failing_func()
assert len(calls) == 3
+734
View File
@@ -0,0 +1,734 @@
import os
import shutil
import json
import sqlite3
import subprocess
import pytest
import time
import sys
import shlex
from pathlib import Path
# Helper to run mutation on agent-sessions.yaml using atomic_dump_yaml in bash
def run_mutation(mam_sandbox, mutation_str, env=None):
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
cmd_str = f"source {lib_path} && atomic_dump_yaml {yaml_path}"
run_env = dict(os.environ)
if env:
run_env.update(env)
res = subprocess.run(["bash", "-c", cmd_str], input=mutation_str, capture_output=True, text=True, env=run_env)
return res
def get_mqtt_common(mam_sandbox):
script_path = str(mam_sandbox / ".agents" / "skills" / "multi-agent-mux-delegate-job" / "scripts")
if script_path not in sys.path:
sys.path.insert(0, script_path)
import mqtt_common
return mqtt_common
# ==============================================================================
# FEATURE 1: Create Session (5 Test Cases)
# ==============================================================================
def test_comp_create_schema_validation(mam_sandbox):
"""Verify that atomic_dump_yaml validates the schema and rejects malformed formats."""
# Try setting herdr_sessions to a dictionary instead of list
mutation = "d['herdr_sessions'] = {}"
res = run_mutation(mam_sandbox, mutation)
assert "VALIDATE: herdr_sessions is not a list" in res.stderr
# Try setting status to an invalid state
mutation_invalid_status = """
d['herdr_sessions'] = [{
'name': 'session1',
'status': 'invalidstatus',
'pane': {}
}]
"""
res2 = run_mutation(mam_sandbox, mutation_invalid_status)
assert "VALIDATE:" in res2.stderr
def test_comp_create_role_immutability(mam_sandbox):
"""Verify that once a role is defined in the sessions, it cannot be mutated."""
# 1. Setup session with a role
mutation_setup = """
d['herdr_sessions'] = [{
'name': 'session-imm-role',
'status': 'running',
'role': 'Creator',
'pane': {}
}]
"""
res = run_mutation(mam_sandbox, mutation_setup)
assert res.returncode == 0
# 2. Try mutating the role
mutation_bad = """
for s in d.get('herdr_sessions', []):
if s.get('name') == 'session-imm-role':
s['role'] = 'Reviewer'
"""
res_bad = run_mutation(mam_sandbox, mutation_bad)
assert res_bad.returncode != 0
assert "role of session 'session-imm-role' cannot be modified" in res_bad.stderr
def test_comp_create_duplicate_running_ids(mam_sandbox):
"""Verify that duplicate running conversation IDs across sessions are rejected."""
mutation = """
d['herdr_sessions'] = [
{
'name': 'sess1',
'status': 'running',
'claude_session_id_own': 'uuid-1234',
'pane': {}
},
{
'name': 'sess2',
'status': 'running',
'claude_session_id_own': 'uuid-1234',
'pane': {}
}
]
"""
res = run_mutation(mam_sandbox, mutation)
assert res.returncode != 0
assert "Duplicate running conversation ID" in res.stderr
def test_comp_create_isolation_folder_setup(mam_sandbox):
"""Verify that provision_isolation correctly creates directories and symlinks."""
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
iso_root = mam_sandbox / "iso_home_test"
# Mock global claude credentials
claude_cred = mam_sandbox / ".claude"
claude_cred.mkdir(parents=True, exist_ok=True)
(claude_cred / ".credentials.json").write_text('{"token": "xyz"}')
cmd_str = f"source {lib_path} && provision_isolation claude {iso_root}"
res = subprocess.run(["bash", "-c", cmd_str], capture_output=True, text=True)
assert res.returncode == 0
# Verify symlink exists and points to the credentials
cred_sym = iso_root / ".credentials.json"
assert cred_sym.is_symlink()
assert cred_sym.read_text() == '{"token": "xyz"}'
def test_comp_create_sqlite_tables_created(mam_sandbox, mock_herdr, mock_agents):
"""Verify that tables exist and contain records after a full create_session.sh run."""
script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"
cmd = [
"bash", str(script_path),
"--workspace", str(mam_sandbox),
"--agent", "claude",
"--role", "Creator"
]
res = subprocess.run(cmd, capture_output=True, text=True)
assert res.returncode == 0
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
assert db_path.exists()
# Connect directly to the SQLite DB
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Check tables
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
tables = [row[0] for row in cursor.fetchall()]
assert "state" in tables
assert "sessions" in tables
# Check entries in sessions table
cursor.execute("SELECT name, status, pane_cwd FROM sessions;")
rows = cursor.fetchall()
assert len(rows) == 1
assert rows[0][0].endswith("-creator-claude")
assert rows[0][1] == "running"
conn.close()
# ==============================================================================
# FEATURE 2: Resume Session (5 Test Cases)
# ==============================================================================
def test_comp_resume_config_restore(mam_sandbox):
"""Verify that isolation details are preserved and can be read from the DB registry."""
# Write a test session with isolation details
mutation = """
d['herdr_sessions'] = [{
'name': 'test-session-iso',
'status': 'stopped',
'pane': {'cwd': '/tmp'},
'isolation': {
'uuid': 'iso-uuid-789',
'root': '/tmp/iso_root',
'lever': 'claude_config_dir',
'seeded': []
}
}]
"""
res = run_mutation(mam_sandbox, mutation)
assert res.returncode == 0
# Read the configuration via resume_session.sh isolation resolution check
script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-resume" / "scripts" / "resume_session.sh"
# We run in dry-run/mock mode or just execute python command checking block
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
# Call the python parsing block from resume_session.sh directly
python_block = """
import os, json, sqlite3
name = "test-session-iso"
db_path = "DB_PATH_PLACEHOLDER"
conn = sqlite3.connect(db_path)
row = conn.execute('SELECT data FROM sessions WHERE name=?', (name,)).fetchone()
if row:
import json
s = json.loads(row[0])
print(json.dumps(s.get('isolation') or {}))
""".replace("DB_PATH_PLACEHOLDER", str(db_path))
res_py = subprocess.run(["python3", "-c", python_block], capture_output=True, text=True)
assert res_py.returncode == 0
data = json.loads(res_py.stdout)
assert data["uuid"] == "iso-uuid-789"
assert data["root"] == "/tmp/iso_root"
def test_comp_resume_metadata_read_integrity(mam_sandbox):
"""Verify metadata read integrity from the SQLite DB by changing state outside YAML."""
# Write initial YAML state
mutation = """
d['herdr_sessions'] = [{
'name': 'sess-integrity',
'status': 'stopped',
'pane': {'cwd': '/tmp'}
}]
"""
run_mutation(mam_sandbox, mutation)
# Update SQLite directly to terminated
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
conn = sqlite3.connect(str(db_path))
conn.execute("UPDATE sessions SET status='terminated' WHERE name='sess-integrity'")
conn.commit()
conn.close()
# Read via atomic_dump_yaml check (which pulls from DB sessions table)
# If the DB reading has integrity, d will contain sess-integrity as status 'terminated'
mutation_check = """
for s in d.get('herdr_sessions', []):
if s.get('name') == 'sess-integrity':
print(f"STATUS={s['status']}")
"""
res = run_mutation(mam_sandbox, mutation_check)
assert "STATUS=terminated" in res.stdout
def test_comp_resume_update_yaml(mam_sandbox):
"""Verify that update_yaml_resumed.sh cleans up stop fields and marks status running."""
# Seed a stopped session with stop metadata
mutation = """
d['herdr_sessions'] = [{
'name': 'test-resumed-session-creator-claude',
'status': 'stopped',
'stopped_at': '2026-07-19T00:00:00Z',
'stop_reason': 'manual_stop',
'pane': {'cwd': '/tmp'}
}]
"""
run_mutation(mam_sandbox, mutation)
script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-resume" / "scripts" / "update_yaml_resumed.sh"
res = subprocess.run(["bash", str(script_path), "--session", "test-resumed-session-creator-claude", "--uuid", "new-uuid-999"], capture_output=True, text=True)
assert res.returncode == 0
# Verify stopped fields are popped
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT data FROM sessions WHERE name='test-resumed-session-creator-claude'").fetchone()
s = json.loads(row[0])
assert s["status"] == "running"
assert "stopped_at" not in s
assert "stop_reason" not in s
assert s["claude_session_id_own"] == "new-uuid-999"
conn.close()
def test_comp_resume_find_workspace_uuid_tier1_own_id(mam_sandbox):
"""Verify find_workspace_uuid resolves the per-row own ID properly."""
# Write a running session with a claude_session_id_own
mutation = """
d['herdr_sessions'] = [{
'name': 'ws-own-creator-claude',
'status': 'running',
'claude_session_id_own': 'own-uuid-111',
'pane': {'cwd': 'WS_PLACEHOLDER'}
}]
""".replace("WS_PLACEHOLDER", str(mam_sandbox))
run_mutation(mam_sandbox, mutation)
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
# Target session mode scopes resolution to the row
cmd_str = f"source {lib_path} && find_workspace_uuid {mam_sandbox} claude ws-own-creator-claude"
# Since we are mock-checking, own_exists function mock will query disk format.
# Claude disk format requires a project file: projects/<key>/<uuid>.jsonl
# Let's create it.
key = str(mam_sandbox).replace('/', '-').replace('_', '-')
proj_dir = mam_sandbox / ".claude" / "projects" / key
proj_dir.mkdir(parents=True, exist_ok=True)
(proj_dir / "own-uuid-111.jsonl").write_text('{"sessionId": "own-uuid-111"}')
res = subprocess.run(["bash", "-c", cmd_str], capture_output=True, text=True)
assert res.returncode == 0
# Should resolve to own-uuid-111 because it exists in projects folder
assert res.stdout.strip() == "own-uuid-111"
def test_comp_resume_find_workspace_uuid_tier2_disk_scan_claude(mam_sandbox):
"""Verify find_workspace_uuid scans workspace on disk if own ID isn't set in the row."""
# Seed session without own ID
mutation = """
d['herdr_sessions'] = [{
'name': 'ws-scan-creator-claude',
'status': 'stopped',
'pane': {'cwd': 'WS_PLACEHOLDER'}
}]
""".replace("WS_PLACEHOLDER", str(mam_sandbox))
run_mutation(mam_sandbox, mutation)
# Create JSONL file
key = str(mam_sandbox).replace('/', '-').replace('_', '-')
proj_dir = mam_sandbox / ".claude" / "projects" / key
proj_dir.mkdir(parents=True, exist_ok=True)
(proj_dir / "scanned-uuid.jsonl").write_text('{"sessionId": "scanned-uuid"}')
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
cmd_str = f"source {lib_path} && find_workspace_uuid {mam_sandbox} claude"
res = subprocess.run(["bash", "-c", cmd_str], capture_output=True, text=True)
assert res.returncode == 0
assert res.stdout.strip() == "scanned-uuid"
# ==============================================================================
# FEATURE 3: Stop Session (5 Test Cases)
# ==============================================================================
def test_comp_stop_safe_path_checking(mam_sandbox):
"""Verify path guards block directory deletion if isolation path check fails."""
# Mock a terminated session where isolation root is set outside .mam folder
mutation = """
d['herdr_sessions'] = [{
'name': 'test-purge-guard-creator-claude',
'status': 'running',
'pane': {'cwd': 'WS_PLACEHOLDER'},
'isolation': {
'uuid': 'some-uuid',
'root': '/tmp/unauthorized_path_outside_mam'
}
}]
""".replace("WS_PLACEHOLDER", str(mam_sandbox))
run_mutation(mam_sandbox, mutation)
script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
# Attempt to purge. The python script should print "WARN: isolated home path check failed" and NOT crash
res = subprocess.run(["bash", str(script_path), "--session", "test-purge-guard-creator-claude", "--purge-conversation", "--yes"], capture_output=True, text=True)
assert res.returncode == 0
assert "WARN: isolated home path check failed" in res.stdout
def test_comp_stop_sqlite_state_update(mam_sandbox):
"""Verify that stop_session.sh updates state in SQLite to stopped."""
mutation = """
d['herdr_sessions'] = [{
'name': 'test-stop-sqlite-creator-claude',
'status': 'running',
'pane': {'cwd': 'WS_PLACEHOLDER'}
}]
""".replace("WS_PLACEHOLDER", str(mam_sandbox))
run_mutation(mam_sandbox, mutation)
script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
res = subprocess.run(["bash", str(script_path), "--session", "test-stop-sqlite-creator-claude"], capture_output=True, text=True)
assert res.returncode == 0
# Query database directly
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT status, data FROM sessions WHERE name='test-stop-sqlite-creator-claude'").fetchone()
assert row[0] == "stopped"
s = json.loads(row[1])
assert s["status"] == "stopped"
assert "stopped_at" in s
assert s["stop_reason"] == "manual_stop"
conn.close()
def test_comp_stop_purge_record_removal(mam_sandbox):
"""Verify that purge completely removes the session record from DB."""
mutation = """
d['herdr_sessions'] = [{
'name': 'test-purge-record-creator-claude',
'status': 'running',
'pane': {'cwd': 'WS_PLACEHOLDER'}
}]
""".replace("WS_PLACEHOLDER", str(mam_sandbox))
run_mutation(mam_sandbox, mutation)
script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
res = subprocess.run(["bash", str(script_path), "--session", "test-purge-record-creator-claude", "--purge-conversation", "--yes"], capture_output=True, text=True)
assert res.returncode == 0
# Check SQLite
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT COUNT(*) FROM sessions WHERE name='test-purge-record-creator-claude'").fetchone()
assert row[0] == 0
conn.close()
def test_comp_stop_lock_release(mam_sandbox):
"""Verify SQLite lock is released after running atomic_dump_yaml."""
mutation = """
d['herdr_sessions'] = [{
'name': 'test-lock-release',
'status': 'stopped',
'pane': {'cwd': '/tmp'}
}]
"""
res = run_mutation(mam_sandbox, mutation)
assert res.returncode == 0
# Since res has returned, the lock should be free.
# We test this by immediately acquiring a direct SQLite write connection
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
conn = sqlite3.connect(str(db_path), timeout=0.1)
try:
conn.execute("BEGIN IMMEDIATE")
conn.execute("UPDATE sessions SET status='archived' WHERE name='test-lock-release'")
conn.commit()
except sqlite3.OperationalError as e:
pytest.fail(f"Could not acquire SQLite lock: {e}")
finally:
conn.close()
def test_comp_stop_graceful_kill_chain(mam_sandbox, mock_herdr, mock_agents):
"""Verify that the graceful stop command fallback chain runs when herdr has-session is alive."""
# Write a running session in YAML
mutation = """
d['herdr_sessions'] = [{
'name': 'test-graceful-chain-creator-claude',
'status': 'running',
'pane': {'cwd': 'WS_PLACEHOLDER', 'pid': 12345}
}]
""".replace("WS_PLACEHOLDER", str(mam_sandbox))
run_mutation(mam_sandbox, mutation)
# Pre-populate herdr mock with the running session so it passes has-session check
with open(mock_herdr, 'r') as f:
state = json.load(f)
state["agents"]["test-graceful-chain-creator-claude"] = {
"status": "running",
"agent": "claude",
"cwd": str(mam_sandbox),
"pid": 12345,
"pane_id": "w1:p1",
"command": "claude",
"buffer": "Anthropic Claude Ready"
}
with open(mock_herdr, 'w') as f:
json.dump(state, f, indent=2)
script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
# Run stop_session.sh. This should call send-keys first
res = subprocess.run(["bash", str(script_path), "--session", "test-graceful-chain-creator-claude"], capture_output=True, text=True)
assert res.returncode == 0
with open(mock_herdr, 'r') as f:
state = json.load(f)
# Verify mock herdr calls recorded the keys "/exit" sent
calls = state.get("calls", [])
# Should see send-keys call
assert any("send" in call and "/exit" in call for call in calls)
# ==============================================================================
# FEATURE 4: Status Query (5 Test Cases)
# ==============================================================================
def test_comp_status_read_lock(mam_sandbox):
"""Verify status.sh accesses SQLite database with a read lock, not writing."""
# Seed DB first so it exists
run_mutation(mam_sandbox, "d['herdr_sessions'] = []")
script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
# Run status.sh in a background sub-process
res = subprocess.run(["bash", str(script_path), "--json"], capture_output=True, text=True)
assert res.returncode == 0
# Verify no writing occurred by comparing DB file modified time
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
t1 = os.path.getmtime(db_path)
# Run status.sh again
subprocess.run(["bash", str(script_path), "--json"], capture_output=True, text=True)
t2 = os.path.getmtime(db_path)
assert t1 == t2
def test_comp_status_drift_class_parsing(mam_sandbox):
"""Verify status.sh correctly parses drifts returned by reconcile.sh."""
# Write reconcile mock logic that outputs a drift JSON
reconcile_script = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
drift_output = {
"timestamp": "2026-07-19T00:00:00Z",
"yaml_path": str(mam_sandbox / ".mam" / "agent-sessions.yaml"),
"herdr_sessions_alive": ["test-session|default"],
"herdr_confirmed": True,
"drifts": [{"class": "A", "name": "test-session", "msg": "drift detected"}],
"actions": []
}
# Overwrite reconcile.sh to print this JSON directly
reconcile_script.write_text(f"""#!/usr/bin/env bash
echo '{json.dumps(drift_output)}'
""")
# Seed YAML
mutation = """
d['herdr_sessions'] = [{
'name': 'test-session',
'status': 'running',
'pane': {'cwd': '/tmp'}
}]
"""
run_mutation(mam_sandbox, mutation)
status_script = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
res = subprocess.run(["bash", str(status_script), "--json"], capture_output=True, text=True)
assert res.returncode == 0
data = json.loads(res.stdout)
assert data["sessions_detail"][0]["drift_classes"] == ["A"]
def test_comp_status_job_candidates_parsing(mam_sandbox):
"""Verify status.sh reads job statuses from audit logs or registry JSONs."""
# Write YAML session
mutation = """
d['herdr_sessions'] = [{
'name': 'test-job-session-creator-claude',
'status': 'running',
'delegate_job_id': 'job12345',
'pane': {'cwd': 'WS_PLACEHOLDER'}
}]
""".replace("WS_PLACEHOLDER", str(mam_sandbox))
run_mutation(mam_sandbox, mutation)
# Create the status.json in both candidate folders to cover path resolutions
job_logs_dir1 = mam_sandbox / ".mam" / "delegate_job_logs" / "job12345"
job_logs_dir1.mkdir(parents=True, exist_ok=True)
(job_logs_dir1 / "status.json").write_text('{"status": "completed"}')
job_logs_dir2 = mam_sandbox.parent / ".mam" / "delegate_job_logs" / "job12345"
job_logs_dir2.mkdir(parents=True, exist_ok=True)
(job_logs_dir2 / "status.json").write_text('{"status": "completed"}')
status_script = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
res = subprocess.run(["bash", str(status_script), "--json"], capture_output=True, text=True)
assert res.returncode == 0
data = json.loads(res.stdout)
assert data["sessions_detail"][0]["job_status"] == "completed"
def test_comp_status_no_side_effects(mam_sandbox):
"""Verify that running status.sh leaves YAML and DB unchanged."""
# Seed DB first so it exists
run_mutation(mam_sandbox, "d['herdr_sessions'] = []")
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
y_content_1 = yaml_path.read_text()
conn = sqlite3.connect(str(db_path))
d_content_1 = conn.execute("SELECT * FROM sessions").fetchall()
conn.close()
status_script = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
subprocess.run(["bash", str(status_script)], capture_output=True, text=True)
y_content_2 = yaml_path.read_text()
conn = sqlite3.connect(str(db_path))
d_content_2 = conn.execute("SELECT * FROM sessions").fetchall()
conn.close()
assert y_content_1 == y_content_2
assert d_content_1 == d_content_2
def test_comp_status_yaml_to_json_structure(mam_sandbox):
"""Verify YAML-to-JSON structure translation preserves all nested maps."""
mutation = """
d['herdr_sessions'] = [{
'name': 'test-struct-creator-claude',
'status': 'running',
'pane': {
'index': 0,
'pid': 1234,
'cmd': 'claude',
'cwd': '/tmp'
}
}]
"""
run_mutation(mam_sandbox, mutation)
status_script = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
res = subprocess.run(["bash", str(status_script), "--json"], capture_output=True, text=True)
assert res.returncode == 0
data = json.loads(res.stdout)
session = data["sessions_detail"][0]
assert session["pane_pid"] == 1234
assert session["pane_cwd"] == "/tmp"
# ==============================================================================
# FEATURE 5: Monitor/Reconcile (6 Test Cases)
# ==============================================================================
def test_comp_monitor_concurrency_lock(mam_sandbox):
"""Verify that multiple subscriber reconciles cannot run concurrently."""
# Reconcile subscribe script tries to acquire .mam/monitor.lock
# Let's write a mock subscriber that acquires the lock
workspace_root = mam_sandbox
lock_file_path = workspace_root / ".mam" / "monitor.lock"
# Hold lock in Python
import fcntl
lock_file = open(lock_file_path, 'w')
fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
# Now execute reconcile subscribe loop, it should exit immediately
reconcile_script = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
res = subprocess.run(["bash", str(reconcile_script), "--subscribe", "--idle-timeout", "1"], capture_output=True, text=True, cwd=str(mam_sandbox))
assert res.returncode == 0
assert "MQTT Monitor: another subscriber is already running" in res.stdout
lock_file.close()
def test_comp_monitor_sqlite_journal_wal(mam_sandbox):
"""Verify that SQLite database defaults to WAL mode on standard filesystems."""
mutation = """
d['herdr_sessions'] = []
"""
# Runs atomic_dump_yaml which sets journal mode
res = run_mutation(mam_sandbox, mutation)
assert res.returncode == 0
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
conn = sqlite3.connect(str(db_path))
mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
conn.close()
assert mode.lower() == "wal"
def test_comp_monitor_sqlite_journal_delete_nfs(mam_sandbox):
"""Verify that SQLite database falls back to DELETE mode on NFS filesystems."""
mutation = """
d['herdr_sessions'] = []
"""
# Set MAM_IS_NFS env var
res = run_mutation(mam_sandbox, mutation, env={"MAM_IS_NFS": "true"})
assert res.returncode == 0
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
conn = sqlite3.connect(str(db_path))
mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
conn.close()
assert mode.lower() == "delete"
def test_comp_monitor_db_schema_auto_update(mam_sandbox):
"""Verify that table structure and indexes are automatically created if DB is deleted."""
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
if db_path.exists():
db_path.unlink()
mutation = """
d['herdr_sessions'] = []
"""
res = run_mutation(mam_sandbox, mutation)
assert res.returncode == 0
assert db_path.exists()
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
tables = [row[0] for row in cursor.fetchall()]
assert "state" in tables
assert "sessions" in tables
cursor.execute("SELECT name FROM sqlite_master WHERE type='index';")
indexes = [row[0] for row in cursor.fetchall()]
assert "idx_sessions_pane_cwd" in indexes
conn.close()
def test_comp_monitor_monotonic_seq_updates(mam_sandbox):
"""Verify sequence monotonic updates in mqtt_common."""
mqtt_common = get_mqtt_common(mam_sandbox)
# Setup job JSON in jobs registry dir
registry_dir = mam_sandbox / ".mam" / "jobs"
registry_dir.mkdir(parents=True, exist_ok=True)
job_id = "job-seq-123"
job_data = {
"job_id": job_id,
"status": "running",
"last_seq": 0
}
with open(registry_dir / f"{job_id}.json", "w") as f:
json.dump(job_data, f)
# Get sequence multiple times
s1 = mqtt_common.next_seq(job_id, str(registry_dir))
s2 = mqtt_common.next_seq(job_id, str(registry_dir))
s3 = mqtt_common.next_seq(job_id, str(registry_dir))
assert s1 == 1
assert s2 == 2
assert s3 == 3
# Reload and check
with open(registry_dir / f"{job_id}.json", "r") as f:
loaded = json.load(f)
assert loaded["last_seq"] == 3
def test_comp_monitor_auto_state_recovery(mam_sandbox, mock_herdr):
"""Verify that reconcile.sh auto-terminates a session if herdr is confirmed dead."""
# Write a running session in YAML
mutation = """
d['herdr_sessions'] = [{
'name': 'test-autorecovery-creator-claude',
'status': 'running',
'pane': {'cwd': 'WS_PLACEHOLDER'}
}]
""".replace("WS_PLACEHOLDER", str(mam_sandbox))
run_mutation(mam_sandbox, mutation)
# Run reconcile.sh --once --emit-diff (which runs the dry-run, which does NOT write but reports)
reconcile_script = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
res = subprocess.run(["bash", str(reconcile_script), "--once", "--emit-diff", "--dry-run"], capture_output=True, text=True, cwd=str(mam_sandbox))
assert res.returncode == 0
data = json.loads(res.stdout)
drifts = data["drifts"]
# Should detect drift class A: herdr gone
assert any(dr["class"] == "A" and dr["name"] == "test-autorecovery-creator-claude" for dr in drifts)
# Now run reconcile.sh --once --emit-diff (without --dry-run) to trigger write
res_write = subprocess.run(["bash", str(reconcile_script), "--once", "--emit-diff"], capture_output=True, text=True, cwd=str(mam_sandbox))
assert res_write.returncode == 0
# Verify session is marked terminated in the SQLite DB
db_path = mam_sandbox / ".mam" / "agent-sessions.db"
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT status FROM sessions WHERE name='test-autorecovery-creator-claude'").fetchone()
assert row[0] == "terminated"
conn.close()
+401
View File
@@ -0,0 +1,401 @@
import os
import subprocess
import json
import sqlite3
import pytest
import shutil
import yaml
from pathlib import Path
# Helper to run mutation on agent-sessions.yaml using atomic_dump_yaml in bash
def run_mutation(mam_sandbox, mutation_str, env=None):
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
cmd_str = f"source {lib_path} && atomic_dump_yaml {yaml_path}"
run_env = dict(os.environ)
if env:
run_env.update(env)
res = subprocess.run(["bash", "-c", cmd_str], input=mutation_str, capture_output=True, text=True, env=run_env)
return res
def test_integration_create_options_combination(mam_sandbox, mock_herdr, mock_agents, monkeypatch):
"""
Tier 3: Create Session Integration
Covers pairwise combination: --onboard + --submit-job + --wrapper + HERDR_SERVER_NAME env override
"""
tmp_path = mam_sandbox
script_path = tmp_path / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"
# Set HERDR_SERVER_NAME env var to test environment override combination
monkeypatch.setenv("HERDR_SERVER_NAME", "custom_server")
cmd = [
"bash", str(script_path),
"--workspace", str(tmp_path),
"--agent", "claude",
"--role", "Creator",
"--onboard",
"--submit-job", "Test onboard prompt",
"--wrapper"
]
res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(tmp_path))
if res.returncode != 0:
with open(mock_herdr, 'r') as f:
state = json.load(f)
debug_info = (
f"MOCK HERDR CALLS: {state.get('calls', [])}\n"
f"MOCK HERDR AGENTS: {state.get('agents', {})}\n"
)
assert False, f"Stdout: {res.stdout}\nStderr: {res.stderr}\nDebug:\n{debug_info}"
assert res.returncode == 0
# Verify it is registered in yaml
yaml_path = tmp_path / ".mam" / "agent-sessions.yaml"
assert yaml_path.exists()
with open(yaml_path, 'r') as f:
reg = yaml.safe_load(f)
sessions = reg.get("herdr_sessions", [])
assert len(sessions) == 1
session = sessions[0]
assert session["role"] == "Creator"
assert session["herdr_server"] == "custom_server"
assert session["delegate_job_id"] is not None
assert session["status"] == "running"
# Verify delegate job JSON file exists and contains correct info
job_id = session["delegate_job_id"]
job_file = tmp_path / ".mam" / "jobs" / f"{job_id}.json"
assert job_file.exists()
with open(job_file, 'r') as jf:
job_data = json.load(jf)
assert job_data["job_id"] == job_id
assert "Test onboard prompt" in job_data["prompt"]
assert job_data["agent_session"] == f"herdr:{session['name']}"
def test_integration_stop_purge_combination(mam_sandbox, mock_herdr, mock_agents):
"""
Tier 3: Stop Session Integration
Covers pairwise combination: --purge-conversation + --yes + HOME override
"""
tmp_path = mam_sandbox
# 1. Run create_session to register a session with isolation enabled
create_script = tmp_path / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"
cmd_create = [
"bash", str(create_script),
"--workspace", str(tmp_path),
"--agent", "claude",
"--role", "Creator"
]
res_create = subprocess.run(cmd_create, capture_output=True, text=True, cwd=str(tmp_path))
assert res_create.returncode == 0, f"Stdout: {res_create.stdout}\nStderr: {res_create.stderr}"
# Get session name and isolation uuid
yaml_path = tmp_path / ".mam" / "agent-sessions.yaml"
with open(yaml_path, 'r') as f:
reg = yaml.safe_load(f)
session = reg["herdr_sessions"][0]
session_name = session["name"]
iso_uuid = session["isolation"]["uuid"]
iso_root = Path(session["isolation"]["root"])
# Ensure isolation home root folder is provisioned
assert iso_root.exists()
# Locate dynamically generated conversation file in claude projects
key = str(tmp_path).replace('/', '-').replace('_', '-')
proj_dir = iso_root / "projects" / key
assert proj_dir.exists(), f"Expected isolated projects directory {proj_dir} to exist"
jsonls = list(proj_dir.glob("*.jsonl"))
assert len(jsonls) == 1, f"Expected exactly 1 jsonl file in {proj_dir}, found {jsonls}"
jsonl_file = jsonls[0]
assert jsonl_file.exists()
# Update mock herdr's state to match the running agent so the stop kill-chain works gracefully
with open(mock_herdr, 'r') as f:
state = json.load(f)
state["agents"][session_name] = {
"status": "running",
"agent": "claude",
"cwd": str(tmp_path),
"pid": 12345,
"pane_id": "w1:p1",
"command": "claude",
"buffer": "Anthropic Claude Ready"
}
with open(mock_herdr, 'w') as f:
json.dump(state, f, indent=2)
# 2. Calling stop_session without --yes fails for --purge-conversation
stop_script = tmp_path / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
cmd_stop_no_yes = [
"bash", str(stop_script),
"--session", session_name,
"--purge-conversation"
]
res_stop_no_yes = subprocess.run(cmd_stop_no_yes, capture_output=True, text=True, cwd=str(tmp_path))
assert res_stop_no_yes.returncode == 3
assert "DANGER: --purge-conversation" in res_stop_no_yes.stdout
# Ensure nothing was deleted yet
assert iso_root.exists()
assert jsonl_file.exists()
# 3. Run stop_session with --yes
cmd_stop_yes = [
"bash", str(stop_script),
"--session", session_name,
"--purge-conversation",
"--yes"
]
res_stop_yes = subprocess.run(cmd_stop_yes, capture_output=True, text=True, cwd=str(tmp_path))
assert res_stop_yes.returncode == 0, f"Stdout: {res_stop_yes.stdout}\nStderr: {res_stop_yes.stderr}"
# Assertions:
# - Isolation directory deleted
assert not iso_root.exists()
# - Conversation files deleted
assert not jsonl_file.exists()
# - Session completely removed from YAML/DB registry
with open(yaml_path, 'r') as f:
reg = yaml.safe_load(f)
assert not any(s["name"] == session_name for s in reg.get("herdr_sessions", []))
db_path = tmp_path / ".mam" / "agent-sessions.db"
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM sessions WHERE name=?", (session_name,))
assert cursor.fetchone()[0] == 0
conn.close()
def test_integration_resume_fallbacks(mam_sandbox, mock_herdr, mock_agents):
"""
Tier 3: Resume Session Fallbacks
Covers boundary checks, invalid / missing UUIDs, and workspace disk scan fallback resolution.
"""
tmp_path = mam_sandbox
# 1. Calling resume_session.sh with missing arguments
resume_script = tmp_path / "skills" / "multi-agent-mux-resume" / "scripts" / "resume_session.sh"
cmd_missing = ["bash", str(resume_script), "--session", "test-session"]
res_missing = subprocess.run(cmd_missing, capture_output=True, text=True, cwd=str(tmp_path))
assert res_missing.returncode == 2
assert "ERROR:" in res_missing.stderr
# 2. Calling with non-existent session
cmd_nonexistent = ["bash", str(resume_script), "--workspace", str(tmp_path), "--agent", "claude", "--session", "non-existent"]
res_nonexistent = subprocess.run(cmd_nonexistent, capture_output=True, text=True, cwd=str(tmp_path))
assert res_nonexistent.returncode == 1
assert "ERROR: No saved session for" in res_nonexistent.stderr
# 3. Fallback resolution: set up a stopped session in YAML with NO own ID
session_name = "test-fallback-creator-claude"
mutation = """
d['herdr_sessions'] = [{
'name': 'test-fallback-creator-claude',
'status': 'stopped',
'role': 'Creator',
'pane': {'cwd': 'WS_PLACEHOLDER'}
}]
""".replace("WS_PLACEHOLDER", str(tmp_path))
res_mut = run_mutation(tmp_path, mutation)
assert res_mut.returncode == 0
# Seed a jsonl file representing a conversation under the workspace path
key = str(tmp_path).replace('/', '-').replace('_', '-')
proj_dir = tmp_path / ".claude" / "projects" / key
proj_dir.mkdir(parents=True, exist_ok=True)
scanned_uuid = "scanned-uuid-xyz-123"
(proj_dir / f"{scanned_uuid}.jsonl").write_text(json.dumps({"sessionId": scanned_uuid}) + "\n")
# Run resume_session.sh
cmd_resume = ["bash", str(resume_script), "--workspace", str(tmp_path), "--agent", "claude", "--session", session_name]
res_resume = subprocess.run(cmd_resume, capture_output=True, text=True, cwd=str(tmp_path))
assert res_resume.returncode == 0, f"Stdout: {res_resume.stdout}\nStderr: {res_resume.stderr}"
# Verify the session resumed using the resolved UUID
with open(mock_herdr, 'r') as f:
state = json.load(f)
calls = state.get("calls", [])
# Find the agent start/new-session call
new_sess_call = None
for call in calls:
if "agent" in call and "start" in call and session_name in call:
new_sess_call = call
break
assert new_sess_call is not None, f"Calls: {calls}"
# Verify the argument contains the resolved scanned_uuid
assert any(scanned_uuid in arg for arg in new_sess_call), f"Resume arguments: {new_sess_call}"
def test_integration_reconcile_diff_formats(mam_sandbox, mock_herdr):
"""
Tier 3: Reconcile Drift Detection and Diff Formatting
Covers outputs of reconcile.sh --once --emit-diff under dry-run and actual mutation.
"""
tmp_path = mam_sandbox
reconcile_script = tmp_path / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
# Seed a running session in YAML/DB that is NOT in herdr (Drift class A)
session_name = "drift-a-creator-claude"
mutation = """
d['herdr_sessions'] = [{
'name': 'drift-a-creator-claude',
'status': 'running',
'role': 'Creator',
'pane': {
'cwd': 'WS_PLACEHOLDER',
'pid': 9876,
'cmd': 'claude',
'cmd_full': 'claude'
}
}]
""".replace("WS_PLACEHOLDER", str(tmp_path))
res_mut = run_mutation(tmp_path, mutation)
assert res_mut.returncode == 0
db_path = tmp_path / ".mam" / "agent-sessions.db"
assert db_path.exists()
# Verify DB content
conn = sqlite3.connect(str(db_path))
db_rows = conn.execute("SELECT * FROM sessions").fetchall()
db_state = conn.execute("SELECT * FROM state").fetchall()
conn.close()
# Verify YAML content
yaml_path = tmp_path / ".mam" / "agent-sessions.yaml"
with open(yaml_path, 'r') as f:
yaml_content = f.read()
# Verify load_state_json output
lib_path = tmp_path / ".agents" / "skills" / "lib.sh"
res_load = subprocess.run(["bash", "-c", f"source {lib_path} && load_state_json"], capture_output=True, text=True)
# Verify herdr ls -F output
res_ls = subprocess.run(["herdr", "ls", "-F", "#{session_name}|#{session_created}"], capture_output=True, text=True)
# 1. Run reconcile with --dry-run
cmd_dry = ["bash", str(reconcile_script), "--once", "--emit-diff", "--dry-run"]
res_dry = subprocess.run(cmd_dry, capture_output=True, text=True, cwd=str(tmp_path))
assert res_dry.returncode == 0, f"Stderr: {res_dry.stderr}"
# Verify stdout is valid JSON and reports class A drift
try:
data_dry = json.loads(res_dry.stdout)
except Exception as e:
assert False, f"Failed to parse JSON. stdout: {res_dry.stdout}, stderr: {res_dry.stderr}, error: {e}"
assert "drifts" in data_dry, f"JSON: {data_dry}"
drifts_dry = data_dry["drifts"]
# Debug print on failure
if not any(d["class"] == "A" and d["name"] == session_name for d in drifts_dry):
debug_info = (
f"DB_ROWS: {db_rows}\n"
f"DB_STATE: {db_state}\n"
f"YAML: {yaml_content}\n"
f"LOAD_STATE_JSON STDOUT: {res_load.stdout}\n"
f"LOAD_STATE_JSON STDERR: {res_load.stderr}\n"
f"HERDR LS STDOUT: {res_ls.stdout}\n"
f"HERDR LS STDERR: {res_ls.stderr}\n"
f"AGENT_SESSIONS_YAML ENV: {os.environ.get('AGENT_SESSIONS_YAML')}\n"
)
# Execute Python sub-process trace
py_code = """
import os, json, glob, subprocess, time, sqlite3
from datetime import datetime, timezone
import yaml
yaml_path = os.environ['YAML_PATH']
home = os.environ['HOME_DIR']
d = {}
ws_root = os.environ.get('WORKSPACE_ROOT')
print("WORKSPACE_ROOT:", ws_root)
script = f"source '{ws_root}/.agents/skills/lib.sh' && load_state_json"
out = subprocess.check_output(['bash', '-c', script])
print("LOAD STATE JSON OUT:", out.decode('utf-8'))
"""
env = {
"YAML_PATH": str(yaml_path),
"HOME_DIR": str(tmp_path),
"CLAUDE_PROJECT_DIR": str(tmp_path / ".claude" / "projects"),
"LOCAL_BIN": str(tmp_path / ".local" / "bin"),
"WORKSPACE_ROOT": str(tmp_path),
"AGENT_SESSIONS_YAML": str(yaml_path),
"PATH": os.environ.get("PATH", "")
}
res_test = subprocess.run(["python3", "-c", py_code], env=env, capture_output=True, text=True)
debug_info += f"TEST_CODE_STDOUT: {res_test.stdout}\nTEST_CODE_STDERR: {res_test.stderr}\n"
assert False, f"Drift class A not found in drifts. Output JSON: {json.dumps(data_dry, indent=2)}\nStderr: {res_dry.stderr}\nDebug:\n{debug_info}"
# Verify dry-run made no state changes in SQLite
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT status FROM sessions WHERE name=?", (session_name,)).fetchone()
assert row[0] == "running"
conn.close()
# 2. Run reconcile without --dry-run
cmd_real = ["bash", str(reconcile_script), "--once", "--emit-diff"]
res_real = subprocess.run(cmd_real, capture_output=True, text=True, cwd=str(tmp_path))
assert res_real.returncode == 0, f"Stderr: {res_real.stderr}"
# Verify DB has been updated to terminated
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT status FROM sessions WHERE name=?", (session_name,)).fetchone()
assert row[0] == "terminated"
conn.close()
def test_integration_invalid_arguments_exit_statuses(mam_sandbox, mock_herdr, mock_agents):
"""
Tier 3: Invalid Arguments & Exit Status Checks
Verifies that all scripts correctly handle invalid options and combinations with correct exit statuses.
"""
tmp_path = mam_sandbox
# 1. create_session.sh with missing arguments
create_script = tmp_path / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"
res = subprocess.run(["bash", str(create_script)], capture_output=True, text=True, cwd=str(tmp_path))
assert res.returncode == 2
assert "ERROR: --workspace required" in res.stderr
res = subprocess.run(["bash", str(create_script), "--workspace", str(tmp_path)], capture_output=True, text=True, cwd=str(tmp_path))
assert res.returncode == 2
assert "ERROR: --agent required" in res.stderr
# 2. stop_session.sh with missing arguments
stop_script = tmp_path / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
res = subprocess.run(["bash", str(stop_script)], capture_output=True, text=True, cwd=str(tmp_path))
assert res.returncode == 2
assert "ERROR: --session required" in res.stderr
# 3. stop_session.sh cannot infer agent from weird session name
res = subprocess.run(["bash", str(stop_script), "--session", "bad-name"], capture_output=True, text=True, cwd=str(tmp_path))
assert res.returncode == 2
assert "ERROR: cannot infer agent" in res.stderr
# 4. stop_session.sh on non-existent session
res = subprocess.run(["bash", str(stop_script), "--session", "nonexistent-creator-claude"], capture_output=True, text=True, cwd=str(tmp_path))
assert res.returncode == 1
assert "ERROR: session 'nonexistent-creator-claude' not in" in res.stderr
# 5. reconcile.sh with invalid flag
reconcile_script = tmp_path / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
res = subprocess.run(["bash", str(reconcile_script), "--invalid-flag"], capture_output=True, text=True, cwd=str(tmp_path))
assert res.returncode == 2
assert "ERROR: unknown arg" in res.stderr
+424
View File
@@ -0,0 +1,424 @@
import os
import subprocess
import json
import sqlite3
import pytest
import shutil
import yaml
import time
import concurrent.futures
import threading
from pathlib import Path
# Helper to run mutation on agent-sessions.yaml using atomic_dump_yaml in bash
def run_mutation(mam_sandbox, mutation_str, env=None):
lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh"
yaml_path = mam_sandbox / ".mam" / "agent-sessions.yaml"
cmd_str = f"source {lib_path} && atomic_dump_yaml {yaml_path}"
run_env = dict(os.environ)
if env:
run_env.update(env)
res = subprocess.run(["bash", "-c", cmd_str], input=mutation_str, capture_output=True, text=True, env=run_env)
return res
def test_e2e_scenario1_standard_lifecycle(mam_sandbox, mock_herdr, mock_agents):
"""
Scenario 1: Standard Agent Session Lifecycle
Spawn session, check registry status, query status via status.sh, and stop session.
"""
tmp_path = mam_sandbox
create_script = tmp_path / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"
status_script = tmp_path / "skills" / "multi-agent-mux-status" / "scripts" / "status.sh"
stop_script = tmp_path / "skills" / "multi-agent-mux-stop" / "scripts" / "stop_session.sh"
session_name = "e2e-sess1-creator-claude"
# 1. Spawn session using create_session.sh
cmd_create = [
"bash", str(create_script),
"--workspace", str(tmp_path),
"--agent", "claude",
"--role", "Creator",
"--session", session_name
]
res_create = subprocess.run(cmd_create, capture_output=True, text=True, cwd=str(tmp_path))
assert res_create.returncode == 0, f"Stderr: {res_create.stderr}"
# Verify mock herdr registers it
with open(mock_herdr, 'r') as f:
state = json.load(f)
assert session_name in state["agents"]
assert state["agents"][session_name]["status"] == "running"
# 2. Check status via status.sh --json
cmd_status = ["bash", str(status_script), "--json"]
res_status = subprocess.run(cmd_status, capture_output=True, text=True, cwd=str(tmp_path))
assert res_status.returncode == 0
status_data = json.loads(res_status.stdout)
sessions_detail = status_data["sessions_detail"]
assert any(s["name"] == session_name and s["status"] == "running" for s in sessions_detail)
# 3. Stop it via stop_session.sh
cmd_stop = ["bash", str(stop_script), "--session", session_name]
res_stop = subprocess.run(cmd_stop, capture_output=True, text=True, cwd=str(tmp_path))
assert res_stop.returncode == 0
# Verify status in YAML/DB becomes stopped
yaml_path = tmp_path / ".mam" / "agent-sessions.yaml"
with open(yaml_path, 'r') as f:
reg = yaml.safe_load(f)
session_entry = [s for s in reg.get("herdr_sessions", []) if s["name"] == session_name][0]
assert session_entry["status"] == "stopped"
def test_e2e_scenario2_disconnect_resume(mam_sandbox, mock_herdr, mock_agents):
"""
Scenario 2: Session Disconnect and Resume
Spawn session, simulate process death in herdr state, run resume_session.sh, and assert resume UUID.
"""
tmp_path = mam_sandbox
create_script = tmp_path / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"
resume_script = tmp_path / "skills" / "multi-agent-mux-resume" / "scripts" / "resume_session.sh"
session_name = "e2e-sess2-creator-claude"
# 1. Spawn session
cmd_create = [
"bash", str(create_script),
"--workspace", str(tmp_path),
"--agent", "claude",
"--role", "Creator",
"--session", session_name
]
res_create = subprocess.run(cmd_create, capture_output=True, text=True, cwd=str(tmp_path))
assert res_create.returncode == 0
# Get own UUID from YAML
yaml_path = tmp_path / ".mam" / "agent-sessions.yaml"
with open(yaml_path, 'r') as f:
reg = yaml.safe_load(f)
orig_session = reg["herdr_sessions"][0]
# Simulate first message creating own ID (materialized own ID)
own_uuid = "e2e-own-uuid-111"
iso_root = Path(orig_session["isolation"]["root"])
key = str(tmp_path).replace('/', '-').replace('_', '-')
proj_dir = iso_root / "projects" / key
proj_dir.mkdir(parents=True, exist_ok=True)
(proj_dir / f"{own_uuid}.jsonl").write_text(json.dumps({"sessionId": own_uuid}) + "\n")
mutation = f"""
for s in d.get('herdr_sessions', []):
if s.get('name') == '{session_name}':
s['status'] = 'stopped'
s['claude_session_id_own'] = '{own_uuid}'
"""
res_mut = run_mutation(tmp_path, mutation)
assert res_mut.returncode == 0
# 2. Simulate process death in mock herdr state
with open(mock_herdr, 'r') as f:
state = json.load(f)
if session_name in state["agents"]:
del state["agents"][session_name]
with open(mock_herdr, 'w') as f:
json.dump(state, f, indent=2)
# Clear herdr calls to isolate assertions
with open(mock_herdr, 'r') as f:
state = json.load(f)
state["calls"] = []
with open(mock_herdr, 'w') as f:
json.dump(state, f, indent=2)
# 3. Run resume_session.sh
cmd_resume = [
"bash", str(resume_script),
"--workspace", str(tmp_path),
"--agent", "claude",
"--session", session_name
]
res_resume = subprocess.run(cmd_resume, capture_output=True, text=True, cwd=str(tmp_path))
assert res_resume.returncode == 0, f"Stderr: {res_resume.stderr}"
# 4. Assert that it resumes using the correct UUID in the start command arguments
with open(mock_herdr, 'r') as f:
state = json.load(f)
calls = state.get("calls", [])
resume_call = None
for call in calls:
if "agent" in call and "start" in call and session_name in call:
resume_call = call
break
assert resume_call is not None, f"Could not find resume agent start call in calls: {calls}"
assert any(own_uuid in arg for arg in resume_call), f"Expected UUID {own_uuid} to be in resume command: {resume_call}"
def test_e2e_scenario3_drift_auto_reconciliation(mam_sandbox, mock_herdr):
"""
Scenario 3: Drift Detection and Auto-Reconciliation
Setup drift states (running in herdr but not in YAML registry, and running in YAML registry but terminated in herdr).
Verify reconcile.sh automatically reconciles both.
"""
tmp_path = mam_sandbox
reconcile_script = tmp_path / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
# Drift 1: Running in herdr but not in YAML
drift_herdr_only = "drift-herdr-only-creator-claude"
with open(mock_herdr, 'r') as f:
state = json.load(f)
state["agents"][drift_herdr_only] = {
"status": "running",
"agent": "claude",
"cwd": str(tmp_path),
"pid": 5555,
"pane_id": "w1:p1",
"command": "claude",
"buffer": "Anthropic Claude Ready"
}
with open(mock_herdr, 'w') as f:
json.dump(state, f, indent=2)
# Drift 2: Running in YAML registry but terminated in herdr
drift_yaml_only = "drift-yaml-only-creator-claude"
mutation = f"""
d['herdr_sessions'] = [{{
'name': '{drift_yaml_only}',
'status': 'running',
'role': 'Creator',
'pane': {{
'cwd': 'WS_PLACEHOLDER',
'pid': 6666,
'cmd': 'claude',
'cmd_full': 'claude'
}}
}}]
""".replace("WS_PLACEHOLDER", str(tmp_path))
res_mut = run_mutation(tmp_path, mutation)
assert res_mut.returncode == 0
# Run reconcile.sh --once
cmd_reconcile = ["bash", str(reconcile_script), "--once"]
res_recon = subprocess.run(cmd_reconcile, capture_output=True, text=True, cwd=str(tmp_path))
assert res_recon.returncode == 0, f"Stderr: {res_recon.stderr}"
# Verify YAML/DB states
yaml_path = tmp_path / ".mam" / "agent-sessions.yaml"
with open(yaml_path, 'r') as f:
reg = yaml.safe_load(f)
sessions = reg.get("herdr_sessions", [])
# drift-herdr-only-creator-claude should have been auto-registered as running
sess_herdr_only = [s for s in sessions if s["name"] == drift_herdr_only]
assert len(sess_herdr_only) == 1
assert sess_herdr_only[0]["status"] == "running"
# drift-yaml-only-creator-claude should have been auto-terminated
sess_yaml_only = [s for s in sessions if s["name"] == drift_yaml_only]
assert len(sess_yaml_only) == 1
assert sess_yaml_only[0]["status"] == "terminated"
def test_e2e_scenario4_parallel_flock_locking(mam_sandbox, mock_herdr, mock_agents):
"""
Scenario 4: Parallel Session Operations with flock Locking
Run multiple concurrent session creation scripts to verify SQLite locking prevents database corruption.
"""
tmp_path = mam_sandbox
create_script = tmp_path / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"
num_sessions = 6
def run_create(i):
session_name = f"parallel-sess-{i}-creator-claude"
cmd = [
"bash", str(create_script),
"--workspace", str(tmp_path),
"--agent", "claude",
"--role", "Creator",
"--session", session_name
]
res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(tmp_path))
return res
# Execute in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=num_sessions) as executor:
futures = [executor.submit(run_create, i) for i in range(num_sessions)]
results = [f.result() for f in futures]
# Verify that all succeeded
for i, res in enumerate(results):
assert res.returncode == 0, f"Session {i} failed. Stdout: {res.stdout}\nStderr: {res.stderr}"
# Verify all 6 sessions are present in YAML registry
yaml_path = tmp_path / ".mam" / "agent-sessions.yaml"
with open(yaml_path, 'r') as f:
reg = yaml.safe_load(f)
sessions = reg.get("herdr_sessions", [])
registered_names = {s["name"] for s in sessions}
for i in range(num_sessions):
assert f"parallel-sess-{i}-creator-claude" in registered_names
def test_e2e_scenario5_multi_agent_review_loop(mam_sandbox, mock_herdr, mock_agents):
"""
Scenario 5: Multi-Agent Review Loop
Run run_loop.sh under simulated conditions where reviewer output is mocked.
Verify loop terminates correctly with expected PASS and NOT PASS verdicts.
"""
tmp_path = mam_sandbox
loop_script = tmp_path / ".agents" / "skills" / "multi-agent-mux-loop" / "scripts" / "run_loop.sh"
# 1. Seed sessions in registry for the worker, reviewer, and planner
worker_name = "test-worker-creator-claude"
reviewer_name = "test-reviewer-creator-claude"
planner_name = "test-planner-creator-claude"
mutation = f"""
d['herdr_sessions'] = [
{{
'name': '{worker_name}',
'status': 'running',
'role': 'worker',
'pane': {{'cwd': 'WS_PLACEHOLDER'}}
}},
{{
'name': '{reviewer_name}',
'status': 'running',
'role': 'reviewer',
'pane': {{'cwd': 'WS_PLACEHOLDER'}}
}},
{{
'name': '{planner_name}',
'status': 'running',
'role': 'planner',
'pane': {{'cwd': 'WS_PLACEHOLDER'}}
}}
]
""".replace("WS_PLACEHOLDER", str(tmp_path))
res_mut = run_mutation(tmp_path, mutation)
assert res_mut.returncode == 0
# Seed mock herdr state with these running sessions to satisfy has-session checks
with open(mock_herdr, 'r') as f:
herdr_state = json.load(f)
for name in [worker_name, reviewer_name, planner_name]:
herdr_state["agents"][name] = {
"status": "running",
"agent": "claude",
"cwd": str(tmp_path),
"pid": 9999,
"pane_id": "w1:p1",
"command": "claude",
"buffer": "Anthropic Claude Ready"
}
with open(mock_herdr, 'w') as f:
json.dump(herdr_state, f, indent=2)
# Define mock reviewer and planner outputs
# Let's mock a scenario:
# Critique: worker challenge
# Refinement: refined plan
# Review: First try NOT PASS, Second try PASS
job_responses = {
"Planner": "Refined Plan:\n1. Implement X\n2. Verify X",
"critique": "Creator Critique: Plan has 1 edge case.",
"Worker": "Creator Output: Code updated.",
"Reviewer": "Reviewer verdict:\n\n[VERDICT: PASS]" # will override inside simulator to test iteration logic
}
# We will run a background simulator thread to resolve jobs
stop_event = threading.Event()
def simulate_delegate_jobs():
jobs_dir = tmp_path / ".mam" / "jobs"
iteration = 1
while not stop_event.is_set():
if not jobs_dir.exists():
time.sleep(0.1)
continue
for job_file in jobs_dir.glob("*.json"):
try:
with open(job_file, 'r+') as f:
job = json.load(f)
if job.get("status") == "pending":
job_id = job["job_id"]
role = job.get("role", "Worker")
prompt = job.get("prompt", "")
# Determine response text
response_text = ""
if role == "Planner":
response_text = job_responses["Planner"]
elif "Challenge" in prompt or "Critique" in prompt:
response_text = job_responses["critique"]
elif role == "Worker":
response_text = job_responses["Worker"]
elif role == "Reviewer":
# For Reviewer, fail the first time, pass the second time
if iteration == 1:
response_text = "Review report:\nSome lint issues found.\n\n[VERDICT: NOT PASS]"
iteration += 1
else:
response_text = "Review report:\nAll clean.\n\n[VERDICT: PASS]"
# Write final report
job_work_dir = jobs_dir / job_id
job_work_dir.mkdir(parents=True, exist_ok=True)
(job_work_dir / "report-final.md").write_text(response_text)
# Complete job
job["status"] = "completed"
f.seek(0)
json.dump(job, f, indent=2)
f.truncate()
# Publish completed event over MQTT using publish_event.py
pub_script = tmp_path / ".agents" / "skills" / "multi-agent-mux-delegate-job" / "scripts" / "publish_event.py"
cmd_pub = [
sys.executable, str(pub_script),
"--registry-dir", str(jobs_dir),
"--job", job_id,
"--event", "completed",
"--detail", f"{role} finished work"
]
subprocess.run(cmd_pub, capture_output=True, text=True)
except Exception:
pass
time.sleep(0.1)
sim_thread = threading.Thread(target=simulate_delegate_jobs)
sim_thread.daemon = True
sim_thread.start()
try:
# Run run_loop.sh
cmd_loop = [
"bash", str(loop_script),
"--target-agent", worker_name,
"--reviewer", reviewer_name,
"--plan",
"--plan-talk", "1",
"--max-loop", "3",
"--task", "Implement feature X and verify"
]
import sys
run_env = dict(os.environ)
run_env["DELEGATE_JOB_PYTHON"] = sys.executable
res_loop = subprocess.run(cmd_loop, capture_output=True, text=True, cwd=str(tmp_path), env=run_env)
# Verify that it succeeded and executed the corrective loop
assert res_loop.returncode == 0, f"Loop failed. Stdout: {res_loop.stdout}\nStderr: {res_loop.stderr}"
assert "Reviewer 'test-reviewer-creator-claude': NOT PASS" in res_loop.stdout or "Reviewer 'test-reviewer-creator-claude': NOT PASS" in res_loop.stderr or "NOT PASS" in res_loop.stdout
assert "Reviewer 'test-reviewer-creator-claude': PASS" in res_loop.stdout
assert "Mux loop finished with 100% PASS verdicts." in res_loop.stdout
finally:
stop_event.set()
sim_thread.join(timeout=1.0)