# test_c1_tui_readiness.py — Unit and integration tests for TUI readiness, # 2-tier readiness model, modal/hint token separation, and session diagnostics/preservation. import os, subprocess, time, json from pathlib import Path import pytest _LIB_SH = str(Path(__file__).resolve().parent.parent / ".agents" / "skills" / "lib.sh") _CREATE_SH = str(Path(__file__).resolve().parent.parent / ".agents" / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh") def _run_lib_helpers(script_body, env_extra=None): import tempfile, shutil tmp_dir = tempfile.mkdtemp(prefix="test_tui_ready_") try: env = dict(os.environ) if env_extra: env.update(env_extra) env["WORKSPACE_ROOT"] = tmp_dir wrapper = f""" set -euo pipefail cd "{tmp_dir}" source "{_LIB_SH}" _init_herdr_isolation() {{ :; }} {script_body} """ return subprocess.run(["bash", "-c", wrapper], capture_output=True, text=True, env=env, cwd=tmp_dir) finally: shutil.rmtree(tmp_dir, ignore_errors=True) def _lib_helper_src(): content = open(_LIB_SH, encoding="utf-8").read() start = content.find("_herdr_ws_id_file() {") end = content.find('cmd="${1:-}"', start) assert start != -1 and end != -1 return content[start:end] def test_c1_hint_tokens_only_dialog_closed(): """C-1: Hint tokens (Use arrow keys / Esc to cancel / Press Enter) alone must return dialog CLOSED.""" screen = """Select an option: 1. Option A 2. Option B Use arrow keys to navigate · Esc to cancel """ script = f""" _pane_capture() {{ printf '%s' '{screen}'; }} if _pane_dialog_open dummy; then echo "DIALOG_OPEN"; else echo "DIALOG_CLOSED"; fi """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "DIALOG_CLOSED" in res.stdout def test_c1b_hint_press_enter_sends_enter(): """C-1b (B-3): 'Press Enter to continue' alone must return dialog CLOSED, and wait_for_tui_ready must send Enter exactly once per iteration (30 total across 30 iterations, not 60).""" screen = """Some startup banner Press Enter to continue """ script = f""" _pane_capture() {{ printf '%s' '{screen}'; }} KEYS=/tmp/mam-press-enter-keys.$$ : > "$KEYS" _sks_herdr() {{ if [ "${{1:-}}" = "send-keys" ]; then echo "$*" >> "$KEYS"; fi return 0 }} sleep() {{ :; }} if _pane_dialog_open dummy; then echo "DIALOG_OPEN"; else echo "DIALOG_CLOSED"; fi export MAM_STRONG_READY_TOKENS='NonExistentStrong' export MAM_WEAK_READY_TOKENS='' export MAM_INPUT_PLACEHOLDER='' rc=0 wait_for_tui_ready dummy-sess custom || rc=$? echo "RC=$rc" echo "KEYS_COUNT=$(grep -c "send-keys -t dummy-sess Enter" "$KEYS" || echo 0)" rm -f "$KEYS" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "DIALOG_CLOSED" in res.stdout assert "KEYS_COUNT=30" in res.stdout def test_c1c_transition_to_upsell_modal_sends_escape_not_enter(): """C-1c: Screen transition from Press Enter to Upsell Modal (Yes, try it) sends Escape, not duplicate Enter.""" state_file = f"/tmp/mam-screen-state.{os.getpid()}" script = f""" echo "1" > "{state_file}" _pane_capture() {{ local s s=$(cat "{state_file}" 2>/dev/null || echo 1) if [ "$s" = "1" ]; then printf 'Some banner\\nPress Enter to continue\\n' else printf 'Claude Code\\nYes, try it\\n' fi }} KEYS=/tmp/mam-transition-keys.{os.getpid()} : > "$KEYS" _sks_herdr() {{ if [ "${{1:-}}" = "send-keys" ]; then echo "$*" >> "$KEYS" # Transition to screen 2 on first Enter if [ "${{4:-}}" = "Enter" ]; then echo "2" > "{state_file}" fi fi return 0 }} sleep() {{ :; }} export MAM_STRONG_READY_TOKENS='AnthropicReady' export MAM_WEAK_READY_TOKENS='' export MAM_INPUT_PLACEHOLDER='' export MAM_TUI_TIMEOUT=2 rc=0 wait_for_tui_ready dummy-sess claude 2 || rc=$? echo "RC=$rc" echo "KEYS_CONTENT=$(tr '\\n' '|' < "$KEYS")" rm -f "$KEYS" "{state_file}" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "send-keys -t dummy-sess Enter" in res.stdout assert "send-keys -t dummy-sess Escape" in res.stdout assert "send-keys -t dummy-sess Enter|send-keys -t dummy-sess Escape|" in res.stdout def test_c2_modal_tokens_dialog_open(): """C-2: High-confidence modal tokens (e.g. Do you trust the files) must return dialog OPEN.""" screen = """Security Alert Do you trust the files in this repository? """ script = f""" _pane_capture() {{ printf '%s' '{screen}'; }} if _pane_dialog_open dummy; then echo "DIALOG_OPEN"; else echo "DIALOG_CLOSED"; fi """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "DIALOG_OPEN" in res.stdout def test_c3_strong_token_and_hint_token_readiness_succeeds(): """C-3: Strong ready token + hint token together must detect readiness immediately (rc 0).""" screen = """MockApp v1.0.0 Use arrow keys to select a tool ❯ """ script = f""" mock_py() {{ if [ "$1" = "-m" ] && [ "$2" = "lib_py.agents" ] && [ "$3" = "facts" ]; then echo "MAM_AGENT_NAME='mocktiered'" echo "MAM_OWN_KEY='mock_id_own'" echo "MAM_INPUT_PROMPT='❯'" echo "MAM_INPUT_PLACEHOLDER=''" echo "MAM_INPUT_RULE_PATTERN=''" echo "MAM_READY_TOKENS='MockApp|Use arrow keys|Ready'" echo "MAM_STRONG_READY_TOKENS='MockApp'" echo "MAM_WEAK_READY_TOKENS='Use arrow keys'" echo "MAM_MODAL_TOKENS=''" echo "MAM_EXIT_KEY='/exit'" echo "MAM_DELEGATE_AGENT_KEY='mock-agent'" return 0 fi }} _delegate_py_bin() {{ echo "mock_py"; }} _pane_capture() {{ printf '%s' '{screen}'; }} sleep() {{ :; }} wait_for_tui_ready dummy-sess mocktiered """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "TUI detected ready" in res.stdout def test_c4_placeholder_only_readiness_succeeds(): """C-4: Long placeholder (Ask anything..., len>=8) alone satisfies S-tier readiness.""" screen = """Welcome to the AI Assistant Ask anything... """ script = f""" _pane_capture() {{ printf '%s' '{screen}'; }} sleep() {{ :; }} # Override ready tokens so strong tokens do not match export MAM_STRONG_READY_TOKENS='NonExistentStrongToken' export MAM_WEAK_READY_TOKENS='' export MAM_INPUT_PLACEHOLDER='Ask anything...' wait_for_tui_ready dummy-sess custom """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "TUI detected ready" in res.stdout def test_c4b_box_modal_does_not_succeed(): """C-4b: Box modal frame (────── + solo ❯) with no strong token or placeholder must NOT satisfy readiness.""" screen = """┌───────────────────────────────────┐ │ Please select authentication mode │ │ ───────────────────────────────── │ │ 1. API Key │ │ 2. OAuth │ └───────────────────────────────────┘ ❯ """ script = f""" _pane_capture() {{ printf '%s' '{screen}'; }} sleep() {{ :; }} export MAM_STRONG_READY_TOKENS='NonExistentStrongToken' export MAM_WEAK_READY_TOKENS='Chat|history' export MAM_INPUT_PLACEHOLDER='Ask anything...' export MAM_INPUT_PROMPT='❯' export MAM_INPUT_RULE_PATTERN='─{{10,}}' _MAM_MODAL_TOKENS='NON_MATCHING_MODAL' rc=0 wait_for_tui_ready dummy-sess custom || rc=$? echo "RC=$rc" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "RC=1" in res.stdout def test_c4c_prompt_and_rule_corroboration_alone_cannot_satisfy_readiness(): """C-4c: Neither prompt alone, rule alone, nor prompt+rule alone can satisfy readiness (C-tier cannot promote alone).""" screen = """──────────────────────────── ❯ """ script = f""" _pane_capture() {{ printf '%s' '{screen}'; }} sleep() {{ :; }} export MAM_STRONG_READY_TOKENS='NonExistentToken' export MAM_WEAK_READY_TOKENS='NonExistentWeak' export MAM_INPUT_PLACEHOLDER='NonExistentPlaceholder' export MAM_INPUT_PROMPT='❯' export MAM_INPUT_RULE_PATTERN='─{{10,}}' rc=0 wait_for_tui_ready dummy-sess custom || rc=$? echo "RC=$rc" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "RC=1" in res.stdout def test_c4d_weak_token_requires_corroboration(): """C-4d: Weak token (Chat) alone fails; Weak token + prompt in last 5 lines succeeds.""" # 1. Weak token alone without prompt in last 5 lines screen_weak_alone = """Chat history from earlier session: line 1 line 2 line 3 line 4 line 5 line 6 """ script1 = f""" _pane_capture() {{ printf '%s' '{screen_weak_alone}'; }} sleep() {{ :; }} export MAM_STRONG_READY_TOKENS='NonExistentStrong' export MAM_WEAK_READY_TOKENS='Chat|history' export MAM_INPUT_PLACEHOLDER='' export MAM_INPUT_PROMPT='❯' export MAM_INPUT_RULE_PATTERN='' rc=0 wait_for_tui_ready dummy-sess custom || rc=$? echo "RC=$rc" """ res1 = _run_lib_helpers(script1) assert res1.returncode == 0, res1.stderr + res1.stdout assert "RC=1" in res1.stdout # 2. Weak token + prompt in last 5 lines screen_weak_corroborated = """Chat Session Active Ready for input ❯ """ script2 = f""" _pane_capture() {{ printf '%s' '{screen_weak_corroborated}'; }} sleep() {{ :; }} export MAM_STRONG_READY_TOKENS='NonExistentStrong' export MAM_WEAK_READY_TOKENS='Chat|history' export MAM_INPUT_PLACEHOLDER='' export MAM_INPUT_PROMPT='❯' export MAM_INPUT_RULE_PATTERN='' rc=0 wait_for_tui_ready dummy-sess custom || rc=$? echo "RC=$rc" """ res2 = _run_lib_helpers(script2) assert res2.returncode == 0, res2.stderr + res2.stdout assert "RC=0" in res2.stdout def test_c4e_short_placeholder_not_promoted_to_strong(): """C-4e: Placeholder with length < 8 chars must NOT be promoted to strong S-tier.""" screen = """Ask """ script = f""" _pane_capture() {{ printf '%s' '{screen}'; }} sleep() {{ :; }} export MAM_STRONG_READY_TOKENS='NonExistentStrong' export MAM_WEAK_READY_TOKENS='' export MAM_INPUT_PLACEHOLDER='Ask' rc=0 wait_for_tui_ready dummy-sess custom || rc=$? echo "RC=$rc" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "RC=1" in res.stdout def test_c5_hermes_fallback_to_strong_only(): """C-5: When input_* are all empty (Hermes), strong_ready_tokens alone satisfies readiness.""" screen = """Hermes Agent v1.0 System initialized. """ script = f""" _pane_capture() {{ printf '%s' '{screen}'; }} sleep() {{ :; }} wait_for_tui_ready dummy-sess hermes """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "TUI detected ready" in res.stdout def test_c6_consecutive_empty_captures_unobservable_rc2(): """C-6: Consecutive empty captures give up as unobservable/headless (rc 2) without burning 30s.""" script = """ _pane_capture() { echo ""; } sleep() { :; } export MAM_READY_EMPTY_GIVEUP=3 rc=0 wait_for_tui_ready dummy-sess claude || rc=$? echo "RC=$rc" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "RC=2" in res.stdout assert "unobservable/headless pane" in res.stderr def test_c7_readiness_checks_decoded_text_not_raw_json(): """C-7: Raw JSON field names (e.g. Chat in JSON envelope) do not cause false positives.""" raw_json = json.dumps({"result": {"read": {"text": "Just raw text with no ready tokens", "Chat": 1}}}) script = f""" _sks_herdr() {{ if [ "${{1:-}}" = "capture-pane" ]; then printf '%s' '{raw_json}' fi return 0 }} sleep() {{ :; }} export MAM_STRONG_READY_TOKENS='NonExistentStrong' export MAM_WEAK_READY_TOKENS='Chat' export MAM_INPUT_PLACEHOLDER='' export MAM_INPUT_PROMPT='❯' export MAM_INPUT_RULE_PATTERN='' rc=0 wait_for_tui_ready dummy-sess custom || rc=$? echo "RC=$rc" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "RC=1" in res.stdout def test_c8_mam_dialog_tokens_is_union_of_modal_and_hint(): """C-8: _MAM_DIALOG_TOKENS must remain the union of _MAM_MODAL_TOKENS and _MAM_HINT_TOKENS.""" script = """ echo "MODAL=$_MAM_MODAL_TOKENS" echo "HINT=$_MAM_HINT_TOKENS" echo "DIALOG=$_MAM_DIALOG_TOKENS" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout lines = dict(line.split("=", 1) for line in res.stdout.strip().splitlines()) expected_dialog = f"{lines['MODAL']}|{lines['HINT']}" assert lines["DIALOG"] == expected_dialog def test_c8b_agent_modal_tokens_merged_into_dialog_open(): """C-8b (T-2d / F-8): Per-agent modal_tokens from adapter are merged into _pane_dialog_open and block send_keys_safe.""" # 1. Direct _pane_dialog_open test screen = """Try the new fullscreen renderer? """ script1 = f""" _pane_capture() {{ printf '%s' '{screen}'; }} export MAM_MODAL_TOKENS='Try the new fullscreen renderer\\?' if _pane_dialog_open dummy; then echo "DIALOG_OPEN"; else echo "DIALOG_CLOSED"; fi """ res1 = _run_lib_helpers(script1) assert res1.returncode == 0, res1.stderr + res1.stdout assert "DIALOG_OPEN" in res1.stdout # 2. Dynamic facts resolution via send_keys_safe on claude session script2 = f""" _pane_quiescent() {{ return 0; }} _pane_capture() {{ printf '%s' 'Try the new fullscreen renderer? (y/n)'; }} _sks_herdr() {{ return 0; }} rc=0 SKS_DIALOG_TIMEOUT=1 send_keys_safe "my-project-worker-claude-01" "some text" || rc=$? echo "RC=$rc" """ res2 = _run_lib_helpers(script2) assert res2.returncode == 0, res2.stderr + res2.stdout assert "RC=2" in res2.stdout, f"Expected dialog blocking (RC=2) for claude modal: {res2.stdout}" def test_c9_create_session_timeout_alive_pid_preserves_session(mam_sandbox, mock_herdr, mock_agents): """C-9 & C-11: create_session with readiness timeout + alive PID preserves session with timeout status (exit 0) and dumps diagnostic.""" script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh" lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh" diag_dir = mam_sandbox / ".mam" / "diagnostics" yaml_file = mam_sandbox / ".mam" / "agent-sessions.yaml" # Make wait_for_tui_ready immediately return 1 in sandboxed lib.sh lib_content = lib_path.read_text() lib_path.write_text(lib_content.replace("wait_for_tui_ready() {", "wait_for_tui_ready() { return 1; #")) # In sandbox create_session.sh, simulate alive process using $$ create_content = script_path.read_text() script_path.write_text(create_content.replace( 'PANE_PID_CHECK=$(_herdr list-panes', 'PANE_PID_CHECK=$$ # $(_herdr list-panes' )) cmd = [ "bash", str(script_path), "--workspace", str(mam_sandbox), "--agent", "claude", "--role", "creator", "--session", "test-timeout-sess", "--onboard" ] res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(mam_sandbox)) assert res.returncode == 0, res.stderr + res.stdout assert "WARNING: agent TUI never became ready within timeout, but process" in res.stderr # Check diagnostic dump file created (C-11) dump_files = list(diag_dir.glob("test-timeout-sess-*.txt")) assert len(dump_files) == 1 dump_content = dump_files[0].read_text() assert "ReturnCode: 1" in dump_content # Check YAML registration with tui-ready-timeout and NO delegate job (N-C) assert yaml_file.exists() yaml_content = yaml_file.read_text() assert "tui-ready-timeout" in yaml_content assert "delegate_job_id: null" in yaml_content or "delegate_job_id: None" in yaml_content or "delegate_job_id:" not in yaml_content or "delegate_job_id: ''" in yaml_content def test_c10_create_session_timeout_dead_pid_kills_session(mam_sandbox, mock_herdr, mock_agents): """C-10: create_session with readiness timeout + dead PID triggers rollback kill and exits with code 42.""" script_path = mam_sandbox / ".agents" / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh" lib_path = mam_sandbox / ".agents" / "skills" / "lib.sh" diag_dir = mam_sandbox / ".mam" / "diagnostics" # Make wait_for_tui_ready return 1 and mock list-panes to return dead PID lib_content = lib_path.read_text() create_content = script_path.read_text() script_path.write_text(create_content.replace( 'PANE_PID_CHECK=$(_herdr list-panes', 'PANE_PID_CHECK=99999999 # $(_herdr list-panes' )) lib_path.write_text(lib_content.replace("wait_for_tui_ready() {", "wait_for_tui_ready() { return 1; #")) cmd = [ "bash", str(script_path), "--workspace", str(mam_sandbox), "--agent", "claude", "--role", "creator", "--session", "test-dead-sess" ] res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(mam_sandbox)) assert res.returncode == 42, res.stderr + res.stdout assert "ERROR: agent TUI never became ready" in res.stderr # Check diagnostic dump file created dump_files = list(diag_dir.glob("test-dead-sess-*.txt")) assert len(dump_files) == 1 assert "ReturnCode: 1" in dump_files[0].read_text() def test_adapter_strong_weak_partition(): """N-D: Verify strong_ready_tokens and weak_ready_tokens partition ready_tokens for all adapters.""" from lib_py.agents.registry import get_adapter for agent in ('claude', 'agy', 'hermes', 'grok'): adapter = get_adapter(agent) assert adapter is not None ready_set = set(t for t in adapter.ready_tokens.split('|') if t) strong_set = set(t for t in adapter.strong_ready_tokens.split('|') if t) weak_set = set(t for t in adapter.weak_ready_tokens.split('|') if t) assert strong_set | weak_set == ready_set, f"Mismatch in {agent}: {strong_set} | {weak_set} != {ready_set}" assert strong_set.isdisjoint(weak_set), f"Overlap in {agent}: {strong_set & weak_set}" def test_sks_agent_suffix_resolution_not_substring(): """Verify send_keys_safe resolves agent by strict suffix without substring false positives.""" script = """ _pane_quiescent() { return 0; } _pane_capture() { echo "prompt > "; } _sks_herdr() { return 0; } RESOLVED="" mock_py() { if [ "$1" = "-m" ] && [ "$2" = "lib_py.agents" ] && [ "$3" = "facts" ]; then echo "echo RESOLVED_AGENT=$4" fi } _delegate_py_bin() { echo "mock_py"; } check_sks() { local sess="$1" unset MAM_MODAL_TOKENS # Call send_keys_safe and capture output send_keys_safe "$sess" "hello" 2>&1 || true } echo "3: $(check_sks 'reviewer-creator-grok-01')" echo "4: $(check_sks 'planner-reviewer-claude-01')" echo "5: $(check_sks 'test-worker-hermes-02')" echo "6: $(check_sks 'workspace-creator-agy-05')" echo "7: $(check_sks 'claude')" echo "9: $(check_sks 'grok-02')" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "3: RESOLVED_AGENT=grok" in res.stdout assert "4: RESOLVED_AGENT=claude" in res.stdout assert "5: RESOLVED_AGENT=hermes" in res.stdout assert "6: RESOLVED_AGENT=agy" in res.stdout assert "7: RESOLVED_AGENT=claude" in res.stdout assert "9: RESOLVED_AGENT=grok" in res.stdout def test_f2_upsell_modal_defense_with_real_facts_and_banner(): """F-2/F-6: Real Claude Code banner with upsell modal must trigger modal defense, not return ready.""" script = """ SENT_KEYS="" _sks_herdr() { if [ "$1" = "send-keys" ]; then SENT_KEYS="$SENT_KEYS $*" fi return 0 } COUNT_FILE=$(mktemp) echo "0" > "$COUNT_FILE" _pane_capture() { local c c=$(cat "$COUNT_FILE") c=$((c + 1)) echo "$c" > "$COUNT_FILE" if [ "$c" -le 2 ]; then # First 2 captures (loop iter 1 capture + tail check): Claude Code banner with Fullscreen Modal cat << 'EOF' Claude Code (Sonnet 3.5) Try the new fullscreen renderer? Flicker-free output Selected text auto-copies to your clipboard Yes, try it EOF else # Subsequent captures (after Escape sent): modal dismissed, normal prompt cat << 'EOF' Claude Code (Sonnet 3.5) ❯ EOF fi } rc=0 wait_for_tui_ready "my-claude-sess" "claude" || rc=$? rm -f "$COUNT_FILE" echo "RC=$rc" echo "KEYS=$SENT_KEYS" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "Escape" in res.stdout, f"Expected Escape to be sent for upsell modal: {res.stdout}" assert "RC=0" in res.stdout def test_r1_resolve_pane_id_fail_closed_on_multiple_same_kind_panes(): """R-1: If fallback finds multiple panes of same agent kind, fail closed (exit 1).""" helper = _lib_helper_src() script = helper + """ # Mock real herdr pane list returning 2 claude panes _real_herdr() { cat << 'EOF' {"result": {"panes": [ {"pane_id": "w1:p1", "label": "unrelated-1", "agent": "claude"}, {"pane_id": "w1:p2", "label": "unrelated-2", "agent": "claude"} ]}} EOF } # Create a dummy yaml with unknown session mapped to claude cmd mkdir -p .mam cat << 'EOF' > .mam/agent-sessions.yaml herdr_sessions: - name: my-unmatched-session pane: cmd: claude EOF rc=0 pid=$(_resolve_herdr_pane_id "my-unmatched-session") || rc=$? echo "RC=$rc|PID=$pid" """ res = _run_lib_helpers(script) assert res.returncode == 0, res.stderr + res.stdout assert "RC=1|PID=" in res.stdout, f"Expected fail-closed RC=1 and empty PID: {res.stdout}"