Godopu
ec61511699
- Introduce finalize_unfinished in multi-agent-mux-delegate-job for fail-closed atomic transitions - Enforce CAS dispatch_token and attempt_id pinning across engine, CLI, and lifecycle scripts - Integrate Herdr wait_for_agent_idle turn handshake to prevent prompt loss and race conditions - Resolve bash 3.2 set -u empty array expansion compatibility in lib.sh - Preserve Issue #5 R2 contract in CAS transition engine (engine.py, publish_event.py, service.py) - Add comprehensive regression suites (test_issue6_fail_closed.py, test_issue6_turn_handshake.py) - Bump lockstep framework and skills to v4.4.1 across lib.sh, eager.py, VERSIONS.md, 8 SKILL.md - Synchronize documentation (IMPROVEMENTS.md, MESSAGING.md, state-transitions.md, task.md, phase5-e2e.md) - Unanimous PASS achieved across 4 reviewer team leaders (Claude, Codex, OpenCode, Grok) Closes #6
437 lines
18 KiB
Python
437 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Issue #6 Phase 2 — TUI Idle Handshake & Submission Hardening regression test suite.
|
|
|
|
Tests T2-1 through T2-11 according to docs/contracts/issue-6/implementation_plan.md (Rev.2).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
SKILLS = REPO_ROOT / ".agents" / "skills"
|
|
LIB_SH = SKILLS / "lib.sh"
|
|
WRAPPER = SKILLS / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
|
|
|
|
|
|
def _run_lib_sh(snippet, env=None, cwd=None, timeout=60):
|
|
run_env = dict(os.environ)
|
|
run_env["DELEGATE_JOB_PYTHON"] = sys.executable
|
|
run_env["AGENT_PYTHON_BIN"] = sys.executable
|
|
if cwd:
|
|
run_env["WORKSPACE_ROOT"] = str(cwd)
|
|
run_env["HOME"] = str(cwd)
|
|
if env:
|
|
run_env.update({k: str(v) for k, v in env.items()})
|
|
return subprocess.run(
|
|
["bash", "-c", f'source "{LIB_SH}" >/dev/null 2>&1; {snippet}'],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd=str(cwd or REPO_ROOT),
|
|
env=run_env,
|
|
timeout=timeout,
|
|
)
|
|
|
|
|
|
def _run_wrapper(args, env=None, cwd=None, timeout=60):
|
|
run_env = dict(os.environ)
|
|
run_env["DELEGATE_JOB_PYTHON"] = sys.executable
|
|
run_env["AGENT_PYTHON_BIN"] = sys.executable
|
|
if cwd:
|
|
run_env["WORKSPACE_ROOT"] = str(cwd)
|
|
run_env["HOME"] = str(cwd)
|
|
if env:
|
|
run_env.update({k: str(v) for k, v in env.items()})
|
|
return subprocess.run(
|
|
[str(WRAPPER)] + list(args),
|
|
capture_output=True,
|
|
text=True,
|
|
cwd=str(cwd or REPO_ROOT),
|
|
env=run_env,
|
|
timeout=timeout,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-1: status_sequence=["working", "done"] -> agent wait 호출 후 agent prompt 호출, Enter/C-m 0회
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_1_status_sequence_wait_then_prompt(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt1"] = {
|
|
"agent": "claude",
|
|
"agent_status": "working",
|
|
"status_sequence": ["working", "done"],
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh(
|
|
'wait_for_agent_idle "agt1" 5000 && send_keys_safe "agt1" "test prompt" "job1"',
|
|
env={"SKS_IDLE_RETRY_BACKOFF_S": "0"},
|
|
cwd=tmp_path,
|
|
)
|
|
assert res.returncode == 0, f"Expected rc=0, got {res.returncode}. Stderr: {res.stderr}"
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
calls = final_state.get("calls", [])
|
|
|
|
wait_calls = [c for c in calls if len(c) >= 2 and c[0] == "agent" and c[1] == "wait"]
|
|
prompt_calls = [c for c in calls if len(c) >= 2 and c[0] == "agent" and c[1] == "prompt"]
|
|
|
|
assert len(wait_calls) >= 1, "Expected at least 1 agent wait call"
|
|
assert len(prompt_calls) == 1, "Expected exactly 1 agent prompt call"
|
|
|
|
wait_idx = calls.index(wait_calls[0])
|
|
prompt_idx = calls.index(prompt_calls[0])
|
|
assert wait_idx < prompt_idx, "agent wait must be called before agent prompt"
|
|
|
|
send_key_calls = [c for c in calls if (c and c[0] in ("send-keys", "send-text")) or (len(c) >= 2 and c[0] == "pane" and c[1] in ("send-keys", "send-text"))]
|
|
assert len(send_key_calls) == 0, f"No legacy send-keys expected, got: {send_key_calls}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-2: working 고정, SKS_IDLE_TIMEOUT_MS=1000 -> rc=1, agent prompt 미호출
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_2_working_fixed_timeout(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_busy"] = {
|
|
"agent": "claude",
|
|
"agent_status": "working",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh(
|
|
'wait_for_agent_idle "agt_busy" 500',
|
|
env={"SKS_IDLE_RETRIES": "1", "SKS_IDLE_RETRY_BACKOFF_S": "0"},
|
|
cwd=tmp_path,
|
|
)
|
|
assert res.returncode == 1, f"Expected rc=1, got {res.returncode}"
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
prompt_calls = [c for c in final_state.get("calls", []) if len(c) >= 2 and c[0] == "agent" and c[1] == "prompt"]
|
|
assert len(prompt_calls) == 0, "agent prompt must NOT be called when idle wait fails"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-3: unknown -> agent wait 미호출, _pane_quiescent 폴백 경로 진입
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_3_unknown_status_fallback(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_unk"] = {
|
|
"agent": "claude",
|
|
"agent_status": "unknown",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh('wait_for_agent_idle "agt_unk"', cwd=tmp_path)
|
|
assert res.returncode == 3, f"Expected rc=3 (unknown), got {res.returncode}"
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
wait_calls = [c for c in final_state.get("calls", []) if len(c) >= 2 and c[0] == "agent" and c[1] == "wait"]
|
|
assert len(wait_calls) == 0, "agent wait must NOT be called when status is unknown"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-4: blocked -> rc=2, 주입 없음
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_4_blocked_status(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_block"] = {
|
|
"agent": "claude",
|
|
"agent_status": "blocked",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh('wait_for_agent_idle "agt_block"', cwd=tmp_path)
|
|
assert res.returncode == 2, f"Expected rc=2 (blocked), got {res.returncode}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-5: done 즉시 -> 대기 0회, 즉시 주입
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_5_done_immediate(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_done"] = {
|
|
"agent": "claude",
|
|
"agent_status": "done",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh('wait_for_agent_idle "agt_done"', cwd=tmp_path)
|
|
assert res.returncode == 0, f"Expected rc=0, got {res.returncode}"
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
wait_calls = [c for c in final_state.get("calls", []) if len(c) >= 2 and c[0] == "agent" and c[1] == "wait"]
|
|
assert len(wait_calls) == 0, "agent wait must NOT be called when already done"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-6: 폴백 경로, pre_submit에 esc to interrupt 존재 + 페이스트 드롭 -> rc=4
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_6_fallback_pre_submit_busy_token_drop(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_drop"] = {
|
|
"agent": "claude",
|
|
"agent_status": "unknown",
|
|
"pane_id": "w1:p1",
|
|
"prompt_wait_result": "timeout",
|
|
"buffer": "esc to interrupt\ntest drop",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh(
|
|
'send_keys_safe "agt_drop" "test drop" "job_drop"',
|
|
env={"SKS_PROMPT_WAIT": "0", "SKS_IDLE_HANDSHAKE": "0"},
|
|
cwd=tmp_path,
|
|
)
|
|
assert res.returncode == 4, f"Expected rc=4, got {res.returncode}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-7: prompt_wait_result=agent_prompt_stalled + 이후 working -> rc=0, prompt 호출 1회
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_7_prompt_wait_stalled_recovery(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_stall"] = {
|
|
"agent": "claude",
|
|
"agent_status": "working",
|
|
"prompt_wait_result": "agent_prompt_stalled",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh(
|
|
'send_keys_safe "agt_stall" "test stalled" "job_stall"',
|
|
env={"SKS_IDLE_HANDSHAKE": "0"},
|
|
cwd=tmp_path,
|
|
)
|
|
assert res.returncode == 0, f"Expected rc=0, got {res.returncode}. Stderr: {res.stderr}"
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
prompt_calls = [c for c in final_state.get("calls", []) if len(c) >= 2 and c[0] == "agent" and c[1] == "prompt"]
|
|
assert len(prompt_calls) == 1, f"Expected prompt called exactly once without re-submission, got {len(prompt_calls)}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-8: 래퍼 순서: idle 대기 로그가 subscriber pid: 로그보다 앞
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_8_wrapper_idle_before_subscriber_log(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_order"] = {
|
|
"agent": "claude",
|
|
"agent_status": "idle",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
reg_dir = tmp_path / "jobs"
|
|
reg_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
res = _run_wrapper(
|
|
[
|
|
"submit",
|
|
"--agent", "claude",
|
|
"--agent-session", "herdr:agt_order",
|
|
"--prompt", "test T2-8 order",
|
|
"--registry-dir", str(reg_dir),
|
|
"--workdir", str(tmp_path),
|
|
"--timeout", "4",
|
|
"--idle-timeout", "2",
|
|
],
|
|
cwd=tmp_path,
|
|
timeout=60,
|
|
)
|
|
|
|
stdout = res.stdout
|
|
assert "subscriber pid:" in stdout
|
|
final_state = json.loads(state_file.read_text())
|
|
calls = final_state.get("calls", [])
|
|
get_calls = [c for c in calls if "agent" in c and "get" in c]
|
|
assert len(get_calls) >= 1, f"agent get should be called during idle handshake, got: {calls}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-9: working 고정 + SKS_IDLE_RETRIES=1, SKS_IDLE_TIMEOUT_MS=500 -> agent wait 2회, 래퍼 rc=5, cancelled
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_9_wrapper_idle_timeout_session_busy(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_busy2"] = {
|
|
"agent": "claude",
|
|
"agent_status": "working",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
reg_dir = tmp_path / "jobs"
|
|
reg_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
res = _run_wrapper(
|
|
[
|
|
"submit",
|
|
"--agent", "claude",
|
|
"--agent-session", "herdr:agt_busy2",
|
|
"--prompt", "test T2-9 busy",
|
|
"--registry-dir", str(reg_dir),
|
|
"--workdir", str(tmp_path),
|
|
"--timeout", "6",
|
|
"--idle-timeout", "2",
|
|
],
|
|
env={"SKS_IDLE_RETRIES": "1", "SKS_IDLE_RETRY_BACKOFF_S": "0", "SKS_IDLE_TIMEOUT_MS": "200"},
|
|
cwd=tmp_path,
|
|
timeout=60,
|
|
)
|
|
assert res.returncode == 5, f"Expected rc=5, got {res.returncode}. Stderr: {res.stderr}"
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
calls = final_state.get("calls", [])
|
|
wait_calls = [c for c in calls if "agent" in c and "wait" in c]
|
|
assert len(wait_calls) == 2, f"Expected 2 agent wait calls, got {len(wait_calls)}"
|
|
|
|
job_files = list(reg_dir.glob("*.json"))
|
|
assert len(job_files) == 1
|
|
job_data = json.loads(job_files[0].read_text())
|
|
assert job_data.get("status") == "cancelled"
|
|
assert job_data.get("detail") == "session_busy"
|
|
|
|
sub_logs = list(reg_dir.glob("*.subscriber.out"))
|
|
assert len(sub_logs) == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-10: HERDR_SESSION_NAME=iso-x -> mock 호출 기록에 herdr --session iso-x agent wait
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_10_watcher_session_argv(tmp_path, mock_herdr):
|
|
from lib_py.loop.watcher import build_argv
|
|
|
|
argv_default = build_argv("agt", session="default")
|
|
assert "--session" not in argv_default
|
|
assert argv_default[:3] == ["herdr", "agent", "wait"]
|
|
|
|
argv_iso = build_argv("agt", session="iso-x")
|
|
assert argv_iso[:5] == ["herdr", "--session", "iso-x", "agent", "wait"]
|
|
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_iso"] = {
|
|
"agent": "claude",
|
|
"agent_status": "working",
|
|
"status_sequence": ["done"],
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh(
|
|
'wait_for_agent_idle "agt_iso"',
|
|
env={"HERDR_SESSION_NAME": "iso-x", "SKS_IDLE_RETRY_BACKOFF_S": "0"},
|
|
cwd=tmp_path,
|
|
)
|
|
assert res.returncode == 0
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
calls = final_state.get("calls", [])
|
|
iso_calls = [c for c in calls if len(c) >= 4 and "iso-x" in c and "agent" in c and "wait" in c]
|
|
assert len(iso_calls) >= 1, f"Expected call with --session iso-x agent wait, got calls: {calls}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-11: 정적 린트: bare python -m 또는 python3 -m lib_py.loop 패턴 0건
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_11_static_lint_no_bare_python_in_lib_sh_and_wrapper():
|
|
pattern = re.compile(r'^\s*python3?\s+-m\s+lib_py\.loop', re.MULTILINE)
|
|
|
|
lib_sh_text = LIB_SH.read_text()
|
|
matches_lib = pattern.findall(lib_sh_text)
|
|
assert len(matches_lib) == 0, f"Found bare python -m in lib.sh: {matches_lib}"
|
|
|
|
wrapper_text = WRAPPER.read_text()
|
|
matches_wrap = pattern.findall(wrapper_text)
|
|
assert len(matches_wrap) == 0, f"Found bare python -m in wrapper: {matches_wrap}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-12: _SKS_IDLE_RC=3 + busy 화면 + SKS_BUSY_TIMEOUT=0 -> native prompt 전 rc=4 반환
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_12_busy_timeout_expiration_returns_rc4(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_busy"] = {
|
|
"agent": "claude",
|
|
"agent_status": "working",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
# Mock pane tail returning "Working... esc to interrupt"
|
|
res = _run_lib_sh(
|
|
'_SKS_IDLE_RC=3 SKS_BUSY_TIMEOUT=0 send_keys_safe "agt_busy" "test prompt" "job_b1"',
|
|
env={"SKS_BUSY_TIMEOUT": "0"},
|
|
cwd=tmp_path,
|
|
)
|
|
assert res.returncode == 4, f"Expected rc=4 on busy expiration, got {res.returncode}"
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
prompt_calls = [c for c in final_state.get("calls", []) if len(c) >= 2 and c[0] == "agent" and c[1] == "prompt"]
|
|
assert len(prompt_calls) == 0, "agent prompt must NOT be called when busy gate expires"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-13: agent prompt 모호한 RPC 실패 -> paste-buffer 폴백 금지, rc=4 반환 (단일 주입 계약)
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_13_ambiguous_rpc_failure_no_paste_duplicate(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_rpc"] = {
|
|
"agent": "claude",
|
|
"agent_status": "done",
|
|
"prompt_wait_result": "transport_error",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh(
|
|
'send_keys_safe "agt_rpc" "test prompt" "job_rpc1"',
|
|
cwd=tmp_path,
|
|
)
|
|
assert res.returncode == 4, f"Expected rc=4 on ambiguous RPC failure, got {res.returncode}"
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
calls = final_state.get("calls", [])
|
|
paste_calls = [c for c in calls if len(c) >= 2 and (c[0] in ("paste-buffer", "set-buffer") or c[1] in ("paste-buffer", "set-buffer"))]
|
|
assert len(paste_calls) == 0, f"Must not fall back to paste buffer on ambiguous RPC error, got: {paste_calls}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# T2-14: agent prompt 빈 응답(empty output) 실패 -> paste-buffer 폴백 금지, rc=4 반환 (단일 주입 계약)
|
|
# ---------------------------------------------------------------------------
|
|
def test_t2_14_empty_rpc_failure_no_paste_fallback(tmp_path, mock_herdr):
|
|
state_file = Path(mock_herdr)
|
|
state = json.loads(state_file.read_text())
|
|
state["agents"]["agt_empty"] = {
|
|
"agent": "claude",
|
|
"agent_status": "done",
|
|
"prompt_wait_result": "empty_output",
|
|
}
|
|
state_file.write_text(json.dumps(state, indent=2))
|
|
|
|
res = _run_lib_sh(
|
|
'send_keys_safe "agt_empty" "test prompt" "job_empty1"',
|
|
cwd=tmp_path,
|
|
)
|
|
assert res.returncode == 4, f"Expected rc=4 on empty RPC failure, got {res.returncode}"
|
|
|
|
final_state = json.loads(state_file.read_text())
|
|
calls = final_state.get("calls", [])
|
|
prompt_calls = [c for c in calls if len(c) >= 2 and c[0] == "agent" and c[1] == "prompt"]
|
|
paste_calls = [c for c in calls if len(c) >= 2 and (c[0] in ("paste-buffer", "set-buffer") or c[1] in ("paste-buffer", "set-buffer"))]
|
|
assert len(prompt_calls) == 1, f"Expected prompt called exactly once, got {len(prompt_calls)}"
|
|
assert len(paste_calls) == 0, f"Must not fall back to paste buffer on empty RPC failure, got: {paste_calls}"
|