fix(issue-9,14): bound the R1 grace by the wall clock and keep other ls failures unknown

Round-2 review of d5a191c (planner-reviewer-claude-01, planner-reviewer-codex-01).

- delegate wrapper: the R1 idle grace and the broker_unavailable re-check
  counted only their sleeps against TIMEOUT, so python/herdr probe time plus
  fresh disk heartbeats let a job run far past --timeout (55 s for a 20 s
  budget in the new test). Both loops now end at START_TIME + TIMEOUT read
  off the clock on every pass; heartbeats extend only the idle window, and
  the herdr probe is cut off at the deadline (_within_deadline)
- document the window policy d5a191c actually implements: an agent with no
  evidence is closed ~2 x IDLE_TIMEOUT after its last wire event (Issue #6
  promised IDLE_TIMEOUT + 5 s) and heartbeats may be ~2 x IDLE_TIMEOUT apart;
  correct the claim that publish_event records a heartbeat on disk before
  the wire publish - it records right after the publish attempt
- shim ls: every failed `agent list` other than a structured
  server_not_running now exits nonzero with a "state unknown" diagnostic
  instead of reaching the list parser, where an error document on stdout
  read as an empty list and let drift A terminate live rows
- tests: stdout variants for down and unrelated errors, reconcile keeping
  herdr_confirmed False on an unrelated failure, and a wall-clock upper bound
  under continuous heartbeats and slow probes; the three new checks fail on
  d5a191c
- VERSIONS.md v4.4.2: items 3 and 6 and the test count (19)

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HCGpJXhSZasJLU38ksDghe

.agents/skills/lib.sh

@@ -1249,7 +1249,9 @@ except Exception:
     # error.code == "server_not_running" means "down". A successful list can
     # legitimately contain that text (an agent named server_not_running-codex),
     # and reading it as "down" made reconcile terminate every live row on that
-    # server. Any other failure falls through to the unknown path below.
+    # server. Any other failed call is "unknown": it exits nonzero below
+    # without its output ever reaching the list parser, so an error document on
+    # stdout cannot pass for an empty list either.
     _ls_err=$(mktemp 2>/dev/null || echo "/tmp/mam_ls_err_$$")
     _ls_rc=0
     _ls_raw="$(_real_herdr agent list 2>"$_ls_err")" || _ls_rc=$?
@@ -1281,6 +1283,13 @@ print(down)
       echo "no server running (herdr session ${_sess:-default})" >&2
       exit 1
     fi
+    if [ "$_ls_rc" -ne 0 ]; then
+      # Unknown, never empty: keep the nonzero status and word the diagnostic
+      # so it matches none of reconcile.sh's empty-server phrases ("no server
+      # running", "no sessions", "failed to connect").
+      echo "herdr agent list returned rc=$_ls_rc (herdr session ${_sess:-default}); state unknown" >&2
+      exit 1
+    fi
     printf '%s\n' "$_ls_raw" | \
       MAM_LS_FORMAT="$format" MAM_REAL_HERDR="$REAL_HERDR" MAM_LS_SESSION="$_sess" \
       TZ=UTC LC_ALL=C python3 -c '

.agents/skills/multi-agent-mux-delegate-job/multi-agent-mux-delegate-job

@@ -599,12 +599,35 @@ _agent_still_working() {
   _pane_tail "$sess" 8 2>/dev/null | grep -Fq "esc to interrupt"
 }
 
+# _within_deadline <deadline_epoch> <cmd...>
+# Run a probe that can block (a herdr RPC, a pane capture) without letting it
+# outlive a wall-clock deadline: once the clock reaches <deadline> the probe
+# and its children are killed and the result is rc 124, i.e. "no evidence".
+_within_deadline() {
+  local deadline="$1" pid
+  shift
+  "$@" &
+  pid=$!
+  while kill -0 "$pid" 2>/dev/null; do
+    if [ "$(date +%s)" -ge "$deadline" ]; then
+      pkill -P "$pid" 2>/dev/null || true
+      kill "$pid" 2>/dev/null || true
+      wait "$pid" 2>/dev/null || true
+      return 124
+    fi
+    sleep 0.1
+  done
+  wait "$pid"
+}
+
 # ISSUE-14: agent-agnostic liveness. publish_event appends every event to the
-# job's on-disk events.log before the wire publish, so a `progress` heartbeat
-# that the subscriber never saw (it had already idled out, or the broker
-# dropped it) is still visible here. An agent running its tests in a
-# background shell looks idle to herdr and shows no busy token, yet keeps
-# heartbeating - this is the only evidence that covers that case.
+# job's on-disk events.log right after its wire publish attempt, whether that
+# publish succeeded or not, so a `progress` heartbeat the subscriber never saw
+# (it had already idled out, or the broker dropped it) is still visible here -
+# late by at most its publish retries while the broker is failing. An agent
+# running its tests in a background shell looks idle to herdr and shows no
+# busy token, yet keeps heartbeating - this is the only evidence that covers
+# that case.
 _recent_disk_heartbeat() {
   local job_id="$1" window="$2" f="$REGISTRY_DIR/$job_id.events.log"
   [ -f "$f" ] || return 1
@@ -672,19 +695,18 @@ finalize_unfinished() {
   else
     FINAL_JOB_STATUS="$job_status"
   fi
-  # broker_unavailable: re-observe disk for remaining deadline at 3s intervals
+  # broker_unavailable: re-observe the disk at 3 s intervals until this
+  # dispatch's wall-clock deadline (START_TIME + TIMEOUT). The deadline is read
+  # off the clock on every pass; summing the sleeps alone ignores the time the
+  # disk probes themselves take and overruns TIMEOUT.
   if [ "$job_status" = "broker_unavailable" ]; then
-    local now elapsed remaining
-    now=$(date +%s)
-    elapsed=$(( now - ${START_TIME:-now} ))
-    remaining=$(( TIMEOUT - elapsed ))
-    [ $remaining -lt 0 ] && remaining=0
-    while [ $remaining -gt 0 ]; do
-      local wait_interval=3
-      [ $remaining -lt $wait_interval ] && wait_interval=$remaining
-      sleep "$wait_interval"
-      remaining=$(( remaining - wait_interval ))
-      local disk_st
+    local deadline left disk_st
+    deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
+    while :; do
+      left=$(( deadline - $(date +%s) ))
+      [ "$left" -le 0 ] && break
+      [ "$left" -gt 3 ] && left=3
+      sleep "$left"
       disk_st="$(_disk_status "$job_id")"
       if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
         job_status="$disk_st"
@@ -703,21 +725,23 @@ finalize_unfinished() {
     target_agent=$(_sanitize_herdr_agent_name "$sess")
 
     # ISSUE-14: the subscriber's idle clock only counts events it received over
-    # the wire. Keep re-observing the disk while ANY liveness evidence is at
-    # most IDLE_TIMEOUT old - herdr `working`, the pane busy token, or a fresh
-    # heartbeat in events.log (see _recent_disk_heartbeat). Each sighting opens
-    # a new idle window; an agent that leaves no evidence for a whole window is
-    # closed, still inside the wall clock. Before this, one heartbeat gap
-    # longer than the idle timeout closed a live review as deadline_timeout
-    # and its later `completed` was refused.
-    local now elapsed remaining evidence_until
-    now=$(date +%s)
-    elapsed=$(( now - ${START_TIME:-now} ))
-    remaining=$(( TIMEOUT - elapsed ))
-    [ $remaining -lt 0 ] && remaining=0
-    evidence_until=$(( now + IDLE_TIMEOUT ))
-    while [ $remaining -gt 0 ]; do
-      local disk_st
+    # the wire. Keep re-observing the disk while liveness evidence keeps
+    # turning up - herdr `working`, the pane busy token, or a heartbeat in
+    # events.log at most IDLE_TIMEOUT old (see _recent_disk_heartbeat).
+    # Window policy: entering this grace opens one IDLE_TIMEOUT window and
+    # every sighting re-opens it from the time of that poll. An agent that
+    # leaves no evidence at all is therefore closed about 2 x IDLE_TIMEOUT
+    # after its last wire event (the subscriber's idle wait plus this window;
+    # it was IDLE_TIMEOUT before disk heartbeats counted), and heartbeats may
+    # be up to about 2 x IDLE_TIMEOUT apart, since one still counts until it
+    # is IDLE_TIMEOUT old. Evidence only ever extends the idle window, never
+    # this dispatch's wall clock: the loop ends at START_TIME + TIMEOUT read
+    # off the clock on every pass, and the herdr probe is cut off at that
+    # deadline too (_within_deadline).
+    local deadline evidence_until left disk_st
+    deadline=$(( ${START_TIME:-$(date +%s)} + TIMEOUT ))
+    evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
+    while [ "$(date +%s)" -lt "$deadline" ]; do
       disk_st="$(_disk_status "$job_id")"
       if [ "$disk_st" = "completed" ] || [ "$disk_st" = "error" ] || [ "$disk_st" = "cancelled" ]; then
         job_status="$disk_st"
@@ -725,16 +749,15 @@ finalize_unfinished() {
         [ "$disk_st" = "completed" ] && return 0
         break
       fi
-      if { [ "$sess" != "human" ] && [ -n "$sess" ] && _agent_still_working "$sess" "$target_agent"; } \
+      if { [ "$sess" != "human" ] && [ -n "$sess" ] && _within_deadline "$deadline" _agent_still_working "$sess" "$target_agent"; } \
          || _recent_disk_heartbeat "$job_id" "$IDLE_TIMEOUT"; then
         evidence_until=$(( $(date +%s) + IDLE_TIMEOUT ))
       elif [ "$(date +%s)" -ge "$evidence_until" ]; then
         break
       fi
-      local wait_interval=1
-      [ $remaining -lt $wait_interval ] && wait_interval=$remaining
-      sleep "$wait_interval"
-      remaining=$(( remaining - wait_interval ))
+      left=$(( deadline - $(date +%s) ))
+      [ "$left" -le 0 ] && break
+      sleep 1
     done
   fi
 

VERSIONS.md

@@ -44,14 +44,14 @@
 #### 🌟 주요 개선 사항
 1. **herdr 서버 기동 env 마커 denylist (Issue #9, `lib.sh` 쉼)**: 쉼이 `herdr --session <name> server` 를 부트스트랩할 때 세션 식별 마커만 제거한 환경을 넘긴다 — `CLAUDECODE`, `CLAUDE_CODE_CHILD_SESSION`, `CLAUDE_CODE_SESSION_ID`, `CLAUDE_CODE_SESSION_ATTENDED`, `CLAUDE_CODE_ENTRYPOINT`, `CLAUDE_PID`, `CLAUDE_CODE_MESSAGING_TOKEN`, `CLAUDE_CODE_MESSAGING_SOCKET`, `CLAUDE_CODE_BRIDGE_SESSION_ID`, `ANTIGRAVITY_CONVERSATION_ID`, `HERMES_SESSION_ID`, `GROK_SESSION_ID`, `OPENCODE_SESSION_ID`. `CLAUDE_PROJECT_DIR`·`CLAUDE_CONFIG_DIR` 등 사용자 설정은 그대로 상속된다. 상속된 `CLAUDE_CODE_CHILD_SESSION` 때문에 서브에이전트 claude 의 transcript 저장이 꺼져 resume 이 불가능하던 증상을 해소.
 2. **조회 경로 자동 기동 금지 `MAM_NO_AUTOSTART=1` (Issue #9)**: 이 값이 설정되면 쉼은 서버를 띄우지 않고 호출만 전달한다. `status.sh`·`reconcile.sh` 가 이를 export 하므로 read-only `multi-agent-mux-status` 조회가 더 이상 서버 데몬을 기동하지 않는다.
-3. **쉼 `ls` 의 "no server running" 번역 (Issue #9)**: 서버가 내려가 있으면 쉼의 `ls` 가 `no server running (herdr session <name>)` 으로 보고하고, `reconcile.sh` 는 이를 빈 서버로 판정해 `herdr_confirmed` 를 유지한다(파싱 실패로 모든 서버의 drift 감지가 멈추는 회귀 방지). 번역은 `agent list` 가 **실패**하고 그 오류 봉투의 `error.code` 가 정확히 `server_not_running` 일 때만 적용한다. 성공한 목록에 같은 문자열이 들어 있거나(예: 에이전트 이름 `server_not_running-codex`) 다른 실패인 경우는 번역하지 않으므로, 살아 있는 서버가 빈 서버로 오판되어 drift A 로 전 세션이 종료 처리되지 않고 그 밖의 실패는 종전대로 unknown(`herdr_confirmed=False`)으로 남는다.
+3. **쉼 `ls` 의 "no server running" 번역 (Issue #9)**: 서버가 내려가 있으면 쉼의 `ls` 가 `no server running (herdr session <name>)` 으로 보고하고, `reconcile.sh` 는 이를 빈 서버로 판정해 `herdr_confirmed` 를 유지한다(파싱 실패로 모든 서버의 drift 감지가 멈추는 회귀 방지). 번역은 `agent list` 가 **실패**하고 그 오류 봉투(stderr·stdout 어느 쪽이든)의 `error.code` 가 정확히 `server_not_running` 일 때만 적용한다. 성공한 목록에 같은 문자열이 들어 있으면(예: 에이전트 이름 `server_not_running-codex`) 번역하지 않으므로 살아 있는 서버가 빈 서버로 오판되어 drift A 로 전 세션이 종료 처리되지 않는다. 그 밖의 실패는 출력이 목록 파서에 닿기 전에 비-0 으로 끝나(`… state unknown`, reconcile 의 빈 서버 문구와 겹치지 않음) unknown(`herdr_confirmed=False`)으로 남는다 — stdout 으로 온 오류 문서도 더 이상 빈 목록으로 읽히지 않는다.
 4. **accepted 제출의 전달 확인 `SKS_DELIVERY_GRACE_S` (Issue #14, `send_keys_safe`)**: `agent prompt --wait` 결과가 `timeout`/`agent_prompt_stalled` 이면 herdr 가 입력을 이미 수락한 것으로 보고, `SKS_DELIVERY_GRACE_S`(기본 15초) 동안 `agent_status` working, 제출 전 화면에 없던 busy 토큰(`esc to interrupt`), 제출 전 스냅샷 대비 pane 변화 중 하나를 전달 확인 신호로 인정한다. 신호가 없으면 종전대로 rc 4 로 fail-closed 하고, 그 밖의 오류(`transport_error` 등)는 accepted 로 취급하지 않는다. 단일 주입 계약(재전송 없음)은 유지된다.
 5. **래퍼 레지스트리 ack 유예 `MAM_INJECT_ACK_GRACE_S` (Issue #14, `multi-agent-mux-delegate-job`)**: `send_keys_safe` 가 전달을 확인하지 못해도 잡을 `agent_blocked` 로 닫기 전에 `MAM_INJECT_ACK_GRACE_S`(기본 10초) 동안 레지스트리를 관측하고, `pending` 이 아닌 상태(에이전트의 `started` 등)가 보이면 전달된 것으로 보고 진행한다(`_inject_acknowledged`).
-6. **R1 유예의 pane busy 토큰·디스크 heartbeat 인정 (Issue #14, `_agent_still_working`, `_recent_disk_heartbeat`)**: 구독자 idle 타임아웃 후 R1 유예가 herdr `working` 외에 pane 최근 8줄의 `esc to interrupt` 토큰, 그리고 잡 `events.log` 의 최근 이벤트(`publish_event` 가 와이어 발행 전에 로컬에 기록하는 `progress` heartbeat)도 작업 중 근거로 받아들인다. 근거가 관측될 때마다 `IDLE_TIMEOUT` 만큼의 새 창이 열리고, 한 창 내내 아무 근거도 없으면 종전대로 닫는다(벽시계 `TIMEOUT` 내). idle 타임아웃보다 긴 codex 리뷰, heartbeat 간격이 한 번 idle 창을 넘긴 리뷰, 백그라운드 셸로 테스트를 돌려 herdr 에 idle 로 보이는 creator 가 턴 도중 `deadline_timeout` 으로 닫히고 이후 `completed` 가 거부되던 문제를 방지.
+6. **R1 유예의 pane busy 토큰·디스크 heartbeat 인정 (Issue #14, `_agent_still_working`, `_recent_disk_heartbeat`)**: 구독자 idle 타임아웃 후 R1 유예가 herdr `working` 외에 pane 최근 8줄의 `esc to interrupt` 토큰, 그리고 잡 `events.log` 의 최근 이벤트(`publish_event` 가 와이어 발행 시도 **뒤**에 성공 여부와 무관하게 로컬에 기록하는 `progress` heartbeat — 브로커 장애 시에는 발행 재시도 시간만큼 늦게 기록된다)도 작업 중 근거로 받아들인다. 유예에 들어가면 `IDLE_TIMEOUT` 창 하나가 열리고, 근거가 관측될 때마다 그 시점부터 창이 다시 열린다. 따라서 **아무 근거도 없는 무응답 에이전트는 마지막 와이어 이벤트 후 약 2×`IDLE_TIMEOUT` 에 닫히고**(디스크 heartbeat 를 인정하기 전에는 `IDLE_TIMEOUT`; Issue #6 수용 기준 "≤ `IDLE_TIMEOUT`+5s 내 `error`"는 "≤ 2×`IDLE_TIMEOUT` + 여유"로 바뀐다), **heartbeat 간격은 약 2×`IDLE_TIMEOUT` 까지 허용된다**(heartbeat 하나가 `IDLE_TIMEOUT` 만큼 오래될 때까지 근거로 인정되므로). 근거는 idle 창만 늘릴 뿐 벽시계는 늘리지 않는다: 루프는 매 반복 시계로 다시 읽은 `START_TIME + TIMEOUT` 에서 끝나고, herdr 관측 호출도 그 시각에 끊는다(`_within_deadline`). `broker_unavailable` 디스크 재관측 루프도 같은 시계 기준이다. 이로써 idle 타임아웃보다 긴 codex 리뷰, heartbeat 간격이 한 번 idle 창을 넘긴 리뷰, 백그라운드 셸로 테스트를 돌려 herdr 에 idle 로 보이는 creator 가 턴 도중 `deadline_timeout` 으로 닫히고 이후 `completed` 가 거부되던 문제를 막는다.
 
 #### ✅ 품질 검증
 - **T1-7 트리거 변경** (`tests/test_issue6_fail_closed.py`): 주입 실패 유발값을 `prompt_wait_result=timeout` → `transport_error` 로 변경(Issue #14 이후 `timeout` 은 accepted 제출이므로 더 이상 확정 실패가 아님)하고 `MAM_INJECT_ACK_GRACE_S=1` 로 대기를 단축.
-- **신규 테스트 17개**: `tests/test_issue9_server_env.py` 8개(마커 제거, `MAM_NO_AUTOSTART` 부트스트랩 생략·호출 전달, status 조회 무기동, 정적 opt-out, `ls` 번역(최소·herdr 0.9 실제 오류 봉투), 성공 목록 속 `server_not_running` 문자열 비오판, 그 밖의 실패 unknown 유지, reconcile drift A 비발생), `tests/test_issue14_delivery_ack.py` 9개(busy 토큰·pane 변화 확인, 무신호 fail-closed, 선행 busy 토큰 배제, `transport_error` 비수용, mock herdr timeout 확인, 래퍼 레지스트리 ack, idle 타임아웃 pane busy 유예, idle 타임아웃 디스크 heartbeat 유예).
+- **신규 테스트 19개**: `tests/test_issue9_server_env.py` 9개(마커 제거, `MAM_NO_AUTOSTART` 부트스트랩 생략·호출 전달, status 조회 무기동, 정적 opt-out, `ls` 번역(최소·herdr 0.9 실제 오류 봉투·stdout 봉투), 성공 목록 속 `server_not_running` 문자열 비오판, 그 밖의 실패 unknown 유지(stderr·stdout), reconcile drift A 비발생, 다른 실패 시 reconcile `herdr_confirmed=False`), `tests/test_issue14_delivery_ack.py` 10개(busy 토큰·pane 변화 확인, 무신호 fail-closed, 선행 busy 토큰 배제, `transport_error` 비수용, mock herdr timeout 확인, 래퍼 레지스트리 ack, idle 타임아웃 pane busy 유예, idle 타임아웃 디스크 heartbeat 유예, heartbeat 가 이어져도 벽시계 `TIMEOUT` 안에서 종료).
 - **테스트 기대값 정정**: `tests/test_a4_adapter_contract.py` 가 `4d444ce` 의 agy `exit_key=/exit` 변경을 반영하도록 수정(그동안 전체 스위트에서 1건 실패하던 문제 해소).
 - **실측 (커밋 `497633f` 기록)**: 새로 부트스트랩한 서버에 마커가 없고 claude pane 의 transcript 저장이 복구됨, codex 잡 `d04ebcb5` 의 `started`/`completed` 가 `agent_blocked` 없이 수락됨.
 

tests/test_issue14_delivery_ack.py

@@ -366,3 +366,77 @@ def test_wrapper_idle_timeout_grace_accepts_disk_heartbeat(tmp_path):
     job_data = json.loads((reg_dir / f"{jid}.json").read_text())
     assert job_data.get("status") == "completed", f"status={job_data.get('status')}\n{stdout}\n{stderr}"
     assert proc.returncode == 0, f"expected rc=0 while heartbeats kept landing, got {proc.returncode}\n{stdout}\n{stderr}"
+
+
+def test_wrapper_idle_grace_never_outlives_the_wall_clock(tmp_path):
+    """Heartbeats may extend the idle window, never the overall --timeout.
+    Fresh heartbeats land in events.log every second for the whole run, and
+    every python probe the wrapper makes is slowed down (herdr-RPC-like
+    latency). Counting sleeps instead of reading the clock let the grace loop
+    run about twice as long as TIMEOUT; the job must instead close as
+    deadline_timeout within TIMEOUT plus a small close/publish margin."""
+    import threading
+
+    timeout_s, idle_s = 20, 4
+    reg_dir = tmp_path / "jobs"
+    reg_dir.mkdir(parents=True, exist_ok=True)
+    slow_py = tmp_path / "slow-python"
+    slow_py.write_text(f'#!/bin/sh\nsleep 0.7\nexec "{sys.executable}" "$@"\n')
+    slow_py.chmod(0o755)
+    env = _base_env(tmp_path, {"DELEGATE_JOB_PYTHON": slow_py})
+
+    started = time.time()
+    proc = subprocess.Popen(
+        [
+            "bash", str(WRAPPER), "submit",
+            "--agent", "human",
+            "--prompt", "issue 14 wall clock",
+            "--registry-dir", str(reg_dir),
+            "--workdir", str(tmp_path),
+            "--timeout", str(timeout_s),
+            "--idle-timeout", str(idle_s),
+        ],
+        cwd=str(tmp_path), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+    )
+
+    jid = ""
+    for _ in range(200):
+        job_files = list(reg_dir.glob("*.json"))
+        if job_files:
+            jid = job_files[0].stem
+            break
+        time.sleep(0.1)
+    assert jid, "job was not registered"
+    events_log = reg_dir / f"{jid}.events.log"
+
+    stop = threading.Event()
+
+    def keep_heartbeating():
+        # The same line shape publish_event appends; only its timestamp matters
+        # to _recent_disk_heartbeat.
+        seq = 10_000
+        while not stop.is_set():
+            seq += 1
+            line = {"schema_version": 1, "seq": seq, "job_id": jid, "event": "progress",
+                    "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                    "detail": "still working", "data": {}}
+            with open(events_log, "a") as fh:
+                fh.write(json.dumps(line) + "\n")
+            stop.wait(1)
+
+    beat = threading.Thread(target=keep_heartbeating, daemon=True)
+    beat.start()
+    try:
+        stdout, stderr = proc.communicate(timeout=120)
+    finally:
+        stop.set()
+    elapsed = time.time() - started
+
+    job_data = json.loads((reg_dir / f"{jid}.json").read_text())
+    assert proc.returncode == 2, f"expected the deadline_timeout path (rc=2), got {proc.returncode}\n{stderr}"
+    assert job_data.get("status") == "error", f"status={job_data.get('status')}\n{stderr}"
+    # The heartbeats were honoured: the job was not closed after the first idle
+    # window, it stayed open up to the wall clock ...
+    assert elapsed >= timeout_s - 1, f"closed before the wall clock: {elapsed:.1f}s\n{stderr}"
+    # ... and not a moment past it beyond closing and publishing the error.
+    assert elapsed <= timeout_s + 12, f"idle grace outlived --timeout {timeout_s}s: {elapsed:.1f}s\n{stderr}"

tests/test_issue9_server_env.py

@@ -195,18 +195,32 @@ def test_reconcile_and_status_opt_out_of_autostart_statically():
             f"{script.name} must opt out of the shim's server bootstrap"
 
 
-@pytest.mark.parametrize("envelope", [
-    {"error": {"code": "server_not_running"}},
-    REAL_DOWN_ENVELOPE,
-], ids=["minimal", "herdr-0.9"])
-def test_ls_reports_down_server_as_no_server_running(mam_sandbox, envelope):
+def _fail_with(text: str, stream: str) -> str:
+    """Fake `agent list` body: write `text` to `stream` and exit 1."""
+    return f'sys.{stream}.write({json.dumps(text)} + "\\n"); sys.exit(1)'
+
+
+# reconcile.sh reads a nonzero `ls` whose stderr contains any of these as a
+# confirmed-EMPTY server (`is_empty`); anything else as unknown.
+RECONCILE_EMPTY_PHRASES = ("no server running", "no sessions", "failed to connect")
+
+# An unrelated structured failure whose message happens to mention the code.
+TRANSPORT_ERROR = json.dumps({"error": {"code": "transport_error",
+                                        "message": "socket reset while probing server_not_running"}})
+
+
+@pytest.mark.parametrize("envelope,stream", [
+    ({"error": {"code": "server_not_running"}}, "stderr"),
+    (REAL_DOWN_ENVELOPE, "stderr"),
+    ({"error": {"code": "server_not_running"}}, "stdout"),
+], ids=["minimal", "herdr-0.9", "minimal-on-stdout"])
+def test_ls_reports_down_server_as_no_server_running(mam_sandbox, envelope, stream):
     """reconcile.sh treats a nonzero `ls` as 'empty' only when stderr says
     'no server running'; anything else flips herdr_confirmed=False for every
     server. The shim must therefore translate server_not_running explicitly."""
     env = _install_fake_herdr(mam_sandbox)
     env["MAM_NO_AUTOSTART"] = "1"
-    _fake_agent_list(mam_sandbox,
-                     f'sys.stderr.write({json.dumps(json.dumps(envelope))} + "\\n"); sys.exit(1)')
+    _fake_agent_list(mam_sandbox, _fail_with(json.dumps(envelope), stream))
 
     res = _shim_ls(mam_sandbox, env, "down-srv")
     assert res.returncode != 0
@@ -229,51 +243,75 @@ def test_ls_lists_agents_whose_names_contain_server_not_running(mam_sandbox):
     assert names == ["server_not_running-codex", "creator-claude-01"], res.stdout
 
 
-@pytest.mark.parametrize("stderr_text", [
-    json.dumps({"error": {"code": "transport_error",
-                          "message": "socket reset while probing server_not_running"}}),
-    "warning: server_not_running cache entry is stale",
-    "",
-], ids=["other-code", "plain-text", "empty"])
-def test_ls_leaves_other_failures_unknown(mam_sandbox, stderr_text):
-    """Only the structured code means "down". Any other failure keeps the
-    unknown path (nonzero exit without 'no server running'), so reconcile
-    drops herdr_confirmed instead of reading a confirmed-empty server."""
+@pytest.mark.parametrize("text,stream", [
+    (TRANSPORT_ERROR, "stderr"),
+    (TRANSPORT_ERROR, "stdout"),
+    ("warning: server_not_running cache entry is stale", "stderr"),
+    ("", "stderr"),
+], ids=["other-code", "other-code-on-stdout", "plain-text", "empty"])
+def test_ls_leaves_other_failures_unknown(mam_sandbox, text, stream):
+    """Only the structured code means "down". Any other failure must stay
+    unknown - nonzero exit, no empty-server phrase - wherever its error
+    document lands. On stdout it used to reach the list parser, which read a
+    JSON error without result.agents as a successful EMPTY list (rc 0)."""
     env = _install_fake_herdr(mam_sandbox)
     env["MAM_NO_AUTOSTART"] = "1"
-    _fake_agent_list(mam_sandbox,
-                     f'sys.stderr.write({json.dumps(stderr_text)} + "\\n"); sys.exit(1)')
+    _fake_agent_list(mam_sandbox, _fail_with(text, stream))
 
     res = _shim_ls(mam_sandbox, env, "flaky-srv")
-    assert res.returncode != 0
-    assert "no server running" not in res.stderr, res.stderr
+    assert res.returncode != 0, f"a failed list must not exit 0: {res.stdout!r} {res.stderr!r}"
+    for phrase in RECONCILE_EMPTY_PHRASES:
+        assert phrase not in res.stderr.lower(), res.stderr
     assert res.stdout.strip() == ""
 
 
+def _write_running_rows(sandbox: Path, names, server: str = "live-srv") -> None:
+    rows = "".join(
+        f"  - name: {name}\n"
+        "    status: running\n"
+        f"    herdr_session: {server}\n"
+        "    pane:\n"
+        "      cwd: /tmp/nonexistent-workspace\n"
+        for name in names
+    )
+    (sandbox / ".mam" / "agent-sessions.yaml").write_text("herdr_sessions:\n" + rows)
+
+
+def _reconcile_dry_run(sandbox: Path, env: dict) -> dict:
+    reconcile = sandbox / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
+    res = subprocess.run(["bash", str(reconcile), "--once", "--emit-diff", "--dry-run"],
+                         capture_output=True, text=True, cwd=str(sandbox), env=env, timeout=120)
+    assert res.returncode == 0, res.stderr
+    return json.loads(res.stdout)
+
+
 def test_reconcile_keeps_live_rows_when_a_name_contains_server_not_running(mam_sandbox):
     """End to end through reconcile.sh: the server answers a successful list in
     which the down-server code appears as data. Both registered rows must stay
     alive with herdr_confirmed intact, so drift A cannot mark them terminated."""
     env = _install_fake_herdr(mam_sandbox)
     _fake_agent_list(mam_sandbox, f"print({json.dumps(json.dumps(LISTING_WITH_SUBSTRING))})")
-    rows = "".join(
-        f"  - name: {name}\n"
-        "    status: running\n"
-        "    herdr_session: live-srv\n"
-        "    pane:\n"
-        "      cwd: /tmp/nonexistent-workspace\n"
-        for name in ("server_not_running-codex", "creator-claude-01")
-    )
-    (mam_sandbox / ".mam" / "agent-sessions.yaml").write_text("herdr_sessions:\n" + rows)
-    reconcile = mam_sandbox / "skills" / "multi-agent-mux-monitor" / "scripts" / "reconcile.sh"
+    _write_running_rows(mam_sandbox, ("server_not_running-codex", "creator-claude-01"))
 
-    res = subprocess.run(["bash", str(reconcile), "--once", "--emit-diff", "--dry-run"],
-                         capture_output=True, text=True, cwd=str(mam_sandbox), env=env, timeout=120)
-    assert res.returncode == 0, res.stderr
-
-    report = json.loads(res.stdout)
+    report = _reconcile_dry_run(mam_sandbox, env)
     assert report["herdr_confirmed"] is True, report
     alive = set(report["herdr_sessions_alive"])
     assert {"server_not_running-codex|live-srv", "creator-claude-01|live-srv"} <= alive, alive
     drift_a = [d for d in report["drifts"] if d.get("class") == "A"]
     assert not drift_a, drift_a
+
+
+def test_reconcile_treats_other_list_failures_as_unknown(mam_sandbox):
+    """End to end through reconcile.sh: a server whose `agent list` fails with
+    an unrelated structured error on stdout must drop herdr_confirmed, so
+    drift A never marks its live row terminated."""
+    env = _install_fake_herdr(mam_sandbox)
+    _fake_agent_list(mam_sandbox, _fail_with(TRANSPORT_ERROR, "stdout"))
+    _write_running_rows(mam_sandbox, ("creator-claude-01",))
+
+    report = _reconcile_dry_run(mam_sandbox, env)
+    assert report["herdr_confirmed"] is False, report
+    drift_a = [d for d in report["drifts"] if d.get("class") == "A"]
+    assert not drift_a, drift_a
+    terminated = [a for a in report.get("actions", []) if str(a).startswith("terminated")]
+    assert not terminated, terminated