feat(issue-30): a stall watcher that only notifies
lib_py/stall_watch.py runs beside the wrapper's subscriber for a local herdr job (started after delivery, stopped when the subscriber ends; MAM_STALL_WATCH=0 turns it off). It follows `herdr agent wait` transitions and publishes only `progress` events through one publisher; it has no path that closes a job. - blocked: `stalled:blocked <detection line>` once per episode, never answered. - idle for MAM_STALL_IDLE_S (600) with no background descendants of the agent pid (pane process-info + /proc; adapter resident_children and the heartbeat timer excluded) and an empty input box: `stalled:idle` and one nudge. - input box judged from adapter facts measured on herdr 0.9.1 (codex '›', grok frame '│', opencode '┃' block + stop pattern); the whole-pane fallback is gone. No facts: withheld + `stall_watch:input_check_unavailable` once; never empty for 3 x S: `stall_watch:input_never_empty` once. - herdr failures are not "no event": warning + `stall_watch:herdr_unavailable: <code>` (or agent_not_found), backoff, rc 2 after MAM_STALL_HERDR_FAIL_S (120); the wrapper prints the watcher's last lines. - remote and human jobs are skipped with one log line. - agent_status_reliable is an adapter field; false turns idle judgement off. - the loop runner relays stalled:/stall_watch: notices once each. - scoped_env drops a pane-inherited workspace scope for name lookups (#34 rule). The module sits at the top of lib_py: delegation and loop may not import the agent adapters (their sibling-import rules). Idle pane captures of the five CLIs are fixtures; the mock herdr answers process-info in the real shape. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
GodopuandClaude Opus 5.5 committed
1 parent
ec9586bb9b
commit
622c1f8a9d
18 files changed
+1245
-30
No files matched your search
@@ -59,11 +59,16 @@ def main():
|
||||
print(f"MAM_INPUT_PROMPT={q(adapter.input_prompt or '')}")
|
||||
print(f"MAM_INPUT_PLACEHOLDER={q(adapter.input_placeholder or '')}")
|
||||
print(f"MAM_INPUT_RULE_PATTERN={q(adapter.input_rule_pattern or '')}")
|
||||
print(f"MAM_INPUT_FRAME_CHARS={q(adapter.input_frame_chars)}")
|
||||
print(f"MAM_INPUT_BLOCK_PREFIX={q(adapter.input_block_prefix or '')}")
|
||||
print(f"MAM_INPUT_STOP_PATTERN={q(adapter.input_stop_pattern or '')}")
|
||||
print(f"MAM_READY_TOKENS={q(adapter.ready_tokens)}")
|
||||
print(f"MAM_STRONG_READY_TOKENS={q(adapter.strong_ready_tokens)}")
|
||||
print(f"MAM_WEAK_READY_TOKENS={q(adapter.weak_ready_tokens)}")
|
||||
print(f"MAM_MODAL_TOKENS={q(adapter.modal_tokens or '')}")
|
||||
print(f"MAM_STARTUP_DIALOGS={q(json.dumps(adapter.startup_dialogs))}")
|
||||
print(f"MAM_RESIDENT_CHILDREN={q(json.dumps(list(adapter.resident_children)))}")
|
||||
print(f"MAM_AGENT_STATUS_RELIABLE={1 if adapter.agent_status_reliable else 0}")
|
||||
print(f"MAM_EXIT_KEY={q(adapter.exit_key)}")
|
||||
print(f"MAM_DELEGATE_AGENT_KEY={q(adapter.delegate_agent_key)}")
|
||||
|
||||
|
||||
@@ -42,10 +42,20 @@ class CodexAgentAdapter(BaseAgentAdapter):
|
||||
def identity_cache_fields(self) -> tuple:
|
||||
return ('session_id',)
|
||||
|
||||
@property
|
||||
def input_prompt(self) -> Optional[str]:
|
||||
# herdr 0.9.1 capture: one `› Ask Codex to do anything` line, no rules.
|
||||
return '›'
|
||||
|
||||
@property
|
||||
def input_placeholder(self) -> Optional[str]:
|
||||
return 'Ask Codex to do anything'
|
||||
|
||||
@property
|
||||
def resident_children(self) -> tuple:
|
||||
# Kept alive while idle (measured 2026-09-25).
|
||||
return ('codex-code-mode-host',)
|
||||
|
||||
def _codex_home(self, ctx: DiscoveryContext) -> str:
|
||||
return os.environ.get('CODEX_HOME') or f"{ctx.home_dir}/.codex"
|
||||
|
||||
|
||||
@@ -49,6 +49,11 @@ class GrokAgentAdapter(BaseAgentAdapter):
|
||||
def input_rule_pattern(self) -> str:
|
||||
return '─{10,}'
|
||||
|
||||
@property
|
||||
def input_frame_chars(self) -> str:
|
||||
# The input box is drawn as ╭─╮ / │ ❯ │ / ╰─╯ (herdr 0.9.1 capture).
|
||||
return '│'
|
||||
|
||||
def _ws_dir(self, ctx: DiscoveryContext) -> str:
|
||||
return quote(os.path.realpath(ctx.cwd), safe='')
|
||||
|
||||
|
||||
@@ -45,8 +45,20 @@ class OpenCodeAgentAdapter(BaseAgentAdapter):
|
||||
|
||||
@property
|
||||
def input_rule_pattern(self) -> Optional[str]:
|
||||
# Kept for lib.sh's readiness corroboration; the input box itself is a
|
||||
# ┃ block (input_block_prefix), which the extractor reads first.
|
||||
return '─{10,}'
|
||||
|
||||
@property
|
||||
def input_block_prefix(self) -> Optional[str]:
|
||||
return '┃'
|
||||
|
||||
@property
|
||||
def input_stop_pattern(self) -> Optional[str]:
|
||||
# The input area ends at the right-aligned cwd line (a long run of
|
||||
# blanks after the ┃) or the `Build auto · <model>` status line.
|
||||
return r'^\s{20,}\S|^\s*(Build|Plan)\b.*·'
|
||||
|
||||
def _resolve_db(self, ctx: DiscoveryContext) -> Optional[str]:
|
||||
# 1. Try opencode CLI "opencode db path"
|
||||
try:
|
||||
|
||||
@@ -152,6 +152,34 @@ class BaseAgentAdapter:
|
||||
def input_rule_pattern(self) -> Optional[str]:
|
||||
return None
|
||||
|
||||
@property
|
||||
def input_frame_chars(self) -> str:
|
||||
"""Box-border characters stripped from both ends of an input line (#30)."""
|
||||
return ''
|
||||
|
||||
@property
|
||||
def input_block_prefix(self) -> Optional[str]:
|
||||
"""Line prefix of an input box drawn as a block instead of between rules (#30)."""
|
||||
return None
|
||||
|
||||
@property
|
||||
def input_stop_pattern(self) -> Optional[str]:
|
||||
"""Regex of the status line that ends the input area inside such a block (#30)."""
|
||||
return None
|
||||
|
||||
@property
|
||||
def resident_children(self) -> tuple:
|
||||
"""Process names this CLI keeps as children while idle (#30). The stall
|
||||
watcher does not count them as background work."""
|
||||
return ()
|
||||
|
||||
@property
|
||||
def agent_status_reliable(self) -> bool:
|
||||
"""Whether herdr's agent_status for this CLI can drive the idle judgement
|
||||
(#30). On herdr 0.9.1 claude, agy, grok, opencode and codex all report
|
||||
working/idle transitions; a False here turns the idle nudge off."""
|
||||
return True
|
||||
|
||||
@property
|
||||
def modal_tokens(self) -> Optional[str]:
|
||||
return None
|
||||
|
||||
@@ -5,33 +5,60 @@ from lib_py.agents.registry import get_adapter
|
||||
|
||||
def extract_input_region(pane_text: str, agent_name: str) -> str:
|
||||
"""
|
||||
Extracts the input area text between horizontal rule separators (─{10,}) for a given agent.
|
||||
If rules are absent, falls back to full pane_text to support mocks and unruled panes.
|
||||
Strips leading prompt tokens ('>', '❯') and placeholders ('Ask anything...').
|
||||
Raises ValueError for unsupported agents (like hermes with no input area facts).
|
||||
Extracts the input area text of a pane, using only the agent adapter's facts
|
||||
(issue #30, measured on herdr 0.9.1 captures in tests/fixtures/panes):
|
||||
|
||||
1. an `input_block_prefix` -> the last contiguous block of lines starting
|
||||
with it, prefix removed, cut at the first `input_stop_pattern` line;
|
||||
2. two or more `input_rule_pattern` rules -> the lines between the last two;
|
||||
3. an `input_prompt` -> the last line starting with the prompt.
|
||||
|
||||
Then `input_frame_chars`, the prompt and the placeholder are stripped. There
|
||||
is no whole-pane fallback: a pane with none of these shapes has no input
|
||||
region and yields a non-empty "unknown" result rather than a guess.
|
||||
Raises ValueError for agents without input area facts (like hermes' peers).
|
||||
"""
|
||||
adapter = get_adapter(agent_name)
|
||||
if not adapter or not adapter.input_prompt:
|
||||
raise ValueError(f"Agent {agent_name!r} has no input area facts")
|
||||
|
||||
rule_pattern = adapter.input_rule_pattern or r'─{10,}'
|
||||
rule_re = re.compile(rule_pattern)
|
||||
lines = pane_text.splitlines()
|
||||
rule_indices = [i for i, line in enumerate(lines) if rule_re.search(line)]
|
||||
prompt = adapter.input_prompt
|
||||
region_lines = None
|
||||
|
||||
if len(rule_indices) >= 2:
|
||||
# Exclude the rule lines themselves: lines between rule_indices[-2] + 1 and rule_indices[-1]
|
||||
region_lines = lines[rule_indices[-2] + 1:rule_indices[-1]]
|
||||
elif len(rule_indices) == 1:
|
||||
region_lines = lines[rule_indices[0] + 1:]
|
||||
else:
|
||||
region_lines = lines
|
||||
if adapter.input_block_prefix:
|
||||
prefix = adapter.input_block_prefix
|
||||
block = []
|
||||
for line in reversed(lines):
|
||||
if line.lstrip().startswith(prefix):
|
||||
block.insert(0, line.lstrip()[len(prefix):])
|
||||
elif block:
|
||||
break
|
||||
stop_re = re.compile(adapter.input_stop_pattern) if adapter.input_stop_pattern else None
|
||||
for i, line in enumerate(block):
|
||||
if stop_re and stop_re.search(line):
|
||||
block = block[:i]
|
||||
break
|
||||
# No block on screen is not an empty box: fall through to the other shapes.
|
||||
region_lines = block or None
|
||||
|
||||
if region_lines is None and adapter.input_rule_pattern:
|
||||
rule_re = re.compile(adapter.input_rule_pattern)
|
||||
rule_indices = [i for i, line in enumerate(lines) if rule_re.search(line)]
|
||||
if len(rule_indices) >= 2:
|
||||
region_lines = lines[rule_indices[-2] + 1:rule_indices[-1]]
|
||||
|
||||
if region_lines is None:
|
||||
starts = [line for line in lines if line.lstrip().startswith(prompt)]
|
||||
# No input line at all: report the whole tail as "not empty" (unknown).
|
||||
region_lines = starts[-1:] if starts else [pane_text or '?']
|
||||
|
||||
frame = adapter.input_frame_chars
|
||||
if frame:
|
||||
region_lines = [line.strip().strip(frame) for line in region_lines]
|
||||
|
||||
region_text = "\n".join(region_lines)
|
||||
|
||||
prompt = adapter.input_prompt
|
||||
if prompt:
|
||||
region_text = re.sub(r'^\s*' + re.escape(prompt) + r'\s*', '', region_text, flags=re.MULTILINE)
|
||||
region_text = re.sub(r'^\s*' + re.escape(prompt) + r'\s*', '', region_text, flags=re.MULTILINE)
|
||||
|
||||
placeholder = adapter.input_placeholder
|
||||
if placeholder:
|
||||
|
||||
@@ -106,6 +106,7 @@ class LoopRunner:
|
||||
self.round_roots: List[str] = []
|
||||
self.wait_open_jobs = False
|
||||
self.failed_submit_jobs: Set[str] = set()
|
||||
self._stall_seen: Dict[str, Tuple[int, Set[str]]] = {}
|
||||
self.current_plan = ""
|
||||
self.plan_job_id: Optional[str] = None
|
||||
self.base_commit = ""
|
||||
@@ -637,6 +638,7 @@ class LoopRunner:
|
||||
log_error(f"Job '{job_id}' record unreadable / status unknown; aborting wait.")
|
||||
return False
|
||||
|
||||
self._relay_stall_warnings(job_id)
|
||||
if status == "completed":
|
||||
if self.verbose:
|
||||
log_success(f"Job '{job_id}' completed successfully.")
|
||||
@@ -650,6 +652,30 @@ class LoopRunner:
|
||||
log_error(f"Job '{job_id}' timed out after {max_wait}s.")
|
||||
return False
|
||||
|
||||
def _relay_stall_warnings(self, job_id: str) -> None:
|
||||
"""Issue #30: surface the stall watcher's progress events (detail
|
||||
`stalled:...` / `stall_watch:...`) in the loop log, once per detail.
|
||||
Reads events.log incrementally; never closes the job."""
|
||||
offset, seen = self._stall_seen.get(job_id, (0, set()))
|
||||
try:
|
||||
with open(self.jobs_dir / f"{job_id}.events.log", "r", encoding="utf-8") as fh:
|
||||
fh.seek(offset)
|
||||
lines = fh.readlines()
|
||||
offset = fh.tell()
|
||||
except OSError:
|
||||
return
|
||||
for line in lines:
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
detail = str(ev.get("detail") or "")
|
||||
if ev.get("event") == "progress" and detail.startswith(("stalled:", "stall_watch:")) \
|
||||
and detail not in seen:
|
||||
seen.add(detail)
|
||||
log_warn(f"job {job_id}: {detail}")
|
||||
self._stall_seen[job_id] = (offset, seen)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Issue #34: a creator job's sub-jobs (registered with --parent-job)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -74,21 +74,39 @@ def caller_server(environ=None):
|
||||
return ""
|
||||
|
||||
|
||||
def scoped_env(session=None, environ=None):
|
||||
"""The environment for a wait aimed at `session`.
|
||||
def _names_agent(argv) -> bool:
|
||||
"""True for a herdr call that looks an agent up by name (lib.sh
|
||||
_herdr_scope_is_pane_inherited): `agent ...`, has-session, kill-session."""
|
||||
args = list(argv or [])
|
||||
if args and os.path.basename(args[0]) == "herdr":
|
||||
args = args[1:]
|
||||
while len(args) >= 2 and args[0] in ("-L", "--session"):
|
||||
args = args[2:]
|
||||
return bool(args) and args[0] in ("agent", "has-session", "kill-session")
|
||||
|
||||
Unchanged unless this process runs inside a pane of another server: only
|
||||
then are the caller's own workspace/pane/tab ids meaningless for the call
|
||||
and dropped. No pane and same-server keep every value.
|
||||
|
||||
def scoped_env(session=None, environ=None, argv=None):
|
||||
"""The environment for a herdr call aimed at `session`.
|
||||
|
||||
Inside a pane of another server the caller's own workspace/pane/tab ids are
|
||||
meaningless for the call and dropped (#23). On the same server, a call that
|
||||
looks an agent up by name (`argv`) drops a HERDR_WORKSPACE_ID that is just
|
||||
the caller pane's own workspace (#34, as mam_herdr does): agent names are
|
||||
unique per server, and a worktree pane must reach an agent of mam-agent.
|
||||
An id that disagrees with the pane, or one set outside a pane, still scopes.
|
||||
"""
|
||||
src = os.environ if environ is None else environ
|
||||
caller = caller_server(src)
|
||||
target = (session or "").strip()
|
||||
# `default` is the same sentinel build_argv reads: no --session flag, so
|
||||
# herdr answers on the ambient server - the caller's own.
|
||||
if not caller or not target or target == "default" or caller == target:
|
||||
return dict(src)
|
||||
return dict((k, v) for k, v in src.items() if k not in CALLER_SCOPE_VARS)
|
||||
if caller and target and target != "default" and caller != target:
|
||||
return dict((k, v) for k, v in src.items() if k not in CALLER_SCOPE_VARS)
|
||||
ws = src.get("HERDR_WORKSPACE_ID") or ""
|
||||
pane = src.get("HERDR_PANE_ID") or ""
|
||||
if ws and pane and pane.split(":", 1)[0] == ws and _names_agent(argv):
|
||||
return dict((k, v) for k, v in src.items() if k != "HERDR_WORKSPACE_ID")
|
||||
return dict(src)
|
||||
|
||||
|
||||
class WaitOutcome(object):
|
||||
@@ -144,7 +162,7 @@ def wait_once(target: str, until: Sequence[str] = DEFAULT_UNTIL,
|
||||
"""One barrier call. `run` is injectable so tests never touch a real herdr."""
|
||||
argv = build_argv(target, until, timeout_ms, session=session)
|
||||
try:
|
||||
proc = run(argv, capture_output=True, text=True, env=scoped_env(session))
|
||||
proc = run(argv, capture_output=True, text=True, env=scoped_env(session, argv=argv))
|
||||
except Exception as exc:
|
||||
return WaitOutcome(TRANSPORT_ERROR, detail="herdr could not be executed: {}".format(exc))
|
||||
|
||||
|
||||
@@ -0,0 +1,367 @@
|
||||
"""Issue #30: a stall watcher that only notifies.
|
||||
|
||||
Runs next to the delegate wrapper's subscriber for a local herdr job and tells
|
||||
people when the agent looks stuck. It never closes the job: its only output is
|
||||
a `progress` event (through `_notify`, the single publisher) plus log lines,
|
||||
and one short nudge prompt per job. Closing stays with the protocol events and
|
||||
the two clocks of #18/#33.
|
||||
|
||||
* blocked -> `progress stalled:blocked <dialog line>` once per episode, no reply.
|
||||
* idle for MAM_STALL_IDLE_S (600) with no background children of the agent
|
||||
process and an empty input box -> `progress stalled:idle` and one nudge.
|
||||
* herdr failing (TRANSPORT_ERROR / AGENT_NOT_FOUND) is never read as "no
|
||||
event": warning + `progress stall_watch:herdr_unavailable:<code>`, backoff,
|
||||
and exit 2 after MAM_STALL_HERDR_FAIL_S (120) without recovery.
|
||||
|
||||
Signals are collected in one place (`HerdrSource.collect`) and judged by the
|
||||
pure `judge`, so a future remote gateway can hand over the same `Signals`.
|
||||
|
||||
Exit codes: 0 the job reached a terminal status, 2 herdr stayed unavailable.
|
||||
|
||||
It lives at the top of lib_py, not in `delegation` or `loop`: it needs the
|
||||
agent adapters, which neither package may import (their sibling-import rules),
|
||||
and it runs as its own process, never inside a frozen loop.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional
|
||||
|
||||
from lib_py.agents.input_region import extract_input_region
|
||||
from lib_py.agents.registry import agent_of_row, get_adapter
|
||||
from lib_py.delegation.engine import TERMINAL_STATUSES
|
||||
from lib_py.loop import watcher
|
||||
|
||||
log = logging.getLogger("stall_watch")
|
||||
|
||||
RC_TERMINAL = 0
|
||||
RC_HERDR_UNAVAILABLE = 2
|
||||
|
||||
NOTIFY_BLOCKED = "notify_blocked"
|
||||
NOTIFY_IDLE_AND_NUDGE = "notify_idle_and_nudge"
|
||||
IDLE_STATES = ("idle", "done")
|
||||
HEARTBEAT_MARK = "lib_py.delegation.heartbeat"
|
||||
SKILLS_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
NUDGE = ("Job {job}: you look idle with no background work. If the job is done, publish "
|
||||
"completed or error with the command in your brief; otherwise continue where you left off.")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Signals:
|
||||
agent_status: Optional[str] = None
|
||||
state_change_seq: Optional[int] = None
|
||||
pane_id: Optional[str] = None
|
||||
blocked_text: str = ""
|
||||
agent_pid: Optional[int] = None
|
||||
background_children: List[str] = field(default_factory=list)
|
||||
input_empty: Optional[bool] = None
|
||||
job_status: Optional[str] = None
|
||||
idle_for: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class WatchState:
|
||||
idle_s: float = 600.0
|
||||
reliable: bool = True
|
||||
nudged: bool = False
|
||||
blocked_notified: bool = False
|
||||
|
||||
|
||||
def judge(sig: Signals, st: WatchState) -> Optional[str]:
|
||||
"""Pure decision over one set of signals. Never an action that closes a job."""
|
||||
if sig.job_status in TERMINAL_STATUSES:
|
||||
return None
|
||||
if sig.agent_status == "blocked":
|
||||
return None if st.blocked_notified else NOTIFY_BLOCKED
|
||||
if (st.reliable and not st.nudged and sig.agent_status in IDLE_STATES
|
||||
and sig.idle_for >= st.idle_s and sig.agent_pid
|
||||
and not sig.background_children and sig.input_empty is True):
|
||||
return NOTIFY_IDLE_AND_NUDGE
|
||||
return None
|
||||
|
||||
|
||||
def _proc_table():
|
||||
"""pid -> (ppid, name, cmdline) from /proc (Linux); `ps` elsewhere."""
|
||||
table = {}
|
||||
if os.path.isdir("/proc/self"):
|
||||
for d in os.listdir("/proc"):
|
||||
if not d.isdigit():
|
||||
continue
|
||||
try:
|
||||
with open("/proc/%s/stat" % d) as f:
|
||||
stat = f.read()
|
||||
with open("/proc/%s/cmdline" % d, "rb") as f:
|
||||
argv = [a.decode("utf-8", "replace") for a in f.read().split(b"\0") if a]
|
||||
except OSError:
|
||||
continue
|
||||
name = stat[stat.index("(") + 1:stat.rindex(")")]
|
||||
ppid = int(stat[stat.rindex(")") + 2:].split()[1])
|
||||
table[int(d)] = (ppid, name, argv)
|
||||
return table
|
||||
out = subprocess.run(["ps", "-eo", "pid=,ppid=,comm=,args="], capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
parts = line.split(None, 3)
|
||||
if len(parts) >= 3:
|
||||
table[int(parts[0])] = (int(parts[1]), os.path.basename(parts[2]), (parts[3] if len(parts) > 3 else parts[2]).split())
|
||||
return table
|
||||
|
||||
|
||||
def background_children(agent_pid: int, resident=()) -> List[str]:
|
||||
"""Descendants of the agent process, minus the adapter's resident children
|
||||
and the heartbeat timer (decision 9: it must never count as work)."""
|
||||
table = _proc_table()
|
||||
kids = {}
|
||||
for pid, (ppid, _, _) in table.items():
|
||||
kids.setdefault(ppid, []).append(pid)
|
||||
found, todo = [], list(kids.get(agent_pid, []))
|
||||
while todo:
|
||||
pid = todo.pop()
|
||||
todo.extend(kids.get(pid, []))
|
||||
_, name, argv = table[pid]
|
||||
base = os.path.basename(argv[0]) if argv else ""
|
||||
if name in resident or base in resident or name[:15] in [r[:15] for r in resident]:
|
||||
continue
|
||||
if HEARTBEAT_MARK in " ".join(argv):
|
||||
continue
|
||||
found.append("%d:%s" % (pid, name))
|
||||
return found
|
||||
|
||||
|
||||
def resolve_agent(agent, target):
|
||||
"""Adapter name for the wrapper's --agent (its default is `claude-code`),
|
||||
else from the session name; None when no adapter matches."""
|
||||
for name in (agent, (agent or "").split("-")[0], agent_of_row({}, session_name=target)):
|
||||
if name and get_adapter(name):
|
||||
return get_adapter(name).name
|
||||
return None
|
||||
|
||||
|
||||
class HerdrError(Exception):
|
||||
def __init__(self, code):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
class HerdrSource(object):
|
||||
"""Every herdr and OS read the judgement needs, in one place."""
|
||||
|
||||
def __init__(self, target, agent, session=None):
|
||||
self.target, self.session = target, session or None
|
||||
self.agent = resolve_agent(agent, target)
|
||||
adapter = get_adapter(self.agent) if self.agent else None
|
||||
self.resident = tuple(adapter.resident_children) if adapter else ()
|
||||
|
||||
def _herdr(self, *args):
|
||||
argv = ["herdr"] + (["--session", self.session] if self.session and self.session != "default" else []) + list(args)
|
||||
try:
|
||||
proc = subprocess.run(argv, capture_output=True, text=True, timeout=30,
|
||||
env=watcher.scoped_env(self.session, argv=argv))
|
||||
except Exception as exc:
|
||||
raise HerdrError("exec_failed: %s" % exc)
|
||||
if proc.returncode != 0:
|
||||
try:
|
||||
code = json.loads(proc.stderr)["error"]["code"]
|
||||
except Exception:
|
||||
code = "rc%d" % proc.returncode
|
||||
raise HerdrError(code)
|
||||
return proc.stdout
|
||||
|
||||
def agent_get(self):
|
||||
payload = json.loads(self._herdr("agent", "get", self.target) or "{}")
|
||||
return (payload.get("result") or {}).get("agent") or {}
|
||||
|
||||
def agent_pid(self, pane_id):
|
||||
info = json.loads(self._herdr("pane", "process-info", "--pane", pane_id) or "{}")
|
||||
procs = ((info.get("result") or {}).get("process_info") or {}).get("foreground_processes") or []
|
||||
# Not shell_pid: the shell is not the agent; without a pid we withhold.
|
||||
return procs[0].get("pid") if procs else None
|
||||
|
||||
def blocked_text(self, pane_id):
|
||||
text = self._herdr("pane", "read", pane_id, "--source", "detection", "--lines", "8")
|
||||
for line in text.splitlines():
|
||||
line = re.sub(r"[^\x20-\x7e]+", " ", line).strip()
|
||||
line = re.sub(r"\s+", " ", line)
|
||||
if line:
|
||||
return line[:160]
|
||||
return ""
|
||||
|
||||
def input_empty(self):
|
||||
"""True/False from the adapter's input-box facts, None when it has none."""
|
||||
if not self.agent:
|
||||
return None
|
||||
try:
|
||||
return extract_input_region(self._herdr("agent", "read", self.target), self.agent) == ""
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def prompt(self, text):
|
||||
self._herdr("agent", "prompt", self.target, text)
|
||||
|
||||
def collect(self, sig: Signals) -> Signals:
|
||||
"""Fill the process and input signals of an idle agent."""
|
||||
agent = self.agent_get()
|
||||
sig.agent_status = agent.get("agent_status", sig.agent_status)
|
||||
sig.state_change_seq = agent.get("state_change_seq", sig.state_change_seq)
|
||||
sig.pane_id = agent.get("pane_id") or sig.pane_id
|
||||
sig.agent_pid = self.agent_pid(sig.pane_id) if sig.pane_id else None
|
||||
sig.background_children = background_children(sig.agent_pid, self.resident) if sig.agent_pid else []
|
||||
sig.input_empty = self.input_empty()
|
||||
return sig
|
||||
|
||||
|
||||
class StallWatch(object):
|
||||
def __init__(self, args, source=None):
|
||||
self.args = args
|
||||
self.src = source or HerdrSource(args.target, args.agent, args.herdr_session)
|
||||
name = resolve_agent(args.agent, args.target)
|
||||
adapter = get_adapter(name) if name else None
|
||||
reliable = bool(adapter.agent_status_reliable) if adapter else True
|
||||
if os.environ.get("MAM_TEST_UNRELIABLE_AGENT") == args.agent:
|
||||
reliable = False
|
||||
self.st = WatchState(idle_s=float(os.environ.get("MAM_STALL_IDLE_S", "600")), reliable=reliable)
|
||||
self.fail_s = float(os.environ.get("MAM_STALL_HERDR_FAIL_S", "120"))
|
||||
self.once = set()
|
||||
|
||||
def _notify(self, detail):
|
||||
"""The only publisher: a `progress` event, nothing else."""
|
||||
cmd = [sys.executable, "-m", "lib_py.delegation.publish_event", "--registry-dir", self.args.registry_dir,
|
||||
"--job", self.args.job, "--event", "progress", "--detail", detail, "--attempts", "1"]
|
||||
if self.args.expect_dispatch_token:
|
||||
cmd += ["--expect-dispatch-token", self.args.expect_dispatch_token]
|
||||
if self.args.attempt_id is not None:
|
||||
cmd += ["--attempt-id", str(self.args.attempt_id)]
|
||||
env = dict(os.environ)
|
||||
env["PYTHONPATH"] = SKILLS_DIR + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
|
||||
# rc 2 = broker unreachable after the local record and events.log were written.
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, env=env, timeout=120)
|
||||
if res.returncode not in (0, 2):
|
||||
log.warning("stall_watch: publishing %r failed (rc %d): %s", detail, res.returncode, res.stderr.strip()[-200:])
|
||||
|
||||
def _notify_once(self, key, detail, message):
|
||||
if key in self.once:
|
||||
return
|
||||
self.once.add(key)
|
||||
log.warning(message)
|
||||
self._notify(detail)
|
||||
|
||||
def _job_status(self):
|
||||
try:
|
||||
with open(os.path.join(self.args.registry_dir, self.args.job + ".json")) as f:
|
||||
return json.load(f).get("status")
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def run(self) -> int:
|
||||
st, sig, agent = self.st, Signals(), self.args.agent
|
||||
if not st.reliable:
|
||||
log.warning("stall_watch: idle judgement off for %s (agent_status unreliable)", agent)
|
||||
backoff = watcher.ReconnectBackoff()
|
||||
fail_since, idle_since = None, None
|
||||
while True:
|
||||
sig.job_status = self._job_status()
|
||||
if sig.job_status in TERMINAL_STATUSES:
|
||||
return RC_TERMINAL
|
||||
if sig.agent_status is None:
|
||||
until, tmo = watcher.VALID_STATES, 1000 # learn the current state
|
||||
elif sig.agent_status in IDLE_STATES:
|
||||
left = st.idle_s - (time.monotonic() - idle_since)
|
||||
# Past S the judgement repeats every 30 s, not on every tick.
|
||||
until, tmo = ("working", "blocked"), int(1000 * (min(left, 30.0) if left > 0 else 30.0))
|
||||
tmo = max(tmo, 1000)
|
||||
elif sig.agent_status == "blocked":
|
||||
until, tmo = ("idle", "done", "working"), 60000
|
||||
else:
|
||||
until, tmo = ("idle", "done", "blocked"), 60000
|
||||
out = watcher.wait_once(self.args.target, until=until, timeout_ms=tmo, session=self.args.herdr_session)
|
||||
if out.kind in (watcher.TRANSPORT_ERROR, watcher.AGENT_NOT_FOUND):
|
||||
code = "agent_not_found" if out.kind == watcher.AGENT_NOT_FOUND else (out.error_code or "transport_error")
|
||||
now = time.monotonic()
|
||||
if fail_since is None:
|
||||
fail_since = now
|
||||
log.warning("stall_watch: herdr unavailable: %s %s", code, out.detail)
|
||||
self._notify("stall_watch:agent_not_found" if code == "agent_not_found"
|
||||
else "stall_watch:herdr_unavailable:%s" % code)
|
||||
if now - fail_since >= self.fail_s:
|
||||
log.warning("stall_watch: herdr unavailable for %ds; stall watch stops (%s)", int(now - fail_since), code)
|
||||
return RC_HERDR_UNAVAILABLE
|
||||
time.sleep(min(backoff.record_failure(), max(0.1, self.fail_s - (now - fail_since))))
|
||||
sig.agent_status = None # re-observe after a gap
|
||||
continue
|
||||
backoff.record_success()
|
||||
fail_since = None
|
||||
if out.kind == watcher.MATCHED:
|
||||
new = out.agent.get("agent_status")
|
||||
sig.pane_id = out.agent.get("pane_id") or sig.pane_id
|
||||
if new in IDLE_STATES and sig.agent_status not in IDLE_STATES:
|
||||
idle_since, sig.state_change_seq = time.monotonic(), out.agent.get("state_change_seq")
|
||||
if new != "blocked":
|
||||
st.blocked_notified = False
|
||||
sig.agent_status = new
|
||||
try:
|
||||
self._act(sig, idle_since)
|
||||
except HerdrError as exc:
|
||||
log.warning("stall_watch: herdr call failed while judging: %s", exc.code)
|
||||
|
||||
def _act(self, sig, idle_since):
|
||||
st, agent = self.st, self.args.agent
|
||||
if sig.agent_status == "blocked" and not st.blocked_notified:
|
||||
sig.blocked_text = self.src.blocked_text(sig.pane_id or self.src.agent_get().get("pane_id", ""))
|
||||
if judge(sig, st) == NOTIFY_BLOCKED:
|
||||
log.warning("stall_watch: agent %s is blocked: %s", self.args.target, sig.blocked_text)
|
||||
self._notify(("stalled:blocked " + sig.blocked_text).strip())
|
||||
st.blocked_notified = True
|
||||
return
|
||||
if sig.agent_status not in IDLE_STATES or not st.reliable or st.nudged:
|
||||
return
|
||||
sig.idle_for = time.monotonic() - idle_since
|
||||
if sig.idle_for < st.idle_s:
|
||||
return
|
||||
seq = sig.state_change_seq
|
||||
self.src.collect(sig)
|
||||
if seq is not None and sig.state_change_seq is not None and sig.state_change_seq != seq:
|
||||
return # a transition we did not see: the next wait re-observes
|
||||
if sig.background_children:
|
||||
log.debug("stall_watch: idle but background children %s", sig.background_children)
|
||||
elif sig.input_empty is None:
|
||||
self._notify_once("input_unavailable", "stall_watch:input_check_unavailable:%s" % agent,
|
||||
"stall_watch: input check unavailable for %s; idle nudge withheld" % agent)
|
||||
elif sig.input_empty is False:
|
||||
log.info("stall_watch: input box of %s is not empty; nudge withheld", self.args.target)
|
||||
if sig.idle_for >= 3 * st.idle_s:
|
||||
self._notify_once("never_empty", "stall_watch:input_never_empty:%s" % agent,
|
||||
"stall_watch: input box of %s never looked empty in %ds" % (agent, int(sig.idle_for)))
|
||||
if judge(sig, st) == NOTIFY_IDLE_AND_NUDGE:
|
||||
log.warning("stall_watch: agent %s idle for %ds with no background work; nudging once",
|
||||
self.args.target, int(sig.idle_for))
|
||||
self._notify("stalled:idle")
|
||||
st.nudged = True
|
||||
self.src.prompt(NUDGE.format(job=self.args.job))
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
p = argparse.ArgumentParser(description="notify-only stall watcher for one local herdr job (#30)")
|
||||
p.add_argument("--job", required=True)
|
||||
p.add_argument("--registry-dir", required=True)
|
||||
p.add_argument("--target", required=True, help="herdr agent name of the job's session")
|
||||
p.add_argument("--agent", required=True, help="agent CLI (adapter name)")
|
||||
p.add_argument("--herdr-session", default="")
|
||||
p.add_argument("--expect-dispatch-token", default=None)
|
||||
p.add_argument("--attempt-id", type=int, default=None)
|
||||
args = p.parse_args(argv)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s stall_watch %(levelname)s %(message)s")
|
||||
# The wrapper stops us with SIGTERM once its subscriber ends: leave quietly.
|
||||
signal.signal(signal.SIGTERM, lambda *_: os._exit(RC_TERMINAL))
|
||||
return StallWatch(args).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -365,6 +365,7 @@ EOF
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 1
|
||||
fi
|
||||
start_stall_watch "$AGENT_SESSION" "$AGENT" "$_disp_tok" "$_attempt_id"
|
||||
|
||||
# 6) optional validation hook
|
||||
if [[ -n "$VALIDATE" ]]; then
|
||||
@@ -379,6 +380,7 @@ EOF
|
||||
|
||||
local sub_rc=0
|
||||
wait "$sub_pid" || sub_rc=$?
|
||||
stop_stall_watch
|
||||
echo "subscriber output:"; cat "$logf" || true
|
||||
|
||||
local job_status
|
||||
@@ -548,10 +550,12 @@ EOF
|
||||
echo "$logs_root/$JOB_ID"
|
||||
exit 1
|
||||
fi
|
||||
start_stall_watch "$current_session" "$cur_agent" "$_disp_tok" "$attempt"
|
||||
|
||||
# Wait for subscriber
|
||||
local sub_rc=0
|
||||
wait "$sub_pid" || sub_rc=$?
|
||||
stop_stall_watch
|
||||
echo "subscriber output:"; cat "$logf" || true
|
||||
|
||||
local job_status
|
||||
@@ -672,6 +676,47 @@ reap_subscriber() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Issue #30: a notify-only stall watcher for a job with a local herdr pane
|
||||
# (lib_py.stall_watch decides; it never closes the job). A remote
|
||||
# or human job has no pane to watch. Sets STALL_PID ("" when none started).
|
||||
start_stall_watch() {
|
||||
local session="$1" agent="$2" tok="${3:-}" att="${4:-}"
|
||||
STALL_PID=""
|
||||
[ "${MAM_STALL_WATCH:-1}" = "0" ] && return 0
|
||||
if _is_remote "$session"; then
|
||||
echo "stall watch: remote job, no local pane - stall detection off" >&2; return 0
|
||||
fi
|
||||
if [ "$agent" = "human" ] || [ "${session#herdr:}" = "human" ]; then
|
||||
echo "stall watch: human job, no local pane - stall detection off" >&2; return 0
|
||||
fi
|
||||
local args=(--job "$JOB_ID" --registry-dir "$REGISTRY_DIR" --target "${session#herdr:}"
|
||||
--agent "$agent" --herdr-session "${HERDR_SESSION_NAME:-}")
|
||||
[ -n "$tok" ] && args+=(--expect-dispatch-token "$tok")
|
||||
[ -n "$att" ] && args+=(--attempt-id "$att")
|
||||
PYTHONPATH="$SCRIPT_DIR/..${PYTHONPATH:+:$PYTHONPATH}" "$PY" -m lib_py.stall_watch "${args[@]}" \
|
||||
>"$REGISTRY_DIR/$JOB_ID.stall.out" 2>&1 &
|
||||
STALL_PID=$!
|
||||
echo "stall watch pid: $STALL_PID (log: $REGISTRY_DIR/$JOB_ID.stall.out)"
|
||||
}
|
||||
|
||||
# A watcher that already ended on its own with an error (herdr unavailable)
|
||||
# is not left silent: its last lines go to this wrapper's stderr.
|
||||
stop_stall_watch() {
|
||||
[ -n "${STALL_PID:-}" ] || return 0
|
||||
local rc=0
|
||||
if kill -0 "$STALL_PID" 2>/dev/null; then
|
||||
kill "$STALL_PID" 2>/dev/null || true
|
||||
wait "$STALL_PID" 2>/dev/null || true
|
||||
else
|
||||
wait "$STALL_PID" 2>/dev/null || rc=$?
|
||||
fi
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "stall watch for $JOB_ID exited rc=$rc:" >&2
|
||||
tail -n 3 "$REGISTRY_DIR/$JOB_ID.stall.out" >&2 2>/dev/null || true
|
||||
fi
|
||||
STALL_PID=""
|
||||
}
|
||||
|
||||
_disk_status() {
|
||||
local jid="$1"
|
||||
"$PY" -c "import json, os; p=os.path.join('$REGISTRY_DIR', '$jid.json'); print(json.load(open(p)).get('status','')) if os.path.exists(p) else print('')" 2>/dev/null || true
|
||||
@@ -818,6 +863,11 @@ finalize_unfinished() {
|
||||
if [ "$job_status" = "completed" ]; then
|
||||
return 0
|
||||
fi
|
||||
# Issue #30: what the stall watcher saw goes with an unfinished job's close.
|
||||
if [ -s "$REGISTRY_DIR/$job_id.stall.out" ]; then
|
||||
echo "stall watch log for $job_id (last 3 lines):" >&2
|
||||
tail -n 3 "$REGISTRY_DIR/$job_id.stall.out" >&2 || true
|
||||
fi
|
||||
|
||||
local cur_disk_st="$(_disk_status "$job_id")"
|
||||
if [ "$cur_disk_st" = "completed" ] || [ "$cur_disk_st" = "error" ] || [ "$cur_disk_st" = "cancelled" ]; then
|
||||
|
||||
Reference in new issue
Block a user