feat(issue-30): a stall watcher that only notifies
lib_py/stall_watch.py runs beside the wrapper's subscriber for a local herdr job (started after delivery, stopped when the subscriber ends; MAM_STALL_WATCH=0 turns it off). It follows `herdr agent wait` transitions and publishes only `progress` events through one publisher; it has no path that closes a job. - blocked: `stalled:blocked <detection line>` once per episode, never answered. - idle for MAM_STALL_IDLE_S (600) with no background descendants of the agent pid (pane process-info + /proc; adapter resident_children and the heartbeat timer excluded) and an empty input box: `stalled:idle` and one nudge. - input box judged from adapter facts measured on herdr 0.9.1 (codex '›', grok frame '│', opencode '┃' block + stop pattern); the whole-pane fallback is gone. No facts: withheld + `stall_watch:input_check_unavailable` once; never empty for 3 x S: `stall_watch:input_never_empty` once. - herdr failures are not "no event": warning + `stall_watch:herdr_unavailable: <code>` (or agent_not_found), backoff, rc 2 after MAM_STALL_HERDR_FAIL_S (120); the wrapper prints the watcher's last lines. - remote and human jobs are skipped with one log line. - agent_status_reliable is an adapter field; false turns idle judgement off. - the loop runner relays stalled:/stall_watch: notices once each. - scoped_env drops a pane-inherited workspace scope for name lookups (#34 rule). The module sits at the top of lib_py: delegation and loop may not import the agent adapters (their sibling-import rules). Idle pane captures of the five CLIs are fixtures; the mock herdr answers process-info in the real shape. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ec9586bb9b
commit
622c1f8a9d
18 files changed
+1245
-30
No files matched your search
@@ -106,6 +106,7 @@ class LoopRunner:
|
||||
self.round_roots: List[str] = []
|
||||
self.wait_open_jobs = False
|
||||
self.failed_submit_jobs: Set[str] = set()
|
||||
self._stall_seen: Dict[str, Tuple[int, Set[str]]] = {}
|
||||
self.current_plan = ""
|
||||
self.plan_job_id: Optional[str] = None
|
||||
self.base_commit = ""
|
||||
@@ -637,6 +638,7 @@ class LoopRunner:
|
||||
log_error(f"Job '{job_id}' record unreadable / status unknown; aborting wait.")
|
||||
return False
|
||||
|
||||
self._relay_stall_warnings(job_id)
|
||||
if status == "completed":
|
||||
if self.verbose:
|
||||
log_success(f"Job '{job_id}' completed successfully.")
|
||||
@@ -650,6 +652,30 @@ class LoopRunner:
|
||||
log_error(f"Job '{job_id}' timed out after {max_wait}s.")
|
||||
return False
|
||||
|
||||
def _relay_stall_warnings(self, job_id: str) -> None:
|
||||
"""Issue #30: surface the stall watcher's progress events (detail
|
||||
`stalled:...` / `stall_watch:...`) in the loop log, once per detail.
|
||||
Reads events.log incrementally; never closes the job."""
|
||||
offset, seen = self._stall_seen.get(job_id, (0, set()))
|
||||
try:
|
||||
with open(self.jobs_dir / f"{job_id}.events.log", "r", encoding="utf-8") as fh:
|
||||
fh.seek(offset)
|
||||
lines = fh.readlines()
|
||||
offset = fh.tell()
|
||||
except OSError:
|
||||
return
|
||||
for line in lines:
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
detail = str(ev.get("detail") or "")
|
||||
if ev.get("event") == "progress" and detail.startswith(("stalled:", "stall_watch:")) \
|
||||
and detail not in seen:
|
||||
seen.add(detail)
|
||||
log_warn(f"job {job_id}: {detail}")
|
||||
self._stall_seen[job_id] = (offset, seen)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Issue #34: a creator job's sub-jobs (registered with --parent-job)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -74,21 +74,39 @@ def caller_server(environ=None):
|
||||
return ""
|
||||
|
||||
|
||||
def scoped_env(session=None, environ=None):
|
||||
"""The environment for a wait aimed at `session`.
|
||||
def _names_agent(argv) -> bool:
|
||||
"""True for a herdr call that looks an agent up by name (lib.sh
|
||||
_herdr_scope_is_pane_inherited): `agent ...`, has-session, kill-session."""
|
||||
args = list(argv or [])
|
||||
if args and os.path.basename(args[0]) == "herdr":
|
||||
args = args[1:]
|
||||
while len(args) >= 2 and args[0] in ("-L", "--session"):
|
||||
args = args[2:]
|
||||
return bool(args) and args[0] in ("agent", "has-session", "kill-session")
|
||||
|
||||
Unchanged unless this process runs inside a pane of another server: only
|
||||
then are the caller's own workspace/pane/tab ids meaningless for the call
|
||||
and dropped. No pane and same-server keep every value.
|
||||
|
||||
def scoped_env(session=None, environ=None, argv=None):
|
||||
"""The environment for a herdr call aimed at `session`.
|
||||
|
||||
Inside a pane of another server the caller's own workspace/pane/tab ids are
|
||||
meaningless for the call and dropped (#23). On the same server, a call that
|
||||
looks an agent up by name (`argv`) drops a HERDR_WORKSPACE_ID that is just
|
||||
the caller pane's own workspace (#34, as mam_herdr does): agent names are
|
||||
unique per server, and a worktree pane must reach an agent of mam-agent.
|
||||
An id that disagrees with the pane, or one set outside a pane, still scopes.
|
||||
"""
|
||||
src = os.environ if environ is None else environ
|
||||
caller = caller_server(src)
|
||||
target = (session or "").strip()
|
||||
# `default` is the same sentinel build_argv reads: no --session flag, so
|
||||
# herdr answers on the ambient server - the caller's own.
|
||||
if not caller or not target or target == "default" or caller == target:
|
||||
return dict(src)
|
||||
return dict((k, v) for k, v in src.items() if k not in CALLER_SCOPE_VARS)
|
||||
if caller and target and target != "default" and caller != target:
|
||||
return dict((k, v) for k, v in src.items() if k not in CALLER_SCOPE_VARS)
|
||||
ws = src.get("HERDR_WORKSPACE_ID") or ""
|
||||
pane = src.get("HERDR_PANE_ID") or ""
|
||||
if ws and pane and pane.split(":", 1)[0] == ws and _names_agent(argv):
|
||||
return dict((k, v) for k, v in src.items() if k != "HERDR_WORKSPACE_ID")
|
||||
return dict(src)
|
||||
|
||||
|
||||
class WaitOutcome(object):
|
||||
@@ -144,7 +162,7 @@ def wait_once(target: str, until: Sequence[str] = DEFAULT_UNTIL,
|
||||
"""One barrier call. `run` is injectable so tests never touch a real herdr."""
|
||||
argv = build_argv(target, until, timeout_ms, session=session)
|
||||
try:
|
||||
proc = run(argv, capture_output=True, text=True, env=scoped_env(session))
|
||||
proc = run(argv, capture_output=True, text=True, env=scoped_env(session, argv=argv))
|
||||
except Exception as exc:
|
||||
return WaitOutcome(TRANSPORT_ERROR, detail="herdr could not be executed: {}".format(exc))
|
||||
|
||||
|
||||
Reference in new issue
Block a user