multi-agent-mux/.agents/skills/lib_py/loop/watcher.py

"""Wraps `herdr agent wait`. Observes; never concludes.

Live contract (herdr 0.8.2, re-confirmed while planning):

    herdr agent wait <TARGET> --until <STATUS>... --timeout <MS>

  * `--timeout` is MILLISECONDS. The parameter here is named `timeout_ms` so the
    unit cannot be misread at a call site; there is deliberately no seconds form.
  * `--until` is always passed explicitly. Omitting it matches idle/done/blocked,
    silently excluding `working`, which is a state the watcher cares about.
  * success -> exit 0, snapshot JSON on STDOUT.
  * timeout and unknown target -> exit 1, error JSON on STDERR, stdout empty.
    The stream split is the contract: an error is never read from stdout.

A single wait is one synchronous barrier returning one final snapshot, so
continuous observation is built by repeating it, not by streaming.
"""
from __future__ import annotations

import json
import logging
import subprocess
from typing import Any, Dict, Optional, Sequence

from lib_py.loop.evidence import Observation

logger = logging.getLogger("loop.watcher")

DEFAULT_UNTIL = ("idle", "done", "blocked")
VALID_STATES = ("idle", "working", "blocked", "done", "unknown")

# Outcome kinds. `timeout` is an ordinary no-event result, not a failure.
MATCHED = "matched"
TIMEOUT = "timeout"
AGENT_NOT_FOUND = "agent_not_found"
TRANSPORT_ERROR = "transport_error"

BACKOFF_START_S = 1.0
BACKOFF_MAX_S = 16.0


class WaitOutcome(object):
    __slots__ = ("kind", "agent", "error_code", "detail", "raw")

    def __init__(self, kind, agent=None, error_code=None, detail="", raw=None):
        self.kind = kind
        self.agent = agent
        self.error_code = error_code
        self.detail = detail
        self.raw = raw

    @property
    def healthy(self) -> bool:
        """A timeout is healthy: the socket answered, the state just did not change."""
        return self.kind in (MATCHED, TIMEOUT, AGENT_NOT_FOUND)

    def __repr__(self):  # pragma: no cover - debugging aid
        return "WaitOutcome({!r}, error_code={!r})".format(self.kind, self.error_code)


def build_argv(target: str, until: Sequence[str] = DEFAULT_UNTIL,
               timeout_ms: Optional[int] = None) -> list:
    if not target:
        raise ValueError("wait target is required")
    argv = ["herdr", "agent", "wait", target]
    for state in until:
        if state not in VALID_STATES:
            raise ValueError("unsupported --until state: {!r}".format(state))
        argv += ["--until", state]
    if timeout_ms is not None:
        if int(timeout_ms) <= 0:
            raise ValueError("timeout_ms must be positive milliseconds")
        argv += ["--timeout", str(int(timeout_ms))]
    return argv


def _parse(stream: str) -> Optional[Dict[str, Any]]:
    try:
        payload = json.loads(stream)
        return payload if isinstance(payload, dict) else None
    except Exception:
        return None


def wait_once(target: str, until: Sequence[str] = DEFAULT_UNTIL,
              timeout_ms: Optional[int] = None, run=subprocess.run) -> WaitOutcome:
    """One barrier call. `run` is injectable so tests never touch a real herdr."""
    argv = build_argv(target, until, timeout_ms)
    try:
        proc = run(argv, capture_output=True, text=True)
    except Exception as exc:
        return WaitOutcome(TRANSPORT_ERROR, detail="herdr could not be executed: {}".format(exc))

    if proc.returncode == 0:
        payload = _parse(proc.stdout or "")
        agent = ((payload or {}).get("result") or {}).get("agent")
        if not isinstance(agent, dict):
            return WaitOutcome(TRANSPORT_ERROR,
                               detail="exit 0 without a parseable agent snapshot",
                               raw=proc.stdout)
        return WaitOutcome(MATCHED, agent=agent, raw=payload)

    # Non-zero: the payload is on stderr by contract, and stdout is empty.
    payload = _parse(proc.stderr or "")
    code = ((payload or {}).get("error") or {}).get("code")
    if code == "timeout":
        return WaitOutcome(TIMEOUT, error_code=code, raw=payload)
    if code == "agent_not_found":
        return WaitOutcome(AGENT_NOT_FOUND, error_code=code, raw=payload)
    return WaitOutcome(TRANSPORT_ERROR, error_code=code,
                       detail="unrecognised failure (rc={})".format(proc.returncode),
                       raw=proc.stderr)


def observation_from(outcome: WaitOutcome, capture_ref=None) -> Optional[Observation]:
    """Turn a matched snapshot into evidence. Non-matches carry no agent state."""
    if outcome.kind != MATCHED or not outcome.agent:
        return None
    agent = outcome.agent
    return Observation(
        "herdr",
        {
            "agent_status": agent.get("agent_status"),
            "interactive_ready": agent.get("interactive_ready"),
            "pane_id": agent.get("pane_id"),
            "state_change_seq": agent.get("state_change_seq"),
        },
        capture_ref=capture_ref,
    )


class ReconnectBackoff(object):
    """Doubling backoff for transport failures, reset by any healthy answer.

    The snapshot cache is dropped on failure on purpose: after a socket gap the
    watcher must re-observe rather than reason from a state that may have moved
    while it was blind. That is what makes `hold_reconnect` the honest decision.
    """

    def __init__(self, start=BACKOFF_START_S, maximum=BACKOFF_MAX_S):
        self.start = start
        self.maximum = maximum
        self.current = 0.0
        self.consecutive_failures = 0
        self.cached_snapshot = None

    def record_success(self, snapshot=None) -> None:
        self.consecutive_failures = 0
        self.current = 0.0
        if snapshot is not None:
            self.cached_snapshot = snapshot

    def record_failure(self) -> float:
        self.consecutive_failures += 1
        self.cached_snapshot = None  # stale evidence must not survive a gap
        self.current = self.start if self.current == 0.0 else min(self.current * 2.0, self.maximum)
        return self.current

    def sequence(self, n: int) -> list:
        """The first n delays, for documentation and for tests to pin."""
        saved = (self.current, self.consecutive_failures, self.cached_snapshot)
        try:
            self.current, self.consecutive_failures = 0.0, 0
            return [self.record_failure() for _ in range(n)]
        finally:
            self.current, self.consecutive_failures, self.cached_snapshot = saved