multi-agent-mux/.agents/skills/lib_py/delegation/notifier.py

"""Durable notification intent, carried in the job record.

The contract forbids a separate outbox store, so the intent rides in the record
the transition already writes: one atomic commit records both the new state and
the fact that the world has not been told yet. Publishing then happens outside
the registry lock, and settling the intent is itself an ordinary transition.

That ordering is the point. If the process dies between the commit and the
publish, the record still says a notification is owed, and startup recovery
re-sends it. Duplicates are safe: the subscriber already drops any event whose
sequence is not greater than the highest it has accepted.
"""
from __future__ import annotations

import logging
from typing import Any, Dict, List, Optional

from lib_py.delegation import mqtt_common

logger = logging.getLogger("delegate_job.notifier")

FIELD = "pending_notification"


def intent_for(event: str, seq: int, timestamp: str, detail: str = "") -> Dict[str, Any]:
    return {"event": event, "seq": seq, "timestamp": timestamp, "detail": detail}


def pending(job_id: str, registry_dir: str) -> Optional[Dict[str, Any]]:
    try:
        return mqtt_common.load_job(job_id, registry_dir).get(FIELD)
    except FileNotFoundError:
        return None


def jobs_with_pending(registry_dir: str) -> List[str]:
    """Records still owing a notification. The input to restart recovery."""
    from lib_py.delegation import registry  # local: keeps module import order flat

    out = []
    for record in registry.list_jobs(registry_dir):
        if record.get(FIELD):
            out.append(record["job_id"])
    return out


def settle(job_id: str, registry_dir: str, sent_intent=None,
           source: str = "notifier") -> bool:
    """Clear the intent that was actually sent, and only that one.

    Goes through the transition boundary like every other record write, using a
    request that carries no status: settling is about a terminal, so a request
    the terminal state would refuse could never be used to settle one.

    `sent_intent` is compared under the lock. Publishing happens outside it, so a
    newer intent can legitimately replace the old one mid-flight; clearing
    unconditionally would drop a notification nobody has sent.
    """
    from lib_py.delegation import engine

    result = engine.transition(job_id, "notification_settled", registry_dir=registry_dir,
                               fields={FIELD: None}, source=source,
                               expected_notification=sent_intent)
    if not result.committed and "changed while it was being sent" in (result.reason or ""):
        logger.info("intent for %s was replaced during send; leaving the newer one", job_id)
    return result.committed


def flush(job_id: str, registry_dir: str, publish=None, attempts: int = 3) -> bool:
    """Publish an owed notification and settle it. Safe to call repeatedly.

    Returns True when nothing is owed afterwards. A failure leaves the intent in
    place on purpose: an unsent notification that looks sent is worse than one
    that is retried.
    """
    intent = pending(job_id, registry_dir)
    if not intent:
        return True

    if publish is None:
        import json as _json

        from lib_py.delegation import publish_event as _pe

        def publish(job, intent_, registry_dir_):
            # The wire event gets a fresh registry sequence, so the intent's own
            # identity is carried in the payload instead. A consumer dedupes on
            # `notification_intent`, not on the transport sequence -- the
            # transport number cannot identify a retry of the same intent.
            identity = _json.dumps({"notification_intent": {
                "seq": intent_.get("seq"), "event": intent_.get("event"),
                "timestamp": intent_.get("timestamp")}})
            return _pe.main(["--registry-dir", registry_dir_, "--job", job,
                             "--event", intent_["event"], "--detail",
                             intent_.get("detail", ""), "--data", identity,
                             "--attempts", str(attempts)])

    try:
        rc = publish(job_id, intent, registry_dir)
    except Exception as exc:
        logger.warning("notification publish failed for %s: %s", job_id, exc)
        return False
    if rc != 0:
        logger.warning("notification for %s not accepted (rc=%s); leaving it pending",
                       job_id, rc)
        return False
    return settle(job_id, registry_dir, sent_intent=intent)


def recover(registry_dir: str, publish=None) -> Dict[str, bool]:
    """Re-send every owed notification. Called at startup after a crash."""
    results = {}
    for job_id in jobs_with_pending(registry_dir):
        results[job_id] = flush(job_id, registry_dir, publish=publish)
    return results