feat(issue-18): decide liveness and delivery by the event protocol, not the screen

Contract (no new event types): heartbeat every N s (automatic timer),
unresponsive after K x N s without an event, `started` within T s of the
injection, retained per-job state, and a remote transport.

- registry: heartbeat_sec / heartbeat_misses / ack_timeout_sec / transport
  fields; idle_timeout_sec derives K x N unless explicit (defaults 30/4/60
  keep the historical 120 s idle window).
- heartbeat.py + publish_event --heartbeat N: detached fixed-cadence timer
  publishing signed `progress` heartbeats (threaded wire publish, local
  record first); stopped by the terminal event via pidfile, by re-dispatch
  and by lifetime. Terminal events refuse --heartbeat.
- publish_event: retained `<prefix>/state` (status, last_event, seq, HMAC)
  sent on the same connection after every event (--no-state to skip);
  publisher.publish_raw takes extra messages.
- job_subscriber: subscribes the state topic, recovers terminal status from
  retained state, resets idle on any accepted message, records seq_gap and
  reconnected in the audit log, idle budget before the first event is
  T + K x N (--ack-timeout), --remote mirrors started/terminal events into
  the registry through the transition boundary.
- lib.sh send_keys_safe: an accepted-but-unconfirmed submission is confirmed
  by agent_status working or the job's own `started` (_sks_job_started);
  pane change and busy token are no longer delivery signals.
- wrapper: --heartbeat/--heartbeat-misses/--ack-timeout (MAM_* env),
  injected start command arms the timer, brief states the contract,
  _wait_started (registry / events.log / subscriber log, bounded by T and
  the wall clock) replaces the 10 s inject grace, R1 grace uses only the
  on-disk event age (herdr/pane logged once as a diagnostic),
  remote:<name> sessions skip preflight/idle/pane and dispatch over the
  broker (remote_dispatch).
- remote.py (+ scripts shim): signed retained dispatch (MAM_DISPATCH_SECRET)
  and a runner that mirrors the job into its own registry with the
  orchestrator's identity, publishes started --heartbeat, runs --exec and
  publishes the terminal event.
- loop runner: passes MAM_HEARTBEAT_SEC / MAM_HEARTBEAT_MISSES /
  MAM_ACK_TIMEOUT_S through; YAML rows with transport: remote delegate as
  remote:<name>.
- docs: MESSAGING.md 2.5 contract + remote transport, delegate-job SKILL.md,
  registry.md, loop SKILL.md.
- tests: tests/test_issue18_liveness_contract.py (16); issue 14 pane-signal
  tests flipped to started-based; issue 6 working-agent tests acknowledge
  with started; CLI snapshot/help captures refreshed.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VCgu9Fa9iT7TbJUSzdq4Ff

.agents/skills/lib_py/delegation/publish_event.py

@@ -29,6 +29,7 @@ import logging
 import sys
 from typing import Any, Dict, Optional
 
+from lib_py.delegation import heartbeat as heartbeat_timer
 from lib_py.delegation import mqtt_common
 from lib_py.delegation import registry
 from lib_py.delegation.mqtt_common import (
@@ -92,15 +93,49 @@ def build_payload(
     return payload
 
 
-def _publish_once(config, topic: str, body: bytes, retain: bool) -> None:
+def _publish_once(config, topic: str, body: bytes, retain: bool, extra=()) -> None:
     """Connect, publish one QoS-1 message, wait for the broker ACK, disconnect.
 
     Delegates to the transport module so the CLI path and the service path share
     one implementation. Keeping a second copy here is what let the persistent
     transport's handshake and not-connected bugs exist in only one of them.
     Raises on any failure so ``with_retry`` can re-run the whole sequence.
+
+    ``extra`` (Issue #18): further ``(topic, body, retain)`` messages sent over
+    the same connection after the event - the retained job-state message, so
+    it costs no second broker handshake.
     """
-    OneShotPublisher().publish_raw(config, topic, body, retain)
+    OneShotPublisher().publish_raw(config, topic, body, retain, extra=extra)
+
+
+def state_topic_for(job: Dict[str, Any]) -> str:
+    prefix = job.get("topic_prefix") or mqtt_common.topic_prefix_for(job["job_id"])
+    return f"{prefix}/state"
+
+
+def build_state_payload(job: Dict[str, Any], prepared: "PreparedEvent") -> Dict[str, Any]:
+    """The retained job-state message published next to every event (Issue #18).
+
+    A subscriber that (re)connects receives it at once and learns the job's
+    status and the last seq without waiting for the next heartbeat; a seq it
+    never saw in between is a detected loss, not a silent one. Signed like an
+    event so a forged state cannot close a job.
+    """
+    status = EVENT_TO_STATUS.get(prepared.event) or "running"
+    data = {k: v for k, v in (prepared.payload.get("data") or {}).items()
+            if k in ("dispatch_token", "attempt_id")}
+    payload = build_payload(job_id=prepared.job_id, seq=prepared.seq, event="state",
+                            detail=prepared.event, data=data, auth_token=job.get("auth_token"))
+    payload["status"] = status
+    payload["last_event"] = prepared.event
+    if job.get("auth_token"):
+        # re-sign: `status`/`last_event` were added after build_payload signed
+        payload["data"].pop("hmac_sig", None)
+        sign_payload = {k: v for k, v in payload.items() if k != "data"}
+        sign_payload["data"] = dict(payload["data"])
+        msg = json.dumps(sign_payload, sort_keys=True, separators=(",", ":")).encode()
+        payload["data"]["hmac_sig"] = hmac.new(job["auth_token"].encode(), msg, hashlib.sha256).hexdigest()
+    return payload
 
 
 class PreparedEvent(object):
@@ -256,9 +291,19 @@ def main(argv=None) -> int:
                              "job's current iteration)")
     parser.add_argument("--expect-dispatch-token", default=None,
                         help="expected dispatch token for CAS verification")
+    parser.add_argument("--heartbeat", type=int, default=None, metavar="N",
+                        help="after publishing, start a detached timer that publishes a "
+                             "`progress` heartbeat every N seconds until the job ends "
+                             "(Issue #18; not with a terminal event)")
+    parser.add_argument("--no-state", action="store_true",
+                        help="do not publish the retained <topic_prefix>/state message")
     parser.add_argument("-v", "--verbose", action="store_true")
     args = parser.parse_args(argv)
 
+    if args.heartbeat is not None and (args.event in TERMINAL_EVENTS or args.heartbeat < 1):
+        logger.error("--heartbeat needs a positive period and a non-terminal event")
+        return 1
+
     mqtt_common.ensure_env_loaded()
     mqtt_common.setup_logging(logging.DEBUG if args.verbose else logging.WARNING)
 
@@ -341,6 +386,12 @@ def main(argv=None) -> int:
                      check_token, actual_tok)
         return 4
 
+    # Issue #18: a terminal event ends the heartbeat obligation. Stop the timer
+    # before taking this event's seq so no heartbeat can follow `completed`.
+    if args.event in TERMINAL_EVENTS:
+        if heartbeat_timer.stop_timer(job_id, args.registry_dir):
+            logger.info("stopped the heartbeat timer for job %s", job_id)
+
     try:
         prepared = prepare_event(
             job_id=job_id,
@@ -356,6 +407,12 @@ def main(argv=None) -> int:
         logger.error("%s", exc)
         return 1
 
+    extra_messages = []
+    if not args.no_state and rec is not None:
+        state_payload = build_state_payload(rec, prepared)
+        extra_messages.append((state_topic_for(rec),
+                               json.dumps(state_payload, ensure_ascii=False).encode("utf-8"), True))
+
     # Local first: the event reaches events.log before the publish can block on
     # the broker ACK and its retries (see record_local_event).
     record_local_event(prepared, args.registry_dir)
@@ -370,7 +427,7 @@ def main(argv=None) -> int:
     publish_ok = True
     publish_error: Optional[str] = None
     try:
-        publish(prepared.config, prepared.topic, prepared.body, prepared.retain)
+        publish(prepared.config, prepared.topic, prepared.body, prepared.retain, extra_messages)
     except Exception as exc:
         publish_ok = False
         publish_error = str(exc)
@@ -388,6 +445,22 @@ def main(argv=None) -> int:
                    expected_attempt=sync_expected_attempt,
                    local_event_recorded=True)
 
+    # Issue #18: the automatic heartbeat. Started whether or not the wire
+    # publish succeeded - the agent is working either way, and each beat lands
+    # in events.log before its own publish, which is what local liveness reads.
+    if args.heartbeat is not None:
+        hb_token = sync_expected_token
+        hb_attempt = sync_expected_attempt
+        if rec is not None:
+            hb_token = hb_token or rec.get("dispatch_token")
+            hb_attempt = hb_attempt if hb_attempt is not None else rec.get("attempt_id", rec.get("iteration", 1))
+        try:
+            pid = heartbeat_timer.start_timer(job_id, args.registry_dir, args.heartbeat,
+                                              dispatch_token=hb_token, attempt_id=hb_attempt)
+            logger.info("heartbeat timer started (pid %d, every %ss)", pid, args.heartbeat)
+        except OSError as exc:
+            logger.error("could not start the heartbeat timer: %s", exc)
+
     if not publish_ok:
         return 2