feat(messaging): implement Track 0 fault tolerance (B-14, B-15) and add 10 regression guards (G-1 to G-10)
This commit is contained in:
@@ -473,3 +473,292 @@ def test_b9_logs_dir_stays_discoverable(mam_sandbox):
|
||||
assert dir(mq).count("LOGS_DIR") == 1 # set-based __dir__ must not duplicate
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Track 0: Fault Tolerance & Local Fallback Regression Guards (G-1 to G-10)
|
||||
# ==============================================================================
|
||||
|
||||
def _save_job_for_test(job_rec, registry_dir):
|
||||
p = os.path.join(registry_dir, f"{job_rec['job_id']}.json")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
json.dump(job_rec, f, indent=2)
|
||||
|
||||
|
||||
def test_g1_publish_failure_persists_completed_status(mam_sandbox, monkeypatch):
|
||||
"""G-1: When publish fails due to broker unreachable, registry status is still updated."""
|
||||
monkeypatch.chdir(mam_sandbox)
|
||||
script_dir = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts"
|
||||
sys.path.insert(0, str(script_dir))
|
||||
import registry, publish_event
|
||||
|
||||
reg_dir = str(mam_sandbox / ".mam" / "jobs")
|
||||
os.makedirs(reg_dir, exist_ok=True)
|
||||
job_id = registry.register_job("Test G-1", registry_dir=reg_dir, job_id="g1test01")
|
||||
job = registry.load_job(job_id, reg_dir)
|
||||
# Point broker to non-routable dummy port
|
||||
job["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
||||
_save_job_for_test(job, reg_dir)
|
||||
|
||||
rc = publish_event.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--job", "g1test01",
|
||||
"--event", "completed",
|
||||
"--detail", "finished work",
|
||||
"--attempts", "1"
|
||||
])
|
||||
assert rc == 2, f"Expected rc=2 on network failure, got {rc}"
|
||||
loaded = registry.load_job("g1test01", reg_dir)
|
||||
assert loaded["status"] == "completed", "Registry status must be completed despite network publish failure"
|
||||
|
||||
|
||||
def test_g2_publish_failure_records_audit_log_error(mam_sandbox, monkeypatch):
|
||||
"""G-2: Audit log records published=False and publish_error when broker is unreachable."""
|
||||
monkeypatch.chdir(mam_sandbox)
|
||||
script_dir = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts"
|
||||
sys.path.insert(0, str(script_dir))
|
||||
import registry, publish_event, mqtt_common
|
||||
|
||||
reg_dir = str(mam_sandbox / ".mam" / "jobs")
|
||||
os.makedirs(reg_dir, exist_ok=True)
|
||||
job_id = registry.register_job("Test G-2", registry_dir=reg_dir, job_id="g2test01")
|
||||
job = registry.load_job(job_id, reg_dir)
|
||||
job["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
||||
_save_job_for_test(job, reg_dir)
|
||||
|
||||
rc = publish_event.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--job", "g2test01",
|
||||
"--event", "completed",
|
||||
"--attempts", "1"
|
||||
])
|
||||
assert rc == 2
|
||||
|
||||
log_file = os.path.join(str(mam_sandbox), ".mam", "delegate_job_logs", "g2test01", "events.ndjson")
|
||||
assert os.path.exists(log_file), "Audit log file must exist"
|
||||
with open(log_file, "r", encoding="utf-8") as f:
|
||||
events = [json.loads(line) for line in f if line.strip()]
|
||||
|
||||
pub_events = [e for e in events if e.get("event") == "published" and e.get("source_event") == "completed"]
|
||||
assert len(pub_events) == 1
|
||||
assert pub_events[0]["published"] is False
|
||||
assert pub_events[0]["publish_error"] is not None
|
||||
|
||||
|
||||
def test_g3_publish_success_records_published_true(mam_sandbox, monkeypatch):
|
||||
"""G-3: When publish succeeds, rc=0, status=completed, and published=True."""
|
||||
monkeypatch.chdir(mam_sandbox)
|
||||
script_dir = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts"
|
||||
sys.path.insert(0, str(script_dir))
|
||||
import registry, publish_event, mqtt_common
|
||||
|
||||
reg_dir = str(mam_sandbox / ".mam" / "jobs")
|
||||
os.makedirs(reg_dir, exist_ok=True)
|
||||
registry.register_job("Test G-3", registry_dir=reg_dir, job_id="g3test01")
|
||||
|
||||
# Mock _publish_once to simulate broker success
|
||||
monkeypatch.setattr(publish_event, "_publish_once", lambda *args, **kwargs: None)
|
||||
|
||||
rc = publish_event.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--job", "g3test01",
|
||||
"--event", "completed",
|
||||
"--attempts", "1"
|
||||
])
|
||||
assert rc == 0
|
||||
loaded = registry.load_job("g3test01", reg_dir)
|
||||
assert loaded["status"] == "completed"
|
||||
|
||||
log_file = os.path.join(str(mam_sandbox), ".mam", "delegate_job_logs", "g3test01", "events.ndjson")
|
||||
assert os.path.exists(log_file), f"Audit log file {log_file} must exist"
|
||||
with open(log_file, "r", encoding="utf-8") as f:
|
||||
events = [json.loads(line) for line in f if line.strip()]
|
||||
pub_events = [e for e in events if e.get("event") == "published" and e.get("source_event") == "completed"]
|
||||
assert len(pub_events) == 1
|
||||
assert pub_events[0]["published"] is True
|
||||
assert pub_events[0]["publish_error"] is None
|
||||
|
||||
|
||||
def test_g4_publish_failure_advances_sequence(mam_sandbox, monkeypatch):
|
||||
"""G-4: A failed publish consumes sequence number, and subsequent publish uses strictly higher seq."""
|
||||
monkeypatch.chdir(mam_sandbox)
|
||||
script_dir = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts"
|
||||
sys.path.insert(0, str(script_dir))
|
||||
import registry, publish_event
|
||||
|
||||
reg_dir = str(mam_sandbox / ".mam" / "jobs")
|
||||
os.makedirs(reg_dir, exist_ok=True)
|
||||
job_id = registry.register_job("Test G-4", registry_dir=reg_dir, job_id="g4test01")
|
||||
job = registry.load_job(job_id, reg_dir)
|
||||
job["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
||||
_save_job_for_test(job, reg_dir)
|
||||
|
||||
# First attempt fails
|
||||
rc1 = publish_event.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--job", "g4test01",
|
||||
"--event", "progress",
|
||||
"--attempts", "1"
|
||||
])
|
||||
assert rc1 == 2
|
||||
j1 = registry.load_job("g4test01", reg_dir)
|
||||
assert int(j1["last_seq"]) == 1
|
||||
|
||||
# Second attempt succeeds (mocked)
|
||||
monkeypatch.setattr(publish_event, "_publish_once", lambda *args, **kwargs: None)
|
||||
rc2 = publish_event.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--job", "g4test01",
|
||||
"--event", "completed",
|
||||
"--attempts", "1"
|
||||
])
|
||||
assert rc2 == 0
|
||||
j2 = registry.load_job("g4test01", reg_dir)
|
||||
assert int(j2["last_seq"]) == 2
|
||||
|
||||
|
||||
def test_g5_subscriber_disk_fallback_on_broker_down_exits_0(mam_sandbox, monkeypatch):
|
||||
"""G-5: When broker is down and disk has status=completed, job_subscriber exits 0 via fallback."""
|
||||
monkeypatch.chdir(mam_sandbox)
|
||||
script_dir = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts"
|
||||
sys.path.insert(0, str(script_dir))
|
||||
import registry, job_subscriber
|
||||
|
||||
reg_dir = str(mam_sandbox / ".mam" / "jobs")
|
||||
os.makedirs(reg_dir, exist_ok=True)
|
||||
job_id = registry.register_job("Test G-5", registry_dir=reg_dir, job_id="g5test01")
|
||||
job = registry.load_job(job_id, reg_dir)
|
||||
job["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
||||
job["status"] = "completed"
|
||||
_save_job_for_test(job, reg_dir)
|
||||
|
||||
rc = job_subscriber.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--job", "g5test01",
|
||||
"--timeout", "5",
|
||||
"--idle-timeout", "2"
|
||||
])
|
||||
assert rc == 0
|
||||
|
||||
|
||||
def test_g6_subscriber_disk_fallback_outputs_source_tag(mam_sandbox, capsys, monkeypatch):
|
||||
"""G-6: When resolving via disk fallback, stdout contains 'disk-fallback' tag."""
|
||||
monkeypatch.chdir(mam_sandbox)
|
||||
script_dir = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts"
|
||||
sys.path.insert(0, str(script_dir))
|
||||
import registry, job_subscriber
|
||||
|
||||
reg_dir = str(mam_sandbox / ".mam" / "jobs")
|
||||
os.makedirs(reg_dir, exist_ok=True)
|
||||
job_id = registry.register_job("Test G-6", registry_dir=reg_dir, job_id="g6test01")
|
||||
job = registry.load_job(job_id, reg_dir)
|
||||
job["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
||||
job["status"] = "completed"
|
||||
_save_job_for_test(job, reg_dir)
|
||||
|
||||
rc = job_subscriber.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--job", "g6test01",
|
||||
"--timeout", "5"
|
||||
])
|
||||
assert rc == 0
|
||||
captured = capsys.readouterr()
|
||||
assert "disk-fallback" in captured.out
|
||||
|
||||
|
||||
def test_g7_subscriber_disk_fallback_error_status_exits_1(mam_sandbox, monkeypatch):
|
||||
"""G-7: When broker is down and disk has status=error, job_subscriber exits 1."""
|
||||
monkeypatch.chdir(mam_sandbox)
|
||||
script_dir = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts"
|
||||
sys.path.insert(0, str(script_dir))
|
||||
import registry, job_subscriber
|
||||
|
||||
reg_dir = str(mam_sandbox / ".mam" / "jobs")
|
||||
os.makedirs(reg_dir, exist_ok=True)
|
||||
job_id = registry.register_job("Test G-7", registry_dir=reg_dir, job_id="g7test01")
|
||||
job = registry.load_job(job_id, reg_dir)
|
||||
job["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
||||
job["status"] = "error"
|
||||
_save_job_for_test(job, reg_dir)
|
||||
|
||||
rc = job_subscriber.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--job", "g7test01",
|
||||
"--timeout", "5"
|
||||
])
|
||||
assert rc == 1
|
||||
|
||||
|
||||
def test_g8_subscriber_wait_any_does_not_exit_early_if_partial_pending(mam_sandbox, monkeypatch):
|
||||
"""G-8: Multi-job wait does not terminate early when only 1 of 2 jobs is terminal on disk."""
|
||||
monkeypatch.chdir(mam_sandbox)
|
||||
script_dir = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts"
|
||||
sys.path.insert(0, str(script_dir))
|
||||
import registry, job_subscriber, mqtt_common
|
||||
|
||||
reg_dir = str(mam_sandbox / ".mam" / "jobs")
|
||||
os.makedirs(reg_dir, exist_ok=True)
|
||||
job_id1 = registry.register_job("Test G-8 Job 1", registry_dir=reg_dir, job_id="g8test01")
|
||||
job_id2 = registry.register_job("Test G-8 Job 2", registry_dir=reg_dir, job_id="g8test02")
|
||||
job1 = registry.load_job(job_id1, reg_dir)
|
||||
job2 = registry.load_job(job_id2, reg_dir)
|
||||
job1["status"] = "completed"
|
||||
job2["status"] = "running"
|
||||
_save_job_for_test(job1, reg_dir)
|
||||
_save_job_for_test(job2, reg_dir)
|
||||
|
||||
# Mock client connection to succeed with empty queue
|
||||
class FakeClient:
|
||||
on_message = None
|
||||
on_connect = None
|
||||
on_disconnect = None
|
||||
on_subscribe = None
|
||||
def reconnect_delay_set(self, **kwargs): pass
|
||||
def connect(self, *args, **kwargs): pass
|
||||
def loop_start(self): pass
|
||||
def loop_stop(self): pass
|
||||
def disconnect(self): pass
|
||||
def subscribe(self, *args, **kwargs): pass
|
||||
|
||||
monkeypatch.setattr(job_subscriber, "make_client", lambda *args, **kwargs: FakeClient())
|
||||
monkeypatch.setattr(mqtt_common, "with_retry", lambda fn, **kwargs: fn)
|
||||
|
||||
rc = job_subscriber.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--wait-any",
|
||||
"--timeout", "0.5",
|
||||
"--idle-timeout", "0.5"
|
||||
])
|
||||
assert rc == 2, "Must timeout waiting for incomplete job2 rather than exiting 0"
|
||||
|
||||
|
||||
def test_g9_subscriber_broker_down_no_disk_terminal_exits_3(mam_sandbox, monkeypatch):
|
||||
"""G-9: When broker is unreachable and no terminal state exists on disk, subscriber exits with rc=3."""
|
||||
monkeypatch.chdir(mam_sandbox)
|
||||
script_dir = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "scripts"
|
||||
sys.path.insert(0, str(script_dir))
|
||||
import registry, job_subscriber
|
||||
|
||||
reg_dir = str(mam_sandbox / ".mam" / "jobs")
|
||||
os.makedirs(reg_dir, exist_ok=True)
|
||||
job_id = registry.register_job("Test G-9", registry_dir=reg_dir, job_id="g9test01")
|
||||
job = registry.load_job(job_id, reg_dir)
|
||||
job["broker"] = {"host": "127.0.0.1", "port": 65432, "tls": False}
|
||||
job["status"] = "running"
|
||||
_save_job_for_test(job, reg_dir)
|
||||
|
||||
rc = job_subscriber.main([
|
||||
"--registry-dir", reg_dir,
|
||||
"--job", "g9test01",
|
||||
"--timeout", "5"
|
||||
])
|
||||
assert rc == 3, f"Expected rc=3 on infrastructure broker down without disk terminal, got {rc}"
|
||||
|
||||
|
||||
def test_g10_delegate_job_rc3_not_mistaken_for_error(mam_sandbox):
|
||||
"""G-10: multi-agent-mux-delegate-job maps rc=3 to broker_unavailable and checks disk status."""
|
||||
delegate_script = mam_sandbox / "skills" / "multi-agent-mux-delegate-job" / "multi-agent-mux-delegate-job"
|
||||
content = delegate_script.read_text()
|
||||
assert "elif [[ $sub_rc -eq 3 ]]; then" in content
|
||||
assert 'job_status="broker_unavailable"' in content
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user