fix(create): one pane claim at a time per herdr server; a create keeps the pane the monitor registered for it

Three `create --onboard` at once into one folder (the control root, so no
control-plane prefix reaches the pane and the shim reuses an idle pane):
the shim's new-session chooses a pane - an idle one in a workspace on the
same cwd, a split, a fresh workspace's root pane - and then claims it with
`agent start`; two shims could choose the pane one of them had just made,
and the later start failed with agent_pane_busy (one create in three).

mam_herdr now runs every new-session call holding one lock per server: a
flock on fd 9, opened for the block and taken by lib_py.pane_lock, which
also bounds the wait (MAM_PANE_CLAIM_WAIT_S, default 120). It is released
with the block or with the process, however it ends. The shim heredoc is
unchanged (S11). Other servers' creates are not held.

Serialising the claims exposed a second race: a create's end-of-run
`reconcile.sh --once` adopted the pane of another create that had not yet
written its row, and that create refused its own name as already running
(exit 4). Such a row - registered by the monitor, no job, for a pane
created after this create began - is now replaced, not refused.

Tests: real herdr - shim-level and create-level three-way races into one
folder (the real herdr's pane-creating replies held back so the window is
hit on every run; both fail on the base with the orchestrator's error),
the different-folder test kept; pane_lock unit tests (per-server file,
timeout, kill -9 holder, handover, knob); the adoption rule and its three
refusals; shim contract h26 (held lock: no pane calls). The herdr box gets
a ~/.zshrc: without it every first `agent start` typed into zsh's first-run
menu and timed out at 30 s.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/data/skill_code_blocks.json

@@ -88,8 +88,8 @@
   },
   {
    "skill": "multi-agent-mux-create",
-   "start": 202,
-   "end": 206,
+   "start": 204,
+   "end": 208,
    "lang": "bash",
    "sha": "38836d31bf38",
    "class": "c",
@@ -103,8 +103,8 @@
   },
   {
    "skill": "multi-agent-mux-create",
-   "start": 228,
-   "end": 250,
+   "start": 230,
+   "end": 252,
    "lang": "bash",
    "sha": "d90cfd2555fd",
    "class": "b",
@@ -617,7 +617,7 @@
     "other"
    ],
    "passthrough": {
-    "4": "multi-agent-mux-create/scripts/create_session.sh:532"
+    "4": "multi-agent-mux-create/scripts/create_session.sh:546"
    }
   }
  },

tests/test_herdr_shim_contract.py

@@ -604,3 +604,37 @@ kill "$_mam_server_pid" 2>/dev/null || true
         f"daemon child stayed in the caller's process group (not detached): {fields}"
     )
 
+
+
+def test_h26_new_session_claims_its_pane_under_the_servers_lock(mam_sandbox, mock_herdr, mock_agents):
+    """Issue #50 follow-up: from the pane choice to `agent start` the shim holds
+    one lock per herdr server (lib_py.pane_lock, a flock on fd 9). While
+    another shim holds it, this one waits MAM_PANE_CLAIM_WAIT_S and then gives
+    up without creating, splitting or starting anything; once it is free the
+    same call goes through as before."""
+    import glob
+    import os
+    tmp_path = mam_sandbox
+    res = _run_lib(tmp_path, f'herdr new-session -d -s "test-creator-claude" -c "{tmp_path}" "claude --dangerously-skip-permissions"')
+    assert res.returncode == 0, res.stderr
+    locks = glob.glob(str(tmp_path / ".local" / "state" / "mam" / "pane-lock" / "*.lock"))
+    assert len(locks) == 1, locks                     # one file per server, under the sandbox HOME
+    n_calls = len(_load_mock_state(mock_herdr).get("calls", []))
+    holder = subprocess.Popen(["bash", "-c", 'exec 9>>"$1"; python3 -m lib_py.pane_lock acquire --fd 9; sleep 600',
+                               "holder", locks[0]], stdout=subprocess.PIPE, text=True,
+                              env=dict(os.environ, PYTHONPATH=str(tmp_path / ".agents" / "skills")))
+    try:
+        assert holder.stdout.readline().strip() == "locked"
+        os.environ["MAM_PANE_CLAIM_WAIT_S"] = "1"
+        res = _run_lib(tmp_path, f'herdr new-session -d -s "test-reviewer-claude" -c "{tmp_path}" "claude --dangerously-skip-permissions"')
+        assert res.returncode == 1 and "did not end within MAM_PANE_CLAIM_WAIT_S" in res.stderr, (res.returncode, res.stderr)
+        new_calls = [c[2:] if c[:1] == ["--session"] else c for c in _load_mock_state(mock_herdr).get("calls", [])[n_calls:]]
+        assert not [c for c in new_calls if c[:2] in (["workspace", "create"], ["pane", "split"], ["agent", "start"])], new_calls
+        assert not [n for n in _load_mock_state(mock_herdr).get("agents", {}) if "test-reviewer-claude" in n]
+    finally:
+        os.environ.pop("MAM_PANE_CLAIM_WAIT_S", None)
+        holder.kill()
+        holder.wait()
+    res = _run_lib(tmp_path, f'herdr new-session -d -s "test-reviewer-claude" -c "{tmp_path}" "claude --dangerously-skip-permissions"')
+    assert res.returncode == 0, res.stderr
+    assert [n for n in _load_mock_state(mock_herdr).get("agents", {}) if "test-reviewer-claude" in n]

tests/test_issue50_dispatch_herdr.py

@@ -196,6 +196,9 @@ def _make_box(sess: str) -> Box:
     for d in (box.bin, box.ws, box.ctl / ".agents", box.jobs, box.ctl / ".mam" / "delegate_job_logs"):
         d.mkdir(parents=True)
     (box.ctl / ".agents" / "skills").symlink_to(SKILLS)
+    # a fresh HOME without ~/.zshrc puts zsh's first-run menu in every new
+    # pane: the first `agent start` would type into it and time out (30 s)
+    (home / ".zshrc").touch()
     (box.bin / "claude").write_text(FAKE_CLAUDE)
     (box.bin / "claude").chmod(0o755)
     box.yaml.write_text("herdr_sessions:\n" + _row(sess, ws=str(box.ws)))
@@ -608,32 +611,71 @@ def _drive_onboardings(box: Box, names: list, delays: list, result: dict):
             _after(delay, box.publish, jid, "completed", "Onboarding complete; aligned with role creator")
 
 
+def _hold_pane_creation(b: Box, seconds: int) -> None:
+    """The real herdr on the box's PATH, with the reply of a pane-creating call
+    (workspace create, pane split) held back `seconds`. The shim's window -
+    a pane exists idle, `agent start` has not claimed it yet - is the one the
+    orchestrator hit; this makes it long enough for a shim started a moment
+    later to walk into it on every run, not one run in three."""
+    (b.bin / "herdr").write_text(f'''#!/usr/bin/env bash
+case " $* " in
+  *" workspace create "*|*" pane split "*)
+    out=$("{HERDR}" "$@"); rc=$?
+    sleep {seconds}
+    printf '%s\\n' "$out"; exit $rc ;;
+esac
+exec "{HERDR}" "$@"
+''')
+    (b.bin / "herdr").chmod(0o755)
+
+
+def _three_creates(b: Box, names: list, workspaces: list, stagger: float = 0.0):
+    """create --onboard x3 at once in one server; (outcome by name, driver result, elapsed)."""
+    procs = []
+    for name, ws in zip(names, workspaces):
+        if procs and stagger:
+            time.sleep(stagger)
+        out = tempfile.TemporaryFile("w+", dir=str(b.home))
+        err = tempfile.TemporaryFile("w+", dir=str(b.home))
+        procs.append((name, subprocess.Popen(
+            ["bash", str(CREATE_SH), "--workspace", str(ws), "--agent", "claude", "--role", "creator",
+             "--session", name, "--onboard", "--herdr-session", b.sess],
+            stdout=out, stderr=err, text=True, cwd=str(b.ctl), env=b.env(MAM_ONBOARD_WAIT_S="120")), out, err))
+    t0 = time.time()
+    result = {}
+    drv = threading.Thread(target=_drive_onboardings, args=(b, names, [10, 15, 20], result), daemon=True)
+    drv.start()
+    outcome = {}
+    for name, proc, out, err in procs:
+        rc = proc.wait(timeout=240)
+        out.seek(0); err.seek(0)
+        outcome[name] = (rc, out.read(), err.read())
+    elapsed = time.time() - t0
+    drv.join(timeout=5)
+    return outcome, result, elapsed
+
+
+def _agent_panes(b: Box) -> dict:
+    """agent name -> (pane_id, pane cwd) from the server."""
+    panes = {p["pane_id"]: p for p in json.loads(b.herdr("pane", "list").stdout)["result"]["panes"]}
+    out = {}
+    for a in json.loads(b.herdr("agent", "list").stdout)["result"]["agents"]:
+        pid = a.get("pane_id") or (a.get("pane") or {}).get("pane_id")
+        out[a["name"]] = (pid, (panes.get(pid) or {}).get("cwd"))
+    return out
+
+
 def test_a_three_concurrent_creates_with_onboarding(local_mqtt_broker):
     b = _make_box(f"t50-{uuid.uuid4().hex[:8]}")
     b.yaml.write_text("herdr_sessions: []\n")
     names = [f"t50-{i}-creator-claude" for i in (1, 2, 3)]
-    procs = []
     try:
-        for i, name in enumerate(names, 1):
-            ws = b.home / f"ws{i}"
-            ws.mkdir()
-            out = tempfile.TemporaryFile("w+", dir=str(b.home))
-            err = tempfile.TemporaryFile("w+", dir=str(b.home))
-            procs.append((name, subprocess.Popen(
-                ["bash", str(CREATE_SH), "--workspace", str(ws), "--agent", "claude", "--role", "creator",
-                 "--session", name, "--onboard", "--herdr-session", b.sess],
-                stdout=out, stderr=err, text=True, cwd=str(b.ctl), env=b.env(MAM_ONBOARD_WAIT_S="120")), out, err))
-        t0 = time.time()
-        result = {}
-        drv = threading.Thread(target=_drive_onboardings, args=(b, names, [10, 15, 20], result), daemon=True)
-        drv.start()
-        outcome = {}
-        for name, proc, out, err in procs:
-            rc = proc.wait(timeout=240)
-            out.seek(0); err.seek(0)
-            outcome[name] = (rc, out.read(), err.read())
-        elapsed = time.time() - t0
-        drv.join(timeout=5)
+        workspaces = []
+        for i in (1, 2, 3):
+            (b.home / f"ws{i}").mkdir()
+            workspaces.append(b.home / f"ws{i}")
+        outcome, result, elapsed = _three_creates(b, names, workspaces)
+        t0 = time.time() - elapsed
         assert all(o[0] == 0 for o in outcome.values()), {n: (o[0], o[2][-1500:]) for n, o in outcome.items()}
         # The three start-ups overlap, and the onboarding waits do too: once the
         # last prompt was delivered (its `started`), the whole run ends with the
@@ -655,6 +697,75 @@ def test_a_three_concurrent_creates_with_onboarding(local_mqtt_broker):
         _teardown(b)
 
 
+def test_a3_three_concurrent_creates_in_one_folder_take_distinct_panes(local_mqtt_broker):
+    """The same --workspace for all three - the control root itself, where
+    several reviewers are made at once: no control-plane binding is forwarded
+    to the pane, so the shim reuses an idle pane when it sees one (BUG-4). The
+    pane choice and its `agent start` are serialised per server, so no two
+    creates claim one pane (agent_pane_busy)."""
+    b = _make_box(f"t50-{uuid.uuid4().hex[:8]}")
+    b.yaml.write_text("herdr_sessions: []\n")
+    _hold_pane_creation(b, 8)
+    names = [f"t50-{i}-creator-claude" for i in (1, 2, 3)]
+    try:
+        assert b.sh("herdr agent list").returncode == 0      # the server is up: the creates race on panes only
+        outcome, result, elapsed = _three_creates(b, names, [b.ctl] * 3, stagger=2.0)
+        assert all(o[0] == 0 for o in outcome.values()), {n: (o[0], o[2][-1500:]) for n, o in outcome.items()}
+        assert "agent_pane_busy" not in "".join(o[1] + o[2] for o in outcome.values())
+        for name in names:
+            jid = result["jobs"][name]
+            assert f"onboarding completed: job {jid} (" in outcome[name][1], outcome[name][1]
+            assert b.record(jid)["status"] == "completed"
+        rows = re.findall(r"^- name: (\S+)", b.yaml.read_text(), re.M)
+        assert sorted(rows) == sorted(names), rows
+        panes = _agent_panes(b)
+        assert sorted(panes) == sorted(names), panes
+        assert len({p for p, _c in panes.values()}) == 3, ("two agents on one pane", panes)
+        assert all(Path(c).resolve() == b.ctl.resolve() for _p, c in panes.values()), panes
+        assert len(b.server_pids()) == 1
+    finally:
+        _teardown(b)
+
+
+def test_s_three_concurrent_new_sessions_in_one_folder_take_distinct_panes():
+    """The shim alone, no create around it: three `new-session` into one cwd on
+    one running server at once. Before the pane claim was serialised, two of
+    them chose the same idle pane and the later `agent start` failed with
+    agent_pane_busy."""
+    b = _make_box(f"t50-{uuid.uuid4().hex[:8]}")
+    _hold_pane_creation(b, 2)
+    names = [f"t50-{i}-creator-claude" for i in (1, 2, 3)]
+    try:
+        assert b.sh("herdr agent list").returncode == 0      # the server is up before the race
+        procs = []
+        for name in names:
+            if procs:
+                time.sleep(0.7)
+            out = tempfile.TemporaryFile("w+", dir=str(b.home))
+            err = tempfile.TemporaryFile("w+", dir=str(b.home))
+            procs.append((name, subprocess.Popen(
+                ["bash", "-c", f'source "{SKILLS}/lib.sh"\nherdr new-session -d -s {name} -c "{b.ws}" '
+                               '"claude --dangerously-skip-permissions"'],
+                stdout=out, stderr=err, text=True, cwd=str(b.ctl), env=b.env()), out, err))
+        outcome = {}
+        for name, proc, out, err in procs:
+            rc = proc.wait(timeout=120)
+            out.seek(0); err.seek(0)
+            outcome[name] = (rc, out.read(), err.read())
+        assert all(o[0] == 0 for o in outcome.values()), {n: (o[0], o[2][-800:]) for n, o in outcome.items()}
+        panes = _agent_panes(b)
+        assert sorted(panes) == sorted(names), panes
+        assert len({p for p, _c in panes.values()}) == 3, ("two agents on one pane", panes)
+        assert all(Path(c).resolve() == b.ws.resolve() for _p, c in panes.values()), panes
+        for name in names:
+            deadline = time.time() + 30
+            while time.time() < deadline and b.agent_status(name) != "idle":
+                time.sleep(0.5)
+            assert b.agent_status(name) == "idle", (name, b.herdr("agent", "list").stdout)
+    finally:
+        _teardown(b)
+
+
 def _git(repo: Path, *args):
     subprocess.run(["git", "-c", "user.email=t@example.com", "-c", "user.name=t", *args], cwd=str(repo), check=True,
                    capture_output=True, text=True)

tests/test_issue50_pane_lock.py

@@ -0,0 +1,162 @@
+"""Issue #50 follow-up: the herdr shim claims one pane at a time per server.
+
+- P: lib_py.pane_lock - the lock file per server, the acquire decision and its
+  bound, a lock that ends with its holder's process (no stale lock after
+  kill -9), the wait knob.
+- H: the race itself, on a real herdr server: test_issue50_dispatch_herdr.py
+  (test_s_three_concurrent_new_sessions_in_one_folder_take_distinct_panes,
+  test_a3_three_concurrent_creates_in_one_folder_take_distinct_panes).
+- the shim's use of the lock against the mock herdr: test_herdr_shim_contract.py h26.
+"""
+from __future__ import annotations
+
+import json
+import os
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+SKILLS = REPO_ROOT / ".agents" / "skills"
+if str(SKILLS) not in sys.path:
+    sys.path.insert(0, str(SKILLS))
+
+from lib_py import pane_lock  # noqa: E402
+
+# a bash holder, as the shim is: fd 9 on the file, the lock taken by the helper, then it sleeps
+HOLDER = 'exec 9>>"$1"; python3 -m lib_py.pane_lock acquire --fd 9; echo held; sleep "${2:-600}"'
+
+
+def _holder(path: Path, seconds: str = "600", **env) -> subprocess.Popen:
+    p = subprocess.Popen(["bash", "-c", HOLDER, "holder", str(path), seconds], stdout=subprocess.PIPE, text=True,
+                         env=dict(os.environ, PYTHONPATH=str(SKILLS), **env))
+    assert p.stdout.readline().strip() == "locked"
+    assert p.stdout.readline().strip() == "held"
+    return p
+
+
+def _acquire(path: Path, **env) -> subprocess.CompletedProcess:
+    return subprocess.run(["bash", "-c", 'exec 9>>"$1"; python3 -m lib_py.pane_lock acquire --fd 9', "x", str(path)],
+                          capture_output=True, text=True, env=dict(os.environ, PYTHONPATH=str(SKILLS), **env), timeout=60)
+
+
+def test_p1_the_lock_file_is_per_server_under_the_sessions_home(tmp_path):
+    env = {"HOME": str(tmp_path)}
+    assert pane_lock.lock_path("t50-x", env) == str(tmp_path / ".local/state/mam/pane-lock/t50-x.lock")
+    assert pane_lock.lock_path("", env) == str(tmp_path / ".local/state/mam/pane-lock/default.lock")
+    assert pane_lock.lock_path("t50-x", {"HOME_DIR": str(tmp_path / "h")}).startswith(str(tmp_path / "h"))
+    assert pane_lock.lock_path("t50-x", env) != pane_lock.lock_path("t50-y", env)
+    res = subprocess.run([sys.executable, "-m", "lib_py.pane_lock", "path", "--session", "t50-x"], capture_output=True,
+                         text=True, env=dict(os.environ, PYTHONPATH=str(SKILLS), HOME=str(tmp_path)), timeout=60)
+    assert res.stdout.strip() == pane_lock.lock_path("t50-x", env), res.stderr
+
+
+def test_p2_a_free_lock_is_taken_at_once_and_a_held_one_times_out(tmp_path):
+    lock = tmp_path / "s.lock"
+    t0 = time.time()
+    assert _acquire(lock).stdout.strip() == "locked" and time.time() - t0 < 5
+    holder = _holder(lock)
+    try:
+        t0 = time.time()
+        res = _acquire(lock, MAM_PANE_CLAIM_WAIT_S="2")
+        assert res.stdout.strip() == "timeout", res.stderr
+        assert 1.5 <= time.time() - t0 < 10
+        assert _acquire(tmp_path / "other-server.lock", MAM_PANE_CLAIM_WAIT_S="2").stdout.strip() == "locked"
+    finally:
+        holder.kill()
+        holder.wait()
+
+
+def test_p3_the_lock_ends_with_its_holders_process(tmp_path):
+    lock = tmp_path / "s.lock"
+    holder = _holder(lock)
+    os.kill(holder.pid, signal.SIGKILL)
+    holder.wait()
+    t0 = time.time()
+    assert _acquire(lock, MAM_PANE_CLAIM_WAIT_S="30").stdout.strip() == "locked"
+    assert time.time() - t0 < 5, "nothing stale to wait out"
+
+
+def test_p4_a_waiter_gets_the_lock_when_the_holder_ends(tmp_path):
+    lock = tmp_path / "s.lock"
+    holder = _holder(lock, "2")
+    t0 = time.time()
+    res = _acquire(lock, MAM_PANE_CLAIM_WAIT_S="30")
+    assert res.stdout.strip() == "locked"
+    assert 1 <= time.time() - t0 < 10
+    holder.wait()
+
+
+def test_p5_the_wait_knob_is_a_decimal_with_a_floor_of_one(monkeypatch):
+    assert pane_lock.wait_s({}) == 120
+    assert pane_lock.wait_s({"MAM_PANE_CLAIM_WAIT_S": "7"}) == 7
+    assert pane_lock.wait_s({"MAM_PANE_CLAIM_WAIT_S": "0700"}) == 700
+    for bad in ("0", "abc", "", "-1", " "):
+        assert pane_lock.wait_s({"MAM_PANE_CLAIM_WAIT_S": bad}) == 120, bad
+
+
+# ---------------------------------------------------------------------------
+# A - create keeps its own pane when the monitor registered it first. With
+# pane claims serialised, one create can reach its end-of-run reconcile while
+# another is between `agent start` and its row write; the adopted row is that
+# create's own pane, not a second live session of its name.
+# ---------------------------------------------------------------------------
+
+
+
+def _seed_row(tmp_path, name, epoch, note="auto-registered by monitor", job=None):
+    row = {"name": name, "status": "running", "role": "creator", "herdr_session_created_at": "2026-10-11T00:00:00Z",
+           "herdr_session_epoch": epoch, "herdr_session": "default", "herdr_server": "default",
+           "pane": {"index": 0, "pid": 9999, "cmd": "claude", "cmd_full": "claude", "cwd": str(tmp_path)},
+           "last_visible_status": "running"}
+    if note:
+        row["last_visible_note"] = note
+    if job:
+        row["delegate_job_id"] = job
+    yaml_path = tmp_path / ".mam" / "agent-sessions.yaml"
+    yaml_path.parent.mkdir(parents=True, exist_ok=True)
+    yaml_path.write_text("herdr_sessions:\n- " + json.dumps(row) + "\n")
+    return yaml_path
+
+
+def _create(tmp_path, name, role="creator"):
+    return subprocess.run(["bash", str(tmp_path / "skills" / "multi-agent-mux-create" / "scripts" / "create_session.sh"),
+                           "--workspace", str(tmp_path), "--agent", "claude", "--role", role, "--session", name,
+                           "--no-onboard"], capture_output=True, text=True, cwd=str(tmp_path), timeout=300)
+
+
+def _rows(yaml_path):
+    import yaml
+    return yaml.safe_load(yaml_path.read_text())["herdr_sessions"]
+
+
+def test_a1_create_replaces_the_monitors_registration_of_its_own_pane(mam_sandbox, mock_herdr, mock_agents):
+    name = "t50-adopted-creator-claude"
+    yaml_path = _seed_row(mam_sandbox, name, int(time.time()))      # registered after this create began
+    res = _create(mam_sandbox, name, role="Creator")               # the monitor guessed "creator" off the name
+    assert res.returncode == 0, res.stdout + res.stderr
+    assert f"NOTE: the monitor registered {name} while it was being created" in res.stdout
+    rows = [r for r in _rows(yaml_path) if r["name"] == name]
+    assert len(rows) == 1 and rows[0]["status"] == "running" and rows[0]["role"] == "Creator"
+    assert rows[0].get("last_visible_note") != "auto-registered by monitor" and rows[0].get("start_command")
+
+
+@pytest.mark.parametrize("kind", ["real", "old-adoption", "adoption-with-job"])
+def test_a2_another_live_session_of_that_name_is_still_refused(mam_sandbox, mock_herdr, mock_agents, kind):
+    name = "t50-taken-creator-claude"
+    now = int(time.time())
+    if kind == "real":
+        yaml_path = _seed_row(mam_sandbox, name, now, note=None)
+    elif kind == "old-adoption":
+        yaml_path = _seed_row(mam_sandbox, name, now - 3600)        # a pane from before this create
+    else:
+        yaml_path = _seed_row(mam_sandbox, name, now, job="deadbeef")
+    res = _create(mam_sandbox, name)
+    assert res.returncode == 4, res.stdout + res.stderr
+    assert f"ERROR: {name} already running in agent-sessions.yaml" in res.stdout
+    rows = [r for r in _rows(yaml_path) if r["name"] == name]
+    assert len(rows) == 1 and rows[0]["status"] == "running"