fix(create): one pane claim at a time per herdr server; a create keeps the pane the monitor registered for it

Three `create --onboard` at once into one folder (the control root, so no
control-plane prefix reaches the pane and the shim reuses an idle pane):
the shim's new-session chooses a pane - an idle one in a workspace on the
same cwd, a split, a fresh workspace's root pane - and then claims it with
`agent start`; two shims could choose the pane one of them had just made,
and the later start failed with agent_pane_busy (one create in three).

mam_herdr now runs every new-session call holding one lock per server: a
flock on fd 9, opened for the block and taken by lib_py.pane_lock, which
also bounds the wait (MAM_PANE_CLAIM_WAIT_S, default 120). It is released
with the block or with the process, however it ends. The shim heredoc is
unchanged (S11). Other servers' creates are not held.

Serialising the claims exposed a second race: a create's end-of-run
`reconcile.sh --once` adopted the pane of another create that had not yet
written its row, and that create refused its own name as already running
(exit 4). Such a row - registered by the monitor, no job, for a pane
created after this create began - is now replaced, not refused.

Tests: real herdr - shim-level and create-level three-way races into one
folder (the real herdr's pane-creating replies held back so the window is
hit on every run; both fail on the base with the orchestrator's error),
the different-folder test kept; pane_lock unit tests (per-server file,
timeout, kill -9 holder, handover, knob); the adoption rule and its three
refusals; shim contract h26 (held lock: no pane calls). The herdr box gets
a ~/.zshrc: without it every first `agent start` typed into zsh's first-run
menu and timed out at 30 s.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/test_issue50_dispatch_herdr.py

@@ -196,6 +196,9 @@ def _make_box(sess: str) -> Box:
     for d in (box.bin, box.ws, box.ctl / ".agents", box.jobs, box.ctl / ".mam" / "delegate_job_logs"):
         d.mkdir(parents=True)
     (box.ctl / ".agents" / "skills").symlink_to(SKILLS)
+    # a fresh HOME without ~/.zshrc puts zsh's first-run menu in every new
+    # pane: the first `agent start` would type into it and time out (30 s)
+    (home / ".zshrc").touch()
     (box.bin / "claude").write_text(FAKE_CLAUDE)
     (box.bin / "claude").chmod(0o755)
     box.yaml.write_text("herdr_sessions:\n" + _row(sess, ws=str(box.ws)))
@@ -608,32 +611,71 @@ def _drive_onboardings(box: Box, names: list, delays: list, result: dict):
             _after(delay, box.publish, jid, "completed", "Onboarding complete; aligned with role creator")
 
 
+def _hold_pane_creation(b: Box, seconds: int) -> None:
+    """The real herdr on the box's PATH, with the reply of a pane-creating call
+    (workspace create, pane split) held back `seconds`. The shim's window -
+    a pane exists idle, `agent start` has not claimed it yet - is the one the
+    orchestrator hit; this makes it long enough for a shim started a moment
+    later to walk into it on every run, not one run in three."""
+    (b.bin / "herdr").write_text(f'''#!/usr/bin/env bash
+case " $* " in
+  *" workspace create "*|*" pane split "*)
+    out=$("{HERDR}" "$@"); rc=$?
+    sleep {seconds}
+    printf '%s\\n' "$out"; exit $rc ;;
+esac
+exec "{HERDR}" "$@"
+''')
+    (b.bin / "herdr").chmod(0o755)
+
+
+def _three_creates(b: Box, names: list, workspaces: list, stagger: float = 0.0):
+    """create --onboard x3 at once in one server; (outcome by name, driver result, elapsed)."""
+    procs = []
+    for name, ws in zip(names, workspaces):
+        if procs and stagger:
+            time.sleep(stagger)
+        out = tempfile.TemporaryFile("w+", dir=str(b.home))
+        err = tempfile.TemporaryFile("w+", dir=str(b.home))
+        procs.append((name, subprocess.Popen(
+            ["bash", str(CREATE_SH), "--workspace", str(ws), "--agent", "claude", "--role", "creator",
+             "--session", name, "--onboard", "--herdr-session", b.sess],
+            stdout=out, stderr=err, text=True, cwd=str(b.ctl), env=b.env(MAM_ONBOARD_WAIT_S="120")), out, err))
+    t0 = time.time()
+    result = {}
+    drv = threading.Thread(target=_drive_onboardings, args=(b, names, [10, 15, 20], result), daemon=True)
+    drv.start()
+    outcome = {}
+    for name, proc, out, err in procs:
+        rc = proc.wait(timeout=240)
+        out.seek(0); err.seek(0)
+        outcome[name] = (rc, out.read(), err.read())
+    elapsed = time.time() - t0
+    drv.join(timeout=5)
+    return outcome, result, elapsed
+
+
+def _agent_panes(b: Box) -> dict:
+    """agent name -> (pane_id, pane cwd) from the server."""
+    panes = {p["pane_id"]: p for p in json.loads(b.herdr("pane", "list").stdout)["result"]["panes"]}
+    out = {}
+    for a in json.loads(b.herdr("agent", "list").stdout)["result"]["agents"]:
+        pid = a.get("pane_id") or (a.get("pane") or {}).get("pane_id")
+        out[a["name"]] = (pid, (panes.get(pid) or {}).get("cwd"))
+    return out
+
+
 def test_a_three_concurrent_creates_with_onboarding(local_mqtt_broker):
     b = _make_box(f"t50-{uuid.uuid4().hex[:8]}")
     b.yaml.write_text("herdr_sessions: []\n")
     names = [f"t50-{i}-creator-claude" for i in (1, 2, 3)]
-    procs = []
     try:
-        for i, name in enumerate(names, 1):
-            ws = b.home / f"ws{i}"
-            ws.mkdir()
-            out = tempfile.TemporaryFile("w+", dir=str(b.home))
-            err = tempfile.TemporaryFile("w+", dir=str(b.home))
-            procs.append((name, subprocess.Popen(
-                ["bash", str(CREATE_SH), "--workspace", str(ws), "--agent", "claude", "--role", "creator",
-                 "--session", name, "--onboard", "--herdr-session", b.sess],
-                stdout=out, stderr=err, text=True, cwd=str(b.ctl), env=b.env(MAM_ONBOARD_WAIT_S="120")), out, err))
-        t0 = time.time()
-        result = {}
-        drv = threading.Thread(target=_drive_onboardings, args=(b, names, [10, 15, 20], result), daemon=True)
-        drv.start()
-        outcome = {}
-        for name, proc, out, err in procs:
-            rc = proc.wait(timeout=240)
-            out.seek(0); err.seek(0)
-            outcome[name] = (rc, out.read(), err.read())
-        elapsed = time.time() - t0
-        drv.join(timeout=5)
+        workspaces = []
+        for i in (1, 2, 3):
+            (b.home / f"ws{i}").mkdir()
+            workspaces.append(b.home / f"ws{i}")
+        outcome, result, elapsed = _three_creates(b, names, workspaces)
+        t0 = time.time() - elapsed
         assert all(o[0] == 0 for o in outcome.values()), {n: (o[0], o[2][-1500:]) for n, o in outcome.items()}
         # The three start-ups overlap, and the onboarding waits do too: once the
         # last prompt was delivered (its `started`), the whole run ends with the
@@ -655,6 +697,75 @@ def test_a_three_concurrent_creates_with_onboarding(local_mqtt_broker):
         _teardown(b)
 
 
+def test_a3_three_concurrent_creates_in_one_folder_take_distinct_panes(local_mqtt_broker):
+    """The same --workspace for all three - the control root itself, where
+    several reviewers are made at once: no control-plane binding is forwarded
+    to the pane, so the shim reuses an idle pane when it sees one (BUG-4). The
+    pane choice and its `agent start` are serialised per server, so no two
+    creates claim one pane (agent_pane_busy)."""
+    b = _make_box(f"t50-{uuid.uuid4().hex[:8]}")
+    b.yaml.write_text("herdr_sessions: []\n")
+    _hold_pane_creation(b, 8)
+    names = [f"t50-{i}-creator-claude" for i in (1, 2, 3)]
+    try:
+        assert b.sh("herdr agent list").returncode == 0      # the server is up: the creates race on panes only
+        outcome, result, elapsed = _three_creates(b, names, [b.ctl] * 3, stagger=2.0)
+        assert all(o[0] == 0 for o in outcome.values()), {n: (o[0], o[2][-1500:]) for n, o in outcome.items()}
+        assert "agent_pane_busy" not in "".join(o[1] + o[2] for o in outcome.values())
+        for name in names:
+            jid = result["jobs"][name]
+            assert f"onboarding completed: job {jid} (" in outcome[name][1], outcome[name][1]
+            assert b.record(jid)["status"] == "completed"
+        rows = re.findall(r"^- name: (\S+)", b.yaml.read_text(), re.M)
+        assert sorted(rows) == sorted(names), rows
+        panes = _agent_panes(b)
+        assert sorted(panes) == sorted(names), panes
+        assert len({p for p, _c in panes.values()}) == 3, ("two agents on one pane", panes)
+        assert all(Path(c).resolve() == b.ctl.resolve() for _p, c in panes.values()), panes
+        assert len(b.server_pids()) == 1
+    finally:
+        _teardown(b)
+
+
+def test_s_three_concurrent_new_sessions_in_one_folder_take_distinct_panes():
+    """The shim alone, no create around it: three `new-session` into one cwd on
+    one running server at once. Before the pane claim was serialised, two of
+    them chose the same idle pane and the later `agent start` failed with
+    agent_pane_busy."""
+    b = _make_box(f"t50-{uuid.uuid4().hex[:8]}")
+    _hold_pane_creation(b, 2)
+    names = [f"t50-{i}-creator-claude" for i in (1, 2, 3)]
+    try:
+        assert b.sh("herdr agent list").returncode == 0      # the server is up before the race
+        procs = []
+        for name in names:
+            if procs:
+                time.sleep(0.7)
+            out = tempfile.TemporaryFile("w+", dir=str(b.home))
+            err = tempfile.TemporaryFile("w+", dir=str(b.home))
+            procs.append((name, subprocess.Popen(
+                ["bash", "-c", f'source "{SKILLS}/lib.sh"\nherdr new-session -d -s {name} -c "{b.ws}" '
+                               '"claude --dangerously-skip-permissions"'],
+                stdout=out, stderr=err, text=True, cwd=str(b.ctl), env=b.env()), out, err))
+        outcome = {}
+        for name, proc, out, err in procs:
+            rc = proc.wait(timeout=120)
+            out.seek(0); err.seek(0)
+            outcome[name] = (rc, out.read(), err.read())
+        assert all(o[0] == 0 for o in outcome.values()), {n: (o[0], o[2][-800:]) for n, o in outcome.items()}
+        panes = _agent_panes(b)
+        assert sorted(panes) == sorted(names), panes
+        assert len({p for p, _c in panes.values()}) == 3, ("two agents on one pane", panes)
+        assert all(Path(c).resolve() == b.ws.resolve() for _p, c in panes.values()), panes
+        for name in names:
+            deadline = time.time() + 30
+            while time.time() < deadline and b.agent_status(name) != "idle":
+                time.sleep(0.5)
+            assert b.agent_status(name) == "idle", (name, b.herdr("agent", "list").stdout)
+    finally:
+        _teardown(b)
+
+
 def _git(repo: Path, *args):
     subprocess.run(["git", "-c", "user.email=t@example.com", "-c", "user.name=t", *args], cwd=str(repo), check=True,
                    capture_output=True, text=True)