GodopuandClaude Opus 5.5
568385c77a
Review round 3 R1 (codex): the registration guard chose the workspace marker from the path's spelling, so a symlink alias of .mam/jobs or a MAM_JOBS_DIR registry was not refused. The reset now writes .reset-in-progress into each registry its open-job check reads (.mam/jobs, MAM_JOBS_DIR, a row's other control_plane.mam_jobs_dir - the latter only for this workspace's sessions), each under that registry's lock and before the check, and removes it on exit; register_job reads the marker inside the registry it writes to. The precheck's separate lock barrier is subsumed by the fence. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
346 lines
13 KiB
Bash
Executable File
346 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# reset.sh — multi-agent-mux-reset (Issue #78): purge the control root's
|
|
# running resident agents and recreate each with the same name, agent, role,
|
|
# herdr server and workspace label, one at a time, with onboarding.
|
|
# Usage:
|
|
# bash .agents/skills/multi-agent-mux-reset/scripts/reset.sh [-y|--yes] [--dry-run]
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SKILL_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
WS="$(cd -P "$SCRIPT_DIR/../../../.." && pwd -P)"
|
|
export WORKSPACE_ROOT="$WS"
|
|
export AGENT_SESSIONS_YAML="${AGENT_SESSIONS_YAML:-$WS/.mam/agent-sessions.yaml}"
|
|
export PYTHONPATH="$SKILL_DIR:${PYTHONPATH:-}"
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: bash .agents/skills/multi-agent-mux-reset/scripts/reset.sh [-y|--yes] [--dry-run]
|
|
|
|
Options:
|
|
-y, --yes Confirm purging and recreating the control root's running agents
|
|
--dry-run Inspect targets, skipped rows, spec progress and blockers without changing anything
|
|
-h, --help Show this help
|
|
USAGE
|
|
}
|
|
|
|
YES=0
|
|
DRY_RUN=0
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-y|--yes) YES=1; shift ;;
|
|
--dry-run) DRY_RUN=1; shift ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) echo "ERROR: unknown arg: $1" >&2; usage >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
if [ "$DRY_RUN" != "1" ] && [ "$YES" != "1" ] && [ ! -t 0 ]; then
|
|
echo "ERROR: not a terminal; run with -y/--yes (or --dry-run to inspect)" >&2
|
|
exit 3
|
|
fi
|
|
|
|
MAM="$WS/.mam"
|
|
MARKER="${MAM_LOOP_MARKER:-$MAM/loop-guard-active}"
|
|
RESET_MARKER="$MAM/reset-in-progress"
|
|
SPEC_FILE="$MAM/reset-spec.json"
|
|
|
|
# shellcheck disable=SC1091
|
|
source "$SKILL_DIR/multi-agent-mux-loop/scripts/loop_lock.sh"
|
|
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
if [ -x "$WS/.venv/bin/python" ]; then
|
|
PY="$WS/.venv/bin/python"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
PY="python3"
|
|
else
|
|
PY="python"
|
|
fi
|
|
DRY_ARGS=()
|
|
if [ -f "$MAM/update-in-progress" ]; then
|
|
upd_st="$(_mam_lock_holder_state "$MAM/update-in-progress")"
|
|
case "$upd_st" in
|
|
stale) echo "NOTE: stale update marker $MAM/update-in-progress (its update is gone); not a reason to refuse" ;;
|
|
*) DRY_ARGS+=(--refusal "an update of $WS is in progress (pid $(_mam_lock_field "$MAM/update-in-progress" pid), holder $upd_st); wait for it to finish") ;;
|
|
esac
|
|
fi
|
|
if [ -f "$MARKER" ]; then
|
|
loop_st="$(_mam_lock_holder_state "$MARKER")"
|
|
case "$loop_st" in
|
|
stale) echo "NOTE: stale loop marker $MARKER (its loop is gone); not a reason to refuse" ;;
|
|
*) DRY_ARGS+=(--refusal "a review loop or another reset is running (pid $(_mam_lock_field "$MARKER" pid), holder $loop_st); wait for it to end") ;;
|
|
esac
|
|
fi
|
|
dry_rc=0
|
|
"$PY" -m lib_py.reset dry-run --workspace "$WS" ${DRY_ARGS[@]+"${DRY_ARGS[@]}"} || dry_rc=$?
|
|
if [ "$dry_rc" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
# shellcheck disable=SC1091
|
|
source "$WS/.agents/skills/lib.sh"
|
|
PY="$(_delegate_py_bin)"
|
|
|
|
_refuse_locked() {
|
|
exec 7>&-
|
|
echo "ERROR: reset refused; nothing was changed:" >&2
|
|
echo " - $1" >&2
|
|
exit 1
|
|
}
|
|
# The update admission lock is taken exclusively, as update.sh does, while the
|
|
# reset publishes its markers: a delegate submit admitted earlier has written
|
|
# its job record (the precheck below sees it), and one admitted later sees
|
|
# .mam/reset-in-progress and refuses. Any other registration (registry.py
|
|
# register, the delegation API) is fenced below.
|
|
mkdir -p "$MAM"
|
|
exec 7>>"$MAM/update-admission.lock"
|
|
case "$("$PY" -m lib_py.pane_lock acquire --fd 7 --wait-env MAM_UPDATE_ADMISSION_WAIT_S)" in
|
|
locked) ;;
|
|
timeout) _refuse_locked "a create, resume, loop, delegate submit, stop or orc-onboard kept the admission lock past MAM_UPDATE_ADMISSION_WAIT_S; try again" ;;
|
|
*) _refuse_locked "the admission lock $MAM/update-admission.lock could not be decided (lib_py.pane_lock)" ;;
|
|
esac
|
|
if [ -f "$MAM/update-in-progress" ]; then
|
|
upd_st="$(_mam_lock_holder_state "$MAM/update-in-progress")"
|
|
if [ "$upd_st" != "stale" ]; then
|
|
_refuse_locked "an update of $WS is in progress (pid $(_mam_lock_field "$MAM/update-in-progress" pid), holder $upd_st); wait for it to finish"
|
|
fi
|
|
fi
|
|
_acquire_rc=0
|
|
mam_acquire_loop_lock "$MARKER" || _acquire_rc=$?
|
|
case "$_acquire_rc" in
|
|
0) ;;
|
|
3) _refuse_locked "a review loop or another reset is running (pid $(_mam_lock_field "$MARKER" "pid" 2>/dev/null || echo "unknown")); wait for it to end" ;;
|
|
*) _refuse_locked "cannot determine whether a review loop or another reset is running (ps unavailable?)" ;;
|
|
esac
|
|
FENCED=()
|
|
_release_reset_lock() {
|
|
local d
|
|
for d in ${FENCED[@]+"${FENCED[@]}"}; do
|
|
[ "$(_mam_lock_field "$d/.reset-in-progress" pid 2>/dev/null || true)" != "$$" ] || rm -f "$d/.reset-in-progress"
|
|
done
|
|
mam_release_loop_lock "$RESET_MARKER" || true
|
|
mam_release_loop_lock "$MARKER" || true
|
|
}
|
|
_on_reset_signal() {
|
|
local sig_rc="$1"
|
|
_release_reset_lock
|
|
exit "$sig_rc"
|
|
}
|
|
trap _release_reset_lock EXIT
|
|
trap '_on_reset_signal 130' INT
|
|
trap '_on_reset_signal 143' TERM
|
|
trap '_on_reset_signal 129' HUP
|
|
if ! mam_acquire_loop_lock "$RESET_MARKER"; then
|
|
_refuse_locked "cannot publish $RESET_MARKER"
|
|
fi
|
|
exec 7>&-
|
|
|
|
STATE_JSON=""
|
|
if ! STATE_JSON="$(load_state_json strict)"; then
|
|
echo "ERROR: reset refused; nothing was changed:" >&2
|
|
echo " - the registry $AGENT_SESSIONS_YAML cannot be read; fix it first" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Every registry the precheck reads gets the reset marker under its own lock
|
|
# (lib_py.reset fence): a registration that read none is written by then, any
|
|
# later one is refused, by whatever path it names that registry.
|
|
fence_out=""
|
|
fence_rc=0
|
|
fence_out="$(MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset fence --workspace "$WS")" || fence_rc=$?
|
|
while IFS= read -r line; do
|
|
[ -n "$line" ] && FENCED+=("$line")
|
|
done <<< "$fence_out"
|
|
if [ "$fence_rc" -ne 0 ]; then
|
|
echo "ERROR: reset refused; nothing was changed:" >&2
|
|
echo " - the job registries could not be fenced against new registrations (lib_py.reset fence rc $fence_rc)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
pre_out=""
|
|
pre_rc=0
|
|
pre_out="$(MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset precheck --workspace "$WS")" || pre_rc=$?
|
|
if [ "$pre_rc" -ne 0 ]; then
|
|
echo "ERROR: reset refused; nothing was changed:" >&2
|
|
while IFS= read -r line; do
|
|
[ -n "$line" ] && echo " - $line" >&2
|
|
done <<< "${pre_out:-lib_py.reset precheck failed (rc $pre_rc)}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ -f "$SPEC_FILE" ]; then
|
|
MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset dry-run --workspace "$WS" || exit 1
|
|
if [ "$YES" != "1" ]; then
|
|
ans=""
|
|
read -r -p "Resume reset from $SPEC_FILE? [y/N]: " ans || ans=""
|
|
case "$ans" in
|
|
y|Y|yes|Yes|YES) ;;
|
|
*) echo "Reset cancelled; nothing was changed"; exit 0 ;;
|
|
esac
|
|
fi
|
|
else
|
|
PLAN_JSON="$(MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset plan --workspace "$WS")" || exit 1
|
|
TARGET_COUNT="$(printf '%s' "$PLAN_JSON" | "$PY" -c 'import json,sys; print(len(json.load(sys.stdin).get("targets", [])))')"
|
|
MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset dry-run --workspace "$WS" || exit 1
|
|
if [ "$TARGET_COUNT" = "0" ]; then
|
|
echo "nothing to reset"
|
|
exit 0
|
|
fi
|
|
if [ "$YES" != "1" ]; then
|
|
ans=""
|
|
read -r -p "Purge and recreate $TARGET_COUNT agents? [y/N]: " ans || ans=""
|
|
case "$ans" in
|
|
y|Y|yes|Yes|YES) ;;
|
|
*) echo "Reset cancelled; nothing was changed"; exit 0 ;;
|
|
esac
|
|
fi
|
|
printf '%s' "$PLAN_JSON" | "$PY" -m lib_py.reset write-spec --workspace "$WS" || exit 1
|
|
fi
|
|
|
|
ENTRIES_OUT="$(MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset entries --workspace "$WS" --warn-new)" || exit 1
|
|
ENTRY_NAMES=()
|
|
while IFS= read -r line; do
|
|
[ -n "$line" ] || continue
|
|
eval "$line"
|
|
ENTRY_NAMES+=("$R_NAME")
|
|
done <<< "$ENTRIES_OUT"
|
|
|
|
SKIPPED_ARGS=()
|
|
FAILED_SPEC=""
|
|
|
|
for target_name in ${ENTRY_NAMES[@]+"${ENTRY_NAMES[@]}"}; do
|
|
if ! STATE_JSON="$(load_state_json strict)"; then
|
|
FAILED_SPEC="$target_name:stop:1"
|
|
break
|
|
fi
|
|
entry_line="$(MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset entries --workspace "$WS" --session "$target_name")" || {
|
|
FAILED_SPEC="$target_name:stop:1"
|
|
break
|
|
}
|
|
eval "$entry_line"
|
|
|
|
case "$R_STATE" in
|
|
done|created)
|
|
continue
|
|
;;
|
|
pending)
|
|
if [ "$R_ROW_AGENT" != "$R_AGENT" ] || [ "$R_ROW_SERVER" != "$R_SERVER" ]; then
|
|
echo "skipped $R_NAME: its row ($R_ROW_AGENT on herdr server $R_ROW_SERVER) no longer matches $SPEC_FILE ($R_AGENT on $R_SERVER); nothing was changed for it; check the row, then rerun the reset"
|
|
SKIPPED_ARGS+=(--skipped "$R_NAME:changed")
|
|
continue
|
|
fi
|
|
jobs_out=""
|
|
jobs_out="$(MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset open-jobs --workspace "$WS" --session "$R_NAME")" || {
|
|
echo "skipped $R_NAME: a job was registered for it after the precheck (${jobs_out//$'\n'/; }); nothing was changed for it; rerun the reset once the job has ended"
|
|
SKIPPED_ARGS+=(--skipped "$R_NAME:open_job")
|
|
continue
|
|
}
|
|
STOP_CMD=(bash "$SKILL_DIR/multi-agent-mux-stop/scripts/stop_session.sh" --session "$R_NAME" --agent "$R_AGENT")
|
|
if [ -n "$R_SERVER" ]; then
|
|
STOP_CMD+=(--herdr-session "$R_SERVER")
|
|
fi
|
|
STOP_CMD+=(--reason reset)
|
|
|
|
echo "reset [$R_NAME]: ${STOP_CMD[*]}"
|
|
stop_rc=0
|
|
"${STOP_CMD[@]}" || stop_rc=$?
|
|
if [ "$stop_rc" -ne 0 ]; then
|
|
FAILED_SPEC="$R_NAME:stop:$stop_rc"
|
|
break
|
|
fi
|
|
|
|
if ! STATE_JSON="$(load_state_json strict)"; then
|
|
FAILED_SPEC="$R_NAME:stop:1"
|
|
break
|
|
fi
|
|
if [ -z "$R_OLD_ID" ]; then
|
|
MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset record-id --workspace "$WS" --session "$R_NAME" || {
|
|
FAILED_SPEC="$R_NAME:stop:1"
|
|
break
|
|
}
|
|
fi
|
|
entry_line="$(MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset entries --workspace "$WS" --session "$R_NAME")" || {
|
|
FAILED_SPEC="$R_NAME:stop:1"
|
|
break
|
|
}
|
|
eval "$entry_line"
|
|
if [ -z "$R_ROW_ID" ]; then
|
|
echo "skipped $R_NAME: no conversation id is recorded, so a purge would drop the row and leave the conversation on disk; resume it, or purge it by hand (stop_session.sh --session $R_NAME --purge-conversation --yes) and rerun the reset to recreate it"
|
|
SKIPPED_ARGS+=(--skipped "$R_NAME:no_id")
|
|
continue
|
|
fi
|
|
if [ "$R_ROW_ID" != "$R_OLD_ID" ]; then
|
|
echo "skipped $R_NAME: the stop left id $R_ROW_ID on its row, not the one recorded before the reset (${R_OLD_ID:-none}); it may be another conversation's, so nothing is purged; check it, purge it by hand if it is this agent's, and rerun the reset"
|
|
SKIPPED_ARGS+=(--skipped "$R_NAME:id_unverified")
|
|
continue
|
|
fi
|
|
if ! MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset id-owned --workspace "$WS" --session "$R_NAME"; then
|
|
echo "skipped $R_NAME: its recorded id $R_ROW_ID does not verify as a conversation of its folder, so it may be another folder's and nothing is purged; check it, purge it by hand if it is this agent's, and rerun the reset"
|
|
SKIPPED_ARGS+=(--skipped "$R_NAME:id_unverified")
|
|
continue
|
|
fi
|
|
|
|
echo "reset [$R_NAME]: ${STOP_CMD[*]} --purge-conversation --yes"
|
|
purge_rc=0
|
|
"${STOP_CMD[@]}" --purge-conversation --yes || purge_rc=$?
|
|
if [ "$purge_rc" -eq 4 ]; then
|
|
echo "skipped $R_NAME: its purge was refused (stop exit 4); the row (now stopped, id recorded) and the conversation are kept; the processes above hold it - they are not killed; rerun the reset once they end"
|
|
SKIPPED_ARGS+=(--skipped "$R_NAME:purge_refused")
|
|
continue
|
|
fi
|
|
if [ "$purge_rc" -ne 0 ]; then
|
|
FAILED_SPEC="$R_NAME:stop:$purge_rc"
|
|
break
|
|
fi
|
|
R_STATE="purged"
|
|
;;
|
|
esac
|
|
|
|
if [ "$R_STATE" = "purged" ]; then
|
|
CREATE_CMD=(bash "$SKILL_DIR/multi-agent-mux-create/scripts/create_session.sh" --workspace "$WS" --agent "$R_AGENT" --role "$R_ROLE" --session "$R_NAME")
|
|
if [ -n "$R_SERVER" ]; then
|
|
CREATE_CMD+=(--herdr-session "$R_SERVER")
|
|
fi
|
|
if [ -n "$R_LABEL" ]; then
|
|
CREATE_CMD+=(--herdr-workspace "$R_LABEL")
|
|
fi
|
|
|
|
echo "reset [$R_NAME]: ${CREATE_CMD[*]}"
|
|
create_rc=0
|
|
# Its onboarding job is the one registration the reset marker lets through.
|
|
MAM_RESET_OWNER_PID="$$" "${CREATE_CMD[@]}" || create_rc=$?
|
|
if [ "$create_rc" -ne 0 ]; then
|
|
FAILED_SPEC="$R_NAME:create:$create_rc"
|
|
break
|
|
fi
|
|
fi
|
|
done
|
|
|
|
STATE_JSON="$(load_state_json strict)" || exit 1
|
|
TABLE_ARGS=(--workspace "$WS")
|
|
if [ ${#SKIPPED_ARGS[@]} -gt 0 ]; then
|
|
TABLE_ARGS+=("${SKIPPED_ARGS[@]}")
|
|
fi
|
|
if [ -n "$FAILED_SPEC" ]; then
|
|
TABLE_ARGS+=(--failed "$FAILED_SPEC")
|
|
fi
|
|
table_rc=0
|
|
MAM_STATE_JSON="$STATE_JSON" "$PY" -m lib_py.reset table "${TABLE_ARGS[@]}" || table_rc=$?
|
|
case "$table_rc" in
|
|
0)
|
|
rm -f "$SPEC_FILE"
|
|
exit 0
|
|
;;
|
|
4)
|
|
exit 4
|
|
;;
|
|
5)
|
|
exit 5
|
|
;;
|
|
*)
|
|
exit 1
|
|
;;
|
|
esac
|