feat(deploy): install update/remove scripts into .mam_deploy/ and refine markdown staging

This commit is contained in:
2026-08-04 22:17:59 +09:00
parent 2ff8b2c4a9
commit 68eff79810
7 changed files with 1092 additions and 157 deletions
+280 -93
View File
@@ -1,22 +1,28 @@
#!/usr/bin/env bash
# ==============================================================================
# install.sh — Multi-Agent Mux (MAM) Orchestration Installer
# ==============================================================================
# Idempotent, robust installer to bootstrap MAM orchestration skills
# and Python backplane dependencies on any local workspace.
# install.sh — Multi-Agent Mux (MAM) Orchestration Installer (Rev.2)
# ==============================================================================
set -euo pipefail
# --- Configuration & Defaults ---
TARGET_DIR=""
FORCE_REFRESH="${MAM_FORCE_REFRESH:-0}"
REFRESH="${MAM_REFRESH:-1}"
OVERWRITE_CUSTOM="${MAM_OVERWRITE_CUSTOM:-0}"
VENV_NAME=".venv"
MIN_PYTHON_VERSION="3.9"
while [[ $# -gt 0 ]]; do
case "$1" in
-f|--force|--refresh-skills)
FORCE_REFRESH=1
REFRESH=1
shift
;;
--no-refresh|--offline)
REFRESH=0
shift
;;
--overwrite-custom)
OVERWRITE_CUSTOM=1
shift
;;
-h|--help)
@@ -24,7 +30,9 @@ while [[ $# -gt 0 ]]; do
Usage: $0 [options] [target_dir]
Options:
-f, --force, --refresh-skills Force fetch and refresh framework skills under .agents/skills/
-f, --force, --refresh-skills Fetch and refresh skills (now default)
--no-refresh, --offline Skip fetching latest assets if assets exist
--overwrite-custom Force overwrite user-modified skill files (backup still created)
-h, --help Show this help message
EOF
exit 0
@@ -64,7 +72,6 @@ check_cmd() {
check_cmd herdr
check_cmd python3
# Verify Python Version
PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
PYTHON_MAJOR="${MIN_PYTHON_VERSION%%.*}"
PYTHON_MINOR="${MIN_PYTHON_VERSION##*.}"
@@ -75,10 +82,8 @@ else
exit 1
fi
# Verify PyYAML (needed by system python3 for atomic state writes)
if ! python3 -c "import yaml" &>/dev/null; then
echo "❌ Error: 'PyYAML' is not installed in the system python3. Please install it first" >&2
echo " (e.g., 'pip3 install PyYAML' or 'sudo apt-get install python3-yaml')." >&2
echo "❌ Error: 'PyYAML' is not installed in system python3." >&2
exit 1
fi
echo "✅ PyYAML (system dependency) detected."
@@ -90,8 +95,6 @@ cd "$TARGET_DIR"
REPO_URL="${MAM_REPO_URL:-https://git.godopu.com/tmpl/multi-agent-mux.git}"
ARCHIVE_URL="${MAM_ARCHIVE_URL:-https://git.godopu.com/tmpl/multi-agent-mux/archive/main.tar.gz}"
# Helper to verify presence of all core runtime files.
# Keying off a set of core files helps detect and recover from partial/interrupted installations.
check_assets_present() {
local dir="${1:-.}"
local core_files=(
@@ -109,7 +112,6 @@ check_assets_present() {
return 0
}
# Helper to classify framework-owned skill definitions vs user-owned project assets
is_framework_owned() {
case "$1" in
.agents/skills/*) return 0 ;;
@@ -117,108 +119,242 @@ is_framework_owned() {
esac
}
# Fetch the orchestration assets if missing or if skill refresh is requested.
#
# Safety model (FW-D1): we NEVER extract the repo archive directly into the
# target. Running inside an existing project must not overwrite the target's
# own files (README.md, FUTURE_WORKS.md, AGENTS.md, MULTI_AGENT_RULES.md) or litter
# it with development docs. Instead we stage the download into a throwaway temp dir,
# verify it, then copy runtime assets: framework skills (.agents/skills/*) are updated,
# while user-owned documents use per-file no-clobber guards so pre-existing target files win.
if [ "$FORCE_REFRESH" -eq 1 ] || ! check_assets_present "."; then
# Fetch orchestration assets if REFRESH=1 or if core assets are missing.
if [ "$REFRESH" -eq 1 ] || [ "${MAM_SKIP_VENV:-0}" -eq 1 ] || ! check_assets_present "."; then
echo "📥 Staging orchestration assets from Gitea repository..."
STAGE_DIR="$(mktemp -d)"
trap 'rm -rf "$STAGE_DIR"' EXIT
if command -v git &>/dev/null; then
FETCH_METHOD="archive"
if [ -d "$REPO_URL" ]; then
echo "🌐 Copying local working tree into a staging area..."
cp -R "$REPO_URL/." "$STAGE_DIR/"
FETCH_METHOD="local"
elif command -v git &>/dev/null; then
echo "🌐 Cloning repository (shallow) into a staging area..."
git clone --depth 1 "$REPO_URL" "$STAGE_DIR"
FETCH_METHOD="git"
elif command -v curl &>/dev/null; then
echo "🌐 Downloading and extracting archive into a staging area..."
curl -fsSL "$ARCHIVE_URL" | tar -xz --strip-components=1 -C "$STAGE_DIR"
curl -fsSL "$ARCHIVE_URL" | tar -xz --strip-components=1 -C "$STAGE_DIR" \
--exclude='*/.agents/reports/*' --exclude='*/.agents/references/*' \
--exclude='*/MESSAGING.md' --exclude='*/BOOTSTRAP.md' --exclude='*/BOOTSTRAP.ko.md' 2>/dev/null || true
FETCH_METHOD="archive"
else
echo "❌ Error: neither 'git' nor 'curl' is available to fetch the skills." >&2
exit 1
fi
# Verify the staged tree before we trust and copy from it.
if ! check_assets_present "$STAGE_DIR"; then
echo "❌ Error: fetched source is missing core runtime assets. Aborting (no files copied)." >&2
echo "❌ Error: fetched source is missing core runtime assets. Aborting." >&2
exit 1
fi
# Create metadata directory and initialize manifest before copying
mkdir -p .mam
MANIFEST_FILE=".mam/install_manifest.txt"
touch "$MANIFEST_FILE"
# Copy runtime assets (.agents/) into the target workspace.
# Framework-owned skill files (.agents/skills/*) are updated/overwritten so that
# latest skill definitions and metadata frontmatter take effect.
# User-owned documents (.agents/MULTI_AGENT_RULES*.md, .agents/INSTALL.md, etc.) use
# explicit no-clobber guards so pre-existing user files are untouched and unmanifested.
# Migrate legacy root layout for remove.sh and update.sh into .mam_deploy/ if owned
mkdir -p .mam_deploy
for legacy_script in remove.sh update.sh; do
if [ -f "$legacy_script" ] && grep -Fqx "$legacy_script" "$MANIFEST_FILE" 2>/dev/null; then
mv -f "$legacy_script" ".mam_deploy/$legacy_script"
python3 -c '
import sys
path = sys.argv[1]
old_s = sys.argv[2]
new_s = sys.argv[3]
with open(path, "r") as f:
lines = f.readlines()
with open(path, "w") as f:
for line in lines:
if line.strip() == old_s:
f.write(new_s + "\n")
else:
f.write(line)
' "$MANIFEST_FILE" "$legacy_script" ".mam_deploy/$legacy_script" 2>/dev/null || true
fi
done
# Safe refresh & fingerprint checking logic
TS=$(date -u +%Y%m%dT%H%M%SZ)
PRESERVED_COUNT=0
MODIFIED_FILES=()
mkdir -p .agents
( cd "$STAGE_DIR/.agents" && find . -type f -print ) | while IFS= read -r rel; do
case "$rel" in
./reports/*|./references/*) continue ;;
*.tmp|*.log|*.pyc|*/__pycache__/*) continue ;;
esac
dest=".agents/${rel#./}"
mkdir -p "$(dirname "$dest")"
if is_framework_owned "$dest"; then
cp -f "$STAGE_DIR/.agents/$rel" "$dest" || { echo "❌ Error: Failed to copy $rel" >&2; exit 1; }
if ! grep -Fqx "$dest" "$MANIFEST_FILE" 2>/dev/null; then
echo "$dest" >> "$MANIFEST_FILE"
fi
# 3-way check using python3 inline
STAGING_FILE="$STAGE_DIR/.agents/$rel"
ACTION=$(python3 - "$dest" "$STAGING_FILE" ".mam/asset_hashes.txt" "$OVERWRITE_CUSTOM" <<'PY'
import sys, hashlib, os
target_path, staging_path, hash_db_path, force_overwrite = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "1"
def file_sha(path):
if not os.path.exists(path):
return None
h = hashlib.sha256()
with open(path, "rb") as f:
while chunk := f.read(65536):
h.update(chunk)
return h.hexdigest()
target_sha = file_sha(target_path)
staging_sha = file_sha(staging_path)
if target_sha is None:
print("COPY_NEW")
sys.exit(0)
if target_sha == staging_sha:
print("NO_OP")
sys.exit(0)
db_sha = None
if os.path.exists(hash_db_path):
with open(hash_db_path, "r") as f:
for line in f:
parts = line.strip().split(None, 1)
if len(parts) == 2 and parts[1] == target_path:
db_sha = parts[0]
break
if db_sha is None:
print("BOOTSTRAP_OVERWRITE")
elif target_sha == db_sha:
print("UPDATE_UNMODIFIED")
else:
if force_overwrite:
print("FORCE_OVERWRITE_CUSTOM")
else:
print("PRESERVE_CUSTOM")
PY
)
case "$ACTION" in
COPY_NEW|UPDATE_UNMODIFIED|BOOTSTRAP_OVERWRITE|FORCE_OVERWRITE_CUSTOM)
if [ "$ACTION" = "BOOTSTRAP_OVERWRITE" ] || [ "$ACTION" = "FORCE_OVERWRITE_CUSTOM" ]; then
BACKUP_DIR=".mam/skill-backups/$TS/$(dirname "$dest")"
mkdir -p "$BACKUP_DIR"
cp "$dest" "$BACKUP_DIR/"
fi
cp -f "$STAGING_FILE" "$dest"
if ! grep -Fqx "$dest" "$MANIFEST_FILE" 2>/dev/null; then
echo "$dest" >> "$MANIFEST_FILE"
fi
;;
PRESERVE_CUSTOM)
BACKUP_DIR=".mam/skill-backups/$TS/$(dirname "$dest")"
mkdir -p "$BACKUP_DIR"
cp "$dest" "$BACKUP_DIR/"
echo "PRESERVED:$dest"
;;
NO_OP)
if ! grep -Fqx "$dest" "$MANIFEST_FILE" 2>/dev/null; then
echo "$dest" >> "$MANIFEST_FILE"
fi
;;
esac
elif [ ! -e "$dest" ]; then
cp "$STAGE_DIR/.agents/$rel" "$dest" || { echo "❌ Error: Failed to copy $rel" >&2; exit 1; }
cp "$STAGE_DIR/.agents/$rel" "$dest"
if ! grep -Fqx "$dest" "$MANIFEST_FILE" 2>/dev/null; then
echo "$dest" >> "$MANIFEST_FILE"
fi
fi
done | while IFS= read -r line; do
if [[ "$line" == PRESERVED:* ]]; then
echo "️ Local modification detected: ${line#PRESERVED:}" >&2
fi
done
# Copy non-dev documents if they don't already exist.
# We skip dev-specific docs like README.md, DONE.md, and FUTURE_WORKS.md.
for doc in MESSAGING.md BOOTSTRAP.md BOOTSTRAP.ko.md AGENTS.md; do
# Re-build asset_hashes.txt for all framework owned files
python3 - .mam/asset_hashes.txt <<'PY'
import os, hashlib, sys
hash_db_path = sys.argv[1]
hashes = []
for root, _, files in os.walk(".agents/skills"):
for file in files:
path = os.path.join(root, file)
h = hashlib.sha256()
with open(path, "rb") as f:
while chunk := f.read(65536):
h.update(chunk)
hashes.append(f"{h.hexdigest()} {path}\n")
with open(hash_db_path, "w") as f:
f.writelines(sorted(hashes))
PY
# Copy root docs (R-2 essential set)
ROOT_DOCS="AGENTS.md"
if [ "${MAM_INSTALL_DOCS:-minimal}" = "full" ]; then
ROOT_DOCS="AGENTS.md MESSAGING.md BOOTSTRAP.md BOOTSTRAP.ko.md"
fi
for doc in $ROOT_DOCS; do
if [ -f "$STAGE_DIR/$doc" ] && [ ! -e "$doc" ]; then
cp "$STAGE_DIR/$doc" . || { echo "❌ Error: Failed to copy $doc" >&2; exit 1; }
cp "$STAGE_DIR/$doc" .
echo "$doc" >> "$MANIFEST_FILE"
fi
done
if [ -f "$STAGE_DIR/deploy/remove.sh" ] && [ ! -e "remove.sh" ]; then
cp "$STAGE_DIR/deploy/remove.sh" remove.sh || { echo "❌ Error: Failed to copy remove.sh" >&2; exit 1; }
chmod +x remove.sh
echo "remove.sh" >> "$MANIFEST_FILE"
# Install remove.sh and update.sh into .mam_deploy/
mkdir -p .mam_deploy
if [ -f "$STAGE_DIR/deploy/remove.sh" ]; then
cp "$STAGE_DIR/deploy/remove.sh" .mam_deploy/remove.sh
chmod 0755 .mam_deploy/remove.sh
if ! grep -Fqx ".mam_deploy/remove.sh" "$MANIFEST_FILE" 2>/dev/null; then
echo ".mam_deploy/remove.sh" >> "$MANIFEST_FILE"
fi
fi
if [ -f "$STAGE_DIR/deploy/update.sh" ] && [ ! -e "update.sh" ]; then
cp "$STAGE_DIR/deploy/update.sh" update.sh || { echo "❌ Error: Failed to copy update.sh" >&2; exit 1; }
chmod +x update.sh
echo "update.sh" >> "$MANIFEST_FILE"
if [ -f "$STAGE_DIR/deploy/update.sh" ]; then
cp "$STAGE_DIR/deploy/update.sh" .mam_deploy/update.sh
chmod 0755 .mam_deploy/update.sh
if ! grep -Fqx ".mam_deploy/update.sh" "$MANIFEST_FILE" 2>/dev/null; then
echo ".mam_deploy/update.sh" >> "$MANIFEST_FILE"
fi
fi
if [ -f "$STAGE_DIR/.mam.env.example" ] && [ ! -e ".mam.env.example" ]; then
cp "$STAGE_DIR/.mam.env.example" . || { echo "❌ Error: Failed to copy .mam.env.example" >&2; exit 1; }
cp "$STAGE_DIR/.mam.env.example" .
echo ".mam.env.example" >> "$MANIFEST_FILE"
fi
# Ship the user manual into the target's .agents/ (consistent with install_mam.sh)
if [ -f "$STAGE_DIR/deploy/INSTALL.md" ]; then
if [ -f "$STAGE_DIR/deploy/INSTALL.md" ] && [ ! -e ".agents/INSTALL.md" ]; then
mkdir -p .agents
if [ ! -e ".agents/INSTALL.md" ]; then
cp "$STAGE_DIR/deploy/INSTALL.md" .agents/INSTALL.md || { echo "❌ Error: Failed to copy INSTALL.md" >&2; exit 1; }
echo ".agents/INSTALL.md" >> "$MANIFEST_FILE"
fi
cp "$STAGE_DIR/deploy/INSTALL.md" .agents/INSTALL.md
echo ".agents/INSTALL.md" >> "$MANIFEST_FILE"
fi
# Record version stamp
STAGE_COMMIT="unknown"
if [ -d "$STAGE_DIR/.git" ]; then
STAGE_COMMIT=$(git -C "$STAGE_DIR" rev-parse HEAD 2>/dev/null || echo "unknown")
fi
cat <<EOF > .mam/version.txt
source=$REPO_URL
commit=$STAGE_COMMIT
fetched_at=$TS
method=$FETCH_METHOD
EOF
rm -rf "$STAGE_DIR"
trap - EXIT
echo "✅ Skills staged into workspace (user documents and custom configs preserved)."
fi
# Sanity check: verify all core files, not just a single one — an empty or
# incomplete layout would yield a silently broken install.
if ! check_assets_present "."; then
echo "❌ Error: Core runtime assets missing after setup. Target layout might be invalid." >&2
echo "❌ Error: Core runtime assets missing after setup." >&2
exit 1
fi
echo "✅ Orchestration skills present."
@@ -226,20 +362,80 @@ echo "✅ Orchestration skills present."
echo "📂 Ensuring metadata directory structure (.mam/)..."
mkdir -p .mam/jobs .mam/delegate_job_logs
# File permission lockdown on database directory (if owned by the current user to prevent multi-user system issues)
if [ -O .mam ]; then
chmod 0700 .mam
fi
# R-4: Manage .gitignore block (never put .gitignore in manifest)
MAM_GI_START="# >>> MAM managed block (managed by install.sh — do not edit) >>>"
MAM_GI_END="# <<< MAM managed block <<<"
if [ "${MAM_SKIP_GITIGNORE:-0}" != "1" ]; then
GI_CREATED=0
[ -e .gitignore ] || { touch .gitignore; GI_CREATED=1; }
if ! grep -q '^gitignore_created=' .mam/install_state 2>/dev/null; then
echo "gitignore_created=$GI_CREATED" >> .mam/install_state
fi
python3 - .gitignore "$MAM_GI_START" "$MAM_GI_END" <<'PY'
import sys, os
gi_path, start_marker, end_marker = sys.argv[1], sys.argv[2], sys.argv[3]
block_lines = [
start_marker + "\n",
"/.venv/\n",
"/.mam/\n",
"/.mam_deploy/\n",
"/.mam.env\n",
"/.mam.env.*\n",
"!/.mam.env.example\n",
"/.cache/multi-agent-mux-monitor/\n",
"/.mam-skill-backup.*/\n",
"CURRENT_JOB.md\n",
end_marker + "\n"
]
lines = []
if os.path.exists(gi_path):
with open(gi_path, "r") as f:
lines = f.readlines()
new_lines = []
in_block = False
block_inserted = False
for line in lines:
if line.strip() == start_marker:
in_block = True
if not block_inserted:
new_lines.extend(block_lines)
block_inserted = True
continue
if line.strip() == end_marker:
in_block = False
continue
if not in_block:
new_lines.append(line)
if not block_inserted:
if new_lines and not new_lines[-1].endswith("\n"):
new_lines[-1] += "\n"
new_lines.extend(block_lines)
with open(gi_path, "w") as f:
f.writelines(new_lines)
PY
fi
# --- 3. Check Network File System (NFS) Warnings ---
echo "💾 Detecting file system mount type..."
if command -v df &>/dev/null && command -v mount &>/dev/null; then
MOUNTPOINT="$(df --output=target . 2>/dev/null | tail -1 || echo "")"
if [ -n "$MOUNTPOINT" ]; then
if mount | grep -q "$MOUNTPOINT.*nfs\|$MOUNTPOINT.*cifs\|$MOUNTPOINT.*fuse.sshfs"; then
echo "⚠️ WARNING: Target directory is on a network filesystem (NFS/CIFS/SSHFS)."
echo " SQLite WAL journaling and file locks are UNRELIABLE on network storage."
echo " The sqlite3 registry will fall back to 'DELETE' journaling instead of WAL."
echo "⚠️ WARNING: Target directory is on a network filesystem."
else
echo "✅ File system supports WAL (Local storage detected)."
fi
@@ -247,38 +443,32 @@ if command -v df &>/dev/null && command -v mount &>/dev/null; then
fi
# --- 4. Python Virtual Environment Setup ---
echo "🐍 Bootstrapping Python virtual environment (.venv)..."
if [ ! -d "$VENV_NAME" ]; then
python3 -m venv "$VENV_NAME"
echo "✅ Virtual environment created."
else
echo "️ Virtual environment (.venv) already exists. Skipping creation."
fi
if [ "${MAM_SKIP_VENV:-0}" != "1" ]; then
echo "🐍 Bootstrapping Python virtual environment (.venv)..."
if [ ! -d "$VENV_NAME" ]; then
python3 -m venv "$VENV_NAME"
echo "✅ Virtual environment created."
else
echo "️ Virtual environment (.venv) already exists. Skipping creation."
fi
# Activate virtual environment
# shellcheck disable=SC1091
source "$VENV_NAME"/bin/activate
source "$VENV_NAME"/bin/activate
pip install --upgrade pip
# Upgrade pip
pip install --upgrade pip
# Install requirements
REQ_FILE=".agents/skills/multi-agent-mux-delegate-job/requirements.txt"
if [ -f "$REQ_FILE" ]; then
echo "📦 Installing backplane dependencies from $REQ_FILE..."
pip install -r "$REQ_FILE"
echo "✅ Dependencies installed successfully."
else
echo "⚠️ WARNING: Could not find requirements file: $REQ_FILE"
echo " Installing default packages (paho-mqtt, pyyaml) manually..."
pip install "paho-mqtt>=2.0.0" pyyaml
REQ_FILE=".agents/skills/multi-agent-mux-delegate-job/requirements.txt"
if [ -f "$REQ_FILE" ]; then
echo "📦 Installing backplane dependencies from $REQ_FILE..."
pip install -r "$REQ_FILE"
echo "✅ Dependencies installed successfully."
else
pip install "paho-mqtt>=2.0.0" pyyaml
fi
fi
# --- 5. Generate Environment Template ---
ENV_FILE=".mam.env"
ENV_EXAMPLE=".mam.env.example"
# M-2: Evidence-based legacy env migration helper
migrate_legacy_env() {
local manifest=".mam/install_manifest.txt"
[ -f ".mam.env" ] && return 0
@@ -314,7 +504,6 @@ with open(path, "w") as f:
migrate_legacy_env
# M-1: Shadowing prevention guard — only create new default config if no legacy env or update tmp exists
if [ ! -f "$ENV_FILE" ] && [ ! -f ".env" ] && [ ! -f ".env.update-tmp" ]; then
if [ -f "$ENV_EXAMPLE" ]; then
echo "📝 Creating configuration from $ENV_EXAMPLE..."
@@ -323,8 +512,7 @@ if [ ! -f "$ENV_FILE" ] && [ ! -f ".env" ] && [ ! -f ".env.update-tmp" ]; then
echo "📝 Creating default $ENV_FILE..."
touch "$ENV_FILE"
fi
# Always append the active defaults to ensure they are set and not commented out
cat <<EOF >> "$ENV_FILE"
# === Installer-applied active defaults ===
@@ -336,8 +524,7 @@ HERDR_SERVER_NAME=default
EOF
chmod 0600 "$ENV_FILE"
echo "✅ Config file .mam.env initialized with chmod 0600."
# Record the newly created .mam.env in the manifest
mkdir -p .mam
touch .mam/install_manifest.txt
echo "$ENV_FILE" >> .mam/install_manifest.txt