runpod/arty.yml

name: "RunPod AI Model Orchestrator"
version: "2.0.0"
description: "Process-based AI model orchestrator for RunPod GPU instances with ComfyUI integration"
author: "valknar@pivoine.art"
license: "MIT"

# Git repositories to clone for a fresh RunPod deployment
references:
  # ComfyUI base installation
  - url: https://github.com/comfyanonymous/ComfyUI.git
    into: $COMFYUI_ROOT
    description: "ComfyUI - Node-based interface for image/video/audio generation"

  # ComfyUI Essential Custom Nodes
  - url: https://github.com/ltdrdata/ComfyUI-Manager.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-Manager
    description: "ComfyUI Manager - Install/manage custom nodes and models"
    essential: true

  - url: https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-VideoHelperSuite
    description: "Video operations and processing"
    essential: true

  - url: https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-AnimateDiff-Evolved
    description: "AnimateDiff for video generation"
    essential: true

  - url: https://github.com/cubiq/ComfyUI_IPAdapter_plus.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI_IPAdapter_plus
    description: "IP-Adapter for style transfer"
    essential: true

  - url: https://github.com/ltdrdata/ComfyUI-Impact-Pack.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-Impact-Pack
    description: "Auto face enhancement and detailer"
    essential: true

  # ComfyUI Optional Custom Nodes
  - url: https://github.com/kijai/ComfyUI-CogVideoXWrapper.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-CogVideoXWrapper
    description: "CogVideoX integration for text-to-video"
    essential: false

  - url: https://github.com/ltdrdata/ComfyUI-Inspire-Pack.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-Inspire-Pack
    description: "Additional inspiration tools"
    essential: false

  - url: https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-Advanced-ControlNet
    description: "Advanced ControlNet features"
    essential: false

  - url: https://github.com/MrForExample/ComfyUI-3D-Pack.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-3D-Pack
    description: "3D asset generation"
    essential: false

  - url: https://github.com/MixLabPro/comfyui-sound-lab.git
    into: $COMFYUI_ROOT/custom_nodes/comfyui-sound-lab
    description: "MusicGen and Stable Audio integration"
    essential: false

  - url: https://github.com/billwuhao/ComfyUI_DiffRhythm.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI_DiffRhythm
    description: "DiffRhythm - Full-length song generation (up to 4m45s) with text/audio conditioning"
    essential: false

  - url: https://github.com/ssitu/ComfyUI_UltimateSDUpscale.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI_UltimateSDUpscale
    description: "Ultimate SD Upscale for high-quality image upscaling"
    essential: false

# Environment profiles for selective repository management
envs:
  # RunPod environment variables
  default:
    AI_ROOT: /workspace/ai
    COMFYUI_ROOT: /workspace/ComfyUI
    HF_CACHE: /workspace/huggingface_cache
    LOGS_DIR: /workspace/logs
    BIN_DIR: /workspace/bin

# Deployment scripts for RunPod instances
scripts:
  #
  # Supervisor Control Scripts
  #
  services/supervisor/start: |
    set -a
    source .env
    set +a
    supervisord -c supervisord.conf

  services/supervisor/stop: |
    supervisorctl -c supervisord.conf shutdown

  services/supervisor/status: |
    supervisorctl -c supervisord.conf status

  services/supervisor/restart: |
    supervisorctl -c supervisord.conf restart all

  #
  # System Setup Scripts
  #
  setup/system-packages: |
    echo "========================================="
    echo "  Installing System Packages"
    echo "========================================="
    echo ""

    # Check GPU availability
    if ! nvidia-smi > /dev/null 2>&1; then
      echo "❌ ERROR: nvidia-smi not found. GPU not available!"
      exit 1
    fi
    echo "✓ GPU detected:"
    nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
    echo ""

    # Update apt cache
    echo "Updating apt cache..."
    sudo apt update

    # Install base system packages
    echo "Installing system packages..."
    sudo apt install -y \
      build-essential \
      python3-dev \
      python3-pip \
      python3-venv \
      git \
      curl \
      wget \
      vim \
      htop \
      tmux \
      net-tools \
      davfs2 \
      ffmpeg \
      libavcodec-dev \
      libavformat-dev \
      libavutil-dev \
      libswscale-dev

    echo ""
    echo "✓ System packages installed successfully"

    # Verify FFmpeg installation
    if ffmpeg -version > /dev/null 2>&1; then
      echo "✓ FFmpeg installed: $(ffmpeg -version | head -1 | cut -d ' ' -f3)"
    else
      echo "❌ WARNING: FFmpeg not found"
    fi

  setup/python-env: |
    echo "========================================="
    echo "  Setting Up Python Environment"
    echo "========================================="
    echo ""

    # Upgrade pip
    echo "Upgrading pip to 23.3.1..."
    sudo pip3 install --upgrade pip==23.3.1

    # Install core Python packages
    echo "Installing core Python packages..."
    if [ -f "$AI_ROOT/core/requirements.txt" ]; then
      sudo pip3 install -r $AI_ROOT/core/requirements.txt
    else
      echo "⚠ Warning: $AI_ROOT/core/requirements.txt not found"
    fi

    # Create virtual environments for services with isolated dependencies
    echo ""
    echo "Creating virtual environments for services..."

    # WebDAV sync service
    if [ -f "webdav-sync/requirements.txt" ]; then
      echo "  - webdav-sync..."
      cd webdav-sync && python3 -m venv venv && source venv/bin/activate && pip install -r requirements.txt && deactivate && cd ..
    fi

    # Model orchestrator service
    if [ -f "model-orchestrator/requirements.txt" ]; then
      echo "  - model-orchestrator..."
      cd model-orchestrator && python3 -m venv venv && source venv/bin/activate && pip install -r requirements.txt && deactivate && cd ..
    fi

    # vLLM service (may require more memory)
    if [ -f "vllm/requirements.txt" ]; then
      echo "  - vllm (this may take a while)..."
      cd vllm && python3 -m venv venv && source venv/bin/activate && pip install -r requirements.txt && deactivate && cd ..
    fi

    echo ""
    echo "✓ Python environment configured successfully"
    echo "  Virtual environments created for: webdav-sync, model-orchestrator, vllm"

  setup/comfyui-base: |
    echo "========================================="
    echo "  Installing ComfyUI Base"
    echo "========================================="
    echo ""

    # Clone ComfyUI if not exists
    if [ ! -d "$COMFYUI_ROOT" ]; then
      echo "Cloning ComfyUI repository..."
      git clone https://github.com/comfyanonymous/ComfyUI.git $COMFYUI_ROOT
    else
      echo "ComfyUI already exists, pulling latest changes..."
      cd $COMFYUI_ROOT && git pull
    fi

    # Create virtual environment for ComfyUI
    echo "Creating virtual environment for ComfyUI..."
    cd $COMFYUI_ROOT
    if [ ! -d "venv" ]; then
      python3 -m venv venv
    fi
    source venv/bin/activate

    # Install ComfyUI dependencies
    echo "Installing ComfyUI dependencies..."
    pip install -r requirements.txt

    # Install additional ComfyUI dependencies
    if [ -f "$AI_ROOT/comfyui/requirements.txt" ]; then
      echo "Installing additional ComfyUI dependencies..."
      pip install -r $AI_ROOT/comfyui/requirements.txt
    fi

    # Install common extension dependencies
    echo "Installing common extension dependencies..."
    pip install GitPython opencv-python-headless diffusers insightface onnxruntime

    deactivate
    cd $AI_ROOT

    # Create model directory structure
    echo "Creating ComfyUI model directories..."
    mkdir -p $COMFYUI_ROOT/models/{checkpoints,unet,vae,loras,clip,clip_vision,controlnet,ipadapter,embeddings,upscale_models,video_models,animatediff_models,animatediff_motion_lora,audio_models,configs,diffusers,diffusion_models,musicgen}

    # Create symlink to huggingface cache
    echo "Creating symlink to HuggingFace cache..."
    ln -sf $HF_CACHE $COMFYUI_ROOT/models/huggingface_cache

    # Make start script executable
    if [ -f "comfyui/start.sh" ]; then
      chmod +x comfyui/start.sh
    fi

    echo ""
    echo "✓ ComfyUI base installed successfully"
    echo "  Directory: $COMFYUI_ROOT"
    echo "  Port: 8188"
    echo "  Start: bash comfyui/start.sh"

  setup/comfyui-nodes: |
    echo "========================================="
    echo "  Installing ComfyUI Custom Nodes"
    echo "========================================="
    echo ""

    # Install system dependencies
    echo "Installing system dependencies..."
    sudo apt-get update -qq
    sudo apt-get install -y -qq espeak-ng
    echo "✓ System dependencies installed (espeak-ng)"
    echo ""

    cd $COMFYUI_ROOT/custom_nodes

    # ComfyUI Manager
    echo "[1/6] Installing ComfyUI-Manager..."
    if [ ! -d "ComfyUI-Manager" ]; then
      git clone https://github.com/ltdrdata/ComfyUI-Manager.git
    fi
    [ -f "ComfyUI-Manager/requirements.txt" ] && sudo pip3 install -r ComfyUI-Manager/requirements.txt

    # VideoHelperSuite
    echo "[2/6] Installing ComfyUI-VideoHelperSuite..."
    if [ ! -d "ComfyUI-VideoHelperSuite" ]; then
      git clone https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git
    fi
    [ -f "ComfyUI-VideoHelperSuite/requirements.txt" ] && sudo pip3 install -r ComfyUI-VideoHelperSuite/requirements.txt

    # AnimateDiff-Evolved
    echo "[3/6] Installing ComfyUI-AnimateDiff-Evolved..."
    if [ ! -d "ComfyUI-AnimateDiff-Evolved" ]; then
      git clone https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved.git
    fi
    [ -f "ComfyUI-AnimateDiff-Evolved/requirements.txt" ] && sudo pip3 install -r ComfyUI-AnimateDiff-Evolved/requirements.txt

    # IPAdapter Plus
    echo "[4/6] Installing ComfyUI_IPAdapter_plus..."
    if [ ! -d "ComfyUI_IPAdapter_plus" ]; then
      git clone https://github.com/cubiq/ComfyUI_IPAdapter_plus.git
    fi
    [ -f "ComfyUI_IPAdapter_plus/requirements.txt" ] && sudo pip3 install -r ComfyUI_IPAdapter_plus/requirements.txt

    # Impact-Pack
    echo "[5/6] Installing ComfyUI-Impact-Pack..."
    if [ ! -d "ComfyUI-Impact-Pack" ]; then
      git clone https://github.com/ltdrdata/ComfyUI-Impact-Pack.git
    fi
    [ -f "ComfyUI-Impact-Pack/requirements.txt" ] && sudo pip3 install -r ComfyUI-Impact-Pack/requirements.txt

    # DiffRhythm
    echo "[6/6] Installing ComfyUI_DiffRhythm..."
    if [ ! -d "ComfyUI_DiffRhythm" ]; then
      git clone https://github.com/billwuhao/ComfyUI_DiffRhythm.git
    fi
    if [ -f "ComfyUI_DiffRhythm/requirements.txt" ]; then
      cd $COMFYUI_ROOT
      source venv/bin/activate
      pip install -r custom_nodes/ComfyUI_DiffRhythm/requirements.txt
      deactivate
      cd custom_nodes
    fi

    # Create DiffRhythm model directories
    echo "Creating DiffRhythm model directories..."
    mkdir -p $COMFYUI_ROOT/models/TTS/DiffRhythm/{MuQ-large-msd-iter,MuQ-MuLan-large,xlm-roberta-base,eval-model}

    # Fix numpy version for vLLM compatibility
    echo "Fixing numpy version..."
    sudo pip3 install 'numpy<2.0.0' --force-reinstall

    echo ""
    echo "✓ Essential custom nodes installed successfully"
    echo "  - ComfyUI-Manager: Install/manage custom nodes"
    echo "  - VideoHelperSuite: Video operations"
    echo "  - AnimateDiff-Evolved: Video generation"
    echo "  - IPAdapter_plus: Style transfer"
    echo "  - Impact-Pack: Face enhancement"
    echo "  - DiffRhythm: Full-length song generation"

  models/diffrhythm-eval: |
    echo "========================================="
    echo "  Downloading DiffRhythm Eval Model"
    echo "========================================="
    echo ""

    # Create eval-model directory
    mkdir -p $COMFYUI_ROOT/models/TTS/DiffRhythm/eval-model
    cd $COMFYUI_ROOT/models/TTS/DiffRhythm/eval-model

    # Download eval.yaml (129 bytes)
    echo "Downloading eval.yaml..."
    curl -L -o eval.yaml "https://huggingface.co/spaces/ASLP-lab/DiffRhythm/resolve/main/pretrained/eval.yaml"

    # Download eval.safetensors (101 MB)
    echo "Downloading eval.safetensors (101 MB)..."
    curl -L -o eval.safetensors "https://huggingface.co/spaces/ASLP-lab/DiffRhythm/resolve/main/pretrained/eval.safetensors"

    # Verify files
    if [ -f "eval.yaml" ] && [ -f "eval.safetensors" ]; then
      echo ""
      echo "✓ DiffRhythm eval-model files downloaded successfully"
      echo "  - eval.yaml: $(du -h eval.yaml | cut -f1)"
      echo "  - eval.safetensors: $(du -h eval.safetensors | cut -f1)"
    else
      echo "❌ ERROR: Failed to download eval-model files"
      exit 1
    fi

  setup/pivoine-nodes: |
    echo "========================================="
    echo "  Linking Pivoine Custom Nodes"
    echo "========================================="
    echo ""

    NODES_SRC="/workspace/ai/comfyui/nodes"
    NODES_DEST="/workspace/ComfyUI/custom_nodes/ComfyUI_Pivoine"

    # Remove existing symlink if present
    if [ -L "$NODES_DEST" ] || [ -d "$NODES_DEST" ]; then
      echo "Removing existing: $NODES_DEST"
      rm -rf "$NODES_DEST"
    fi

    # Create symlink
    ln -s "$NODES_SRC" "$NODES_DEST"

    echo ""
    echo "✓ Pivoine custom nodes linked"
    echo "  Source: $NODES_SRC"
    echo "  Linked: $NODES_DEST"
    echo ""
    echo "Available Pivoine nodes:"
    echo "  🌸 PivoineDiffRhythmRun - DiffRhythm with chunked disabled"
    echo ""
    echo "Category: 🌸Pivoine/Audio"

  fix/diffrhythm-patch: |
    echo "========================================="
    echo "  Apply DiffRhythm LlamaConfig Patch"
    echo "========================================="
    echo ""
    echo "Issue: Tensor dimension mismatch (32 vs 64) in rotary embeddings"
    echo "Solution: Patch DiffRhythm __init__.py to fix LlamaConfig"
    echo ""
    echo "References:"
    echo "  - https://github.com/billwuhao/ComfyUI_DiffRhythm/issues/44"
    echo "  - https://github.com/billwuhao/ComfyUI_DiffRhythm/issues/48"
    echo ""

    DIFF_RHYTHM_DIR="/workspace/ComfyUI/custom_nodes/ComfyUI_DiffRhythm"
    PATCH_FILE="/workspace/ai/comfyui/patches/diffrhythm-llamaconfig-fix.patch"

    if [ ! -d "$DIFF_RHYTHM_DIR" ]; then
      echo "✗ Error: DiffRhythm not found at $DIFF_RHYTHM_DIR"
      exit 1
    fi

    if [ ! -f "$PATCH_FILE" ]; then
      echo "✗ Error: Patch file not found at $PATCH_FILE"
      exit 1
    fi

    cd "$DIFF_RHYTHM_DIR"

    echo "Checking if patch already applied..."
    if grep -q "PatchedLlamaConfig" __init__.py; then
      echo "✓ Patch already applied!"
      exit 0
    fi

    echo "Applying patch..."
    patch -p1 < "$PATCH_FILE"

    if [ $? -eq 0 ]; then
      echo ""
      echo "✓ Patch applied successfully!"
      echo ""
      echo "Next steps:"
      echo "  1. Restart ComfyUI: arty services/comfyui/restart"
      echo "  2. Test DiffRhythm workflows"
    else
      echo ""
      echo "✗ Failed to apply patch"
      echo "You may need to manually apply the patch or check for conflicts"
      exit 1
    fi

  setup/comfyui-extensions-deps: |
    echo "========================================="
    echo "  Installing ComfyUI Extensions Dependencies"
    echo "========================================="
    echo ""

    cd $COMFYUI_ROOT

    # Activate ComfyUI venv
    if [ ! -d "venv" ]; then
      echo "❌ ERROR: ComfyUI venv not found. Run setup/comfyui-base first!"
      exit 1
    fi

    source venv/bin/activate

    # Install dependencies for each custom node
    echo "Installing dependencies for all custom nodes..."
    cd custom_nodes

    installed_count=0
    skipped_count=0

    for dir in */; do
      if [ -f "${dir}requirements.txt" ]; then
        echo ""
        echo "📦 Installing ${dir%/} dependencies..."
        if pip install -r "${dir}requirements.txt"; then
          installed_count=$((installed_count + 1))
          echo "  ✓ ${dir%/} dependencies installed"
        else
          echo "  ⚠ Warning: Some dependencies for ${dir%/} may have failed"
          installed_count=$((installed_count + 1))
        fi
      else
        skipped_count=$((skipped_count + 1))
      fi
    done

    deactivate
    cd $AI_ROOT

    echo ""
    echo "✓ Extension dependencies installation complete"
    echo "  Extensions with dependencies: $installed_count"
    echo "  Extensions without requirements.txt: $skipped_count"

  setup/tailscale: |
    echo "========================================="
    echo "  Installing Tailscale VPN"
    echo "========================================="
    echo ""

    # Check if already installed
    if command -v tailscale > /dev/null 2>&1; then
      echo "✓ Tailscale already installed"
      tailscale version
    else
      echo "Installing Tailscale..."
      curl -fsSL https://tailscale.com/install.sh | sh
      echo "✓ Tailscale installed successfully"
    fi

    echo ""
    echo "To connect Tailscale:"
    echo "  1. Start daemon: tailscaled --tun=userspace-networking --socks5-server=localhost:1055 &"
    echo "  2. Authenticate: tailscale up --advertise-tags=tag:gpu"
    echo "  3. Get IP: tailscale ip -4"
    echo ""
    echo "Note: Authentication requires manual URL visit"

  setup/supervisor: |
    echo "========================================="
    echo "  Installing Supervisor"
    echo "========================================="
    echo ""

    # Install Supervisor
    echo "Installing Supervisor via pip..."
    sudo pip3 install supervisor

    # Create logs directory
    mkdir -p $LOGS_DIR

    # Copy supervisor configuration
    if [ -f "$AI_ROOT/supervisord.conf" ]; then
      echo "Deploying supervisord configuration..."
      cp $AI_ROOT/supervisord.conf /workspace/supervisord.conf
      chmod 644 /workspace/supervisord.conf
    else
      echo "⚠ Warning: supervisord.conf not found at $AI_ROOT/supervisord.conf"
    fi

    echo ""
    echo "✓ Supervisor installed successfully"
    echo ""
    echo "Configuration: /workspace/supervisord.conf"
    echo "Logs: $LOGS_DIR/"
    echo ""
    echo "Services configured:"
    echo "  - comfyui: ComfyUI server (port 8188) - autostart enabled"
    echo "  - orchestrator: Model orchestrator (port 9000) - autostart disabled"
    echo "  - webdav-sync: WebDAV output sync service - autostart enabled"
    echo ""
    echo "To start: supervisord -c /workspace/supervisord.conf"
    echo "To manage: supervisorctl status"
    echo "Web UI: http://localhost:9001 (admin/runpod2024)"

  #
  # Utility Scripts
  #
  setup/validate: |
    echo "========================================="
    echo "  Validating Installation"
    echo "========================================="
    echo ""

    # Check Python packages
    echo "Installed Python packages:"
    pip3 list | grep -E "(fastapi|uvicorn|torch|transformers|diffusers)" || echo "  (some packages not found)"
    echo ""

    # Check GPU memory
    echo "GPU Memory:"
    nvidia-smi --query-gpu=name,memory.free,memory.total --format=csv,noheader
    echo ""

    # Check cache size
    if [ -d "$HF_CACHE" ]; then
      echo "Model cache size:"
      du -sh $HF_CACHE
    else
      echo "⚠ Warning: $HF_CACHE not found"
    fi
    echo ""

    # Verify service scripts
    echo "Service scripts:"
    for script in $AI_ROOT/scripts/*.sh; do
      if [ -f "$script" ]; then
        echo "  ✓ $(basename $script)"
        chmod +x "$script"
      fi
    done
    echo ""

    echo "✓ Validation complete"

  setup/cleanup: |
    echo "========================================="
    echo "  Cleaning Up for Template Creation"
    echo "========================================="
    echo ""

    # Remove sensitive files
    echo "Removing sensitive files..."
    rm -f $AI_ROOT/.env
    rm -f /root/.ssh/known_hosts
    rm -f /root/.bash_history
    rm -f /root/.python_history

    # Clear system logs
    echo "Clearing system logs..."
    sudo find /var/log -type f -name "*.log" -delete 2>/dev/null || true

    # Create template version marker
    cat > /workspace/TEMPLATE_VERSION << 'EOF'
    RunPod Multi-Modal AI Template (Process-Based Architecture)
    Version: 2.0

    Components:
    - Python 3.10
    - Orchestrator (process-based)
    - ComfyUI with 20 production workflows
    - Supervisor process manager
    - Tailscale VPN

    Architecture: No Docker, direct Python execution

    Deployment:
    1. Create .env file with HF_TOKEN
    2. Run model downloads (Ansible)
    3. Start services: arty run services/start

    EOF

    echo ""
    echo "✓ Cleanup complete"
    echo ""
    echo "Next steps in RunPod dashboard:"
    echo "  1. Stop all services"
    echo "  2. My Pods → Select pod → ⋮ → Save as Template"
    echo "  3. Name: multi-modal-ai-v2.0"
    echo "  4. Save and test deployment"

  install/full: |
    echo "========================================="
    echo "  Full Installation"
    echo "========================================="
    echo ""
    echo "Installing: All components + Tailscale"
    echo ""

    arty run setup/system-packages && \
    arty run setup/python-env && \
    arty run setup/comfyui-base && \
    arty run setup/comfyui-nodes && \
    arty run setup/tailscale && \
    arty run setup/supervisor

    echo ""
    echo "✓ Full installation complete"
    echo ""
    echo "Next steps:"
    echo "  1. Download models: ansible-playbook playbook.yml --tags comfyui-models-all"
    echo "  2. Link models: arty run models/link-comfyui"
    echo "  3. Link workflows: arty run workflows/link-comfyui"
    echo "  4. Configure Tailscale (see instructions above)"
    echo "  5. Start services: arty run services/start"

  # Workflow linking (link production workflows with category prefixes)
  workflows/link-comfyui: |
    # Create ComfyUI user workflows directory
    mkdir -p $COMFYUI_ROOT/user/default/workflows
    cd $COMFYUI_ROOT/user/default/workflows

    # Remove old symlinks from previous location
    rm -f $COMFYUI_ROOT/workflows/*.json 2>/dev/null || true

    # Clear existing workflow symlinks to avoid conflicts
    find . -type l -name "*.json" -delete

    # Absolute path to workflows from /workspace/ai
    SOURCE_DIR="$AI_ROOT/comfyui/workflows"

    # Text-to-Image workflows (prefix: t2i_)
    for file in "$SOURCE_DIR/text-to-image"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "t2i_${basename}"
      fi
    done

    # Image-to-Image workflows (prefix: i2i_)
    for file in "$SOURCE_DIR/image-to-image"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "i2i_${basename}"
      fi
    done

    # Image-to-Video workflows (prefix: i2v_)
    for file in "$SOURCE_DIR/image-to-video"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "i2v_${basename}"
      fi
    done

    # Text-to-Music workflows (prefix: t2m_)
    for file in "$SOURCE_DIR/text-to-music"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "t2m_${basename}"
      fi
    done

    # Upscaling workflows (prefix: upscale_)
    for file in "$SOURCE_DIR/upscaling"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "upscale_${basename}"
      fi
    done

    # Advanced workflows (prefix: adv_)
    for file in "$SOURCE_DIR/advanced"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "adv_${basename}"
      fi
    done

    # NSFW workflows (prefix: nsfw_)
    for file in "$SOURCE_DIR/nsfw"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "nsfw_${basename}"
      fi
    done

    # Count workflows
    WORKFLOW_COUNT=$(ls -1 *.json 2>/dev/null | wc -l)

    echo "Production workflows linked to ComfyUI user directory"
    echo "  Total workflows: $WORKFLOW_COUNT"
    echo "  Location: $COMFYUI_ROOT/user/default/workflows/"
    echo ""
    echo "Categories (by prefix):"
    echo "  - t2i_    : 5 text-to-image workflows (FLUX, SDXL, SD3.5, LoRA Fusion)"
    echo "  - i2i_    : 3 image-to-image workflows (IP-Adapter)"
    echo "  - i2v_    : 3 image-to-video workflows (CogVideoX, SVD)"
    echo "  - t2m_    : 4 text-to-music workflows (MusicGen)"
    echo "  - upscale_: 3 upscaling workflows (Ultimate SD, Simple, Face)"
    echo "  - adv_    : 3 advanced workflows (ControlNet, AnimateDiff, Batch)"
    echo "  - nsfw_   : 4 NSFW workflows (LUSTIFY, Pony, RealVisXL, Ultimate Upscale)"
    echo ""
    echo "Total: 25 production workflows"
    echo "Workflows will appear in ComfyUI's workflow browser, sorted by category prefix"

  #
  # Service Management (Supervisor-based)
  #
  # All services
  services/start: supervisorctl -c /workspace/supervisord.conf start ai-services:*
  services/stop: supervisorctl -c /workspace/supervisord.conf stop ai-services:*
  services/restart: supervisorctl -c /workspace/supervisord.conf restart ai-services:*
  services/status: supervisorctl -c /workspace/supervisord.conf status

  # ComfyUI service
  services/comfyui/start: supervisorctl -c /workspace/supervisord.conf start ai-services:comfyui
  services/comfyui/stop: supervisorctl -c /workspace/supervisord.conf stop ai-services:comfyui
  services/comfyui/restart: supervisorctl -c /workspace/supervisord.conf restart ai-services:comfyui
  services/comfyui/status: supervisorctl -c /workspace/supervisord.conf status ai-services:comfyui
  services/comfyui/logs: supervisorctl -c /workspace/supervisord.conf tail -f ai-services:comfyui

  # Orchestrator service
  services/orchestrator/start: supervisorctl -c /workspace/supervisord.conf start ai-services:orchestrator
  services/orchestrator/stop: supervisorctl -c /workspace/supervisord.conf stop ai-services:orchestrator
  services/orchestrator/restart: supervisorctl -c /workspace/supervisord.conf restart ai-services:orchestrator
  services/orchestrator/status: supervisorctl -c /workspace/supervisord.conf status ai-services:orchestrator
  services/orchestrator/logs: supervisorctl -c /workspace/supervisord.conf tail -f ai-services:orchestrator

  # WebDAV Sync service
  services/webdav-sync/start: supervisorctl -c /workspace/supervisord.conf start ai-services:webdav-sync
  services/webdav-sync/stop: supervisorctl -c /workspace/supervisord.conf stop ai-services:webdav-sync
  services/webdav-sync/restart: supervisorctl -c /workspace/supervisord.conf restart ai-services:webdav-sync
  services/webdav-sync/status: supervisorctl -c /workspace/supervisord.conf status ai-services:webdav-sync
  services/webdav-sync/logs: supervisorctl -c /workspace/supervisord.conf tail -f ai-services:webdav-sync

  #
  # Health Checks
  #
  health/orchestrator: curl http://localhost:9000/health
  health/comfyui: curl http://localhost:8188
  health/vllm: curl http://localhost:8000/health

  #
  # System Checks
  #
  check/gpu: nvidia-smi
  check/disk: df -h /workspace
  check/models: du -sh $HF_CACHE
  check/cache: find $HF_CACHE -type d -name 'models--*' -maxdepth 1

# Deployment notes
notes: |
  RunPod AI Model Orchestrator - Quick Start Guide

  ========================================
  FRESH DEPLOYMENT
  ========================================

  1. Clone Git Repositories:
     arty sync --env prod

  2. Run Installation:
     # Minimal (System + Python + ComfyUI base + Supervisor)
     arty run install/minimal

     # Essential (+ Custom nodes)
     arty run install/essential

     # Full (+ Tailscale VPN)
     arty run install/full

  3. Configure Environment:
     cd $AI_ROOT
     cp .env.example .env
     # Edit .env and set HF_TOKEN

  4. Download Models (using Ansible for now):
     # Essential models (~80GB)
     ansible-playbook playbook.yml --tags comfyui-essential

     # All models (~137GB)
     ansible-playbook playbook.yml --tags comfyui-models-all

  5. Link Models and Workflows:
     arty run models/link-comfyui
     arty run workflows/link-comfyui

  6. Start Services:
     arty run services/start

  ========================================
  INSTALLATION SCRIPTS
  ========================================

  System Setup:
  - arty run setup/system-packages  # Install apt packages, verify GPU
  - arty run setup/python-env        # Setup Python, pip, core packages
  - arty run setup/comfyui-base      # Install ComfyUI, create directories
  - arty run setup/comfyui-nodes     # Install 5 essential custom nodes
  - arty run setup/supervisor        # Install Supervisor process manager
  - arty run setup/tailscale         # Install Tailscale VPN (optional)

  Utility Scripts:
  - arty run setup/validate          # Validate installation
  - arty run setup/cleanup           # Cleanup for template creation

  Orchestration (run multiple steps):
  - arty run install/minimal         # Fast minimal setup
  - arty run install/essential       # Recommended for most users
  - arty run install/full            # Everything including Tailscale

  ========================================
  SERVICE MANAGEMENT
  ========================================

  All Services:
  - arty services/start              # Start all services
  - arty services/stop               # Stop all services
  - arty services/restart            # Restart all services
  - arty services/status             # Check all services status

  ComfyUI Service:
  - arty services/comfyui/start      # Start ComfyUI
  - arty services/comfyui/stop       # Stop ComfyUI
  - arty services/comfyui/restart    # Restart ComfyUI
  - arty services/comfyui/status     # Check ComfyUI status
  - arty services/comfyui/logs       # Follow ComfyUI logs

  Orchestrator Service:
  - arty services/orchestrator/start    # Start orchestrator
  - arty services/orchestrator/stop     # Stop orchestrator
  - arty services/orchestrator/restart  # Restart orchestrator
  - arty services/orchestrator/status   # Check orchestrator status
  - arty services/orchestrator/logs     # Follow orchestrator logs

  ========================================
  HEALTH & MONITORING
  ========================================

  Health Checks:
  - arty run health/orchestrator     # curl http://localhost:9000/health
  - arty run health/comfyui          # curl http://localhost:8188
  - arty run health/vllm             # curl http://localhost:8000/health

  System Checks:
  - arty run check/gpu               # nvidia-smi
  - arty run check/disk              # Disk usage
  - arty run check/models            # Model cache size
  - arty run check/cache             # List cached models

  ========================================
  COMFYUI WORKFLOWS
  ========================================

  Location: $COMFYUI_ROOT/workflows/

  Link workflows: arty run workflows/link-comfyui

  25 Production Workflows:
  - Text-to-Image: FLUX Schnell, FLUX Dev, SDXL+Refiner, SD3.5, LoRA Fusion
  - Image-to-Image: IP-Adapter (style, face, composition)
  - Image-to-Video: CogVideoX, SVD, SVD-XT
  - Text-to-Music: MusicGen (small/medium/large/melody)
  - Upscaling: Ultimate SD, Simple, Face-focused
  - Advanced: ControlNet, AnimateDiff, Batch processing
  - NSFW: LUSTIFY Realistic, Pony Diffusion, RealVisXL Lightning, Ultimate Upscale

  Documentation: README.md, WORKFLOW_STANDARDS.md, nsfw/README.md

  ========================================
  ENVIRONMENT PROFILES
  ========================================

  - arty sync --env prod     # Production (essential repos only)
  - arty sync --env dev      # Development (all repos)
  - arty sync --env minimal  # Minimal (orchestrator + ComfyUI)

  ========================================
  IMPORTANT PATHS
  ========================================

  Configuration:
  - $AI_ROOT/arty.yml              # This file
  - $AI_ROOT/.env                  # Environment variables
  - /workspace/supervisord.conf         # Supervisor config

  Code:
  - $AI_ROOT/                      # Project directory
  - $COMFYUI_ROOT/                 # ComfyUI installation
  - $AI_ROOT/scripts/*.sh          # Service scripts

  Models:
  - $HF_CACHE/       # Model cache (~401GB)
  - $COMFYUI_ROOT/models/          # Symlinks to cache

  Workflows:
  - $AI_ROOT/comfyui/workflows/  # Source (git)
  - $COMFYUI_ROOT/workflows/            # Linked workflows

  Logs:
  - $LOGS_DIR/                    # Supervisor logs

  ========================================
  PORTS
  ========================================

  - 9000: Model Orchestrator
  - 8188: ComfyUI
  - 8000+: vLLM servers
  - 9001: Supervisor web UI (admin/runpod2024)

  ========================================
  DOCUMENTATION
  ========================================

  - $AI_ROOT/README.md
  - $AI_ROOT/CLAUDE.md
  - $AI_ROOT/COMFYUI_MODELS.md
  - $AI_ROOT/MODELS_LINKED.md
  - $AI_ROOT/comfyui/workflows/README.md