runpod/arty.yml

name: "RunPod AI Model Orchestrator"
version: "2.0.0"
description: "Process-based AI model orchestrator for RunPod GPU instances with ComfyUI integration"
author: "valknar@pivoine.art"
license: "MIT"

# Git repositories to clone for a fresh RunPod deployment
references:
  # ComfyUI base installation
  - url: https://github.com/comfyanonymous/ComfyUI.git
    into: $COMFYUI_ROOT
    description: "ComfyUI - Node-based interface for image/video/audio generation"

  # ComfyUI Essential Custom Nodes
  - url: https://github.com/ltdrdata/ComfyUI-Manager.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-Manager
    description: "ComfyUI Manager - Install/manage custom nodes and models"
    essential: true

  - url: https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-VideoHelperSuite
    description: "Video operations and processing"
    essential: true

  - url: https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-AnimateDiff-Evolved
    description: "AnimateDiff for video generation"
    essential: true

  - url: https://github.com/cubiq/ComfyUI_IPAdapter_plus.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI_IPAdapter_plus
    description: "IP-Adapter for style transfer"
    essential: true

  - url: https://github.com/ltdrdata/ComfyUI-Impact-Pack.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-Impact-Pack
    description: "Auto face enhancement and detailer"
    essential: true

  # ComfyUI Optional Custom Nodes
  - url: https://github.com/kijai/ComfyUI-CogVideoXWrapper.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-CogVideoXWrapper
    description: "CogVideoX integration for text-to-video"
    essential: false

  - url: https://github.com/ltdrdata/ComfyUI-Inspire-Pack.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-Inspire-Pack
    description: "Additional inspiration tools"
    essential: false

  - url: https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-Advanced-ControlNet
    description: "Advanced ControlNet features"
    essential: false

  - url: https://github.com/MrForExample/ComfyUI-3D-Pack.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI-3D-Pack
    description: "3D asset generation"
    essential: false

  - url: https://github.com/MixLabPro/comfyui-sound-lab.git
    into: $COMFYUI_ROOT/custom_nodes/comfyui-sound-lab
    description: "MusicGen and Stable Audio integration"
    essential: false

  - url: https://github.com/ssitu/ComfyUI_UltimateSDUpscale.git
    into: $COMFYUI_ROOT/custom_nodes/ComfyUI_UltimateSDUpscale
    description: "Ultimate SD Upscale for high-quality image upscaling"
    essential: false

# Environment profiles for selective repository management
envs:
  # Production: Only essential components
  prod:
    - $AI_ROOT
    - $COMFYUI_ROOT
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-Manager
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-VideoHelperSuite
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-AnimateDiff-Evolved
    - $COMFYUI_ROOT/custom_nodes/ComfyUI_IPAdapter_plus
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-Impact-Pack

  # Development: All repositories including optional nodes
  dev:
    - $AI_ROOT
    - $COMFYUI_ROOT
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-Manager
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-VideoHelperSuite
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-AnimateDiff-Evolved
    - $COMFYUI_ROOT/custom_nodes/ComfyUI_IPAdapter_plus
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-Impact-Pack
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-CogVideoXWrapper
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-Inspire-Pack
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-Advanced-ControlNet
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-3D-Pack
    - $COMFYUI_ROOT/custom_nodes/comfyui-sound-lab

  # Minimal: Only orchestrator and ComfyUI base
  minimal:
    - $AI_ROOT
    - $COMFYUI_ROOT
    - $COMFYUI_ROOT/custom_nodes/ComfyUI-Manager

# Deployment scripts for RunPod instances
scripts:
  #
  # System Setup Scripts
  #
  setup/system-packages: |
    echo "========================================="
    echo "  Installing System Packages"
    echo "========================================="
    echo ""

    # Check GPU availability
    if ! nvidia-smi > /dev/null 2>&1; then
      echo "❌ ERROR: nvidia-smi not found. GPU not available!"
      exit 1
    fi
    echo "✓ GPU detected:"
    nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
    echo ""

    # Update apt cache
    echo "Updating apt cache..."
    sudo apt update

    # Install base system packages
    echo "Installing system packages..."
    sudo apt install -y \
      build-essential \
      python3-dev \
      python3-pip \
      python3-venv \
      git \
      curl \
      wget \
      vim \
      htop \
      tmux \
      net-tools \
      davfs2

    echo ""
    echo "✓ System packages installed successfully"

  setup/python-env: |
    echo "========================================="
    echo "  Setting Up Python Environment"
    echo "========================================="
    echo ""

    # Upgrade pip
    echo "Upgrading pip to 23.3.1..."
    sudo pip3 install --upgrade pip==23.3.1

    # Install core Python packages
    echo "Installing core Python packages..."
    if [ -f "$AI_ROOT/core/requirements.txt" ]; then
      sudo pip3 install -r $AI_ROOT/core/requirements.txt
    else
      echo "⚠ Warning: $AI_ROOT/core/requirements.txt not found"
    fi

    # Install vLLM dependencies
    echo "Installing vLLM dependencies..."
    if [ -f "vllm/requirements.txt" ]; then
      sudo pip3 install -r vllm/requirements.txt
    else
      echo "⚠ Warning: vllm/requirements.txt not found"
    fi

    echo ""
    echo "✓ Python environment configured successfully"

  setup/comfyui-base: |
    echo "========================================="
    echo "  Installing ComfyUI Base"
    echo "========================================="
    echo ""

    # Clone ComfyUI if not exists
    if [ ! -d "$COMFYUI_ROOT" ]; then
      echo "Cloning ComfyUI repository..."
      git clone https://github.com/comfyanonymous/ComfyUI.git $COMFYUI_ROOT
    else
      echo "ComfyUI already exists, pulling latest changes..."
      cd $COMFYUI_ROOT && git pull
    fi

    # Install ComfyUI dependencies
    echo "Installing ComfyUI dependencies..."
    sudo pip3 install -r $COMFYUI_ROOT/requirements.txt

    # Install additional ComfyUI dependencies
    if [ -f "comfyui/requirements.txt" ]; then
      echo "Installing additional ComfyUI dependencies..."
      sudo pip3 install -r comfyui/requirements.txt
    fi

    # Create model directory structure
    echo "Creating ComfyUI model directories..."
    mkdir -p $COMFYUI_ROOT/models/{checkpoints,unet,vae,loras,clip,clip_vision,controlnet,ipadapter,embeddings,upscale_models,video_models,animatediff_models,animatediff_motion_lora,audio_models,configs,diffusers,diffusion_models}

    # Create symlink to huggingface cache
    echo "Creating symlink to HuggingFace cache..."
    ln -sf $HF_CACHE $COMFYUI_ROOT/models/huggingface_cache

    # Make start script executable
    if [ -f "comfyui/start.sh" ]; then
      chmod +x comfyui/start.sh
    fi

    echo ""
    echo "✓ ComfyUI base installed successfully"
    echo "  Directory: $COMFYUI_ROOT"
    echo "  Port: 8188"
    echo "  Start: bash comfyui/start.sh"

  setup/comfyui-nodes: |
    echo "========================================="
    echo "  Installing ComfyUI Custom Nodes"
    echo "========================================="
    echo ""

    cd $COMFYUI_ROOT/custom_nodes

    # ComfyUI Manager
    echo "[1/5] Installing ComfyUI-Manager..."
    if [ ! -d "ComfyUI-Manager" ]; then
      git clone https://github.com/ltdrdata/ComfyUI-Manager.git
    fi
    [ -f "ComfyUI-Manager/requirements.txt" ] && sudo pip3 install -r ComfyUI-Manager/requirements.txt

    # VideoHelperSuite
    echo "[2/5] Installing ComfyUI-VideoHelperSuite..."
    if [ ! -d "ComfyUI-VideoHelperSuite" ]; then
      git clone https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git
    fi
    [ -f "ComfyUI-VideoHelperSuite/requirements.txt" ] && sudo pip3 install -r ComfyUI-VideoHelperSuite/requirements.txt

    # AnimateDiff-Evolved
    echo "[3/5] Installing ComfyUI-AnimateDiff-Evolved..."
    if [ ! -d "ComfyUI-AnimateDiff-Evolved" ]; then
      git clone https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved.git
    fi
    [ -f "ComfyUI-AnimateDiff-Evolved/requirements.txt" ] && sudo pip3 install -r ComfyUI-AnimateDiff-Evolved/requirements.txt

    # IPAdapter Plus
    echo "[4/5] Installing ComfyUI_IPAdapter_plus..."
    if [ ! -d "ComfyUI_IPAdapter_plus" ]; then
      git clone https://github.com/cubiq/ComfyUI_IPAdapter_plus.git
    fi
    [ -f "ComfyUI_IPAdapter_plus/requirements.txt" ] && sudo pip3 install -r ComfyUI_IPAdapter_plus/requirements.txt

    # Impact-Pack
    echo "[5/5] Installing ComfyUI-Impact-Pack..."
    if [ ! -d "ComfyUI-Impact-Pack" ]; then
      git clone https://github.com/ltdrdata/ComfyUI-Impact-Pack.git
    fi
    [ -f "ComfyUI-Impact-Pack/requirements.txt" ] && sudo pip3 install -r ComfyUI-Impact-Pack/requirements.txt

    # Fix numpy version for vLLM compatibility
    echo "Fixing numpy version..."
    sudo pip3 install 'numpy<2.0.0' --force-reinstall

    echo ""
    echo "✓ Essential custom nodes installed successfully"
    echo "  - ComfyUI-Manager: Install/manage custom nodes"
    echo "  - VideoHelperSuite: Video operations"
    echo "  - AnimateDiff-Evolved: Video generation"
    echo "  - IPAdapter_plus: Style transfer"
    echo "  - Impact-Pack: Face enhancement"

  setup/tailscale: |
    echo "========================================="
    echo "  Installing Tailscale VPN"
    echo "========================================="
    echo ""

    # Check if already installed
    if command -v tailscale > /dev/null 2>&1; then
      echo "✓ Tailscale already installed"
      tailscale version
    else
      echo "Installing Tailscale..."
      curl -fsSL https://tailscale.com/install.sh | sh
      echo "✓ Tailscale installed successfully"
    fi

    echo ""
    echo "To connect Tailscale:"
    echo "  1. Start daemon: tailscaled --tun=userspace-networking --socks5-server=localhost:1055 &"
    echo "  2. Authenticate: tailscale up --advertise-tags=tag:gpu"
    echo "  3. Get IP: tailscale ip -4"
    echo ""
    echo "Note: Authentication requires manual URL visit"

  setup/supervisor: |
    echo "========================================="
    echo "  Installing Supervisor"
    echo "========================================="
    echo ""

    # Install Supervisor
    echo "Installing Supervisor via pip..."
    sudo pip3 install supervisor

    # Create logs directory
    mkdir -p $LOGS_DIR

    # Copy supervisor configuration
    if [ -f "$AI_ROOT/supervisord.conf" ]; then
      echo "Deploying supervisord configuration..."
      cp $AI_ROOT/supervisord.conf /workspace/supervisord.conf
      chmod 644 /workspace/supervisord.conf
    else
      echo "⚠ Warning: supervisord.conf not found at $AI_ROOT/supervisord.conf"
    fi

    echo ""
    echo "✓ Supervisor installed successfully"
    echo ""
    echo "Configuration: /workspace/supervisord.conf"
    echo "Logs: $LOGS_DIR/"
    echo ""
    echo "Services configured:"
    echo "  - comfyui: ComfyUI server (port 8188) - autostart enabled"
    echo "  - orchestrator: Model orchestrator (port 9000) - autostart disabled"
    echo "  - webdav-sync: WebDAV output sync service - autostart enabled"
    echo ""
    echo "To start: supervisord -c /workspace/supervisord.conf"
    echo "To manage: supervisorctl status"
    echo "Web UI: http://localhost:9001 (admin/runpod2024)"

  setup/webdav: |
    echo "========================================="
    echo "  Setting Up WebDAV Mount (HiDrive)"
    echo "========================================="
    echo ""

    # Install davfs2 if not present
    if ! command -v mount.davfs >/dev/null 2>&1; then
      echo "Installing davfs2..."
      DEBIAN_FRONTEND=noninteractive apt update && DEBIAN_FRONTEND=noninteractive apt install -y davfs2
    fi

    # Create mount point
    echo "Creating mount point..."
    mkdir -p /mnt/hidrive

    # Create davfs2 secrets file
    echo "Configuring WebDAV credentials..."
    mkdir -p /etc/davfs2
    echo "https://webdav.hidrive.ionos.com/ valknar MwRTW4hR.eRbipQ" | tee /etc/davfs2/secrets > /dev/null
    chmod 600 /etc/davfs2/secrets

    # Configure davfs2
    sed -i 's/# use_locks       1/use_locks       0/' /etc/davfs2/davfs2.conf 2>/dev/null || true

    # Mount WebDAV
    echo "Mounting HiDrive WebDAV..."
    if mount -t davfs https://webdav.hidrive.ionos.com/ /mnt/hidrive; then
      echo "✓ HiDrive mounted successfully"
    else
      echo "⚠ Warning: Mount failed, you may need to mount manually"
      echo "  Try: mount -t davfs https://webdav.hidrive.ionos.com/ /mnt/hidrive"
    fi

    # Create ComfyUI output directory
    echo "Creating ComfyUI output directory..."
    mkdir -p /mnt/hidrive/users/valknar/Pictures/AI/ComfyUI

    # Create symlink in ComfyUI
    echo "Creating symlink in ComfyUI..."
    ln -sf /mnt/hidrive/users/valknar/Pictures/AI/ComfyUI $COMFYUI_ROOT/output_hidrive

    echo ""
    echo "✓ WebDAV setup complete"
    echo ""
    echo "Mount point: /mnt/hidrive"
    echo "ComfyUI output: /mnt/hidrive/users/valknar/Pictures/AI/ComfyUI"
    echo "ComfyUI symlink: $COMFYUI_ROOT/output_hidrive"
    echo ""
    echo "To unmount: umount /mnt/hidrive"
    echo "To remount: mount -t davfs https://webdav.hidrive.ionos.com/ /mnt/hidrive"

  #
  # Utility Scripts
  #
  setup/validate: |
    echo "========================================="
    echo "  Validating Installation"
    echo "========================================="
    echo ""

    # Check Python packages
    echo "Installed Python packages:"
    pip3 list | grep -E "(fastapi|uvicorn|torch|transformers|diffusers)" || echo "  (some packages not found)"
    echo ""

    # Check GPU memory
    echo "GPU Memory:"
    nvidia-smi --query-gpu=name,memory.free,memory.total --format=csv,noheader
    echo ""

    # Check cache size
    if [ -d "$HF_CACHE" ]; then
      echo "Model cache size:"
      du -sh $HF_CACHE
    else
      echo "⚠ Warning: $HF_CACHE not found"
    fi
    echo ""

    # Verify service scripts
    echo "Service scripts:"
    for script in $AI_ROOT/scripts/*.sh; do
      if [ -f "$script" ]; then
        echo "  ✓ $(basename $script)"
        chmod +x "$script"
      fi
    done
    echo ""

    echo "✓ Validation complete"

  setup/cleanup: |
    echo "========================================="
    echo "  Cleaning Up for Template Creation"
    echo "========================================="
    echo ""

    # Remove sensitive files
    echo "Removing sensitive files..."
    rm -f $AI_ROOT/.env
    rm -f /root/.ssh/known_hosts
    rm -f /root/.bash_history
    rm -f /root/.python_history

    # Clear system logs
    echo "Clearing system logs..."
    sudo find /var/log -type f -name "*.log" -delete 2>/dev/null || true

    # Create template version marker
    cat > /workspace/TEMPLATE_VERSION << 'EOF'
    RunPod Multi-Modal AI Template (Process-Based Architecture)
    Version: 2.0

    Components:
    - Python 3.10
    - Orchestrator (process-based)
    - ComfyUI with 20 production workflows
    - Supervisor process manager
    - Tailscale VPN

    Architecture: No Docker, direct Python execution

    Deployment:
    1. Create .env file with HF_TOKEN
    2. Run model downloads (Ansible)
    3. Start services: arty run services/start

    EOF

    echo ""
    echo "✓ Cleanup complete"
    echo ""
    echo "Next steps in RunPod dashboard:"
    echo "  1. Stop all services"
    echo "  2. My Pods → Select pod → ⋮ → Save as Template"
    echo "  3. Name: multi-modal-ai-v2.0"
    echo "  4. Save and test deployment"

  #
  # Orchestration Scripts
  #
  install/minimal: |
    echo "========================================="
    echo "  Minimal Installation"
    echo "========================================="
    echo ""
    echo "Installing: System + Python + ComfyUI + Supervisor"
    echo ""

    arty run setup/system-packages && \
    arty run setup/python-env && \
    arty run setup/comfyui-base && \
    arty run setup/supervisor

    echo ""
    echo "✓ Minimal installation complete"
    echo ""
    echo "Next steps:"
    echo "  1. Download models: Use Ansible playbook"
    echo "  2. Link models: arty run models/link-comfyui"
    echo "  3. Start services: arty run services/start"

  install/essential: |
    echo "========================================="
    echo "  Essential Installation"
    echo "========================================="
    echo ""
    echo "Installing: System + Python + ComfyUI + Nodes + Supervisor"
    echo ""

    arty run setup/system-packages && \
    arty run setup/python-env && \
    arty run setup/comfyui-base && \
    arty run setup/comfyui-nodes && \
    arty run setup/supervisor

    echo ""
    echo "✓ Essential installation complete"
    echo ""
    echo "Next steps:"
    echo "  1. Download models: ansible-playbook playbook.yml --tags comfyui-essential"
    echo "  2. Link models: arty run models/link-comfyui"
    echo "  3. Link workflows: arty run workflows/link-comfyui"
    echo "  4. Start services: arty run services/start"

  install/full: |
    echo "========================================="
    echo "  Full Installation"
    echo "========================================="
    echo ""
    echo "Installing: All components + Tailscale"
    echo ""

    arty run setup/system-packages && \
    arty run setup/python-env && \
    arty run setup/comfyui-base && \
    arty run setup/comfyui-nodes && \
    arty run setup/tailscale && \
    arty run setup/supervisor

    echo ""
    echo "✓ Full installation complete"
    echo ""
    echo "Next steps:"
    echo "  1. Download models: ansible-playbook playbook.yml --tags comfyui-models-all"
    echo "  2. Link models: arty run models/link-comfyui"
    echo "  3. Link workflows: arty run workflows/link-comfyui"
    echo "  4. Configure Tailscale (see instructions above)"
    echo "  5. Start services: arty run services/start"

  #
  # Legacy Setup (deprecated - use install/* instead)
  #
  setup/full-legacy: |
    cd $AI_ROOT
    cp .env.example .env
    echo "⚠ DEPRECATED: Use 'arty run install/full' instead"
    echo "Edit .env and set HF_TOKEN, then run: ansible-playbook playbook.yml"

  setup/essential-legacy: |
    cd $AI_ROOT
    cp .env.example .env
    echo "⚠ DEPRECATED: Use 'arty run install/essential' instead"
    echo "Edit .env and set HF_TOKEN, then run: ansible-playbook playbook.yml --tags comfyui-essential"

  # Model linking (run after models are downloaded)
  models/link-comfyui: |
    cd $COMFYUI_ROOT/models/diffusers
    ln -sf $HF_CACHE/models--black-forest-labs--FLUX.1-schnell FLUX.1-schnell
    ln -sf $HF_CACHE/models--black-forest-labs--FLUX.1-dev FLUX.1-dev
    ln -sf $HF_CACHE/models--stabilityai--stable-diffusion-xl-base-1.0 stable-diffusion-xl-base-1.0
    ln -sf $HF_CACHE/models--stabilityai--stable-diffusion-xl-refiner-1.0 stable-diffusion-xl-refiner-1.0
    ln -sf $HF_CACHE/models--stabilityai--stable-diffusion-3.5-large stable-diffusion-3.5-large
    cd $COMFYUI_ROOT/models/clip_vision
    ln -sf $HF_CACHE/models--openai--clip-vit-large-patch14 clip-vit-large-patch14
    ln -sf $HF_CACHE/models--laion--CLIP-ViT-bigG-14-laion2B-39B-b160k CLIP-ViT-bigG-14
    ln -sf $HF_CACHE/models--google--siglip-so400m-patch14-384 siglip-so400m-patch14-384
    cd $COMFYUI_ROOT/models/diffusion_models
    ln -sf $HF_CACHE/models--THUDM--CogVideoX-5b CogVideoX-5b
    ln -sf $HF_CACHE/models--stabilityai--stable-video-diffusion-img2vid stable-video-diffusion-img2vid
    ln -sf $HF_CACHE/models--stabilityai--stable-video-diffusion-img2vid-xt stable-video-diffusion-img2vid-xt
    echo "Models linked to ComfyUI"

  # Workflow linking (link production workflows with category prefixes)
  workflows/link-comfyui: |
    # Create ComfyUI user workflows directory
    mkdir -p $COMFYUI_ROOT/user/default/workflows
    cd $COMFYUI_ROOT/user/default/workflows

    # Remove old symlinks from previous location
    rm -f $COMFYUI_ROOT/workflows/*.json 2>/dev/null || true

    # Clear existing workflow symlinks to avoid conflicts
    find . -type l -name "*.json" -delete

    SOURCE_DIR="comfyui/workflows"

    # Text-to-Image workflows (prefix: t2i_)
    for file in "$SOURCE_DIR/text-to-image"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "t2i_${basename}"
      fi
    done

    # Image-to-Image workflows (prefix: i2i_)
    for file in "$SOURCE_DIR/image-to-image"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "i2i_${basename}"
      fi
    done

    # Image-to-Video workflows (prefix: i2v_)
    for file in "$SOURCE_DIR/image-to-video"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "i2v_${basename}"
      fi
    done

    # Text-to-Music workflows (prefix: t2m_)
    for file in "$SOURCE_DIR/text-to-music"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "t2m_${basename}"
      fi
    done

    # Upscaling workflows (prefix: upscale_)
    for file in "$SOURCE_DIR/upscaling"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "upscale_${basename}"
      fi
    done

    # Advanced workflows (prefix: adv_)
    for file in "$SOURCE_DIR/advanced"/*.json; do
      if [ -f "$file" ]; then
        basename=$(basename "$file")
        ln -sf "$file" "adv_${basename}"
      fi
    done

    # Count workflows
    WORKFLOW_COUNT=$(ls -1 *.json 2>/dev/null | wc -l)

    echo "Production workflows linked to ComfyUI user directory"
    echo "  Total workflows: $WORKFLOW_COUNT"
    echo "  Location: $COMFYUI_ROOT/user/default/workflows/"
    echo ""
    echo "Categories (by prefix):"
    echo "  - t2i_   : 4 text-to-image workflows (FLUX, SDXL, SD3.5)"
    echo "  - i2i_   : 3 image-to-image workflows (IP-Adapter)"
    echo "  - i2v_   : 3 image-to-video workflows (CogVideoX, SVD)"
    echo "  - t2m_   : 4 text-to-music workflows (MusicGen)"
    echo "  - upscale_: 3 upscaling workflows"
    echo "  - adv_   : 3 advanced workflows (ControlNet, AnimateDiff, Batch)"
    echo ""
    echo "Workflows will appear in ComfyUI's workflow browser, sorted by category prefix"

  #
  # Service Management (Supervisor-based)
  #
  # All services
  services/start: supervisorctl -c /workspace/supervisord.conf start ai-services:*
  services/stop: supervisorctl -c /workspace/supervisord.conf stop ai-services:*
  services/restart: supervisorctl -c /workspace/supervisord.conf restart ai-services:*
  services/status: supervisorctl -c /workspace/supervisord.conf status

  # ComfyUI service
  services/comfyui/start: supervisorctl -c /workspace/supervisord.conf start ai-services:comfyui
  services/comfyui/stop: supervisorctl -c /workspace/supervisord.conf stop ai-services:comfyui
  services/comfyui/restart: supervisorctl -c /workspace/supervisord.conf restart ai-services:comfyui
  services/comfyui/status: supervisorctl -c /workspace/supervisord.conf status ai-services:comfyui
  services/comfyui/logs: supervisorctl -c /workspace/supervisord.conf tail -f ai-services:comfyui

  # Orchestrator service
  services/orchestrator/start: supervisorctl -c /workspace/supervisord.conf start ai-services:orchestrator
  services/orchestrator/stop: supervisorctl -c /workspace/supervisord.conf stop ai-services:orchestrator
  services/orchestrator/restart: supervisorctl -c /workspace/supervisord.conf restart ai-services:orchestrator
  services/orchestrator/status: supervisorctl -c /workspace/supervisord.conf status ai-services:orchestrator
  services/orchestrator/logs: supervisorctl -c /workspace/supervisord.conf tail -f ai-services:orchestrator

  # WebDAV Sync service
  services/webdav-sync/start: supervisorctl -c /workspace/supervisord.conf start ai-services:webdav-sync
  services/webdav-sync/stop: supervisorctl -c /workspace/supervisord.conf stop ai-services:webdav-sync
  services/webdav-sync/restart: supervisorctl -c /workspace/supervisord.conf restart ai-services:webdav-sync
  services/webdav-sync/status: supervisorctl -c /workspace/supervisord.conf status ai-services:webdav-sync
  services/webdav-sync/logs: supervisorctl -c /workspace/supervisord.conf tail -f ai-services:webdav-sync

  #
  # Health Checks
  #
  health/orchestrator: curl http://localhost:9000/health
  health/comfyui: curl http://localhost:8188
  health/vllm: curl http://localhost:8000/health

  #
  # System Checks
  #
  check/gpu: nvidia-smi
  check/disk: df -h /workspace
  check/models: du -sh $HF_CACHE
  check/cache: find $HF_CACHE -type d -name 'models--*' -maxdepth 1

# Deployment notes
notes: |
  RunPod AI Model Orchestrator - Quick Start Guide

  ========================================
  FRESH DEPLOYMENT
  ========================================

  1. Clone Git Repositories:
     arty sync --env prod

  2. Run Installation:
     # Minimal (System + Python + ComfyUI base + Supervisor)
     arty run install/minimal

     # Essential (+ Custom nodes)
     arty run install/essential

     # Full (+ Tailscale VPN)
     arty run install/full

  3. Configure Environment:
     cd $AI_ROOT
     cp .env.example .env
     # Edit .env and set HF_TOKEN

  4. Download Models (using Ansible for now):
     # Essential models (~80GB)
     ansible-playbook playbook.yml --tags comfyui-essential

     # All models (~137GB)
     ansible-playbook playbook.yml --tags comfyui-models-all

  5. Link Models and Workflows:
     arty run models/link-comfyui
     arty run workflows/link-comfyui

  6. Start Services:
     arty run services/start

  ========================================
  INSTALLATION SCRIPTS
  ========================================

  System Setup:
  - arty run setup/system-packages  # Install apt packages, verify GPU
  - arty run setup/python-env        # Setup Python, pip, core packages
  - arty run setup/comfyui-base      # Install ComfyUI, create directories
  - arty run setup/comfyui-nodes     # Install 5 essential custom nodes
  - arty run setup/supervisor        # Install Supervisor process manager
  - arty run setup/tailscale         # Install Tailscale VPN (optional)

  Utility Scripts:
  - arty run setup/validate          # Validate installation
  - arty run setup/cleanup           # Cleanup for template creation

  Orchestration (run multiple steps):
  - arty run install/minimal         # Fast minimal setup
  - arty run install/essential       # Recommended for most users
  - arty run install/full            # Everything including Tailscale

  ========================================
  SERVICE MANAGEMENT
  ========================================

  All Services:
  - arty services/start              # Start all services
  - arty services/stop               # Stop all services
  - arty services/restart            # Restart all services
  - arty services/status             # Check all services status

  ComfyUI Service:
  - arty services/comfyui/start      # Start ComfyUI
  - arty services/comfyui/stop       # Stop ComfyUI
  - arty services/comfyui/restart    # Restart ComfyUI
  - arty services/comfyui/status     # Check ComfyUI status
  - arty services/comfyui/logs       # Follow ComfyUI logs

  Orchestrator Service:
  - arty services/orchestrator/start    # Start orchestrator
  - arty services/orchestrator/stop     # Stop orchestrator
  - arty services/orchestrator/restart  # Restart orchestrator
  - arty services/orchestrator/status   # Check orchestrator status
  - arty services/orchestrator/logs     # Follow orchestrator logs

  ========================================
  HEALTH & MONITORING
  ========================================

  Health Checks:
  - arty run health/orchestrator     # curl http://localhost:9000/health
  - arty run health/comfyui          # curl http://localhost:8188
  - arty run health/vllm             # curl http://localhost:8000/health

  System Checks:
  - arty run check/gpu               # nvidia-smi
  - arty run check/disk              # Disk usage
  - arty run check/models            # Model cache size
  - arty run check/cache             # List cached models

  ========================================
  COMFYUI WORKFLOWS
  ========================================

  Location: $COMFYUI_ROOT/workflows/

  Link workflows: arty run workflows/link-comfyui

  20 Production Workflows:
  - Text-to-Image: FLUX Schnell, FLUX Dev, SDXL+Refiner, SD3.5
  - Image-to-Image: IP-Adapter (style, face, composition)
  - Image-to-Video: CogVideoX, SVD, SVD-XT
  - Text-to-Music: MusicGen (small/medium/large/melody)
  - Upscaling: Ultimate SD, Simple, Face-focused
  - Advanced: ControlNet, AnimateDiff, Batch processing

  Documentation: README.md, WORKFLOW_STANDARDS.md

  ========================================
  ENVIRONMENT PROFILES
  ========================================

  - arty sync --env prod     # Production (essential repos only)
  - arty sync --env dev      # Development (all repos)
  - arty sync --env minimal  # Minimal (orchestrator + ComfyUI)

  ========================================
  IMPORTANT PATHS
  ========================================

  Configuration:
  - $AI_ROOT/arty.yml              # This file
  - $AI_ROOT/.env                  # Environment variables
  - /workspace/supervisord.conf         # Supervisor config

  Code:
  - $AI_ROOT/                      # Project directory
  - $COMFYUI_ROOT/                 # ComfyUI installation
  - $AI_ROOT/scripts/*.sh          # Service scripts

  Models:
  - $HF_CACHE/       # Model cache (~401GB)
  - $COMFYUI_ROOT/models/          # Symlinks to cache

  Workflows:
  - $AI_ROOT/comfyui/workflows/  # Source (git)
  - $COMFYUI_ROOT/workflows/            # Linked workflows

  Logs:
  - $LOGS_DIR/                    # Supervisor logs

  ========================================
  PORTS
  ========================================

  - 9000: Model Orchestrator
  - 8188: ComfyUI
  - 8000+: vLLM servers
  - 9001: Supervisor web UI (admin/runpod2024)

  ========================================
  DOCUMENTATION
  ========================================

  - $AI_ROOT/README.md
  - $AI_ROOT/CLAUDE.md
  - $AI_ROOT/COMFYUI_MODELS.md
  - $AI_ROOT/MODELS_LINKED.md
  - $AI_ROOT/comfyui/workflows/README.md