feat: github workflow

.github/workflows/README.md

@@ -8,15 +8,36 @@ This directory contains automated workflows for building, testing, and deploying
 
 **Triggers:**
 - Schedule: Every 6 hours
-- Manual: `workflow_dispatch`
+- Manual: `workflow_dispatch` with options:
+  - `build_mode`: `download` (default) or `build`
+  - `awesome_repo`: Source repository (default: `valknarness/awesome`)
 - Push to main (when db.yml or build-db.js changes)
 
 **Purpose:**
-Builds the SQLite database by scraping awesome lists from GitHub.
+Builds the SQLite database using one of two methods:
+
+1. **Download Mode** (default, fast ~5 min):
+   - Downloads pre-built database from the awesome CLI repository
+   - Uses GitHub Actions artifacts from the upstream repo
+   - Fallback to local build if download fails
+
+2. **Build Mode** (slow ~1-2 hours):
+   - Clones the awesome CLI repository
+   - Runs full indexing using `./awesome index`
+   - Scrapes all awesome lists from GitHub
+
+**Dependencies:**
+- Requires awesome CLI repository (checked out during workflow)
+- GitHub CLI for artifact downloads
+- better-sqlite3 for database operations
 
 **Artifacts:**
 - `awesome.db` - SQLite database file
-- `db-metadata.json` - Metadata about the build (timestamp, hash, counts)
+- `db-metadata.json` - Metadata including:
+  - Build mode used (download/build)
+  - Source repository
+  - Timestamp, hash, counts
+  - Statistics (lists, repos, READMEs)
 
 **Retention:** 90 days
 
@@ -109,7 +130,17 @@ graph LR
 ### Manual Database Build
 
 ```bash
+# Download pre-built database (fast, default)
 gh workflow run db.yml
+
+# Download from specific repository
+gh workflow run db.yml -f awesome_repo=owner/awesome
+
+# Build locally (slow but fresh)
+gh workflow run db.yml -f build_mode=build
+
+# Custom source and build mode
+gh workflow run db.yml -f build_mode=build -f awesome_repo=owner/awesome
 ```
 
 ### Manual Docker Build

.github/workflows/db.yml

@@ -4,7 +4,21 @@ on:
   schedule:
     # Run every 6 hours
     - cron: '0 */6 * * *'
-  workflow_dispatch: # Manual trigger
+  workflow_dispatch:
+    inputs:
+      build_mode:
+        description: 'Build mode: download (from awesome repo) or build (local indexing)'
+        required: false
+        default: 'download'
+        type: choice
+        options:
+          - download
+          - build
+      awesome_repo:
+        description: 'Awesome repository to download from (owner/repo)'
+        required: false
+        default: 'valknarness/awesome'
+        type: string
   push:
     branches:
       - main
@@ -15,12 +29,22 @@ on:
 jobs:
   build-database:
     runs-on: ubuntu-latest
+    timeout-minutes: 180  # 3 hours for local builds
 
     steps:
-      - name: Checkout repository
+      - name: Checkout awesome-app repository
         uses: actions/checkout@v4
+        with:
+          path: awesome-app
 
-      - name: Install pnpm
+      - name: Checkout awesome CLI repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }}
+          path: awesome
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
           version: 10
@@ -30,76 +54,121 @@ jobs:
         with:
           node-version: '22'
           cache: 'pnpm'
+          cache-dependency-path: awesome-app/pnpm-lock.yaml
 
-      - name: Install dependencies
+      - name: Install GitHub CLI
+        run: |
+          # GitHub CLI is pre-installed on ubuntu-latest
+          gh --version
+
+      - name: Install awesome-app dependencies
+        working-directory: awesome-app
         run: pnpm install --frozen-lockfile
 
-      - name: Build SQLite Database
+      - name: Install awesome CLI dependencies
+        working-directory: awesome
+        run: |
+          pnpm install --frozen-lockfile
+          pnpm rebuild better-sqlite3
+          chmod +x awesome
+
+      - name: Build database
+        working-directory: awesome-app
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BUILD_MODE: ${{ github.event.inputs.build_mode || 'download' }}
+          AWESOME_REPO: ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }}
         run: |
+          echo "Build Mode: $BUILD_MODE"
+          echo "Source Repo: $AWESOME_REPO"
+
+          # Run the build script
           node scripts/build-db.js
 
-      - name: Generate database metadata
+      - name: Verify database
+        working-directory: awesome-app
         run: |
+          if [ ! -f awesome.db ]; then
+            echo "❌ Database file not found!"
+            exit 1
+          fi
+
           DB_SIZE=$(du -h awesome.db | cut -f1)
-          DB_HASH=$(sha256sum awesome.db | cut -d' ' -f1)
+          echo "✅ Database created successfully"
-          TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          echo "Size: $DB_SIZE"
 
-          cat > db-metadata.json << EOF
+          # Verify it's a valid SQLite database
-          {
+          if command -v sqlite3 &> /dev/null; then
-            "version": "${GITHUB_SHA}",
+            LISTS_COUNT=$(sqlite3 awesome.db "SELECT COUNT(*) FROM awesome_lists" 2>/dev/null || echo "0")
-            "timestamp": "${TIMESTAMP}",
+            REPOS_COUNT=$(sqlite3 awesome.db "SELECT COUNT(*) FROM repositories" 2>/dev/null || echo "0")
-            "size": "${DB_SIZE}",
+            READMES_COUNT=$(sqlite3 awesome.db "SELECT COUNT(*) FROM readmes" 2>/dev/null || echo "0")
-            "hash": "${DB_HASH}",
-            "lists_count": $(sqlite3 awesome.db "SELECT COUNT(*) FROM awesome_lists"),
-            "repos_count": $(sqlite3 awesome.db "SELECT COUNT(*) FROM repositories"),
-            "readmes_count": $(sqlite3 awesome.db "SELECT COUNT(*) FROM readmes")
-          }
-          EOF
 
-          cat db-metadata.json
+            echo "Lists: $LISTS_COUNT"
+            echo "Repositories: $REPOS_COUNT"
+            echo "READMEs: $READMES_COUNT"
+          fi
 
       - name: Upload database artifact
         uses: actions/upload-artifact@v4
         with:
           name: awesome-database
           path: |
-            awesome.db
+            awesome-app/awesome.db
-            db-metadata.json
+            awesome-app/db-metadata.json
           retention-days: 90
+          compression-level: 9
 
-      - name: Deploy to hosting (optional)
+      - name: Create build summary
-        if: github.ref == 'refs/heads/main'
+        working-directory: awesome-app
         run: |
-          # Upload to your hosting provider
+          cat >> $GITHUB_STEP_SUMMARY <<EOF
-          # Example for S3:
+          # 🎉 Awesome Database Build Complete
-          # aws s3 cp awesome.db s3://your-bucket/awesome.db
-          # aws s3 cp db-metadata.json s3://your-bucket/db-metadata.json
 
-          # Or webhook to your Next.js app
+          ## 📊 Build Information
-          curl -X POST "${{ secrets.WEBHOOK_URL }}" \
-            -H "Content-Type: application/json" \
-            -H "X-GitHub-Secret: ${{ secrets.WEBHOOK_SECRET }}" \
-            -d @db-metadata.json
 
-      - name: Create release (on schedule)
+          | Metric | Value |
-        if: github.event_name == 'schedule'
+          |--------|-------|
-        uses: softprops/action-gh-release@v1
+          | Build Mode | ${{ github.event.inputs.build_mode || 'download' }} |
-        with:
+          | Source Repository | ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }} |
-          tag_name: db-${{ github.run_number }}
+          | Workflow Run | [\#${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) |
-          name: Database Build ${{ github.run_number }}
+          | Commit | \`${{ github.sha }}\` |
-          body: |
-            Automated database build
 
-            **Statistics:**
+          EOF
-            - Lists: $(sqlite3 awesome.db "SELECT COUNT(*) FROM awesome_lists")
-            - Repositories: $(sqlite3 awesome.db "SELECT COUNT(*) FROM repositories")
-            - READMEs: $(sqlite3 awesome.db "SELECT COUNT(*) FROM readmes")
 
-            **Generated:** $(date -u)
+          # Add metadata if available
-          files: |
+          if [ -f db-metadata.json ]; then
-            awesome.db
+            echo "## 📈 Database Statistics" >> $GITHUB_STEP_SUMMARY
-            db-metadata.json
+            echo "" >> $GITHUB_STEP_SUMMARY
-        env:
+            echo "\`\`\`json" >> $GITHUB_STEP_SUMMARY
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            cat db-metadata.json >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          fi
+
+          cat >> $GITHUB_STEP_SUMMARY <<EOF
+
+          ## 📥 Download Instructions
+
+          \`\`\`bash
+          # Using GitHub CLI
+          gh run download ${{ github.run_id }} -n awesome-database
+
+          # Or visit the workflow run page:
+          # ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          \`\`\`
+
+          ## 🐳 Docker Build
+
+          This database will be automatically used in the next Docker image build.
+          EOF
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          cat >> $GITHUB_STEP_SUMMARY <<EOF
+          # ❌ Database Build Failed
+
+          **Build Mode:** ${{ github.event.inputs.build_mode || 'download' }}
+          **Source Repo:** ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }}
+
+          Please check the workflow logs for details.
+          EOF

DATABASE_BUILD.md

@@ -0,0 +1,355 @@
+# Database Build System
+
+This document explains how the awesome-app builds and maintains its SQLite database.
+
+## Overview
+
+The awesome-app uses a flexible database build system with two modes:
+
+1. **Download Mode** (Fast): Downloads pre-built databases from the awesome CLI repository
+2. **Build Mode** (Slow): Builds the database locally using the awesome CLI indexer
+
+## Build Modes
+
+### Download Mode (Default) ⚡
+
+Downloads pre-built databases that are automatically created by the awesome CLI repository.
+
+**Speed:** ~5 minutes
+**Requirements:**
+- GitHub CLI (`gh`)
+- GitHub authentication
+- Access to awesome CLI repository artifacts
+
+**How it works:**
+1. Fetches latest successful workflow run from awesome CLI repo
+2. Downloads database artifact from GitHub Actions
+3. Extracts and validates database
+4. Falls back to build mode if download fails
+
+**Advantages:**
+- Very fast (5 min vs 1-2 hours)
+- No GitHub API rate limits needed
+- Consistent results
+- Reduces infrastructure load
+
+### Build Mode 🔨
+
+Builds the database from scratch using the awesome CLI indexer.
+
+**Speed:** ~1-2 hours
+**Requirements:**
+- awesome CLI repository (cloned during workflow)
+- GitHub token for API access (5000 req/hour)
+- Node.js 22+
+- better-sqlite3
+
+**How it works:**
+1. Clones awesome CLI repository
+2. Installs dependencies
+3. Runs `./awesome index` with full mode
+4. Copies resulting database
+
+**Advantages:**
+- Always fresh data
+- No dependency on upstream artifacts
+- Full control over indexing
+
+## Scripts
+
+### `scripts/build-db.js`
+
+Main build script that handles both modes.
+
+**Environment Variables:**
+- `BUILD_MODE`: `download` or `build` (default: `download`)
+- `AWESOME_REPO`: Repository to download from (default: `valknarness/awesome`)
+- `GITHUB_TOKEN`: GitHub token for API access
+
+**Usage:**
+```bash
+# Download mode (default)
+node scripts/build-db.js
+
+# Build mode
+BUILD_MODE=build node scripts/build-db.js
+
+# Custom repository
+AWESOME_REPO=owner/repo node scripts/build-db.js
+
+# Build mode with custom repo
+BUILD_MODE=build AWESOME_REPO=owner/repo node scripts/build-db.js
+```
+
+**Output:**
+- `awesome.db` - SQLite database file
+- `db-metadata.json` - Build metadata
+
+**Metadata Format:**
+```json
+{
+  "version": "abc123...",
+  "timestamp": "2025-10-26T13:45:00.000Z",
+  "size": "156.42MB",
+  "hash": "sha256...",
+  "lists_count": 450,
+  "repos_count": 15000,
+  "readmes_count": 12500,
+  "build_mode": "download",
+  "source_repo": "valknarness/awesome"
+}
+```
+
+## GitHub Actions Workflow
+
+### `.github/workflows/db.yml`
+
+**Triggers:**
+- Schedule: Every 6 hours
+- Manual: `workflow_dispatch` with parameters
+- Push: Changes to db.yml or build-db.js
+
+**Workflow Inputs:**
+- `build_mode`: Choose download or build
+- `awesome_repo`: Source repository
+
+**Steps:**
+1. Checkout awesome-app repository
+2. Checkout awesome CLI repository
+3. Install dependencies for both
+4. Run build script
+5. Verify database
+6. Upload artifact
+7. Create summary
+
+**Artifacts:**
+- Name: `awesome-database`
+- Retention: 90 days
+- Contents:
+  - awesome.db
+  - db-metadata.json
+
+**Manual Triggering:**
+```bash
+# Download mode (fast)
+gh workflow run db.yml
+
+# Build mode (slow)
+gh workflow run db.yml -f build_mode=build
+
+# Custom repository
+gh workflow run db.yml -f awesome_repo=owner/awesome
+
+# Both options
+gh workflow run db.yml -f build_mode=build -f awesome_repo=owner/awesome
+```
+
+## Integration with Docker
+
+The docker-publish workflow automatically uses the database artifact:
+
+```yaml
+jobs:
+  build-database:
+    uses: ./.github/workflows/db.yml
+    secrets: inherit
+
+  build-and-push:
+    needs: build-database
+    # Downloads artifact and includes in Docker image
+```
+
+**Docker Build Args:**
+- `INCLUDE_DATABASE`: `true` to embed database in image (default: `false`)
+
+## Local Development
+
+### Option 1: Download Database
+
+```bash
+# Install dependencies
+pnpm install
+
+# Build database (download mode)
+node scripts/build-db.js
+
+# Or with environment variables
+BUILD_MODE=download AWESOME_REPO=valknarness/awesome node scripts/build-db.js
+```
+
+### Option 2: Build Database
+
+```bash
+# Clone awesome CLI (sibling directory)
+cd ..
+git clone https://github.com/valknarness/awesome.git
+cd awesome-app
+
+# Build database
+BUILD_MODE=build node scripts/build-db.js
+```
+
+### Option 3: Download from GitHub Actions
+
+```bash
+# Find latest run
+gh run list --workflow=db.yml --limit 1
+
+# Download artifact
+gh run download <run-id> -n awesome-database
+
+# Move to project root
+mv awesome-database/awesome.db .
+mv awesome-database/db-metadata.json .
+```
+
+## Troubleshooting
+
+### Download Mode Issues
+
+**Problem:** No artifacts found
+```bash
+# Solution: Build locally instead
+BUILD_MODE=build node scripts/build-db.js
+```
+
+**Problem:** GitHub CLI not authenticated
+```bash
+# Solution: Authenticate
+gh auth login
+
+# Or use token
+export GITHUB_TOKEN=your_token_here
+```
+
+**Problem:** Download fails
+```bash
+# The script automatically falls back to build mode
+# Check logs for fallback message
+```
+
+### Build Mode Issues
+
+**Problem:** awesome CLI not found
+```bash
+# Solution: Clone to sibling directory
+cd ..
+git clone https://github.com/valknarness/awesome.git
+cd awesome-app
+```
+
+**Problem:** API rate limits
+```bash
+# Solution: Set GitHub token
+export GITHUB_TOKEN=your_token_here
+```
+
+**Problem:** Build timeout
+```bash
+# Solution: Increase timeout or use download mode
+# Build mode takes 1-2 hours for full index
+```
+
+## Best Practices
+
+### For CI/CD
+
+1. **Use download mode** by default (fast, reliable)
+2. **Set proper secrets** (GITHUB_TOKEN for downloads)
+3. **Configure fallback** to build mode if download fails
+4. **Monitor artifacts** retention (90 days)
+5. **Track build time** to detect issues
+
+### For Local Development
+
+1. **Use download mode** for quick setup
+2. **Use build mode** only when needed (testing indexer, custom data)
+3. **Keep awesome CLI updated** (pull latest changes)
+4. **Check metadata** to verify database freshness
+
+### For Production
+
+1. **Schedule regular builds** (every 6 hours recommended)
+2. **Monitor build success** rate
+3. **Verify database integrity** after builds
+4. **Track metadata** for troubleshooting
+5. **Implement alerts** for build failures
+
+## Architecture
+
+```mermaid
+graph TD
+    A[GitHub Actions Trigger] --> B{Build Mode?}
+    B -->|download| C[Download from awesome CLI]
+    B -->|build| D[Clone awesome CLI]
+
+    C --> E{Download Success?}
+    E -->|Yes| F[Extract Database]
+    E -->|No| D
+
+    D --> G[Install Dependencies]
+    G --> H[Run ./awesome index]
+    H --> I[Copy Database]
+
+    F --> J[Generate Metadata]
+    I --> J
+
+    J --> K[Upload Artifact]
+    K --> L[Docker Build Uses Artifact]
+```
+
+## Database Schema
+
+The database is built by the awesome CLI and follows this schema:
+
+**Tables:**
+- `awesome_lists` - Hierarchical awesome lists
+- `repositories` - Individual projects
+- `readmes` - Full README content
+- `readmes_fts` - Full-text search index
+- `bookmarks` - User favorites (web app only)
+- `custom_lists` - User lists (web app only)
+- `reading_history` - Activity tracking (web app only)
+- `tags` - Content tags
+- `categories` - Content categories
+- `settings` - Configuration
+
+**Indexes:**
+- FTS5 index for full-text search
+- Foreign key indexes for relations
+- Performance indexes on common queries
+
+## Monitoring
+
+### Key Metrics
+
+- **Build time** (download: ~5 min, build: ~1-2 hours)
+- **Database size** (~50-200 MB compressed)
+- **Success rate** (target: >95%)
+- **Artifact retention** (90 days)
+- **Download vs build ratio** (target: 90% download, 10% build)
+
+### Health Checks
+
+```bash
+# Check latest build
+gh run list --workflow=db.yml --limit 1
+
+# Download and verify
+gh run download <run-id> -n awesome-database
+sqlite3 awesome.db "SELECT COUNT(*) FROM repositories"
+
+# Check metadata
+cat db-metadata.json | jq .
+```
+
+## Future Improvements
+
+1. **Incremental updates** - Only update changed READMEs
+2. **CDN hosting** - Serve database from CDN for faster downloads
+3. **Multiple regions** - Build and host in different regions
+4. **Compression** - Better compression algorithms
+5. **Checksums** - Verify database integrity
+6. **Notifications** - Alert on build failures
+7. **Metrics** - Track build performance over time
+8. **Caching** - Cache intermediate results

README.md

@@ -30,12 +30,15 @@ This webapp perfectly matches the **beautiful purple/pink/gold theme** from the
    - Typography plugin
 
 3. **GitHub Actions Workflow**
-   - Automated database building
+   - Two build modes:
-   - Runs every 6 hours
+     - **Download Mode**: Fast (~5 min) - downloads pre-built database from awesome CLI repo
-   - Manual trigger support
+     - **Build Mode**: Slow (~1-2 hours) - builds locally using awesome CLI indexer
-   - Artifact upload
+   - Runs every 6 hours (download mode by default)
-   - Release creation
+   - Manual trigger with mode selection
-   - Webhook integration
+   - Configurable source repository
+   - Artifact upload (90-day retention)
+   - Metadata generation with build stats
+   - Fallback to local build if download fails
 
 4. **Web Worker System**
    - Smart polling with exponential backoff
@@ -82,7 +85,7 @@ awesome-web/
 │   ├── manifest.json       ✅ PWA manifest
 │   └── icons/              🔨 Generate from logo
 ├── scripts/
-│   └── build-db.js         ✅ Database builder
+│   └── build-db.js         ✅ Database builder (download/build modes)
 ├── tailwind.config.ts      ✅ Custom theme
 └── next.config.js          ✅ PWA & optimization
 ```

scripts/build-db.js

@@ -2,142 +2,316 @@
 
 /**
  * Build Awesome Database for GitHub Actions
- * This script indexes awesome lists and builds the SQLite database
+ * This script uses the awesome CLI to either download a pre-built database
+ * or build it from scratch using the indexer
  */
 
-const Database = require('better-sqlite3');
+const { execSync, spawn } = require('child_process');
-const axios = require('axios');
 const fs = require('fs');
 const path = require('path');
 
 const DB_PATH = path.join(process.cwd(), 'awesome.db');
+const AWESOME_REPO = process.env.AWESOME_REPO || 'valknarness/awesome';
+const BUILD_MODE = process.env.BUILD_MODE || 'download'; // 'download' or 'build'
 const GITHUB_TOKEN = process.env.GITHUB_TOKEN;
-const RATE_LIMIT_DELAY = 100;
 
-let lastRequestTime = 0;
+// Colors for console output
-let requestCount = 0;
+const colors = {
+  reset: '\x1b[0m',
+  bright: '\x1b[1m',
+  green: '\x1b[32m',
+  yellow: '\x1b[33m',
+  blue: '\x1b[34m',
+  red: '\x1b[31m',
+  cyan: '\x1b[36m'
+};
 
-// Rate-limited request
+function log(message, color = 'reset') {
-async function rateLimitedRequest(url) {
+  console.log(`${colors[color]}${message}${colors.reset}`);
-  const now = Date.now();
+}
-  const timeSinceLastRequest = now - lastRequestTime;
-
-  if (timeSinceLastRequest < RATE_LIMIT_DELAY) {
-    await new Promise(resolve => setTimeout(resolve, RATE_LIMIT_DELAY - timeSinceLastRequest));
-  }
-
-  lastRequestTime = Date.now();
-  requestCount++;
-
-  const headers = {
-    'Accept': 'application/vnd.github.v3+json',
-    'User-Agent': 'awesome-web-builder',
-  };
-
-  if (GITHUB_TOKEN) {
-    headers['Authorization'] = `token ${GITHUB_TOKEN}`;
-  }
 
+function checkCommand(command) {
   try {
-    return await axios.get(url, { headers, timeout: 10000 });
+    execSync(`which ${command}`, { stdio: 'ignore' });
-  } catch (error) {
+    return true;
-    if (error.response?.status === 404) {
+  } catch {
-      return null;
+    return false;
-    }
-    throw error;
   }
 }
 
-// Initialize database
+async function downloadDatabase() {
-function initializeDatabase() {
+  log('\n📥 Downloading pre-built database from GitHub Actions...', 'cyan');
-  console.log('🗄️  Initializing database...');
 
-  const db = new Database(DB_PATH);
+  // Check if gh CLI is installed
-  db.pragma('journal_mode = WAL');
+  if (!checkCommand('gh')) {
-  db.pragma('foreign_keys = ON');
+    log('❌ GitHub CLI (gh) is not installed', 'red');
-
+    log('Install from: https://cli.github.com/', 'yellow');
-  // Create tables
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS awesome_lists (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      name TEXT NOT NULL,
-      url TEXT NOT NULL UNIQUE,
-      description TEXT,
-      category TEXT,
-      stars INTEGER DEFAULT 0,
-      indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP
-    );
-
-    CREATE TABLE IF NOT EXISTS repositories (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      awesome_list_id INTEGER NOT NULL,
-      name TEXT NOT NULL,
-      url TEXT NOT NULL UNIQUE,
-      description TEXT,
-      stars INTEGER DEFAULT 0,
-      language TEXT,
-      topics TEXT,
-      FOREIGN KEY (awesome_list_id) REFERENCES awesome_lists(id)
-    );
-
-    CREATE TABLE IF NOT EXISTS readmes (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      repository_id INTEGER NOT NULL UNIQUE,
-      content TEXT,
-      raw_content TEXT,
-      indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-      FOREIGN KEY (repository_id) REFERENCES repositories(id)
-    );
-
-    CREATE VIRTUAL TABLE IF NOT EXISTS readmes_fts USING fts5(
-      repository_name,
-      description,
-      content,
-      tags,
-      categories,
-      content_rowid UNINDEXED
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_repos_list ON repositories(awesome_list_id);
-    CREATE INDEX IF NOT EXISTS idx_readmes_repo ON readmes(repository_id);
-  `);
-
-  console.log('✅ Database initialized');
-  return db;
-}
-
-// Main build process
-async function build() {
-  console.log('🚀 Starting Awesome Database Build\n');
-
-  const db = initializeDatabase();
-
-  console.log('📥 Fetching main awesome list...');
-  const mainReadme = await rateLimitedRequest(
-    'https://raw.githubusercontent.com/sindresorhus/awesome/main/readme.md'
-  );
-
-  if (!mainReadme) {
-    console.error('❌ Failed to fetch main awesome list');
     process.exit(1);
   }
 
-  console.log('✅ Fetched main list\n');
+  // Authenticate gh CLI if needed
+  try {
+    execSync('gh auth status', { stdio: 'ignore' });
+  } catch {
+    log('⚠️  Not authenticated with GitHub CLI', 'yellow');
+    if (GITHUB_TOKEN) {
+      log('Using GITHUB_TOKEN from environment...', 'blue');
+      process.env.GH_TOKEN = GITHUB_TOKEN;
+    } else {
+      log('❌ No GitHub authentication available', 'red');
+      log('Set GITHUB_TOKEN environment variable or run: gh auth login', 'yellow');
+      process.exit(1);
+    }
+  }
 
-  // Parse markdown and build index
+  try {
-  // For this example, we'll do a simplified version
+    // Get latest successful workflow run
-  // In production, use the full indexer logic from the CLI
+    log(`Fetching latest database build from ${AWESOME_REPO}...`, 'blue');
 
-  console.log('📊 Build Statistics:');
+    const runsOutput = execSync(
-  console.log(`   Total Requests: ${requestCount}`);
+      `gh api -H "Accept: application/vnd.github+json" "/repos/${AWESOME_REPO}/actions/workflows/build-database.yml/runs?per_page=1&status=success"`,
-  console.log(`   Database Size: ${(fs.statSync(DB_PATH).size / 1024 / 1024).toFixed(2)} MB`);
+      { encoding: 'utf-8' }
+    );
 
-  db.close();
+    const runs = JSON.parse(runsOutput);
-  console.log('\n✅ Build Complete!');
+
+    if (!runs.workflow_runs || runs.workflow_runs.length === 0) {
+      log('❌ No successful database builds found', 'red');
+      log('Falling back to local build...', 'yellow');
+      return false;
+    }
+
+    const latestRun = runs.workflow_runs[0];
+    log(`✓ Found build from ${latestRun.created_at}`, 'green');
+
+    // Get artifacts for this run
+    const artifactsOutput = execSync(
+      `gh api -H "Accept: application/vnd.github+json" "/repos/${AWESOME_REPO}/actions/runs/${latestRun.id}/artifacts"`,
+      { encoding: 'utf-8' }
+    );
+
+    const artifacts = JSON.parse(artifactsOutput);
+    const dbArtifact = artifacts.artifacts.find(a => a.name.startsWith('awesome-database'));
+
+    if (!dbArtifact) {
+      log('❌ No database artifact found in latest run', 'red');
+      log('Falling back to local build...', 'yellow');
+      return false;
+    }
+
+    log(`✓ Found artifact: ${dbArtifact.name} (${(dbArtifact.size_in_bytes / 1024 / 1024).toFixed(1)} MB)`, 'green');
+
+    // Download artifact
+    const tempDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'awesome-db-'));
+
+    log('Downloading artifact...', 'blue');
+    execSync(`gh run download ${latestRun.id} -R ${AWESOME_REPO} -D ${tempDir}`, {
+      stdio: 'inherit'
+    });
+
+    // Find and copy database file
+    const files = fs.readdirSync(tempDir, { recursive: true, withFileTypes: true });
+    const dbFile = files.find(f => f.isFile() && f.name.endsWith('.db'));
+
+    if (!dbFile) {
+      log('❌ Database file not found in artifact', 'red');
+      fs.rmSync(tempDir, { recursive: true, force: true });
+      return false;
+    }
+
+    const dbFilePath = path.join(dbFile.path || tempDir, dbFile.name);
+    fs.copyFileSync(dbFilePath, DB_PATH);
+
+    // Copy metadata if available
+    const metadataFile = files.find(f => f.isFile() && f.name === 'metadata.json');
+    if (metadataFile) {
+      const metadataPath = path.join(metadataFile.path || tempDir, metadataFile.name);
+      fs.copyFileSync(metadataPath, path.join(process.cwd(), 'db-metadata.json'));
+    }
+
+    // Cleanup
+    fs.rmSync(tempDir, { recursive: true, force: true });
+
+    const size = fs.statSync(DB_PATH).size;
+    log(`✓ Database downloaded successfully (${(size / 1024 / 1024).toFixed(2)} MB)`, 'green');
+
+    return true;
+  } catch (error) {
+    log(`❌ Download failed: ${error.message}`, 'red');
+    return false;
+  }
 }
 
-// Run build
+async function buildDatabaseLocally() {
-build().catch(error => {
+  log('\n🔨 Building database locally using awesome CLI...', 'cyan');
-  console.error('❌ Build failed:', error);
+
+  // Check if awesome CLI is available
+  const awesomePath = path.join(__dirname, '../../awesome/awesome');
+
+  if (!fs.existsSync(awesomePath)) {
+    log('❌ Awesome CLI not found at: ' + awesomePath, 'red');
+    log('Expected location: /path/to/awesome/awesome', 'yellow');
+    log('Please ensure the awesome repository is checked out as a sibling directory', 'yellow');
+    process.exit(1);
+  }
+
+  try {
+    // Ensure awesome CLI is executable
+    fs.chmodSync(awesomePath, '755');
+
+    log('Installing awesome CLI dependencies...', 'blue');
+    execSync('pnpm install && pnpm rebuild better-sqlite3', {
+      cwd: path.dirname(awesomePath),
+      stdio: 'inherit'
+    });
+
+    // Configure GitHub token if available
+    if (GITHUB_TOKEN) {
+      log('Configuring GitHub token for API access...', 'blue');
+      execSync(`node -e "
+        const db = require('./lib/database');
+        const dbOps = require('./lib/db-operations');
+        db.initialize();
+        dbOps.setSetting('githubToken', '${GITHUB_TOKEN}');
+        db.close();
+      "`, {
+        cwd: path.dirname(awesomePath),
+        stdio: 'inherit'
+      });
+    }
+
+    log('Building index (this may take 1-2 hours)...', 'blue');
+
+    // Run indexer with full mode
+    const buildProcess = spawn(awesomePath, ['index'], {
+      cwd: path.dirname(awesomePath),
+      stdio: ['pipe', 'inherit', 'inherit']
+    });
+
+    // Automatically select 'full' mode
+    buildProcess.stdin.write('full\n');
+    buildProcess.stdin.end();
+
+    await new Promise((resolve, reject) => {
+      buildProcess.on('close', (code) => {
+        if (code === 0) {
+          resolve();
+        } else {
+          reject(new Error(`Build failed with code ${code}`));
+        }
+      });
+      buildProcess.on('error', reject);
+    });
+
+    // Copy database to current directory
+    const awesomeDbPath = path.join(require('os').homedir(), '.awesome', 'awesome.db');
+
+    if (!fs.existsSync(awesomeDbPath)) {
+      throw new Error('Database not created at expected location: ' + awesomeDbPath);
+    }
+
+    fs.copyFileSync(awesomeDbPath, DB_PATH);
+
+    const size = fs.statSync(DB_PATH).size;
+    log(`✓ Database built successfully (${(size / 1024 / 1024).toFixed(2)} MB)`, 'green');
+
+    return true;
+  } catch (error) {
+    log(`❌ Build failed: ${error.message}`, 'red');
+    return false;
+  }
+}
+
+async function generateMetadata() {
+  log('\n📊 Generating metadata...', 'cyan');
+
+  try {
+    const Database = require('better-sqlite3');
+    const db = new Database(DB_PATH, { readonly: true });
+
+    const listsCount = db.prepare('SELECT COUNT(*) as count FROM awesome_lists').get().count;
+    const reposCount = db.prepare('SELECT COUNT(*) as count FROM repositories').get().count;
+    const readmesCount = db.prepare('SELECT COUNT(*) as count FROM readmes').get().count;
+
+    db.close();
+
+    const stats = fs.statSync(DB_PATH);
+    const size = (stats.size / 1024 / 1024).toFixed(2);
+    const hash = require('crypto')
+      .createHash('sha256')
+      .update(fs.readFileSync(DB_PATH))
+      .digest('hex');
+
+    const metadata = {
+      version: process.env.GITHUB_SHA || 'unknown',
+      timestamp: new Date().toISOString(),
+      size: `${size}MB`,
+      hash: hash,
+      lists_count: listsCount,
+      repos_count: reposCount,
+      readmes_count: readmesCount,
+      build_mode: BUILD_MODE,
+      source_repo: AWESOME_REPO
+    };
+
+    fs.writeFileSync(
+      path.join(process.cwd(), 'db-metadata.json'),
+      JSON.stringify(metadata, null, 2)
+    );
+
+    log('✓ Metadata generated', 'green');
+    log(`  Lists: ${listsCount}`, 'blue');
+    log(`  Repositories: ${reposCount}`, 'blue');
+    log(`  READMEs: ${readmesCount}`, 'blue');
+    log(`  Size: ${size} MB`, 'blue');
+
+    return metadata;
+  } catch (error) {
+    log(`⚠️  Failed to generate metadata: ${error.message}`, 'yellow');
+    return null;
+  }
+}
+
+async function main() {
+  log('\n' + '='.repeat(60), 'bright');
+  log('  🚀 AWESOME DATABASE BUILDER', 'bright');
+  log('='.repeat(60) + '\n', 'bright');
+
+  log(`Build Mode: ${BUILD_MODE}`, 'cyan');
+  log(`Source Repo: ${AWESOME_REPO}\n`, 'cyan');
+
+  let success = false;
+
+  if (BUILD_MODE === 'download') {
+    success = await downloadDatabase();
+
+    if (!success) {
+      log('\n⚠️  Download failed, attempting local build...', 'yellow');
+      success = await buildDatabaseLocally();
+    }
+  } else {
+    success = await buildDatabaseLocally();
+  }
+
+  if (!success) {
+    log('\n❌ Database build failed', 'red');
+    process.exit(1);
+  }
+
+  // Generate metadata
+  await generateMetadata();
+
+  log('\n' + '='.repeat(60), 'bright');
+  log('  ✅ BUILD COMPLETE', 'green');
+  log('='.repeat(60) + '\n', 'bright');
+
+  log(`Database: ${DB_PATH}`, 'cyan');
+  log(`Metadata: ${path.join(process.cwd(), 'db-metadata.json')}\n`, 'cyan');
+}
+
+// Run
+main().catch(error => {
+  log(`\n❌ Fatal error: ${error.message}`, 'red');
+  console.error(error);
   process.exit(1);
 });