diff --git a/.github/workflows/README.md b/.github/workflows/README.md index a0b28f3..6a6b96e 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -8,15 +8,36 @@ This directory contains automated workflows for building, testing, and deploying **Triggers:** - Schedule: Every 6 hours -- Manual: `workflow_dispatch` +- Manual: `workflow_dispatch` with options: + - `build_mode`: `download` (default) or `build` + - `awesome_repo`: Source repository (default: `valknarness/awesome`) - Push to main (when db.yml or build-db.js changes) **Purpose:** -Builds the SQLite database by scraping awesome lists from GitHub. +Builds the SQLite database using one of two methods: + +1. **Download Mode** (default, fast ~5 min): + - Downloads pre-built database from the awesome CLI repository + - Uses GitHub Actions artifacts from the upstream repo + - Fallback to local build if download fails + +2. **Build Mode** (slow ~1-2 hours): + - Clones the awesome CLI repository + - Runs full indexing using `./awesome index` + - Scrapes all awesome lists from GitHub + +**Dependencies:** +- Requires awesome CLI repository (checked out during workflow) +- GitHub CLI for artifact downloads +- better-sqlite3 for database operations **Artifacts:** - `awesome.db` - SQLite database file -- `db-metadata.json` - Metadata about the build (timestamp, hash, counts) +- `db-metadata.json` - Metadata including: + - Build mode used (download/build) + - Source repository + - Timestamp, hash, counts + - Statistics (lists, repos, READMEs) **Retention:** 90 days @@ -109,7 +130,17 @@ graph LR ### Manual Database Build ```bash +# Download pre-built database (fast, default) gh workflow run db.yml + +# Download from specific repository +gh workflow run db.yml -f awesome_repo=owner/awesome + +# Build locally (slow but fresh) +gh workflow run db.yml -f build_mode=build + +# Custom source and build mode +gh workflow run db.yml -f build_mode=build -f awesome_repo=owner/awesome ``` ### Manual Docker Build diff --git a/.github/workflows/db.yml b/.github/workflows/db.yml index 195e3af..5a27883 100644 --- a/.github/workflows/db.yml +++ b/.github/workflows/db.yml @@ -4,7 +4,21 @@ on: schedule: # Run every 6 hours - cron: '0 */6 * * *' - workflow_dispatch: # Manual trigger + workflow_dispatch: + inputs: + build_mode: + description: 'Build mode: download (from awesome repo) or build (local indexing)' + required: false + default: 'download' + type: choice + options: + - download + - build + awesome_repo: + description: 'Awesome repository to download from (owner/repo)' + required: false + default: 'valknarness/awesome' + type: string push: branches: - main @@ -15,12 +29,22 @@ on: jobs: build-database: runs-on: ubuntu-latest + timeout-minutes: 180 # 3 hours for local builds steps: - - name: Checkout repository + - name: Checkout awesome-app repository uses: actions/checkout@v4 + with: + path: awesome-app - - name: Install pnpm + - name: Checkout awesome CLI repository + uses: actions/checkout@v4 + with: + repository: ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }} + path: awesome + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup pnpm uses: pnpm/action-setup@v4 with: version: 10 @@ -30,76 +54,121 @@ jobs: with: node-version: '22' cache: 'pnpm' + cache-dependency-path: awesome-app/pnpm-lock.yaml - - name: Install dependencies + - name: Install GitHub CLI + run: | + # GitHub CLI is pre-installed on ubuntu-latest + gh --version + + - name: Install awesome-app dependencies + working-directory: awesome-app run: pnpm install --frozen-lockfile - - name: Build SQLite Database + - name: Install awesome CLI dependencies + working-directory: awesome + run: | + pnpm install --frozen-lockfile + pnpm rebuild better-sqlite3 + chmod +x awesome + + - name: Build database + working-directory: awesome-app env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MODE: ${{ github.event.inputs.build_mode || 'download' }} + AWESOME_REPO: ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }} run: | + echo "Build Mode: $BUILD_MODE" + echo "Source Repo: $AWESOME_REPO" + + # Run the build script node scripts/build-db.js - - name: Generate database metadata + - name: Verify database + working-directory: awesome-app run: | + if [ ! -f awesome.db ]; then + echo "āŒ Database file not found!" + exit 1 + fi + DB_SIZE=$(du -h awesome.db | cut -f1) - DB_HASH=$(sha256sum awesome.db | cut -d' ' -f1) - TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + echo "āœ… Database created successfully" + echo "Size: $DB_SIZE" - cat > db-metadata.json << EOF - { - "version": "${GITHUB_SHA}", - "timestamp": "${TIMESTAMP}", - "size": "${DB_SIZE}", - "hash": "${DB_HASH}", - "lists_count": $(sqlite3 awesome.db "SELECT COUNT(*) FROM awesome_lists"), - "repos_count": $(sqlite3 awesome.db "SELECT COUNT(*) FROM repositories"), - "readmes_count": $(sqlite3 awesome.db "SELECT COUNT(*) FROM readmes") - } - EOF + # Verify it's a valid SQLite database + if command -v sqlite3 &> /dev/null; then + LISTS_COUNT=$(sqlite3 awesome.db "SELECT COUNT(*) FROM awesome_lists" 2>/dev/null || echo "0") + REPOS_COUNT=$(sqlite3 awesome.db "SELECT COUNT(*) FROM repositories" 2>/dev/null || echo "0") + READMES_COUNT=$(sqlite3 awesome.db "SELECT COUNT(*) FROM readmes" 2>/dev/null || echo "0") - cat db-metadata.json + echo "Lists: $LISTS_COUNT" + echo "Repositories: $REPOS_COUNT" + echo "READMEs: $READMES_COUNT" + fi - name: Upload database artifact uses: actions/upload-artifact@v4 with: name: awesome-database path: | - awesome.db - db-metadata.json + awesome-app/awesome.db + awesome-app/db-metadata.json retention-days: 90 + compression-level: 9 - - name: Deploy to hosting (optional) - if: github.ref == 'refs/heads/main' + - name: Create build summary + working-directory: awesome-app run: | - # Upload to your hosting provider - # Example for S3: - # aws s3 cp awesome.db s3://your-bucket/awesome.db - # aws s3 cp db-metadata.json s3://your-bucket/db-metadata.json + cat >> $GITHUB_STEP_SUMMARY <> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`json" >> $GITHUB_STEP_SUMMARY + cat db-metadata.json >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + fi + + cat >> $GITHUB_STEP_SUMMARY <> $GITHUB_STEP_SUMMARY < -n awesome-database + +# Move to project root +mv awesome-database/awesome.db . +mv awesome-database/db-metadata.json . +``` + +## Troubleshooting + +### Download Mode Issues + +**Problem:** No artifacts found +```bash +# Solution: Build locally instead +BUILD_MODE=build node scripts/build-db.js +``` + +**Problem:** GitHub CLI not authenticated +```bash +# Solution: Authenticate +gh auth login + +# Or use token +export GITHUB_TOKEN=your_token_here +``` + +**Problem:** Download fails +```bash +# The script automatically falls back to build mode +# Check logs for fallback message +``` + +### Build Mode Issues + +**Problem:** awesome CLI not found +```bash +# Solution: Clone to sibling directory +cd .. +git clone https://github.com/valknarness/awesome.git +cd awesome-app +``` + +**Problem:** API rate limits +```bash +# Solution: Set GitHub token +export GITHUB_TOKEN=your_token_here +``` + +**Problem:** Build timeout +```bash +# Solution: Increase timeout or use download mode +# Build mode takes 1-2 hours for full index +``` + +## Best Practices + +### For CI/CD + +1. **Use download mode** by default (fast, reliable) +2. **Set proper secrets** (GITHUB_TOKEN for downloads) +3. **Configure fallback** to build mode if download fails +4. **Monitor artifacts** retention (90 days) +5. **Track build time** to detect issues + +### For Local Development + +1. **Use download mode** for quick setup +2. **Use build mode** only when needed (testing indexer, custom data) +3. **Keep awesome CLI updated** (pull latest changes) +4. **Check metadata** to verify database freshness + +### For Production + +1. **Schedule regular builds** (every 6 hours recommended) +2. **Monitor build success** rate +3. **Verify database integrity** after builds +4. **Track metadata** for troubleshooting +5. **Implement alerts** for build failures + +## Architecture + +```mermaid +graph TD + A[GitHub Actions Trigger] --> B{Build Mode?} + B -->|download| C[Download from awesome CLI] + B -->|build| D[Clone awesome CLI] + + C --> E{Download Success?} + E -->|Yes| F[Extract Database] + E -->|No| D + + D --> G[Install Dependencies] + G --> H[Run ./awesome index] + H --> I[Copy Database] + + F --> J[Generate Metadata] + I --> J + + J --> K[Upload Artifact] + K --> L[Docker Build Uses Artifact] +``` + +## Database Schema + +The database is built by the awesome CLI and follows this schema: + +**Tables:** +- `awesome_lists` - Hierarchical awesome lists +- `repositories` - Individual projects +- `readmes` - Full README content +- `readmes_fts` - Full-text search index +- `bookmarks` - User favorites (web app only) +- `custom_lists` - User lists (web app only) +- `reading_history` - Activity tracking (web app only) +- `tags` - Content tags +- `categories` - Content categories +- `settings` - Configuration + +**Indexes:** +- FTS5 index for full-text search +- Foreign key indexes for relations +- Performance indexes on common queries + +## Monitoring + +### Key Metrics + +- **Build time** (download: ~5 min, build: ~1-2 hours) +- **Database size** (~50-200 MB compressed) +- **Success rate** (target: >95%) +- **Artifact retention** (90 days) +- **Download vs build ratio** (target: 90% download, 10% build) + +### Health Checks + +```bash +# Check latest build +gh run list --workflow=db.yml --limit 1 + +# Download and verify +gh run download -n awesome-database +sqlite3 awesome.db "SELECT COUNT(*) FROM repositories" + +# Check metadata +cat db-metadata.json | jq . +``` + +## Future Improvements + +1. **Incremental updates** - Only update changed READMEs +2. **CDN hosting** - Serve database from CDN for faster downloads +3. **Multiple regions** - Build and host in different regions +4. **Compression** - Better compression algorithms +5. **Checksums** - Verify database integrity +6. **Notifications** - Alert on build failures +7. **Metrics** - Track build performance over time +8. **Caching** - Cache intermediate results diff --git a/README.md b/README.md index 85d031b..6359359 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,15 @@ This webapp perfectly matches the **beautiful purple/pink/gold theme** from the - Typography plugin 3. **GitHub Actions Workflow** - - Automated database building - - Runs every 6 hours - - Manual trigger support - - Artifact upload - - Release creation - - Webhook integration + - Two build modes: + - **Download Mode**: Fast (~5 min) - downloads pre-built database from awesome CLI repo + - **Build Mode**: Slow (~1-2 hours) - builds locally using awesome CLI indexer + - Runs every 6 hours (download mode by default) + - Manual trigger with mode selection + - Configurable source repository + - Artifact upload (90-day retention) + - Metadata generation with build stats + - Fallback to local build if download fails 4. **Web Worker System** - Smart polling with exponential backoff @@ -82,7 +85,7 @@ awesome-web/ ā”‚ ā”œā”€ā”€ manifest.json āœ… PWA manifest ā”‚ ā””ā”€ā”€ icons/ šŸ”Ø Generate from logo ā”œā”€ā”€ scripts/ -ā”‚ ā””ā”€ā”€ build-db.js āœ… Database builder +ā”‚ ā””ā”€ā”€ build-db.js āœ… Database builder (download/build modes) ā”œā”€ā”€ tailwind.config.ts āœ… Custom theme ā””ā”€ā”€ next.config.js āœ… PWA & optimization ``` diff --git a/scripts/build-db.js b/scripts/build-db.js index ba21aa9..d689eda 100644 --- a/scripts/build-db.js +++ b/scripts/build-db.js @@ -2,142 +2,316 @@ /** * Build Awesome Database for GitHub Actions - * This script indexes awesome lists and builds the SQLite database + * This script uses the awesome CLI to either download a pre-built database + * or build it from scratch using the indexer */ -const Database = require('better-sqlite3'); -const axios = require('axios'); +const { execSync, spawn } = require('child_process'); const fs = require('fs'); const path = require('path'); const DB_PATH = path.join(process.cwd(), 'awesome.db'); +const AWESOME_REPO = process.env.AWESOME_REPO || 'valknarness/awesome'; +const BUILD_MODE = process.env.BUILD_MODE || 'download'; // 'download' or 'build' const GITHUB_TOKEN = process.env.GITHUB_TOKEN; -const RATE_LIMIT_DELAY = 100; -let lastRequestTime = 0; -let requestCount = 0; +// Colors for console output +const colors = { + reset: '\x1b[0m', + bright: '\x1b[1m', + green: '\x1b[32m', + yellow: '\x1b[33m', + blue: '\x1b[34m', + red: '\x1b[31m', + cyan: '\x1b[36m' +}; -// Rate-limited request -async function rateLimitedRequest(url) { - const now = Date.now(); - const timeSinceLastRequest = now - lastRequestTime; - - if (timeSinceLastRequest < RATE_LIMIT_DELAY) { - await new Promise(resolve => setTimeout(resolve, RATE_LIMIT_DELAY - timeSinceLastRequest)); - } - - lastRequestTime = Date.now(); - requestCount++; - - const headers = { - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'awesome-web-builder', - }; - - if (GITHUB_TOKEN) { - headers['Authorization'] = `token ${GITHUB_TOKEN}`; - } +function log(message, color = 'reset') { + console.log(`${colors[color]}${message}${colors.reset}`); +} +function checkCommand(command) { try { - return await axios.get(url, { headers, timeout: 10000 }); - } catch (error) { - if (error.response?.status === 404) { - return null; - } - throw error; + execSync(`which ${command}`, { stdio: 'ignore' }); + return true; + } catch { + return false; } } -// Initialize database -function initializeDatabase() { - console.log('šŸ—„ļø Initializing database...'); +async function downloadDatabase() { + log('\nšŸ“„ Downloading pre-built database from GitHub Actions...', 'cyan'); - const db = new Database(DB_PATH); - db.pragma('journal_mode = WAL'); - db.pragma('foreign_keys = ON'); - - // Create tables - db.exec(` - CREATE TABLE IF NOT EXISTS awesome_lists ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - url TEXT NOT NULL UNIQUE, - description TEXT, - category TEXT, - stars INTEGER DEFAULT 0, - indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - - CREATE TABLE IF NOT EXISTS repositories ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - awesome_list_id INTEGER NOT NULL, - name TEXT NOT NULL, - url TEXT NOT NULL UNIQUE, - description TEXT, - stars INTEGER DEFAULT 0, - language TEXT, - topics TEXT, - FOREIGN KEY (awesome_list_id) REFERENCES awesome_lists(id) - ); - - CREATE TABLE IF NOT EXISTS readmes ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - repository_id INTEGER NOT NULL UNIQUE, - content TEXT, - raw_content TEXT, - indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (repository_id) REFERENCES repositories(id) - ); - - CREATE VIRTUAL TABLE IF NOT EXISTS readmes_fts USING fts5( - repository_name, - description, - content, - tags, - categories, - content_rowid UNINDEXED - ); - - CREATE INDEX IF NOT EXISTS idx_repos_list ON repositories(awesome_list_id); - CREATE INDEX IF NOT EXISTS idx_readmes_repo ON readmes(repository_id); - `); - - console.log('āœ… Database initialized'); - return db; -} - -// Main build process -async function build() { - console.log('šŸš€ Starting Awesome Database Build\n'); - - const db = initializeDatabase(); - - console.log('šŸ“„ Fetching main awesome list...'); - const mainReadme = await rateLimitedRequest( - 'https://raw.githubusercontent.com/sindresorhus/awesome/main/readme.md' - ); - - if (!mainReadme) { - console.error('āŒ Failed to fetch main awesome list'); + // Check if gh CLI is installed + if (!checkCommand('gh')) { + log('āŒ GitHub CLI (gh) is not installed', 'red'); + log('Install from: https://cli.github.com/', 'yellow'); process.exit(1); } - console.log('āœ… Fetched main list\n'); + // Authenticate gh CLI if needed + try { + execSync('gh auth status', { stdio: 'ignore' }); + } catch { + log('āš ļø Not authenticated with GitHub CLI', 'yellow'); + if (GITHUB_TOKEN) { + log('Using GITHUB_TOKEN from environment...', 'blue'); + process.env.GH_TOKEN = GITHUB_TOKEN; + } else { + log('āŒ No GitHub authentication available', 'red'); + log('Set GITHUB_TOKEN environment variable or run: gh auth login', 'yellow'); + process.exit(1); + } + } - // Parse markdown and build index - // For this example, we'll do a simplified version - // In production, use the full indexer logic from the CLI + try { + // Get latest successful workflow run + log(`Fetching latest database build from ${AWESOME_REPO}...`, 'blue'); - console.log('šŸ“Š Build Statistics:'); - console.log(` Total Requests: ${requestCount}`); - console.log(` Database Size: ${(fs.statSync(DB_PATH).size / 1024 / 1024).toFixed(2)} MB`); + const runsOutput = execSync( + `gh api -H "Accept: application/vnd.github+json" "/repos/${AWESOME_REPO}/actions/workflows/build-database.yml/runs?per_page=1&status=success"`, + { encoding: 'utf-8' } + ); - db.close(); - console.log('\nāœ… Build Complete!'); + const runs = JSON.parse(runsOutput); + + if (!runs.workflow_runs || runs.workflow_runs.length === 0) { + log('āŒ No successful database builds found', 'red'); + log('Falling back to local build...', 'yellow'); + return false; + } + + const latestRun = runs.workflow_runs[0]; + log(`āœ“ Found build from ${latestRun.created_at}`, 'green'); + + // Get artifacts for this run + const artifactsOutput = execSync( + `gh api -H "Accept: application/vnd.github+json" "/repos/${AWESOME_REPO}/actions/runs/${latestRun.id}/artifacts"`, + { encoding: 'utf-8' } + ); + + const artifacts = JSON.parse(artifactsOutput); + const dbArtifact = artifacts.artifacts.find(a => a.name.startsWith('awesome-database')); + + if (!dbArtifact) { + log('āŒ No database artifact found in latest run', 'red'); + log('Falling back to local build...', 'yellow'); + return false; + } + + log(`āœ“ Found artifact: ${dbArtifact.name} (${(dbArtifact.size_in_bytes / 1024 / 1024).toFixed(1)} MB)`, 'green'); + + // Download artifact + const tempDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'awesome-db-')); + + log('Downloading artifact...', 'blue'); + execSync(`gh run download ${latestRun.id} -R ${AWESOME_REPO} -D ${tempDir}`, { + stdio: 'inherit' + }); + + // Find and copy database file + const files = fs.readdirSync(tempDir, { recursive: true, withFileTypes: true }); + const dbFile = files.find(f => f.isFile() && f.name.endsWith('.db')); + + if (!dbFile) { + log('āŒ Database file not found in artifact', 'red'); + fs.rmSync(tempDir, { recursive: true, force: true }); + return false; + } + + const dbFilePath = path.join(dbFile.path || tempDir, dbFile.name); + fs.copyFileSync(dbFilePath, DB_PATH); + + // Copy metadata if available + const metadataFile = files.find(f => f.isFile() && f.name === 'metadata.json'); + if (metadataFile) { + const metadataPath = path.join(metadataFile.path || tempDir, metadataFile.name); + fs.copyFileSync(metadataPath, path.join(process.cwd(), 'db-metadata.json')); + } + + // Cleanup + fs.rmSync(tempDir, { recursive: true, force: true }); + + const size = fs.statSync(DB_PATH).size; + log(`āœ“ Database downloaded successfully (${(size / 1024 / 1024).toFixed(2)} MB)`, 'green'); + + return true; + } catch (error) { + log(`āŒ Download failed: ${error.message}`, 'red'); + return false; + } } -// Run build -build().catch(error => { - console.error('āŒ Build failed:', error); +async function buildDatabaseLocally() { + log('\nšŸ”Ø Building database locally using awesome CLI...', 'cyan'); + + // Check if awesome CLI is available + const awesomePath = path.join(__dirname, '../../awesome/awesome'); + + if (!fs.existsSync(awesomePath)) { + log('āŒ Awesome CLI not found at: ' + awesomePath, 'red'); + log('Expected location: /path/to/awesome/awesome', 'yellow'); + log('Please ensure the awesome repository is checked out as a sibling directory', 'yellow'); + process.exit(1); + } + + try { + // Ensure awesome CLI is executable + fs.chmodSync(awesomePath, '755'); + + log('Installing awesome CLI dependencies...', 'blue'); + execSync('pnpm install && pnpm rebuild better-sqlite3', { + cwd: path.dirname(awesomePath), + stdio: 'inherit' + }); + + // Configure GitHub token if available + if (GITHUB_TOKEN) { + log('Configuring GitHub token for API access...', 'blue'); + execSync(`node -e " + const db = require('./lib/database'); + const dbOps = require('./lib/db-operations'); + db.initialize(); + dbOps.setSetting('githubToken', '${GITHUB_TOKEN}'); + db.close(); + "`, { + cwd: path.dirname(awesomePath), + stdio: 'inherit' + }); + } + + log('Building index (this may take 1-2 hours)...', 'blue'); + + // Run indexer with full mode + const buildProcess = spawn(awesomePath, ['index'], { + cwd: path.dirname(awesomePath), + stdio: ['pipe', 'inherit', 'inherit'] + }); + + // Automatically select 'full' mode + buildProcess.stdin.write('full\n'); + buildProcess.stdin.end(); + + await new Promise((resolve, reject) => { + buildProcess.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`Build failed with code ${code}`)); + } + }); + buildProcess.on('error', reject); + }); + + // Copy database to current directory + const awesomeDbPath = path.join(require('os').homedir(), '.awesome', 'awesome.db'); + + if (!fs.existsSync(awesomeDbPath)) { + throw new Error('Database not created at expected location: ' + awesomeDbPath); + } + + fs.copyFileSync(awesomeDbPath, DB_PATH); + + const size = fs.statSync(DB_PATH).size; + log(`āœ“ Database built successfully (${(size / 1024 / 1024).toFixed(2)} MB)`, 'green'); + + return true; + } catch (error) { + log(`āŒ Build failed: ${error.message}`, 'red'); + return false; + } +} + +async function generateMetadata() { + log('\nšŸ“Š Generating metadata...', 'cyan'); + + try { + const Database = require('better-sqlite3'); + const db = new Database(DB_PATH, { readonly: true }); + + const listsCount = db.prepare('SELECT COUNT(*) as count FROM awesome_lists').get().count; + const reposCount = db.prepare('SELECT COUNT(*) as count FROM repositories').get().count; + const readmesCount = db.prepare('SELECT COUNT(*) as count FROM readmes').get().count; + + db.close(); + + const stats = fs.statSync(DB_PATH); + const size = (stats.size / 1024 / 1024).toFixed(2); + const hash = require('crypto') + .createHash('sha256') + .update(fs.readFileSync(DB_PATH)) + .digest('hex'); + + const metadata = { + version: process.env.GITHUB_SHA || 'unknown', + timestamp: new Date().toISOString(), + size: `${size}MB`, + hash: hash, + lists_count: listsCount, + repos_count: reposCount, + readmes_count: readmesCount, + build_mode: BUILD_MODE, + source_repo: AWESOME_REPO + }; + + fs.writeFileSync( + path.join(process.cwd(), 'db-metadata.json'), + JSON.stringify(metadata, null, 2) + ); + + log('āœ“ Metadata generated', 'green'); + log(` Lists: ${listsCount}`, 'blue'); + log(` Repositories: ${reposCount}`, 'blue'); + log(` READMEs: ${readmesCount}`, 'blue'); + log(` Size: ${size} MB`, 'blue'); + + return metadata; + } catch (error) { + log(`āš ļø Failed to generate metadata: ${error.message}`, 'yellow'); + return null; + } +} + +async function main() { + log('\n' + '='.repeat(60), 'bright'); + log(' šŸš€ AWESOME DATABASE BUILDER', 'bright'); + log('='.repeat(60) + '\n', 'bright'); + + log(`Build Mode: ${BUILD_MODE}`, 'cyan'); + log(`Source Repo: ${AWESOME_REPO}\n`, 'cyan'); + + let success = false; + + if (BUILD_MODE === 'download') { + success = await downloadDatabase(); + + if (!success) { + log('\nāš ļø Download failed, attempting local build...', 'yellow'); + success = await buildDatabaseLocally(); + } + } else { + success = await buildDatabaseLocally(); + } + + if (!success) { + log('\nāŒ Database build failed', 'red'); + process.exit(1); + } + + // Generate metadata + await generateMetadata(); + + log('\n' + '='.repeat(60), 'bright'); + log(' āœ… BUILD COMPLETE', 'green'); + log('='.repeat(60) + '\n', 'bright'); + + log(`Database: ${DB_PATH}`, 'cyan'); + log(`Metadata: ${path.join(process.cwd(), 'db-metadata.json')}\n`, 'cyan'); +} + +// Run +main().catch(error => { + log(`\nāŒ Fatal error: ${error.message}`, 'red'); + console.error(error); process.exit(1); });