feat: github workflow
This commit is contained in:
valknarness
37
.github/workflows/README.md
vendored
37
.github/workflows/README.md
vendored
@@ -8,15 +8,36 @@ This directory contains automated workflows for building, testing, and deploying
|
||||
|
||||
**Triggers:**
|
||||
- Schedule: Every 6 hours
|
||||
- Manual: `workflow_dispatch`
|
||||
- Manual: `workflow_dispatch` with options:
|
||||
- `build_mode`: `download` (default) or `build`
|
||||
- `awesome_repo`: Source repository (default: `valknarness/awesome`)
|
||||
- Push to main (when db.yml or build-db.js changes)
|
||||
|
||||
**Purpose:**
|
||||
Builds the SQLite database by scraping awesome lists from GitHub.
|
||||
Builds the SQLite database using one of two methods:
|
||||
|
||||
1. **Download Mode** (default, fast ~5 min):
|
||||
- Downloads pre-built database from the awesome CLI repository
|
||||
- Uses GitHub Actions artifacts from the upstream repo
|
||||
- Fallback to local build if download fails
|
||||
|
||||
2. **Build Mode** (slow ~1-2 hours):
|
||||
- Clones the awesome CLI repository
|
||||
- Runs full indexing using `./awesome index`
|
||||
- Scrapes all awesome lists from GitHub
|
||||
|
||||
**Dependencies:**
|
||||
- Requires awesome CLI repository (checked out during workflow)
|
||||
- GitHub CLI for artifact downloads
|
||||
- better-sqlite3 for database operations
|
||||
|
||||
**Artifacts:**
|
||||
- `awesome.db` - SQLite database file
|
||||
- `db-metadata.json` - Metadata about the build (timestamp, hash, counts)
|
||||
- `db-metadata.json` - Metadata including:
|
||||
- Build mode used (download/build)
|
||||
- Source repository
|
||||
- Timestamp, hash, counts
|
||||
- Statistics (lists, repos, READMEs)
|
||||
|
||||
**Retention:** 90 days
|
||||
|
||||
@@ -109,7 +130,17 @@ graph LR
|
||||
### Manual Database Build
|
||||
|
||||
```bash
|
||||
# Download pre-built database (fast, default)
|
||||
gh workflow run db.yml
|
||||
|
||||
# Download from specific repository
|
||||
gh workflow run db.yml -f awesome_repo=owner/awesome
|
||||
|
||||
# Build locally (slow but fresh)
|
||||
gh workflow run db.yml -f build_mode=build
|
||||
|
||||
# Custom source and build mode
|
||||
gh workflow run db.yml -f build_mode=build -f awesome_repo=owner/awesome
|
||||
```
|
||||
|
||||
### Manual Docker Build
|
||||
|
||||
171
.github/workflows/db.yml
vendored
171
.github/workflows/db.yml
vendored
@@ -4,7 +4,21 @@ on:
|
||||
schedule:
|
||||
# Run every 6 hours
|
||||
- cron: '0 */6 * * *'
|
||||
workflow_dispatch: # Manual trigger
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_mode:
|
||||
description: 'Build mode: download (from awesome repo) or build (local indexing)'
|
||||
required: false
|
||||
default: 'download'
|
||||
type: choice
|
||||
options:
|
||||
- download
|
||||
- build
|
||||
awesome_repo:
|
||||
description: 'Awesome repository to download from (owner/repo)'
|
||||
required: false
|
||||
default: 'valknarness/awesome'
|
||||
type: string
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
@@ -15,12 +29,22 @@ on:
|
||||
jobs:
|
||||
build-database:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180 # 3 hours for local builds
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
- name: Checkout awesome-app repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: awesome-app
|
||||
|
||||
- name: Install pnpm
|
||||
- name: Checkout awesome CLI repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }}
|
||||
path: awesome
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 10
|
||||
@@ -30,76 +54,121 @@ jobs:
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'pnpm'
|
||||
cache-dependency-path: awesome-app/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
- name: Install GitHub CLI
|
||||
run: |
|
||||
# GitHub CLI is pre-installed on ubuntu-latest
|
||||
gh --version
|
||||
|
||||
- name: Install awesome-app dependencies
|
||||
working-directory: awesome-app
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build SQLite Database
|
||||
- name: Install awesome CLI dependencies
|
||||
working-directory: awesome
|
||||
run: |
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm rebuild better-sqlite3
|
||||
chmod +x awesome
|
||||
|
||||
- name: Build database
|
||||
working-directory: awesome-app
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_MODE: ${{ github.event.inputs.build_mode || 'download' }}
|
||||
AWESOME_REPO: ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }}
|
||||
run: |
|
||||
echo "Build Mode: $BUILD_MODE"
|
||||
echo "Source Repo: $AWESOME_REPO"
|
||||
|
||||
# Run the build script
|
||||
node scripts/build-db.js
|
||||
|
||||
- name: Generate database metadata
|
||||
- name: Verify database
|
||||
working-directory: awesome-app
|
||||
run: |
|
||||
if [ ! -f awesome.db ]; then
|
||||
echo "❌ Database file not found!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DB_SIZE=$(du -h awesome.db | cut -f1)
|
||||
DB_HASH=$(sha256sum awesome.db | cut -d' ' -f1)
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
echo "✅ Database created successfully"
|
||||
echo "Size: $DB_SIZE"
|
||||
|
||||
cat > db-metadata.json << EOF
|
||||
{
|
||||
"version": "${GITHUB_SHA}",
|
||||
"timestamp": "${TIMESTAMP}",
|
||||
"size": "${DB_SIZE}",
|
||||
"hash": "${DB_HASH}",
|
||||
"lists_count": $(sqlite3 awesome.db "SELECT COUNT(*) FROM awesome_lists"),
|
||||
"repos_count": $(sqlite3 awesome.db "SELECT COUNT(*) FROM repositories"),
|
||||
"readmes_count": $(sqlite3 awesome.db "SELECT COUNT(*) FROM readmes")
|
||||
}
|
||||
EOF
|
||||
# Verify it's a valid SQLite database
|
||||
if command -v sqlite3 &> /dev/null; then
|
||||
LISTS_COUNT=$(sqlite3 awesome.db "SELECT COUNT(*) FROM awesome_lists" 2>/dev/null || echo "0")
|
||||
REPOS_COUNT=$(sqlite3 awesome.db "SELECT COUNT(*) FROM repositories" 2>/dev/null || echo "0")
|
||||
READMES_COUNT=$(sqlite3 awesome.db "SELECT COUNT(*) FROM readmes" 2>/dev/null || echo "0")
|
||||
|
||||
cat db-metadata.json
|
||||
echo "Lists: $LISTS_COUNT"
|
||||
echo "Repositories: $REPOS_COUNT"
|
||||
echo "READMEs: $READMES_COUNT"
|
||||
fi
|
||||
|
||||
- name: Upload database artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: awesome-database
|
||||
path: |
|
||||
awesome.db
|
||||
db-metadata.json
|
||||
awesome-app/awesome.db
|
||||
awesome-app/db-metadata.json
|
||||
retention-days: 90
|
||||
compression-level: 9
|
||||
|
||||
- name: Deploy to hosting (optional)
|
||||
if: github.ref == 'refs/heads/main'
|
||||
- name: Create build summary
|
||||
working-directory: awesome-app
|
||||
run: |
|
||||
# Upload to your hosting provider
|
||||
# Example for S3:
|
||||
# aws s3 cp awesome.db s3://your-bucket/awesome.db
|
||||
# aws s3 cp db-metadata.json s3://your-bucket/db-metadata.json
|
||||
cat >> $GITHUB_STEP_SUMMARY <<EOF
|
||||
# 🎉 Awesome Database Build Complete
|
||||
|
||||
# Or webhook to your Next.js app
|
||||
curl -X POST "${{ secrets.WEBHOOK_URL }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-GitHub-Secret: ${{ secrets.WEBHOOK_SECRET }}" \
|
||||
-d @db-metadata.json
|
||||
## 📊 Build Information
|
||||
|
||||
- name: Create release (on schedule)
|
||||
if: github.event_name == 'schedule'
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: db-${{ github.run_number }}
|
||||
name: Database Build ${{ github.run_number }}
|
||||
body: |
|
||||
Automated database build
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Build Mode | ${{ github.event.inputs.build_mode || 'download' }} |
|
||||
| Source Repository | ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }} |
|
||||
| Workflow Run | [\#${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) |
|
||||
| Commit | \`${{ github.sha }}\` |
|
||||
|
||||
**Statistics:**
|
||||
- Lists: $(sqlite3 awesome.db "SELECT COUNT(*) FROM awesome_lists")
|
||||
- Repositories: $(sqlite3 awesome.db "SELECT COUNT(*) FROM repositories")
|
||||
- READMEs: $(sqlite3 awesome.db "SELECT COUNT(*) FROM readmes")
|
||||
EOF
|
||||
|
||||
**Generated:** $(date -u)
|
||||
files: |
|
||||
awesome.db
|
||||
db-metadata.json
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# Add metadata if available
|
||||
if [ -f db-metadata.json ]; then
|
||||
echo "## 📈 Database Statistics" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "\`\`\`json" >> $GITHUB_STEP_SUMMARY
|
||||
cat db-metadata.json >> $GITHUB_STEP_SUMMARY
|
||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
|
||||
cat >> $GITHUB_STEP_SUMMARY <<EOF
|
||||
|
||||
## 📥 Download Instructions
|
||||
|
||||
\`\`\`bash
|
||||
# Using GitHub CLI
|
||||
gh run download ${{ github.run_id }} -n awesome-database
|
||||
|
||||
# Or visit the workflow run page:
|
||||
# ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
\`\`\`
|
||||
|
||||
## 🐳 Docker Build
|
||||
|
||||
This database will be automatically used in the next Docker image build.
|
||||
EOF
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
cat >> $GITHUB_STEP_SUMMARY <<EOF
|
||||
# ❌ Database Build Failed
|
||||
|
||||
**Build Mode:** ${{ github.event.inputs.build_mode || 'download' }}
|
||||
**Source Repo:** ${{ github.event.inputs.awesome_repo || 'valknarness/awesome' }}
|
||||
|
||||
Please check the workflow logs for details.
|
||||
EOF
|
||||
|
||||
355
DATABASE_BUILD.md
Normal file
355
DATABASE_BUILD.md
Normal file
@@ -0,0 +1,355 @@
|
||||
# Database Build System
|
||||
|
||||
This document explains how the awesome-app builds and maintains its SQLite database.
|
||||
|
||||
## Overview
|
||||
|
||||
The awesome-app uses a flexible database build system with two modes:
|
||||
|
||||
1. **Download Mode** (Fast): Downloads pre-built databases from the awesome CLI repository
|
||||
2. **Build Mode** (Slow): Builds the database locally using the awesome CLI indexer
|
||||
|
||||
## Build Modes
|
||||
|
||||
### Download Mode (Default) ⚡
|
||||
|
||||
Downloads pre-built databases that are automatically created by the awesome CLI repository.
|
||||
|
||||
**Speed:** ~5 minutes
|
||||
**Requirements:**
|
||||
- GitHub CLI (`gh`)
|
||||
- GitHub authentication
|
||||
- Access to awesome CLI repository artifacts
|
||||
|
||||
**How it works:**
|
||||
1. Fetches latest successful workflow run from awesome CLI repo
|
||||
2. Downloads database artifact from GitHub Actions
|
||||
3. Extracts and validates database
|
||||
4. Falls back to build mode if download fails
|
||||
|
||||
**Advantages:**
|
||||
- Very fast (5 min vs 1-2 hours)
|
||||
- No GitHub API rate limits needed
|
||||
- Consistent results
|
||||
- Reduces infrastructure load
|
||||
|
||||
### Build Mode 🔨
|
||||
|
||||
Builds the database from scratch using the awesome CLI indexer.
|
||||
|
||||
**Speed:** ~1-2 hours
|
||||
**Requirements:**
|
||||
- awesome CLI repository (cloned during workflow)
|
||||
- GitHub token for API access (5000 req/hour)
|
||||
- Node.js 22+
|
||||
- better-sqlite3
|
||||
|
||||
**How it works:**
|
||||
1. Clones awesome CLI repository
|
||||
2. Installs dependencies
|
||||
3. Runs `./awesome index` with full mode
|
||||
4. Copies resulting database
|
||||
|
||||
**Advantages:**
|
||||
- Always fresh data
|
||||
- No dependency on upstream artifacts
|
||||
- Full control over indexing
|
||||
|
||||
## Scripts
|
||||
|
||||
### `scripts/build-db.js`
|
||||
|
||||
Main build script that handles both modes.
|
||||
|
||||
**Environment Variables:**
|
||||
- `BUILD_MODE`: `download` or `build` (default: `download`)
|
||||
- `AWESOME_REPO`: Repository to download from (default: `valknarness/awesome`)
|
||||
- `GITHUB_TOKEN`: GitHub token for API access
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
# Download mode (default)
|
||||
node scripts/build-db.js
|
||||
|
||||
# Build mode
|
||||
BUILD_MODE=build node scripts/build-db.js
|
||||
|
||||
# Custom repository
|
||||
AWESOME_REPO=owner/repo node scripts/build-db.js
|
||||
|
||||
# Build mode with custom repo
|
||||
BUILD_MODE=build AWESOME_REPO=owner/repo node scripts/build-db.js
|
||||
```
|
||||
|
||||
**Output:**
|
||||
- `awesome.db` - SQLite database file
|
||||
- `db-metadata.json` - Build metadata
|
||||
|
||||
**Metadata Format:**
|
||||
```json
|
||||
{
|
||||
"version": "abc123...",
|
||||
"timestamp": "2025-10-26T13:45:00.000Z",
|
||||
"size": "156.42MB",
|
||||
"hash": "sha256...",
|
||||
"lists_count": 450,
|
||||
"repos_count": 15000,
|
||||
"readmes_count": 12500,
|
||||
"build_mode": "download",
|
||||
"source_repo": "valknarness/awesome"
|
||||
}
|
||||
```
|
||||
|
||||
## GitHub Actions Workflow
|
||||
|
||||
### `.github/workflows/db.yml`
|
||||
|
||||
**Triggers:**
|
||||
- Schedule: Every 6 hours
|
||||
- Manual: `workflow_dispatch` with parameters
|
||||
- Push: Changes to db.yml or build-db.js
|
||||
|
||||
**Workflow Inputs:**
|
||||
- `build_mode`: Choose download or build
|
||||
- `awesome_repo`: Source repository
|
||||
|
||||
**Steps:**
|
||||
1. Checkout awesome-app repository
|
||||
2. Checkout awesome CLI repository
|
||||
3. Install dependencies for both
|
||||
4. Run build script
|
||||
5. Verify database
|
||||
6. Upload artifact
|
||||
7. Create summary
|
||||
|
||||
**Artifacts:**
|
||||
- Name: `awesome-database`
|
||||
- Retention: 90 days
|
||||
- Contents:
|
||||
- awesome.db
|
||||
- db-metadata.json
|
||||
|
||||
**Manual Triggering:**
|
||||
```bash
|
||||
# Download mode (fast)
|
||||
gh workflow run db.yml
|
||||
|
||||
# Build mode (slow)
|
||||
gh workflow run db.yml -f build_mode=build
|
||||
|
||||
# Custom repository
|
||||
gh workflow run db.yml -f awesome_repo=owner/awesome
|
||||
|
||||
# Both options
|
||||
gh workflow run db.yml -f build_mode=build -f awesome_repo=owner/awesome
|
||||
```
|
||||
|
||||
## Integration with Docker
|
||||
|
||||
The docker-publish workflow automatically uses the database artifact:
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
build-database:
|
||||
uses: ./.github/workflows/db.yml
|
||||
secrets: inherit
|
||||
|
||||
build-and-push:
|
||||
needs: build-database
|
||||
# Downloads artifact and includes in Docker image
|
||||
```
|
||||
|
||||
**Docker Build Args:**
|
||||
- `INCLUDE_DATABASE`: `true` to embed database in image (default: `false`)
|
||||
|
||||
## Local Development
|
||||
|
||||
### Option 1: Download Database
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
pnpm install
|
||||
|
||||
# Build database (download mode)
|
||||
node scripts/build-db.js
|
||||
|
||||
# Or with environment variables
|
||||
BUILD_MODE=download AWESOME_REPO=valknarness/awesome node scripts/build-db.js
|
||||
```
|
||||
|
||||
### Option 2: Build Database
|
||||
|
||||
```bash
|
||||
# Clone awesome CLI (sibling directory)
|
||||
cd ..
|
||||
git clone https://github.com/valknarness/awesome.git
|
||||
cd awesome-app
|
||||
|
||||
# Build database
|
||||
BUILD_MODE=build node scripts/build-db.js
|
||||
```
|
||||
|
||||
### Option 3: Download from GitHub Actions
|
||||
|
||||
```bash
|
||||
# Find latest run
|
||||
gh run list --workflow=db.yml --limit 1
|
||||
|
||||
# Download artifact
|
||||
gh run download <run-id> -n awesome-database
|
||||
|
||||
# Move to project root
|
||||
mv awesome-database/awesome.db .
|
||||
mv awesome-database/db-metadata.json .
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Download Mode Issues
|
||||
|
||||
**Problem:** No artifacts found
|
||||
```bash
|
||||
# Solution: Build locally instead
|
||||
BUILD_MODE=build node scripts/build-db.js
|
||||
```
|
||||
|
||||
**Problem:** GitHub CLI not authenticated
|
||||
```bash
|
||||
# Solution: Authenticate
|
||||
gh auth login
|
||||
|
||||
# Or use token
|
||||
export GITHUB_TOKEN=your_token_here
|
||||
```
|
||||
|
||||
**Problem:** Download fails
|
||||
```bash
|
||||
# The script automatically falls back to build mode
|
||||
# Check logs for fallback message
|
||||
```
|
||||
|
||||
### Build Mode Issues
|
||||
|
||||
**Problem:** awesome CLI not found
|
||||
```bash
|
||||
# Solution: Clone to sibling directory
|
||||
cd ..
|
||||
git clone https://github.com/valknarness/awesome.git
|
||||
cd awesome-app
|
||||
```
|
||||
|
||||
**Problem:** API rate limits
|
||||
```bash
|
||||
# Solution: Set GitHub token
|
||||
export GITHUB_TOKEN=your_token_here
|
||||
```
|
||||
|
||||
**Problem:** Build timeout
|
||||
```bash
|
||||
# Solution: Increase timeout or use download mode
|
||||
# Build mode takes 1-2 hours for full index
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### For CI/CD
|
||||
|
||||
1. **Use download mode** by default (fast, reliable)
|
||||
2. **Set proper secrets** (GITHUB_TOKEN for downloads)
|
||||
3. **Configure fallback** to build mode if download fails
|
||||
4. **Monitor artifacts** retention (90 days)
|
||||
5. **Track build time** to detect issues
|
||||
|
||||
### For Local Development
|
||||
|
||||
1. **Use download mode** for quick setup
|
||||
2. **Use build mode** only when needed (testing indexer, custom data)
|
||||
3. **Keep awesome CLI updated** (pull latest changes)
|
||||
4. **Check metadata** to verify database freshness
|
||||
|
||||
### For Production
|
||||
|
||||
1. **Schedule regular builds** (every 6 hours recommended)
|
||||
2. **Monitor build success** rate
|
||||
3. **Verify database integrity** after builds
|
||||
4. **Track metadata** for troubleshooting
|
||||
5. **Implement alerts** for build failures
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[GitHub Actions Trigger] --> B{Build Mode?}
|
||||
B -->|download| C[Download from awesome CLI]
|
||||
B -->|build| D[Clone awesome CLI]
|
||||
|
||||
C --> E{Download Success?}
|
||||
E -->|Yes| F[Extract Database]
|
||||
E -->|No| D
|
||||
|
||||
D --> G[Install Dependencies]
|
||||
G --> H[Run ./awesome index]
|
||||
H --> I[Copy Database]
|
||||
|
||||
F --> J[Generate Metadata]
|
||||
I --> J
|
||||
|
||||
J --> K[Upload Artifact]
|
||||
K --> L[Docker Build Uses Artifact]
|
||||
```
|
||||
|
||||
## Database Schema
|
||||
|
||||
The database is built by the awesome CLI and follows this schema:
|
||||
|
||||
**Tables:**
|
||||
- `awesome_lists` - Hierarchical awesome lists
|
||||
- `repositories` - Individual projects
|
||||
- `readmes` - Full README content
|
||||
- `readmes_fts` - Full-text search index
|
||||
- `bookmarks` - User favorites (web app only)
|
||||
- `custom_lists` - User lists (web app only)
|
||||
- `reading_history` - Activity tracking (web app only)
|
||||
- `tags` - Content tags
|
||||
- `categories` - Content categories
|
||||
- `settings` - Configuration
|
||||
|
||||
**Indexes:**
|
||||
- FTS5 index for full-text search
|
||||
- Foreign key indexes for relations
|
||||
- Performance indexes on common queries
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Key Metrics
|
||||
|
||||
- **Build time** (download: ~5 min, build: ~1-2 hours)
|
||||
- **Database size** (~50-200 MB compressed)
|
||||
- **Success rate** (target: >95%)
|
||||
- **Artifact retention** (90 days)
|
||||
- **Download vs build ratio** (target: 90% download, 10% build)
|
||||
|
||||
### Health Checks
|
||||
|
||||
```bash
|
||||
# Check latest build
|
||||
gh run list --workflow=db.yml --limit 1
|
||||
|
||||
# Download and verify
|
||||
gh run download <run-id> -n awesome-database
|
||||
sqlite3 awesome.db "SELECT COUNT(*) FROM repositories"
|
||||
|
||||
# Check metadata
|
||||
cat db-metadata.json | jq .
|
||||
```
|
||||
|
||||
## Future Improvements
|
||||
|
||||
1. **Incremental updates** - Only update changed READMEs
|
||||
2. **CDN hosting** - Serve database from CDN for faster downloads
|
||||
3. **Multiple regions** - Build and host in different regions
|
||||
4. **Compression** - Better compression algorithms
|
||||
5. **Checksums** - Verify database integrity
|
||||
6. **Notifications** - Alert on build failures
|
||||
7. **Metrics** - Track build performance over time
|
||||
8. **Caching** - Cache intermediate results
|
||||
17
README.md
17
README.md
@@ -30,12 +30,15 @@ This webapp perfectly matches the **beautiful purple/pink/gold theme** from the
|
||||
- Typography plugin
|
||||
|
||||
3. **GitHub Actions Workflow**
|
||||
- Automated database building
|
||||
- Runs every 6 hours
|
||||
- Manual trigger support
|
||||
- Artifact upload
|
||||
- Release creation
|
||||
- Webhook integration
|
||||
- Two build modes:
|
||||
- **Download Mode**: Fast (~5 min) - downloads pre-built database from awesome CLI repo
|
||||
- **Build Mode**: Slow (~1-2 hours) - builds locally using awesome CLI indexer
|
||||
- Runs every 6 hours (download mode by default)
|
||||
- Manual trigger with mode selection
|
||||
- Configurable source repository
|
||||
- Artifact upload (90-day retention)
|
||||
- Metadata generation with build stats
|
||||
- Fallback to local build if download fails
|
||||
|
||||
4. **Web Worker System**
|
||||
- Smart polling with exponential backoff
|
||||
@@ -82,7 +85,7 @@ awesome-web/
|
||||
│ ├── manifest.json ✅ PWA manifest
|
||||
│ └── icons/ 🔨 Generate from logo
|
||||
├── scripts/
|
||||
│ └── build-db.js ✅ Database builder
|
||||
│ └── build-db.js ✅ Database builder (download/build modes)
|
||||
├── tailwind.config.ts ✅ Custom theme
|
||||
└── next.config.js ✅ PWA & optimization
|
||||
```
|
||||
|
||||
@@ -2,142 +2,316 @@
|
||||
|
||||
/**
|
||||
* Build Awesome Database for GitHub Actions
|
||||
* This script indexes awesome lists and builds the SQLite database
|
||||
* This script uses the awesome CLI to either download a pre-built database
|
||||
* or build it from scratch using the indexer
|
||||
*/
|
||||
|
||||
const Database = require('better-sqlite3');
|
||||
const axios = require('axios');
|
||||
const { execSync, spawn } = require('child_process');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const DB_PATH = path.join(process.cwd(), 'awesome.db');
|
||||
const AWESOME_REPO = process.env.AWESOME_REPO || 'valknarness/awesome';
|
||||
const BUILD_MODE = process.env.BUILD_MODE || 'download'; // 'download' or 'build'
|
||||
const GITHUB_TOKEN = process.env.GITHUB_TOKEN;
|
||||
const RATE_LIMIT_DELAY = 100;
|
||||
|
||||
let lastRequestTime = 0;
|
||||
let requestCount = 0;
|
||||
// Colors for console output
|
||||
const colors = {
|
||||
reset: '\x1b[0m',
|
||||
bright: '\x1b[1m',
|
||||
green: '\x1b[32m',
|
||||
yellow: '\x1b[33m',
|
||||
blue: '\x1b[34m',
|
||||
red: '\x1b[31m',
|
||||
cyan: '\x1b[36m'
|
||||
};
|
||||
|
||||
// Rate-limited request
|
||||
async function rateLimitedRequest(url) {
|
||||
const now = Date.now();
|
||||
const timeSinceLastRequest = now - lastRequestTime;
|
||||
|
||||
if (timeSinceLastRequest < RATE_LIMIT_DELAY) {
|
||||
await new Promise(resolve => setTimeout(resolve, RATE_LIMIT_DELAY - timeSinceLastRequest));
|
||||
}
|
||||
|
||||
lastRequestTime = Date.now();
|
||||
requestCount++;
|
||||
|
||||
const headers = {
|
||||
'Accept': 'application/vnd.github.v3+json',
|
||||
'User-Agent': 'awesome-web-builder',
|
||||
};
|
||||
|
||||
if (GITHUB_TOKEN) {
|
||||
headers['Authorization'] = `token ${GITHUB_TOKEN}`;
|
||||
}
|
||||
function log(message, color = 'reset') {
|
||||
console.log(`${colors[color]}${message}${colors.reset}`);
|
||||
}
|
||||
|
||||
function checkCommand(command) {
|
||||
try {
|
||||
return await axios.get(url, { headers, timeout: 10000 });
|
||||
} catch (error) {
|
||||
if (error.response?.status === 404) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
execSync(`which ${command}`, { stdio: 'ignore' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize database
|
||||
function initializeDatabase() {
|
||||
console.log('🗄️ Initializing database...');
|
||||
async function downloadDatabase() {
|
||||
log('\n📥 Downloading pre-built database from GitHub Actions...', 'cyan');
|
||||
|
||||
const db = new Database(DB_PATH);
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('foreign_keys = ON');
|
||||
|
||||
// Create tables
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS awesome_lists (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
category TEXT,
|
||||
stars INTEGER DEFAULT 0,
|
||||
indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS repositories (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
awesome_list_id INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
stars INTEGER DEFAULT 0,
|
||||
language TEXT,
|
||||
topics TEXT,
|
||||
FOREIGN KEY (awesome_list_id) REFERENCES awesome_lists(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS readmes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
repository_id INTEGER NOT NULL UNIQUE,
|
||||
content TEXT,
|
||||
raw_content TEXT,
|
||||
indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (repository_id) REFERENCES repositories(id)
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS readmes_fts USING fts5(
|
||||
repository_name,
|
||||
description,
|
||||
content,
|
||||
tags,
|
||||
categories,
|
||||
content_rowid UNINDEXED
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_repos_list ON repositories(awesome_list_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_readmes_repo ON readmes(repository_id);
|
||||
`);
|
||||
|
||||
console.log('✅ Database initialized');
|
||||
return db;
|
||||
}
|
||||
|
||||
// Main build process
|
||||
async function build() {
|
||||
console.log('🚀 Starting Awesome Database Build\n');
|
||||
|
||||
const db = initializeDatabase();
|
||||
|
||||
console.log('📥 Fetching main awesome list...');
|
||||
const mainReadme = await rateLimitedRequest(
|
||||
'https://raw.githubusercontent.com/sindresorhus/awesome/main/readme.md'
|
||||
);
|
||||
|
||||
if (!mainReadme) {
|
||||
console.error('❌ Failed to fetch main awesome list');
|
||||
// Check if gh CLI is installed
|
||||
if (!checkCommand('gh')) {
|
||||
log('❌ GitHub CLI (gh) is not installed', 'red');
|
||||
log('Install from: https://cli.github.com/', 'yellow');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('✅ Fetched main list\n');
|
||||
// Authenticate gh CLI if needed
|
||||
try {
|
||||
execSync('gh auth status', { stdio: 'ignore' });
|
||||
} catch {
|
||||
log('⚠️ Not authenticated with GitHub CLI', 'yellow');
|
||||
if (GITHUB_TOKEN) {
|
||||
log('Using GITHUB_TOKEN from environment...', 'blue');
|
||||
process.env.GH_TOKEN = GITHUB_TOKEN;
|
||||
} else {
|
||||
log('❌ No GitHub authentication available', 'red');
|
||||
log('Set GITHUB_TOKEN environment variable or run: gh auth login', 'yellow');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse markdown and build index
|
||||
// For this example, we'll do a simplified version
|
||||
// In production, use the full indexer logic from the CLI
|
||||
try {
|
||||
// Get latest successful workflow run
|
||||
log(`Fetching latest database build from ${AWESOME_REPO}...`, 'blue');
|
||||
|
||||
console.log('📊 Build Statistics:');
|
||||
console.log(` Total Requests: ${requestCount}`);
|
||||
console.log(` Database Size: ${(fs.statSync(DB_PATH).size / 1024 / 1024).toFixed(2)} MB`);
|
||||
const runsOutput = execSync(
|
||||
`gh api -H "Accept: application/vnd.github+json" "/repos/${AWESOME_REPO}/actions/workflows/build-database.yml/runs?per_page=1&status=success"`,
|
||||
{ encoding: 'utf-8' }
|
||||
);
|
||||
|
||||
db.close();
|
||||
console.log('\n✅ Build Complete!');
|
||||
const runs = JSON.parse(runsOutput);
|
||||
|
||||
if (!runs.workflow_runs || runs.workflow_runs.length === 0) {
|
||||
log('❌ No successful database builds found', 'red');
|
||||
log('Falling back to local build...', 'yellow');
|
||||
return false;
|
||||
}
|
||||
|
||||
const latestRun = runs.workflow_runs[0];
|
||||
log(`✓ Found build from ${latestRun.created_at}`, 'green');
|
||||
|
||||
// Get artifacts for this run
|
||||
const artifactsOutput = execSync(
|
||||
`gh api -H "Accept: application/vnd.github+json" "/repos/${AWESOME_REPO}/actions/runs/${latestRun.id}/artifacts"`,
|
||||
{ encoding: 'utf-8' }
|
||||
);
|
||||
|
||||
const artifacts = JSON.parse(artifactsOutput);
|
||||
const dbArtifact = artifacts.artifacts.find(a => a.name.startsWith('awesome-database'));
|
||||
|
||||
if (!dbArtifact) {
|
||||
log('❌ No database artifact found in latest run', 'red');
|
||||
log('Falling back to local build...', 'yellow');
|
||||
return false;
|
||||
}
|
||||
|
||||
log(`✓ Found artifact: ${dbArtifact.name} (${(dbArtifact.size_in_bytes / 1024 / 1024).toFixed(1)} MB)`, 'green');
|
||||
|
||||
// Download artifact
|
||||
const tempDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'awesome-db-'));
|
||||
|
||||
log('Downloading artifact...', 'blue');
|
||||
execSync(`gh run download ${latestRun.id} -R ${AWESOME_REPO} -D ${tempDir}`, {
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
// Find and copy database file
|
||||
const files = fs.readdirSync(tempDir, { recursive: true, withFileTypes: true });
|
||||
const dbFile = files.find(f => f.isFile() && f.name.endsWith('.db'));
|
||||
|
||||
if (!dbFile) {
|
||||
log('❌ Database file not found in artifact', 'red');
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
return false;
|
||||
}
|
||||
|
||||
const dbFilePath = path.join(dbFile.path || tempDir, dbFile.name);
|
||||
fs.copyFileSync(dbFilePath, DB_PATH);
|
||||
|
||||
// Copy metadata if available
|
||||
const metadataFile = files.find(f => f.isFile() && f.name === 'metadata.json');
|
||||
if (metadataFile) {
|
||||
const metadataPath = path.join(metadataFile.path || tempDir, metadataFile.name);
|
||||
fs.copyFileSync(metadataPath, path.join(process.cwd(), 'db-metadata.json'));
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
|
||||
const size = fs.statSync(DB_PATH).size;
|
||||
log(`✓ Database downloaded successfully (${(size / 1024 / 1024).toFixed(2)} MB)`, 'green');
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`❌ Download failed: ${error.message}`, 'red');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Run build
|
||||
build().catch(error => {
|
||||
console.error('❌ Build failed:', error);
|
||||
async function buildDatabaseLocally() {
|
||||
log('\n🔨 Building database locally using awesome CLI...', 'cyan');
|
||||
|
||||
// Check if awesome CLI is available
|
||||
const awesomePath = path.join(__dirname, '../../awesome/awesome');
|
||||
|
||||
if (!fs.existsSync(awesomePath)) {
|
||||
log('❌ Awesome CLI not found at: ' + awesomePath, 'red');
|
||||
log('Expected location: /path/to/awesome/awesome', 'yellow');
|
||||
log('Please ensure the awesome repository is checked out as a sibling directory', 'yellow');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure awesome CLI is executable
|
||||
fs.chmodSync(awesomePath, '755');
|
||||
|
||||
log('Installing awesome CLI dependencies...', 'blue');
|
||||
execSync('pnpm install && pnpm rebuild better-sqlite3', {
|
||||
cwd: path.dirname(awesomePath),
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
// Configure GitHub token if available
|
||||
if (GITHUB_TOKEN) {
|
||||
log('Configuring GitHub token for API access...', 'blue');
|
||||
execSync(`node -e "
|
||||
const db = require('./lib/database');
|
||||
const dbOps = require('./lib/db-operations');
|
||||
db.initialize();
|
||||
dbOps.setSetting('githubToken', '${GITHUB_TOKEN}');
|
||||
db.close();
|
||||
"`, {
|
||||
cwd: path.dirname(awesomePath),
|
||||
stdio: 'inherit'
|
||||
});
|
||||
}
|
||||
|
||||
log('Building index (this may take 1-2 hours)...', 'blue');
|
||||
|
||||
// Run indexer with full mode
|
||||
const buildProcess = spawn(awesomePath, ['index'], {
|
||||
cwd: path.dirname(awesomePath),
|
||||
stdio: ['pipe', 'inherit', 'inherit']
|
||||
});
|
||||
|
||||
// Automatically select 'full' mode
|
||||
buildProcess.stdin.write('full\n');
|
||||
buildProcess.stdin.end();
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
buildProcess.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`Build failed with code ${code}`));
|
||||
}
|
||||
});
|
||||
buildProcess.on('error', reject);
|
||||
});
|
||||
|
||||
// Copy database to current directory
|
||||
const awesomeDbPath = path.join(require('os').homedir(), '.awesome', 'awesome.db');
|
||||
|
||||
if (!fs.existsSync(awesomeDbPath)) {
|
||||
throw new Error('Database not created at expected location: ' + awesomeDbPath);
|
||||
}
|
||||
|
||||
fs.copyFileSync(awesomeDbPath, DB_PATH);
|
||||
|
||||
const size = fs.statSync(DB_PATH).size;
|
||||
log(`✓ Database built successfully (${(size / 1024 / 1024).toFixed(2)} MB)`, 'green');
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`❌ Build failed: ${error.message}`, 'red');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function generateMetadata() {
|
||||
log('\n📊 Generating metadata...', 'cyan');
|
||||
|
||||
try {
|
||||
const Database = require('better-sqlite3');
|
||||
const db = new Database(DB_PATH, { readonly: true });
|
||||
|
||||
const listsCount = db.prepare('SELECT COUNT(*) as count FROM awesome_lists').get().count;
|
||||
const reposCount = db.prepare('SELECT COUNT(*) as count FROM repositories').get().count;
|
||||
const readmesCount = db.prepare('SELECT COUNT(*) as count FROM readmes').get().count;
|
||||
|
||||
db.close();
|
||||
|
||||
const stats = fs.statSync(DB_PATH);
|
||||
const size = (stats.size / 1024 / 1024).toFixed(2);
|
||||
const hash = require('crypto')
|
||||
.createHash('sha256')
|
||||
.update(fs.readFileSync(DB_PATH))
|
||||
.digest('hex');
|
||||
|
||||
const metadata = {
|
||||
version: process.env.GITHUB_SHA || 'unknown',
|
||||
timestamp: new Date().toISOString(),
|
||||
size: `${size}MB`,
|
||||
hash: hash,
|
||||
lists_count: listsCount,
|
||||
repos_count: reposCount,
|
||||
readmes_count: readmesCount,
|
||||
build_mode: BUILD_MODE,
|
||||
source_repo: AWESOME_REPO
|
||||
};
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(process.cwd(), 'db-metadata.json'),
|
||||
JSON.stringify(metadata, null, 2)
|
||||
);
|
||||
|
||||
log('✓ Metadata generated', 'green');
|
||||
log(` Lists: ${listsCount}`, 'blue');
|
||||
log(` Repositories: ${reposCount}`, 'blue');
|
||||
log(` READMEs: ${readmesCount}`, 'blue');
|
||||
log(` Size: ${size} MB`, 'blue');
|
||||
|
||||
return metadata;
|
||||
} catch (error) {
|
||||
log(`⚠️ Failed to generate metadata: ${error.message}`, 'yellow');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
log('\n' + '='.repeat(60), 'bright');
|
||||
log(' 🚀 AWESOME DATABASE BUILDER', 'bright');
|
||||
log('='.repeat(60) + '\n', 'bright');
|
||||
|
||||
log(`Build Mode: ${BUILD_MODE}`, 'cyan');
|
||||
log(`Source Repo: ${AWESOME_REPO}\n`, 'cyan');
|
||||
|
||||
let success = false;
|
||||
|
||||
if (BUILD_MODE === 'download') {
|
||||
success = await downloadDatabase();
|
||||
|
||||
if (!success) {
|
||||
log('\n⚠️ Download failed, attempting local build...', 'yellow');
|
||||
success = await buildDatabaseLocally();
|
||||
}
|
||||
} else {
|
||||
success = await buildDatabaseLocally();
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
log('\n❌ Database build failed', 'red');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Generate metadata
|
||||
await generateMetadata();
|
||||
|
||||
log('\n' + '='.repeat(60), 'bright');
|
||||
log(' ✅ BUILD COMPLETE', 'green');
|
||||
log('='.repeat(60) + '\n', 'bright');
|
||||
|
||||
log(`Database: ${DB_PATH}`, 'cyan');
|
||||
log(`Metadata: ${path.join(process.cwd(), 'db-metadata.json')}\n`, 'cyan');
|
||||
}
|
||||
|
||||
// Run
|
||||
main().catch(error => {
|
||||
log(`\n❌ Fatal error: ${error.message}`, 'red');
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user