fix: workflows

2025-10-26 02:05:34 +02:00
parent 6ee26db0ba
commit 7ab9b4f091
7 changed files with 488 additions and 3 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -56,10 +56,13 @@ next-env.d.ts
 .claude

 # SQLite database files (will be mounted as volume)
+# Note: awesome.db is explicitly included when built by CI
 *.db
+!awesome.db
 *.db-journal
 *.sqlite
 *.sqlite3
+!db-metadata.json

 # misc
 .mcp.json
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -0,0 +1,188 @@
+# GitHub Actions Workflows
+
+This directory contains automated workflows for building, testing, and deploying the awesome-app.
+
+## Workflows
+
+### 1. `db.yml` - Build Awesome Database
+
+**Triggers:**
+- Schedule: Every 6 hours
+- Manual: `workflow_dispatch`
+- Push to main (when db.yml or build-db.js changes)
+
+**Purpose:**
+Builds the SQLite database by scraping awesome lists from GitHub.
+
+**Artifacts:**
+- `awesome.db` - SQLite database file
+- `db-metadata.json` - Metadata about the build (timestamp, hash, counts)
+
+**Retention:** 90 days
+
+---
+
+### 2. `docker-publish.yml` - Build and Push Docker Image
+
+**Triggers:**
+- Push to `main` or `develop` branches
+- Git tags matching `v*.*.*`
+- Pull requests to `main`
+- Manual: `workflow_dispatch`
+
+**Dependencies:**
+- Calls `db.yml` workflow first
+- Downloads database artifact before building Docker image
+
+**Features:**
+- Multi-platform builds (linux/amd64, linux/arm64)
+- Automatic semantic versioning from git tags
+- GitHub Container Registry (ghcr.io)
+- Build cache optimization
+- Database metadata embedded in image labels
+- Configurable database inclusion via `INCLUDE_DATABASE` build arg (default: `true` in CI)
+
+**Image Tags:**
+- `latest` - Latest build from main branch
+- `main` - Latest main branch build
+- `develop` - Latest develop branch build
+- `v1.2.3` - Semantic version tags
+- `main-abc1234` - Branch + commit SHA
+
+**Image Labels:**
+- Standard OCI labels (title, description, vendor, source)
+- Database metadata (timestamp, hash, counts)
+
+---
+
+### 3. `docker-scan.yml` - Security Scanning
+
+**Triggers:**
+- Schedule: Daily at 2 AM UTC
+- Push to `main` branch
+- Git tags matching `v*.*.*`
+- Manual: `workflow_dispatch`
+
+**Purpose:**
+Scans Docker images for security vulnerabilities using Trivy.
+
+**Features:**
+- SARIF report upload to GitHub Security tab
+- Scans for CRITICAL, HIGH, and MEDIUM severity issues
+- Automated daily security checks
+
+---
+
+### 4. `cleanup-images.yml` - Cleanup Old Docker Images
+
+**Triggers:**
+- Schedule: Weekly on Sundays at 3 AM UTC
+- Manual: `workflow_dispatch` (configurable retention count)
+
+**Purpose:**
+Removes old untagged Docker images to save storage.
+
+**Configuration:**
+- Default: Keep 10 most recent versions
+- Configurable via workflow_dispatch input
+
+---
+
+## Workflow Integration
+
+The workflows are designed to work together:
+
+```mermaid
+graph LR
+    A[db.yml] --> B[docker-publish.yml]
+    B --> C[docker-scan.yml]
+    D[cleanup-images.yml]
+```
+
+1. **Database Build** (`db.yml`) runs every 6 hours and on-demand
+2. **Docker Build** (`docker-publish.yml`) depends on database build
+3. **Security Scan** (`docker-scan.yml`) runs after image push
+4. **Cleanup** (`cleanup-images.yml`) runs weekly to free storage
+
+## Usage
+
+### Manual Database Build
+
+```bash
+gh workflow run db.yml
+```
+
+### Manual Docker Build
+
+```bash
+gh workflow run docker-publish.yml -f tag=custom-tag
+```
+
+### Manual Security Scan
+
+```bash
+gh workflow run docker-scan.yml
+```
+
+### Manual Cleanup
+
+```bash
+gh workflow run cleanup-images.yml -f keep_count=20
+```
+
+## Environment Variables
+
+Required repository secrets:
+- `GITHUB_TOKEN` - Automatically provided
+- `WEBHOOK_URL` - (Optional) Webhook for database updates
+- `WEBHOOK_SECRET` - (Optional) Secret for webhook authentication
+
+## Docker Image
+
+Pull the latest image:
+
+```bash
+docker pull ghcr.io/valknarness/awesome-app:latest
+```
+
+Run with embedded database (CI builds):
+
+```bash
+docker run -p 3000:3000 ghcr.io/valknarness/awesome-app:latest
+```
+
+Run with volume-mounted database (local builds):
+
+```bash
+docker run -p 3000:3000 -v $(pwd)/data:/app/data ghcr.io/valknarness/awesome-app:latest
+```
+
+### Build Arguments
+
+Control database inclusion when building locally:
+
+```bash
+# Include database in image (like CI)
+docker build --build-arg INCLUDE_DATABASE=true -t awesome-app .
+
+# Exclude database (mount at runtime)
+docker build --build-arg INCLUDE_DATABASE=false -t awesome-app .
+```
+
+Or with docker-compose:
+
+```yaml
+services:
+  awesome-app:
+    build:
+      args:
+        INCLUDE_DATABASE: true  # or false
+```
+
+## Best Practices
+
+1. **Database builds** happen automatically every 6 hours
+2. **Docker images** are built on every push to main/develop
+3. **Security scans** run daily to catch new vulnerabilities
+4. **Old images** are cleaned up weekly to save storage
+5. **Database metadata** is embedded in Docker image labels for traceability
--- a/.github/workflows/db.yml
+++ b/.github/workflows/db.yml
@@ -20,14 +20,19 @@ jobs:
      - name: Checkout repository
        uses: actions/checkout@v4

+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '22'
-          cache: 'npm'
+          cache: 'pnpm'

      - name: Install dependencies
-        run: npm ci
+        run: pnpm install --frozen-lockfile

      - name: Build SQLite Database
        env:
--- a/.github/workflows/docker-publish.yml
+++ b/.github/workflows/docker-publish.yml
@@ -22,7 +22,12 @@ env:
  IMAGE_NAME: valknarness/awesome-app

 jobs:
+  build-database:
+    uses: ./.github/workflows/db.yml
+    secrets: inherit
+
  build-and-push:
+    needs: build-database
    runs-on: ubuntu-latest
    permissions:
      contents: read
@@ -33,6 +38,27 @@ jobs:
    - name: Checkout repository
      uses: actions/checkout@v4

+    - name: Download database artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: awesome-database
+        path: ./
+
+    - name: Verify database artifact
+      run: |
+        ls -lah awesome.db* || echo "No database file found"
+        cat db-metadata.json || echo "No metadata file found"
+
+    - name: Extract database metadata
+      id: db-meta
+      run: |
+        if [ -f db-metadata.json ]; then
+          echo "db_timestamp=$(jq -r '.timestamp' db-metadata.json)" >> $GITHUB_OUTPUT
+          echo "db_hash=$(jq -r '.hash' db-metadata.json)" >> $GITHUB_OUTPUT
+          echo "lists_count=$(jq -r '.lists_count' db-metadata.json)" >> $GITHUB_OUTPUT
+          echo "repos_count=$(jq -r '.repos_count' db-metadata.json)" >> $GITHUB_OUTPUT
+        fi
+
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
      with:
@@ -70,6 +96,10 @@ jobs:
          org.opencontainers.image.description=Next.js application for exploring awesome lists
          org.opencontainers.image.vendor=valknarness
          org.opencontainers.image.source=https://github.com/${{ github.repository }}
+          app.database.timestamp=${{ steps.db-meta.outputs.db_timestamp }}
+          app.database.hash=${{ steps.db-meta.outputs.db_hash }}
+          app.database.lists_count=${{ steps.db-meta.outputs.lists_count }}
+          app.database.repos_count=${{ steps.db-meta.outputs.repos_count }}

    - name: Build and push Docker image
      uses: docker/build-push-action@v5
@@ -83,6 +113,7 @@ jobs:
        cache-to: type=gha,mode=max
        build-args: |
          NODE_ENV=production
+          INCLUDE_DATABASE=false

    - name: Generate image digest
      if: github.event_name != 'pull_request'
@@ -97,6 +128,12 @@ jobs:
        echo "${{ steps.meta.outputs.tags }}" >> $GITHUB_STEP_SUMMARY
        echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "**Database Info:**" >> $GITHUB_STEP_SUMMARY
+        echo "- Built: ${{ steps.db-meta.outputs.db_timestamp }}" >> $GITHUB_STEP_SUMMARY
+        echo "- Lists: ${{ steps.db-meta.outputs.lists_count }}" >> $GITHUB_STEP_SUMMARY
+        echo "- Repositories: ${{ steps.db-meta.outputs.repos_count }}" >> $GITHUB_STEP_SUMMARY
+        echo "- Hash: \`${{ steps.db-meta.outputs.db_hash }}\`" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
        echo "**Pull command:**" >> $GITHUB_STEP_SUMMARY
        echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
        echo "docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> $GITHUB_STEP_SUMMARY
--- a/DOCKER.md
+++ b/DOCKER.md
@@ -0,0 +1,230 @@
+# Docker Deployment Guide
+
+This guide covers building and deploying the awesome-app using Docker.
+
+## Quick Start
+
+### Using Pre-built Image (Recommended)
+
+Pull and run the latest image from GitHub Container Registry:
+
+```bash
+docker pull ghcr.io/valknarness/awesome-app:latest
+docker run -p 3000:3000 ghcr.io/valknarness/awesome-app:latest
+```
+
+The image includes a pre-built database, updated every 6 hours by GitHub Actions.
+
+### Using Docker Compose
+
+```bash
+docker-compose up -d
+```
+
+## Build Options
+
+The Dockerfile supports a build argument `INCLUDE_DATABASE` to control whether the database is embedded in the image or mounted at runtime.
+
+### Option 1: Embedded Database (CI Default)
+
+**Pros:**
+- Self-contained image
+- No external dependencies
+- Faster startup
+- Database version matches image version
+
+**Cons:**
+- Larger image size
+- Database updates require new image build
+
+```bash
+docker build --build-arg INCLUDE_DATABASE=true -t awesome-app .
+docker run -p 3000:3000 awesome-app
+```
+
+### Option 2: Volume-Mounted Database (Local Default)
+
+**Pros:**
+- Smaller image size
+- Database can be updated independently
+- Easier for development
+
+**Cons:**
+- Requires database setup/volume mount
+- Extra configuration needed
+
+```bash
+docker build --build-arg INCLUDE_DATABASE=false -t awesome-app .
+docker run -p 3000:3000 -v $(pwd)/data:/app/data awesome-app
+```
+
+## Docker Compose Configuration
+
+Edit `docker-compose.yml` to control database inclusion:
+
+```yaml
+services:
+  awesome-app:
+    build:
+      args:
+        INCLUDE_DATABASE: false  # Change to true to embed database
+    volumes:
+      - ./data:/app/data  # Only needed when INCLUDE_DATABASE=false
+```
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `NODE_ENV` | `production` | Node.js environment |
+| `PORT` | `3000` | Application port |
+| `HOSTNAME` | `0.0.0.0` | Bind hostname |
+
+## Database Location
+
+- **Embedded mode**: `/app/awesome.db`
+- **Volume mode**: `/app/data/awesome.db` (mounted)
+
+The application will automatically detect and use the database from either location.
+
+## Multi-Platform Support
+
+Images are built for multiple platforms:
+- `linux/amd64` (x86_64)
+- `linux/arm64` (ARM64/Apple Silicon)
+
+Docker will automatically pull the correct architecture for your system.
+
+## Health Checks
+
+The image includes a built-in health check that pings the application every 30 seconds:
+
+```bash
+docker ps  # Check HEALTH status column
+```
+
+## Image Metadata
+
+View database metadata embedded in the image:
+
+```bash
+docker inspect ghcr.io/valknarness/awesome-app:latest | jq '.[0].Config.Labels'
+```
+
+Metadata includes:
+- `app.database.timestamp` - When the database was built
+- `app.database.hash` - SHA256 hash of the database
+- `app.database.lists_count` - Number of awesome lists
+- `app.database.repos_count` - Number of repositories
+
+## Production Deployment
+
+### Using Pre-built Image
+
+```bash
+docker pull ghcr.io/valknarness/awesome-app:latest
+docker run -d \
+  --name awesome-app \
+  -p 3000:3000 \
+  --restart unless-stopped \
+  ghcr.io/valknarness/awesome-app:latest
+```
+
+### With Volume Mount
+
+```bash
+docker run -d \
+  --name awesome-app \
+  -p 3000:3000 \
+  -v awesome-data:/app/data \
+  --restart unless-stopped \
+  ghcr.io/valknarness/awesome-app:latest
+```
+
+### Using Docker Compose
+
+```bash
+docker-compose up -d
+```
+
+## Database Updates
+
+### Embedded Database
+
+Pull the latest image to get an updated database:
+
+```bash
+docker pull ghcr.io/valknarness/awesome-app:latest
+docker-compose up -d  # Recreates container with new image
+```
+
+### Volume-Mounted Database
+
+Update the database file in the mounted volume:
+
+```bash
+# Download latest database
+wget https://github.com/your-repo/releases/latest/download/awesome.db
+
+# Place in volume
+cp awesome.db ./data/
+
+# Restart container
+docker-compose restart
+```
+
+## Troubleshooting
+
+### Database not found
+
+If the application can't find the database:
+
+1. **Embedded mode**: Ensure `INCLUDE_DATABASE=true` was set during build
+2. **Volume mode**: Check that the volume is mounted correctly
+
+```bash
+docker exec awesome-app ls -la /app/awesome.db     # Embedded
+docker exec awesome-app ls -la /app/data/awesome.db # Volume
+```
+
+### Permission issues
+
+Ensure the database file has correct permissions:
+
+```bash
+docker exec awesome-app chown nextjs:nodejs /app/data/awesome.db
+```
+
+### Rebuild from scratch
+
+Remove cached layers and rebuild:
+
+```bash
+docker build --no-cache --build-arg INCLUDE_DATABASE=true -t awesome-app .
+```
+
+## Development
+
+For local development with hot reload:
+
+```bash
+# Use the dev server instead of Docker
+pnpm dev
+```
+
+For testing the production Docker build locally:
+
+```bash
+docker build -t awesome-app-test .
+docker run -p 3000:3000 awesome-app-test
+```
+
+## Security
+
+The container runs as a non-root user (`nextjs:nodejs`) with UID/GID 1001 for enhanced security.
+
+## Support
+
+For issues or questions:
+- GitHub Issues: [your-repo/issues](https://github.com/your-repo/issues)
+- Workflow Docs: [.github/workflows/README.md](.github/workflows/README.md)
--- a/18
+++ b/18
@@ -13,6 +13,10 @@ RUN corepack enable pnpm && pnpm install --frozen-lockfile
 # Rebuild the source code only when needed
 FROM base AS builder
 WORKDIR /app
+
+# Build argument to control database inclusion
+ARG INCLUDE_DATABASE=false
+
 COPY --from=deps /app/node_modules ./node_modules
 COPY . .

@@ -28,6 +32,9 @@ RUN corepack enable pnpm && pnpm run build
 FROM base AS runner
 WORKDIR /app

+# Build argument to control database inclusion (passed from builder)
+ARG INCLUDE_DATABASE=false
+
 ENV NODE_ENV=production
 ENV NEXT_TELEMETRY_DISABLED=1

@@ -52,6 +59,17 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
 # Create directory for SQLite database
 RUN mkdir -p /app/data && chown -R nextjs:nodejs /app/data

+# Conditionally copy pre-built database if INCLUDE_DATABASE=true
+# The database is downloaded by GitHub Actions before Docker build
+RUN if [ "$INCLUDE_DATABASE" = "true" ]; then \
+      echo "Including database in image..."; \
+    else \
+      echo "Database will be mounted at runtime or built on first run"; \
+    fi
+
+COPY --from=builder --chown=nextjs:nodejs /app/awesome.db* /app/ || true
+COPY --from=builder --chown=nextjs:nodejs /app/db-metadata.json /app/ || true
+
 USER nextjs

 EXPOSE 3000
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -3,12 +3,16 @@ services:
    build:
      context: .
      dockerfile: Dockerfile
+      args:
+        # Set to true to include pre-built database in image
+        # Set to false to mount database at runtime
+        INCLUDE_DATABASE: false
    ports:
      - "3000:3000"
    environment:
      - NODE_ENV=production
    volumes:
-      # Mount SQLite database directory
+      # Mount SQLite database directory (used when INCLUDE_DATABASE=false)
      - ./data:/app/data
    restart: unless-stopped
    healthcheck: