feat: make incremental indexing configurable via CLI and workflow

Added full control over incremental indexing behavior:

**CLI Changes:**
- Added `--incremental` flag (default: true)
- Added `--full` flag to disable incremental mode
- Updated interactive prompt to ask about incremental mode

**Function Changes:**
- Updated buildIndex(force, mode, incremental) signature
- Added incremental parameter with default value true
- Conditional logic: if incremental=true, skip unchanged repos; else re-index all
- Added console logging to show incremental mode status

**Workflow Changes:**
- Added `incremental` input (boolean, default: true)
- Passes incremental setting to buildIndex via environment variable
- Defaults to true for scheduled (cron) runs

**Usage Examples:**
```bash
# CLI - incremental mode (default)
./awesome index

# CLI - force full re-index
./awesome index --full

# CLI - explicit incremental
./awesome index --incremental

# Workflow - incremental (default)
gh workflow run build-database.yml

# Workflow - full re-index
gh workflow run build-database.yml -f incremental=false
```

This makes incremental indexing opt-out instead of hardcoded, giving users full control over indexing behavior.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

.github/workflows/build-database.yml

@@ -14,6 +14,11 @@ on:
         options:
           - full
           - sample
+      incremental:
+        description: 'Incremental mode (skip unchanged repos)'
+        required: false
+        default: true
+        type: boolean
 
 permissions:
   contents: read
@@ -49,6 +54,7 @@ jobs:
         env:
           CI: true
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INCREMENTAL: ${{ github.event.inputs.incremental || 'true' }}
         run: |
           # Pass env vars!
           CI=${CI:-false}
@@ -56,9 +62,15 @@ jobs:
           START_TIME=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
           echo "start_time=$START_TIME" >> $GITHUB_OUTPUT
 
-          # Determine index mode
+          # Determine index mode and incremental setting
           INDEX_MODE="${{ github.event.inputs.index_mode || 'full' }}"
+          INCREMENTAL="${{ github.event.inputs.incremental }}"
+          # Default to true if not specified (for scheduled runs)
+          if [ -z "$INCREMENTAL" ]; then
+            INCREMENTAL="true"
+          fi
           echo "Index mode: $INDEX_MODE"
+          echo "Incremental: $INCREMENTAL"
 
           # Build the index in non-interactive mode (350m timeout, job timeout is 360m)
           timeout 350m node -e "
@@ -79,8 +91,9 @@ jobs:
                   console.warn('⚠️  WARNING: No GitHub token found! Rate limit will be 60/hour instead of 5000/hour');
                 }
 
-                // Build index
+                // Build index with incremental flag
-                await indexer.buildIndex(false, '${INDEX_MODE}');
+                const incremental = process.env.INCREMENTAL === 'true';
+                await indexer.buildIndex(false, '${INDEX_MODE}', incremental);
 
                 // Close database
                 db.close();

awesome

@@ -23,9 +23,13 @@ program
   .command('index')
   .description('Build or rebuild the index from awesome lists')
   .option('-f, --force', 'Force rebuild, clearing existing data')
+  .option('-i, --incremental', 'Incremental mode: only update changed repositories (default: true)')
+  .option('--full', 'Full mode: re-index all repositories even if unchanged')
   .action(async (options) => {
     const indexer = require('./lib/indexer');
-    await indexer.buildIndex(options.force);
+    // Determine incremental mode: default true unless --full is specified
+    const incremental = options.full ? false : (options.incremental !== false);
+    await indexer.buildIndex(options.force, null, incremental);
   });
 
 program

lib/indexer.js

@@ -89,7 +89,7 @@ function isAwesomeList(url, name, description) {
 }
 
 // Build the complete index
-async function buildIndex(force = false, mode = null) {
+async function buildIndex(force = false, mode = null, incremental = true) {
   console.clear();
   console.log(purpleGold('\n🚀 AWESOME INDEX BUILDER 🚀\n'));
 
@@ -146,6 +146,7 @@ async function buildIndex(force = false, mode = null) {
   console.log(chalk.green(`✓ Found ${awesomeLists.length} awesome lists!\n`));
 
   let indexChoice = mode;
+  let incrementalChoice = incremental;
 
   // Ask user what to index (only if interactive)
   if (!isNonInteractive) {
@@ -161,9 +162,19 @@ async function buildIndex(force = false, mode = null) {
           { name: '🔍 Select specific categories', value: 'select' },
           { name: '← Back', value: 'cancel' }
         ]
+      },
+      {
+        type: 'confirm',
+        name: 'incremental',
+        message: 'Use incremental mode? (only update changed repos, faster)',
+        default: true,
+        when: (answers) => answers.indexChoice === 'full' || answers.indexChoice === 'sample'
       }
     ]);
     indexChoice = result.indexChoice;
+    if (result.incremental !== undefined) {
+      incrementalChoice = result.incremental;
+    }
   }
 
   if (indexChoice === 'cancel') return;
@@ -171,7 +182,12 @@ async function buildIndex(force = false, mode = null) {
   // Default to 'full' if no mode specified
   if (!indexChoice) indexChoice = 'full';
 
-  console.log(chalk.cyan(`Index mode: ${indexChoice}\n`));
+  console.log(chalk.cyan(`Index mode: ${indexChoice}`));
+  if (indexChoice === 'full' || indexChoice === 'sample') {
+    console.log(chalk.cyan(`Incremental: ${incrementalChoice ? 'Yes (skip unchanged repos)' : 'No (re-index all)'}\n`));
+  } else {
+    console.log('');
+  }
 
   let listsToIndex = awesomeLists;
 
@@ -274,14 +290,20 @@ async function buildIndex(force = false, mode = null) {
             // Determine if we need to fetch README
             let shouldFetchReadme = false;
             if (indexChoice === 'full' || indexChoice === 'sample') {
-              if (!existingRepo) {
+              if (incrementalChoice) {
-                // New repo - fetch README
+                // Incremental mode: only fetch if new or changed
-                shouldFetchReadme = true;
+                if (!existingRepo) {
-              } else if (existingRepo.last_commit !== repoInfo.pushedAt) {
+                  // New repo - fetch README
-                // Repo updated since last index - fetch README
+                  shouldFetchReadme = true;
+                } else if (existingRepo.last_commit !== repoInfo.pushedAt) {
+                  // Repo updated since last index - fetch README
+                  shouldFetchReadme = true;
+                }
+                // else: repo unchanged, skip README fetch
+              } else {
+                // Full re-index mode: always fetch README
                 shouldFetchReadme = true;
               }
-              // else: repo unchanged, skip README fetch
             }
 
             if (shouldFetchReadme) {