From 99cf83330cc03358567dc3c0fd3a43df97e2e02a Mon Sep 17 00:00:00 2001 From: valknarness Date: Tue, 28 Oct 2025 09:59:47 +0100 Subject: [PATCH] feat: make incremental indexing configurable via CLI and workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added full control over incremental indexing behavior: **CLI Changes:** - Added `--incremental` flag (default: true) - Added `--full` flag to disable incremental mode - Updated interactive prompt to ask about incremental mode **Function Changes:** - Updated buildIndex(force, mode, incremental) signature - Added incremental parameter with default value true - Conditional logic: if incremental=true, skip unchanged repos; else re-index all - Added console logging to show incremental mode status **Workflow Changes:** - Added `incremental` input (boolean, default: true) - Passes incremental setting to buildIndex via environment variable - Defaults to true for scheduled (cron) runs **Usage Examples:** ```bash # CLI - incremental mode (default) ./awesome index # CLI - force full re-index ./awesome index --full # CLI - explicit incremental ./awesome index --incremental # Workflow - incremental (default) gh workflow run build-database.yml # Workflow - full re-index gh workflow run build-database.yml -f incremental=false ``` This makes incremental indexing opt-out instead of hardcoded, giving users full control over indexing behavior. šŸ¤– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/build-database.yml | 19 +++++++++++--- awesome | 6 ++++- lib/indexer.js | 38 ++++++++++++++++++++++------ 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-database.yml b/.github/workflows/build-database.yml index cc22e90..98d5439 100644 --- a/.github/workflows/build-database.yml +++ b/.github/workflows/build-database.yml @@ -14,6 +14,11 @@ on: options: - full - sample + incremental: + description: 'Incremental mode (skip unchanged repos)' + required: false + default: true + type: boolean permissions: contents: read @@ -49,6 +54,7 @@ jobs: env: CI: true GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INCREMENTAL: ${{ github.event.inputs.incremental || 'true' }} run: | # Pass env vars! CI=${CI:-false} @@ -56,9 +62,15 @@ jobs: START_TIME=$(date -u +"%Y-%m-%d %H:%M:%S UTC") echo "start_time=$START_TIME" >> $GITHUB_OUTPUT - # Determine index mode + # Determine index mode and incremental setting INDEX_MODE="${{ github.event.inputs.index_mode || 'full' }}" + INCREMENTAL="${{ github.event.inputs.incremental }}" + # Default to true if not specified (for scheduled runs) + if [ -z "$INCREMENTAL" ]; then + INCREMENTAL="true" + fi echo "Index mode: $INDEX_MODE" + echo "Incremental: $INCREMENTAL" # Build the index in non-interactive mode (350m timeout, job timeout is 360m) timeout 350m node -e " @@ -79,8 +91,9 @@ jobs: console.warn('āš ļø WARNING: No GitHub token found! Rate limit will be 60/hour instead of 5000/hour'); } - // Build index - await indexer.buildIndex(false, '${INDEX_MODE}'); + // Build index with incremental flag + const incremental = process.env.INCREMENTAL === 'true'; + await indexer.buildIndex(false, '${INDEX_MODE}', incremental); // Close database db.close(); diff --git a/awesome b/awesome index 367835c..da5fffd 100755 --- a/awesome +++ b/awesome @@ -23,9 +23,13 @@ program .command('index') .description('Build or rebuild the index from awesome lists') .option('-f, --force', 'Force rebuild, clearing existing data') + .option('-i, --incremental', 'Incremental mode: only update changed repositories (default: true)') + .option('--full', 'Full mode: re-index all repositories even if unchanged') .action(async (options) => { const indexer = require('./lib/indexer'); - await indexer.buildIndex(options.force); + // Determine incremental mode: default true unless --full is specified + const incremental = options.full ? false : (options.incremental !== false); + await indexer.buildIndex(options.force, null, incremental); }); program diff --git a/lib/indexer.js b/lib/indexer.js index e62d31f..25a85f2 100644 --- a/lib/indexer.js +++ b/lib/indexer.js @@ -89,7 +89,7 @@ function isAwesomeList(url, name, description) { } // Build the complete index -async function buildIndex(force = false, mode = null) { +async function buildIndex(force = false, mode = null, incremental = true) { console.clear(); console.log(purpleGold('\nšŸš€ AWESOME INDEX BUILDER šŸš€\n')); @@ -146,6 +146,7 @@ async function buildIndex(force = false, mode = null) { console.log(chalk.green(`āœ“ Found ${awesomeLists.length} awesome lists!\n`)); let indexChoice = mode; + let incrementalChoice = incremental; // Ask user what to index (only if interactive) if (!isNonInteractive) { @@ -161,9 +162,19 @@ async function buildIndex(force = false, mode = null) { { name: 'šŸ” Select specific categories', value: 'select' }, { name: 'ā† Back', value: 'cancel' } ] + }, + { + type: 'confirm', + name: 'incremental', + message: 'Use incremental mode? (only update changed repos, faster)', + default: true, + when: (answers) => answers.indexChoice === 'full' || answers.indexChoice === 'sample' } ]); indexChoice = result.indexChoice; + if (result.incremental !== undefined) { + incrementalChoice = result.incremental; + } } if (indexChoice === 'cancel') return; @@ -171,7 +182,12 @@ async function buildIndex(force = false, mode = null) { // Default to 'full' if no mode specified if (!indexChoice) indexChoice = 'full'; - console.log(chalk.cyan(`Index mode: ${indexChoice}\n`)); + console.log(chalk.cyan(`Index mode: ${indexChoice}`)); + if (indexChoice === 'full' || indexChoice === 'sample') { + console.log(chalk.cyan(`Incremental: ${incrementalChoice ? 'Yes (skip unchanged repos)' : 'No (re-index all)'}\n`)); + } else { + console.log(''); + } let listsToIndex = awesomeLists; @@ -274,14 +290,20 @@ async function buildIndex(force = false, mode = null) { // Determine if we need to fetch README let shouldFetchReadme = false; if (indexChoice === 'full' || indexChoice === 'sample') { - if (!existingRepo) { - // New repo - fetch README - shouldFetchReadme = true; - } else if (existingRepo.last_commit !== repoInfo.pushedAt) { - // Repo updated since last index - fetch README + if (incrementalChoice) { + // Incremental mode: only fetch if new or changed + if (!existingRepo) { + // New repo - fetch README + shouldFetchReadme = true; + } else if (existingRepo.last_commit !== repoInfo.pushedAt) { + // Repo updated since last index - fetch README + shouldFetchReadme = true; + } + // else: repo unchanged, skip README fetch + } else { + // Full re-index mode: always fetch README shouldFetchReadme = true; } - // else: repo unchanged, skip README fetch } if (shouldFetchReadme) {