diff --git a/.github/workflows/build-database.yml b/.github/workflows/build-database.yml index 9f0d723..dbfc42d 100644 --- a/.github/workflows/build-database.yml +++ b/.github/workflows/build-database.yml @@ -50,6 +50,8 @@ jobs: CI: true GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + # Pass env vars! + CI=${CI:-false} # Capture start time START_TIME=$(date -u +"%Y-%m-%d %H:%M:%S UTC") echo "start_time=$START_TIME" >> $GITHUB_OUTPUT diff --git a/lib/github-api.js b/lib/github-api.js index 4b9d7de..ce4db0d 100644 --- a/lib/github-api.js +++ b/lib/github-api.js @@ -44,6 +44,70 @@ async function checkRateLimit() { } } +// Wait for rate limit to reset using polling +async function waitForRateLimitReset(targetResetTime) { + const POLL_INTERVAL = 60000; // 60 seconds + const MIN_REMAINING_TO_CONTINUE = 100; // Need at least 100 requests available + const MAX_POLLS = 120; // Max 120 polls = 2 hours + + const estimatedWaitMinutes = Math.ceil(Math.max(0, targetResetTime - Date.now()) / 60000); + + console.log(chalk.cyan('\nšŸ”„ Starting rate limit polling (checks every 60s)...')); + console.log(chalk.gray(` Estimated wait time: ~${estimatedWaitMinutes} minutes`)); + + let pollCount = 0; + + while (true) { + pollCount++; + + // Safety check: prevent infinite polling + if (pollCount > MAX_POLLS) { + console.log(chalk.red(`\nāœ— Exceeded maximum poll count (${MAX_POLLS}). Rate limit may not be resetting properly.`)); + console.log(chalk.yellow(' This could indicate an issue with the GitHub token or API.')); + throw new Error('Rate limit polling timeout'); + } + + // Wait before polling (except first time) + if (pollCount > 1) { + console.log(chalk.gray(` Waiting 60 seconds before next check...`)); + await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL)); + } else { + // First poll: wait a short time to avoid immediate re-check + await new Promise(resolve => setTimeout(resolve, 5000)); + } + + // Check current rate limit status + console.log(chalk.cyan(` Poll #${pollCount}: Checking rate limit status...`)); + + const status = await checkRateLimit(); + + if (!status) { + console.log(chalk.yellow(' āš ļø Could not check rate limit, waiting 60s and retrying...')); + continue; + } + + const now = Date.now(); + const timeUntilReset = Math.max(0, status.reset - now); + const minutesUntilReset = Math.ceil(timeUntilReset / 60000); + + console.log(chalk.gray(` Limit: ${status.limit}/hour | Remaining: ${status.remaining} | Reset in: ${minutesUntilReset}m`)); + + // Check if we have enough requests to continue + if (status.remaining >= MIN_REMAINING_TO_CONTINUE) { + console.log(chalk.green(`\nāœ“ Rate limit reset! ${status.remaining}/${status.limit} requests available.`)); + console.log(chalk.green(` Resuming indexing... (will continue from last successful state)`)); + return; + } + + // Still rate limited + if (status.remaining === 0) { + console.log(chalk.yellow(` Still rate limited (0 remaining). Will check again in 60s.`)); + } else { + console.log(chalk.yellow(` Only ${status.remaining} requests remaining (need ${MIN_REMAINING_TO_CONTINUE}). Will check again in 60s.`)); + } + } +} + // Rate-limited request with better handling async function rateLimitedRequest(url, options = {}) { const now = Date.now(); @@ -53,22 +117,18 @@ async function rateLimitedRequest(url, options = {}) { await new Promise(resolve => setTimeout(resolve, RATE_LIMIT_DELAY - timeSinceLastRequest)); } - // Check rate limit proactively every 50 requests - if (Math.random() < 0.02) { // 2% chance = roughly every 50 requests + // Check rate limit proactively every 100 requests + if (Math.random() < 0.01) { // 1% chance = roughly every 100 requests const rateLimitStatus = await checkRateLimit(); - if (rateLimitStatus && rateLimitStatus.remaining < 10) { - const waitTime = Math.max(0, rateLimitStatus.reset - Date.now()); - const waitMinutes = Math.ceil(waitTime / 60000); - + if (rateLimitStatus && rateLimitStatus.remaining < 200) { console.log(); - console.log(chalk.yellow(`āš ļø Rate limit low: ${rateLimitStatus.remaining}/${rateLimitStatus.limit} remaining`)); - console.log(chalk.yellow(` Proactively waiting ${waitMinutes} minutes to avoid exhausting limit...`)); + console.log(chalk.yellow(`āš ļø Rate limit getting low: ${rateLimitStatus.remaining}/${rateLimitStatus.limit} remaining`)); + console.log(chalk.yellow(` Proactively waiting for rate limit to reset...`)); const isCI = process.env.CI === 'true'; if (isCI) { - console.log(chalk.cyan('šŸ¤– CI mode: automatically waiting...')); - await new Promise(resolve => setTimeout(resolve, waitTime + 30000)); // 30s buffer - console.log(chalk.green('āœ“ Rate limit reset, resuming...')); + console.log(chalk.cyan('šŸ¤– CI mode: starting polling to wait for reset...')); + await waitForRateLimitReset(rateLimitStatus.reset); } } } @@ -150,13 +210,8 @@ async function rateLimitedRequest(url, options = {}) { } else if (action === 'skip') { throw new Error('SKIP_RATE_LIMIT'); // Special error to skip } else { - // Wait with countdown and add 30 second buffer to ensure rate limit has fully reset - const bufferTime = 30000; // 30 seconds - const totalWaitTime = waitTime + bufferTime; - const totalWaitMinutes = Math.ceil(totalWaitTime / 60000); - console.log(chalk.gray(`\nWaiting ${totalWaitMinutes} minutes (including 30s buffer)...`)); - await new Promise(resolve => setTimeout(resolve, totalWaitTime)); - console.log(chalk.green('āœ“ Rate limit should be reset, resuming...')); + // Use polling approach instead of blind waiting + await waitForRateLimitReset(resetTime); return rateLimitedRequest(url, options); } }