fix: suspend

.github/workflows/build-database.yml

@@ -50,6 +50,8 @@ jobs:
           CI: true
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
+          # Pass env vars!
+          CI=${CI:-false}
           # Capture start time
           START_TIME=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
           echo "start_time=$START_TIME" >> $GITHUB_OUTPUT

lib/github-api.js

@@ -44,6 +44,70 @@ async function checkRateLimit() {
   }
 }
 
+// Wait for rate limit to reset using polling
+async function waitForRateLimitReset(targetResetTime) {
+  const POLL_INTERVAL = 60000; // 60 seconds
+  const MIN_REMAINING_TO_CONTINUE = 100; // Need at least 100 requests available
+  const MAX_POLLS = 120; // Max 120 polls = 2 hours
+
+  const estimatedWaitMinutes = Math.ceil(Math.max(0, targetResetTime - Date.now()) / 60000);
+
+  console.log(chalk.cyan('\n🔄 Starting rate limit polling (checks every 60s)...'));
+  console.log(chalk.gray(`   Estimated wait time: ~${estimatedWaitMinutes} minutes`));
+
+  let pollCount = 0;
+
+  while (true) {
+    pollCount++;
+
+    // Safety check: prevent infinite polling
+    if (pollCount > MAX_POLLS) {
+      console.log(chalk.red(`\n✗ Exceeded maximum poll count (${MAX_POLLS}). Rate limit may not be resetting properly.`));
+      console.log(chalk.yellow('   This could indicate an issue with the GitHub token or API.'));
+      throw new Error('Rate limit polling timeout');
+    }
+
+    // Wait before polling (except first time)
+    if (pollCount > 1) {
+      console.log(chalk.gray(`   Waiting 60 seconds before next check...`));
+      await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL));
+    } else {
+      // First poll: wait a short time to avoid immediate re-check
+      await new Promise(resolve => setTimeout(resolve, 5000));
+    }
+
+    // Check current rate limit status
+    console.log(chalk.cyan(`   Poll #${pollCount}: Checking rate limit status...`));
+
+    const status = await checkRateLimit();
+
+    if (!status) {
+      console.log(chalk.yellow('   ⚠️  Could not check rate limit, waiting 60s and retrying...'));
+      continue;
+    }
+
+    const now = Date.now();
+    const timeUntilReset = Math.max(0, status.reset - now);
+    const minutesUntilReset = Math.ceil(timeUntilReset / 60000);
+
+    console.log(chalk.gray(`   Limit: ${status.limit}/hour | Remaining: ${status.remaining} | Reset in: ${minutesUntilReset}m`));
+
+    // Check if we have enough requests to continue
+    if (status.remaining >= MIN_REMAINING_TO_CONTINUE) {
+      console.log(chalk.green(`\n✓ Rate limit reset! ${status.remaining}/${status.limit} requests available.`));
+      console.log(chalk.green(`   Resuming indexing... (will continue from last successful state)`));
+      return;
+    }
+
+    // Still rate limited
+    if (status.remaining === 0) {
+      console.log(chalk.yellow(`   Still rate limited (0 remaining). Will check again in 60s.`));
+    } else {
+      console.log(chalk.yellow(`   Only ${status.remaining} requests remaining (need ${MIN_REMAINING_TO_CONTINUE}). Will check again in 60s.`));
+    }
+  }
+}
+
 // Rate-limited request with better handling
 async function rateLimitedRequest(url, options = {}) {
   const now = Date.now();
@@ -53,22 +117,18 @@ async function rateLimitedRequest(url, options = {}) {
     await new Promise(resolve => setTimeout(resolve, RATE_LIMIT_DELAY - timeSinceLastRequest));
   }
 
-  // Check rate limit proactively every 50 requests
-  if (Math.random() < 0.02) { // 2% chance = roughly every 50 requests
+  // Check rate limit proactively every 100 requests
+  if (Math.random() < 0.01) { // 1% chance = roughly every 100 requests
     const rateLimitStatus = await checkRateLimit();
-    if (rateLimitStatus && rateLimitStatus.remaining < 10) {
-      const waitTime = Math.max(0, rateLimitStatus.reset - Date.now());
-      const waitMinutes = Math.ceil(waitTime / 60000);
-
+    if (rateLimitStatus && rateLimitStatus.remaining < 200) {
       console.log();
-      console.log(chalk.yellow(`⚠️  Rate limit low: ${rateLimitStatus.remaining}/${rateLimitStatus.limit} remaining`));
-      console.log(chalk.yellow(`   Proactively waiting ${waitMinutes} minutes to avoid exhausting limit...`));
+      console.log(chalk.yellow(`⚠️  Rate limit getting low: ${rateLimitStatus.remaining}/${rateLimitStatus.limit} remaining`));
+      console.log(chalk.yellow(`   Proactively waiting for rate limit to reset...`));
 
       const isCI = process.env.CI === 'true';
       if (isCI) {
-        console.log(chalk.cyan('🤖 CI mode: automatically waiting...'));
-        await new Promise(resolve => setTimeout(resolve, waitTime + 30000)); // 30s buffer
-        console.log(chalk.green('✓ Rate limit reset, resuming...'));
+        console.log(chalk.cyan('🤖 CI mode: starting polling to wait for reset...'));
+        await waitForRateLimitReset(rateLimitStatus.reset);
       }
     }
   }
@@ -150,13 +210,8 @@ async function rateLimitedRequest(url, options = {}) {
         } else if (action === 'skip') {
           throw new Error('SKIP_RATE_LIMIT'); // Special error to skip
         } else {
-          // Wait with countdown and add 30 second buffer to ensure rate limit has fully reset
-          const bufferTime = 30000; // 30 seconds
-          const totalWaitTime = waitTime + bufferTime;
-          const totalWaitMinutes = Math.ceil(totalWaitTime / 60000);
-          console.log(chalk.gray(`\nWaiting ${totalWaitMinutes} minutes (including 30s buffer)...`));
-          await new Promise(resolve => setTimeout(resolve, totalWaitTime));
-          console.log(chalk.green('✓ Rate limit should be reset, resuming...'));
+          // Use polling approach instead of blind waiting
+          await waitForRateLimitReset(resetTime);
           return rateLimitedRequest(url, options);
         }
       }