fix: CI indexing

.github/workflows/build-database.yml

@@ -58,8 +58,8 @@ jobs:
           INDEX_MODE="${{ github.event.inputs.index_mode || 'full' }}"
           echo "Index mode: $INDEX_MODE"
 
-          # Build the index in non-interactive mode
+          # Build the index in non-interactive mode (170m timeout, job timeout is 180m)
-          timeout 150m node -e "
+          timeout 170m node -e "
             const db = require('./lib/database');
             const dbOps = require('./lib/db-operations');
             const indexer = require('./lib/indexer');
@@ -73,6 +73,8 @@ jobs:
                 if (process.env.GITHUB_TOKEN) {
                   dbOps.setSetting('githubToken', process.env.GITHUB_TOKEN);
                   console.log('GitHub token configured');
+                } else {
+                  console.warn('⚠️  WARNING: No GitHub token found! Rate limit will be 60/hour instead of 5000/hour');
                 }
 
                 // Build index
@@ -92,7 +94,8 @@ jobs:
           " || {
             EXIT_CODE=$?
             if [ $EXIT_CODE -eq 124 ]; then
-              echo "Index building timed out after 150 minutes"
+              echo "❌ Index building timed out after 170 minutes"
+              echo "This may indicate rate limiting issues or too many lists to index"
             fi
             exit $EXIT_CODE
           }

lib/github-api.js

@@ -13,6 +13,37 @@ function getGitHubToken() {
   return db.getSetting('githubToken', null);
 }
 
+// Check rate limit status proactively
+async function checkRateLimit() {
+  const token = getGitHubToken();
+  const headers = {
+    'Accept': 'application/vnd.github.v3+json',
+    'User-Agent': 'awesome-cli'
+  };
+
+  if (token) {
+    headers['Authorization'] = `token ${token}`;
+  }
+
+  try {
+    const response = await axios.get('https://api.github.com/rate_limit', {
+      timeout: 5000,
+      headers
+    });
+
+    const core = response.data.resources.core;
+    return {
+      limit: core.limit,
+      remaining: core.remaining,
+      reset: core.reset * 1000,
+      used: core.used
+    };
+  } catch (error) {
+    // If we can't check rate limit, continue anyway
+    return null;
+  }
+}
+
 // Rate-limited request with better handling
 async function rateLimitedRequest(url, options = {}) {
   const now = Date.now();
@@ -22,6 +53,26 @@ async function rateLimitedRequest(url, options = {}) {
     await new Promise(resolve => setTimeout(resolve, RATE_LIMIT_DELAY - timeSinceLastRequest));
   }
 
+  // Check rate limit proactively every 50 requests
+  if (Math.random() < 0.02) { // 2% chance = roughly every 50 requests
+    const rateLimitStatus = await checkRateLimit();
+    if (rateLimitStatus && rateLimitStatus.remaining < 10) {
+      const waitTime = Math.max(0, rateLimitStatus.reset - Date.now());
+      const waitMinutes = Math.ceil(waitTime / 60000);
+
+      console.log();
+      console.log(chalk.yellow(`⚠️  Rate limit low: ${rateLimitStatus.remaining}/${rateLimitStatus.limit} remaining`));
+      console.log(chalk.yellow(`   Proactively waiting ${waitMinutes} minutes to avoid exhausting limit...`));
+
+      const isCI = process.env.CI === 'true';
+      if (isCI) {
+        console.log(chalk.cyan('🤖 CI mode: automatically waiting...'));
+        await new Promise(resolve => setTimeout(resolve, waitTime + 30000)); // 30s buffer
+        console.log(chalk.green('✓ Rate limit reset, resuming...'));
+      }
+    }
+  }
+
   lastRequestTime = Date.now();
 
   const token = getGitHubToken();
@@ -50,9 +101,15 @@ async function rateLimitedRequest(url, options = {}) {
       const waitMinutes = Math.ceil(waitTime / 60000);
 
       if (remaining === '0' || remaining === 0) {
+        const rateLimit = error.response.headers['x-ratelimit-limit'];
+        const resetDate = new Date(resetTime).toISOString();
+
         console.log();
         console.log(chalk.red('⚠️  GitHub API Rate Limit Exceeded!'));
+        console.log(chalk.yellow(`   Rate limit: ${rateLimit} requests/hour`));
+        console.log(chalk.yellow(`   Reset at: ${resetDate}`));
         console.log(chalk.yellow(`   Wait time: ${waitMinutes} minutes`));
+        console.log(chalk.gray(`   Token status: ${token ? 'USING TOKEN ✓' : 'NO TOKEN ✗'}`));
 
         if (!token && !rateLimitWarningShown) {
           console.log();
@@ -93,9 +150,13 @@ async function rateLimitedRequest(url, options = {}) {
         } else if (action === 'skip') {
           throw new Error('SKIP_RATE_LIMIT'); // Special error to skip
         } else {
-          // Wait with countdown
+          // Wait with countdown and add 30 second buffer to ensure rate limit has fully reset
-          console.log(chalk.gray(`\nWaiting ${waitMinutes} minutes...`));
+          const bufferTime = 30000; // 30 seconds
-          await new Promise(resolve => setTimeout(resolve, waitTime + 1000));
+          const totalWaitTime = waitTime + bufferTime;
+          const totalWaitMinutes = Math.ceil(totalWaitTime / 60000);
+          console.log(chalk.gray(`\nWaiting ${totalWaitMinutes} minutes (including 30s buffer)...`));
+          await new Promise(resolve => setTimeout(resolve, totalWaitTime));
+          console.log(chalk.green('✓ Rate limit should be reset, resuming...'));
           return rateLimitedRequest(url, options);
         }
       }
@@ -223,5 +284,6 @@ module.exports = {
   getLatestCommit,
   getAwesomeListsIndex,
   parseGitHubUrl,
-  rateLimitedRequest
+  rateLimitedRequest,
+  getRateLimitStatus: checkRateLimit
 };

lib/indexer.js

@@ -202,6 +202,27 @@ async function buildIndex(force = false, mode = null) {
     }
   }
 
+  // Check rate limit status before starting
+  try {
+    const rateLimitInfo = await github.getRateLimitStatus();
+    if (rateLimitInfo) {
+      console.log(chalk.cyan('📊 GitHub API Rate Limit Status:'));
+      console.log(chalk.gray(`   Limit: ${rateLimitInfo.limit} requests/hour`));
+      console.log(chalk.gray(`   Remaining: ${rateLimitInfo.remaining}/${rateLimitInfo.limit}`));
+      console.log(chalk.gray(`   Used: ${rateLimitInfo.used}`));
+      console.log(chalk.gray(`   Resets at: ${new Date(rateLimitInfo.reset).toISOString()}`));
+      console.log();
+
+      if (rateLimitInfo.limit === 60) {
+        console.log(chalk.yellow('⚠️  WARNING: Using unauthenticated rate limit (60/hour)'));
+        console.log(chalk.yellow('   This will likely not be enough to complete indexing'));
+        console.log();
+      }
+    }
+  } catch (error) {
+    console.log(chalk.gray('Could not check rate limit status, continuing...'));
+  }
+
   console.log(pinkPurple(`\n✨ Starting index of ${listsToIndex.length} awesome lists ✨\n`));
 
   // Progress bars