diff --git a/.github/workflows/build-database.yml b/.github/workflows/build-database.yml index e30d20c..9f0d723 100644 --- a/.github/workflows/build-database.yml +++ b/.github/workflows/build-database.yml @@ -58,8 +58,8 @@ jobs: INDEX_MODE="${{ github.event.inputs.index_mode || 'full' }}" echo "Index mode: $INDEX_MODE" - # Build the index in non-interactive mode - timeout 150m node -e " + # Build the index in non-interactive mode (170m timeout, job timeout is 180m) + timeout 170m node -e " const db = require('./lib/database'); const dbOps = require('./lib/db-operations'); const indexer = require('./lib/indexer'); @@ -73,6 +73,8 @@ jobs: if (process.env.GITHUB_TOKEN) { dbOps.setSetting('githubToken', process.env.GITHUB_TOKEN); console.log('GitHub token configured'); + } else { + console.warn('⚠️ WARNING: No GitHub token found! Rate limit will be 60/hour instead of 5000/hour'); } // Build index @@ -92,7 +94,8 @@ jobs: " || { EXIT_CODE=$? if [ $EXIT_CODE -eq 124 ]; then - echo "Index building timed out after 150 minutes" + echo "❌ Index building timed out after 170 minutes" + echo "This may indicate rate limiting issues or too many lists to index" fi exit $EXIT_CODE } diff --git a/lib/github-api.js b/lib/github-api.js index 1185934..4b9d7de 100644 --- a/lib/github-api.js +++ b/lib/github-api.js @@ -13,6 +13,37 @@ function getGitHubToken() { return db.getSetting('githubToken', null); } +// Check rate limit status proactively +async function checkRateLimit() { + const token = getGitHubToken(); + const headers = { + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'awesome-cli' + }; + + if (token) { + headers['Authorization'] = `token ${token}`; + } + + try { + const response = await axios.get('https://api.github.com/rate_limit', { + timeout: 5000, + headers + }); + + const core = response.data.resources.core; + return { + limit: core.limit, + remaining: core.remaining, + reset: core.reset * 1000, + used: core.used + }; + } catch (error) { + // If we can't check rate limit, continue anyway + return null; + } +} + // Rate-limited request with better handling async function rateLimitedRequest(url, options = {}) { const now = Date.now(); @@ -22,6 +53,26 @@ async function rateLimitedRequest(url, options = {}) { await new Promise(resolve => setTimeout(resolve, RATE_LIMIT_DELAY - timeSinceLastRequest)); } + // Check rate limit proactively every 50 requests + if (Math.random() < 0.02) { // 2% chance = roughly every 50 requests + const rateLimitStatus = await checkRateLimit(); + if (rateLimitStatus && rateLimitStatus.remaining < 10) { + const waitTime = Math.max(0, rateLimitStatus.reset - Date.now()); + const waitMinutes = Math.ceil(waitTime / 60000); + + console.log(); + console.log(chalk.yellow(`⚠️ Rate limit low: ${rateLimitStatus.remaining}/${rateLimitStatus.limit} remaining`)); + console.log(chalk.yellow(` Proactively waiting ${waitMinutes} minutes to avoid exhausting limit...`)); + + const isCI = process.env.CI === 'true'; + if (isCI) { + console.log(chalk.cyan('🤖 CI mode: automatically waiting...')); + await new Promise(resolve => setTimeout(resolve, waitTime + 30000)); // 30s buffer + console.log(chalk.green('✓ Rate limit reset, resuming...')); + } + } + } + lastRequestTime = Date.now(); const token = getGitHubToken(); @@ -50,9 +101,15 @@ async function rateLimitedRequest(url, options = {}) { const waitMinutes = Math.ceil(waitTime / 60000); if (remaining === '0' || remaining === 0) { + const rateLimit = error.response.headers['x-ratelimit-limit']; + const resetDate = new Date(resetTime).toISOString(); + console.log(); console.log(chalk.red('⚠️ GitHub API Rate Limit Exceeded!')); + console.log(chalk.yellow(` Rate limit: ${rateLimit} requests/hour`)); + console.log(chalk.yellow(` Reset at: ${resetDate}`)); console.log(chalk.yellow(` Wait time: ${waitMinutes} minutes`)); + console.log(chalk.gray(` Token status: ${token ? 'USING TOKEN ✓' : 'NO TOKEN ✗'}`)); if (!token && !rateLimitWarningShown) { console.log(); @@ -93,9 +150,13 @@ async function rateLimitedRequest(url, options = {}) { } else if (action === 'skip') { throw new Error('SKIP_RATE_LIMIT'); // Special error to skip } else { - // Wait with countdown - console.log(chalk.gray(`\nWaiting ${waitMinutes} minutes...`)); - await new Promise(resolve => setTimeout(resolve, waitTime + 1000)); + // Wait with countdown and add 30 second buffer to ensure rate limit has fully reset + const bufferTime = 30000; // 30 seconds + const totalWaitTime = waitTime + bufferTime; + const totalWaitMinutes = Math.ceil(totalWaitTime / 60000); + console.log(chalk.gray(`\nWaiting ${totalWaitMinutes} minutes (including 30s buffer)...`)); + await new Promise(resolve => setTimeout(resolve, totalWaitTime)); + console.log(chalk.green('✓ Rate limit should be reset, resuming...')); return rateLimitedRequest(url, options); } } @@ -223,5 +284,6 @@ module.exports = { getLatestCommit, getAwesomeListsIndex, parseGitHubUrl, - rateLimitedRequest + rateLimitedRequest, + getRateLimitStatus: checkRateLimit }; diff --git a/lib/indexer.js b/lib/indexer.js index b91b1e2..e400ad2 100644 --- a/lib/indexer.js +++ b/lib/indexer.js @@ -202,6 +202,27 @@ async function buildIndex(force = false, mode = null) { } } + // Check rate limit status before starting + try { + const rateLimitInfo = await github.getRateLimitStatus(); + if (rateLimitInfo) { + console.log(chalk.cyan('📊 GitHub API Rate Limit Status:')); + console.log(chalk.gray(` Limit: ${rateLimitInfo.limit} requests/hour`)); + console.log(chalk.gray(` Remaining: ${rateLimitInfo.remaining}/${rateLimitInfo.limit}`)); + console.log(chalk.gray(` Used: ${rateLimitInfo.used}`)); + console.log(chalk.gray(` Resets at: ${new Date(rateLimitInfo.reset).toISOString()}`)); + console.log(); + + if (rateLimitInfo.limit === 60) { + console.log(chalk.yellow('⚠️ WARNING: Using unauthenticated rate limit (60/hour)')); + console.log(chalk.yellow(' This will likely not be enough to complete indexing')); + console.log(); + } + } + } catch (error) { + console.log(chalk.gray('Could not check rate limit status, continuing...')); + } + console.log(pinkPurple(`\n✨ Starting index of ${listsToIndex.length} awesome lists ✨\n`)); // Progress bars