diff --git a/.github/workflows/build-database.yml b/.github/workflows/build-database.yml index 2b9f191..dbfc42d 100644 --- a/.github/workflows/build-database.yml +++ b/.github/workflows/build-database.yml @@ -22,7 +22,7 @@ permissions: jobs: build-database: runs-on: ubuntu-latest - timeout-minutes: 330 # 5.5 hours max (allows 5-6 rate limit cycles) + timeout-minutes: 180 # 3 hours max steps: - name: Checkout repository @@ -60,8 +60,8 @@ jobs: INDEX_MODE="${{ github.event.inputs.index_mode || 'full' }}" echo "Index mode: $INDEX_MODE" - # Build the index in non-interactive mode (320m timeout, job timeout is 330m) - timeout 320m node -e " + # Build the index in non-interactive mode (170m timeout, job timeout is 180m) + timeout 170m node -e " const db = require('./lib/database'); const dbOps = require('./lib/db-operations'); const indexer = require('./lib/indexer'); @@ -96,7 +96,7 @@ jobs: " || { EXIT_CODE=$? if [ $EXIT_CODE -eq 124 ]; then - echo "❌ Index building timed out after 320 minutes" + echo "❌ Index building timed out after 170 minutes" echo "This may indicate rate limiting issues or too many lists to index" fi exit $EXIT_CODE diff --git a/lib/github-api.js b/lib/github-api.js index 148fd9e..13a1ccd 100644 --- a/lib/github-api.js +++ b/lib/github-api.js @@ -112,6 +112,25 @@ async function waitForRateLimitReset(targetResetTime) { } } +// Direct fetch for raw.githubusercontent.com (does NOT count against API rate limit) +async function fetchRawContent(url) { + try { + const response = await axios.get(url, { + timeout: 10000, + headers: { + 'User-Agent': 'awesome-cli' + } + }); + return response; + } catch (error) { + // Return null for 404s (file not found), throw for other errors + if (error.response?.status === 404) { + return null; + } + throw error; + } +} + // Rate-limited request with better handling async function rateLimitedRequest(url, options = {}) { const now = Date.now(); @@ -300,8 +319,8 @@ async function getReadme(repoUrl) { for (const url of urls) { try { - const response = await rateLimitedRequest(url); - if (response.data) { + const response = await fetchRawContent(url); + if (response && response.data) { return { content: response.data, url: url @@ -344,7 +363,7 @@ async function getLatestCommit(repoUrl) { // Get list of awesome lists from main awesome repo async function getAwesomeListsIndex() { try { - const response = await rateLimitedRequest( + const response = await fetchRawContent( 'https://raw.githubusercontent.com/sindresorhus/awesome/main/readme.md' ); return response.data; @@ -360,5 +379,6 @@ module.exports = { getAwesomeListsIndex, parseGitHubUrl, rateLimitedRequest, + fetchRawContent, getRateLimitStatus: checkRateLimit };