fix: suspend
This commit is contained in:
@@ -7,6 +7,7 @@ const db = require('./db-operations');
|
|||||||
const RATE_LIMIT_DELAY = 100;
|
const RATE_LIMIT_DELAY = 100;
|
||||||
let lastRequestTime = 0;
|
let lastRequestTime = 0;
|
||||||
let rateLimitWarningShown = false;
|
let rateLimitWarningShown = false;
|
||||||
|
let lastRateLimitRecoveryTime = 0; // Track when we last recovered from rate limit
|
||||||
|
|
||||||
// Get GitHub token from settings
|
// Get GitHub token from settings
|
||||||
function getGitHubToken() {
|
function getGitHubToken() {
|
||||||
@@ -96,6 +97,7 @@ async function waitForRateLimitReset(targetResetTime) {
|
|||||||
if (status.remaining >= MIN_REMAINING_TO_CONTINUE) {
|
if (status.remaining >= MIN_REMAINING_TO_CONTINUE) {
|
||||||
console.log(chalk.green(`\n✓ Rate limit reset! ${status.remaining}/${status.limit} requests available.`));
|
console.log(chalk.green(`\n✓ Rate limit reset! ${status.remaining}/${status.limit} requests available.`));
|
||||||
console.log(chalk.green(` Resuming indexing... (will continue from last successful state)`));
|
console.log(chalk.green(` Resuming indexing... (will continue from last successful state)`));
|
||||||
|
lastRateLimitRecoveryTime = Date.now(); // Mark recovery time
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,7 +120,11 @@ async function rateLimitedRequest(url, options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check rate limit proactively every 100 requests
|
// Check rate limit proactively every 100 requests
|
||||||
if (Math.random() < 0.01) { // 1% chance = roughly every 100 requests
|
// BUT skip proactive checks for 10 minutes after recovering from rate limit
|
||||||
|
const timeSinceRecovery = Date.now() - lastRateLimitRecoveryTime;
|
||||||
|
const RECOVERY_GRACE_PERIOD = 10 * 60 * 1000; // 10 minutes
|
||||||
|
|
||||||
|
if (timeSinceRecovery > RECOVERY_GRACE_PERIOD && Math.random() < 0.01) { // 1% chance
|
||||||
const rateLimitStatus = await checkRateLimit();
|
const rateLimitStatus = await checkRateLimit();
|
||||||
if (rateLimitStatus && rateLimitStatus.remaining < 200) {
|
if (rateLimitStatus && rateLimitStatus.remaining < 200) {
|
||||||
console.log();
|
console.log();
|
||||||
@@ -148,11 +154,19 @@ async function rateLimitedRequest(url, options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return await axios.get(url, {
|
const response = await axios.get(url, {
|
||||||
timeout: 10000,
|
timeout: 10000,
|
||||||
headers,
|
headers,
|
||||||
...options
|
...options
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Log successful requests after rate limit recovery (check if we recently polled)
|
||||||
|
const isCI = process.env.CI === 'true';
|
||||||
|
if (isCI && Math.random() < 0.01) { // 1% of requests in CI
|
||||||
|
console.log(chalk.gray(` ✓ Request successful`));
|
||||||
|
}
|
||||||
|
|
||||||
|
return response;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error.response?.status === 403) {
|
if (error.response?.status === 403) {
|
||||||
const remaining = error.response.headers['x-ratelimit-remaining'];
|
const remaining = error.response.headers['x-ratelimit-remaining'];
|
||||||
@@ -212,6 +226,7 @@ async function rateLimitedRequest(url, options = {}) {
|
|||||||
} else {
|
} else {
|
||||||
// Use polling approach instead of blind waiting
|
// Use polling approach instead of blind waiting
|
||||||
await waitForRateLimitReset(resetTime);
|
await waitForRateLimitReset(resetTime);
|
||||||
|
console.log(chalk.gray(` → Retrying failed request: ${url.substring(0, 80)}...`));
|
||||||
return rateLimitedRequest(url, options);
|
return rateLimitedRequest(url, options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user