fix: suspend
This commit is contained in:
@@ -7,6 +7,7 @@ const db = require('./db-operations');
|
||||
const RATE_LIMIT_DELAY = 100;
|
||||
let lastRequestTime = 0;
|
||||
let rateLimitWarningShown = false;
|
||||
let lastRateLimitRecoveryTime = 0; // Track when we last recovered from rate limit
|
||||
|
||||
// Get GitHub token from settings
|
||||
function getGitHubToken() {
|
||||
@@ -96,6 +97,7 @@ async function waitForRateLimitReset(targetResetTime) {
|
||||
if (status.remaining >= MIN_REMAINING_TO_CONTINUE) {
|
||||
console.log(chalk.green(`\n✓ Rate limit reset! ${status.remaining}/${status.limit} requests available.`));
|
||||
console.log(chalk.green(` Resuming indexing... (will continue from last successful state)`));
|
||||
lastRateLimitRecoveryTime = Date.now(); // Mark recovery time
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -118,7 +120,11 @@ async function rateLimitedRequest(url, options = {}) {
|
||||
}
|
||||
|
||||
// Check rate limit proactively every 100 requests
|
||||
if (Math.random() < 0.01) { // 1% chance = roughly every 100 requests
|
||||
// BUT skip proactive checks for 10 minutes after recovering from rate limit
|
||||
const timeSinceRecovery = Date.now() - lastRateLimitRecoveryTime;
|
||||
const RECOVERY_GRACE_PERIOD = 10 * 60 * 1000; // 10 minutes
|
||||
|
||||
if (timeSinceRecovery > RECOVERY_GRACE_PERIOD && Math.random() < 0.01) { // 1% chance
|
||||
const rateLimitStatus = await checkRateLimit();
|
||||
if (rateLimitStatus && rateLimitStatus.remaining < 200) {
|
||||
console.log();
|
||||
@@ -148,11 +154,19 @@ async function rateLimitedRequest(url, options = {}) {
|
||||
}
|
||||
|
||||
try {
|
||||
return await axios.get(url, {
|
||||
const response = await axios.get(url, {
|
||||
timeout: 10000,
|
||||
headers,
|
||||
...options
|
||||
});
|
||||
|
||||
// Log successful requests after rate limit recovery (check if we recently polled)
|
||||
const isCI = process.env.CI === 'true';
|
||||
if (isCI && Math.random() < 0.01) { // 1% of requests in CI
|
||||
console.log(chalk.gray(` ✓ Request successful`));
|
||||
}
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
if (error.response?.status === 403) {
|
||||
const remaining = error.response.headers['x-ratelimit-remaining'];
|
||||
@@ -212,6 +226,7 @@ async function rateLimitedRequest(url, options = {}) {
|
||||
} else {
|
||||
// Use polling approach instead of blind waiting
|
||||
await waitForRateLimitReset(resetTime);
|
||||
console.log(chalk.gray(` → Retrying failed request: ${url.substring(0, 80)}...`));
|
||||
return rateLimitedRequest(url, options);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user