a new start

lib/github-api.js

@@ -0,0 +1,218 @@
+const axios = require('axios');
+const chalk = require('chalk');
+const inquirer = require('inquirer');
+const db = require('./db-operations');
+
+// Rate limiting
+const RATE_LIMIT_DELAY = 100;
+let lastRequestTime = 0;
+let rateLimitWarningShown = false;
+
+// Get GitHub token from settings
+function getGitHubToken() {
+  return db.getSetting('githubToken', null);
+}
+
+// Rate-limited request with better handling
+async function rateLimitedRequest(url, options = {}) {
+  const now = Date.now();
+  const timeSinceLastRequest = now - lastRequestTime;
+
+  if (timeSinceLastRequest < RATE_LIMIT_DELAY) {
+    await new Promise(resolve => setTimeout(resolve, RATE_LIMIT_DELAY - timeSinceLastRequest));
+  }
+
+  lastRequestTime = Date.now();
+
+  const token = getGitHubToken();
+  const headers = {
+    'Accept': 'application/vnd.github.v3+json',
+    'User-Agent': 'awesome-cli',
+    ...options.headers
+  };
+
+  // Add auth token if available
+  if (token) {
+    headers['Authorization'] = `token ${token}`;
+  }
+
+  try {
+    return await axios.get(url, {
+      timeout: 10000,
+      headers,
+      ...options
+    });
+  } catch (error) {
+    if (error.response?.status === 403) {
+      const remaining = error.response.headers['x-ratelimit-remaining'];
+      const resetTime = parseInt(error.response.headers['x-ratelimit-reset']) * 1000;
+      const waitTime = Math.max(0, resetTime - Date.now());
+      const waitMinutes = Math.ceil(waitTime / 60000);
+
+      if (remaining === '0' || remaining === 0) {
+        console.log();
+        console.log(chalk.red('⚠️  GitHub API Rate Limit Exceeded!'));
+        console.log(chalk.yellow(`   Wait time: ${waitMinutes} minutes`));
+
+        if (!token && !rateLimitWarningShown) {
+          console.log();
+          console.log(chalk.hex('#FFD700')('💡 TIP: Add a GitHub Personal Access Token to increase limit from 60/hour to 5000/hour!'));
+          console.log(chalk.gray('   1. Go to: https://github.com/settings/tokens'));
+          console.log(chalk.gray('   2. Generate new token (classic) with "public_repo" scope'));
+          console.log(chalk.gray('   3. Run: awesome settings → Add GitHub token'));
+          rateLimitWarningShown = true;
+        }
+
+        console.log();
+
+        // Ask user what to do
+        const { action } = await inquirer.prompt([
+          {
+            type: 'list',
+            name: 'action',
+            message: 'What would you like to do?',
+            choices: [
+              { name: `Wait ${waitMinutes} minutes and continue`, value: 'wait' },
+              { name: 'Skip remaining items and continue with what we have', value: 'skip' },
+              { name: 'Abort indexing', value: 'abort' }
+            ]
+          }
+        ]);
+
+        if (action === 'abort') {
+          throw new Error('Indexing aborted by user');
+        } else if (action === 'skip') {
+          throw new Error('SKIP_RATE_LIMIT'); // Special error to skip
+        } else {
+          // Wait with countdown
+          console.log(chalk.gray(`\nWaiting ${waitMinutes} minutes...`));
+          await new Promise(resolve => setTimeout(resolve, waitTime + 1000));
+          return rateLimitedRequest(url, options);
+        }
+      }
+    }
+    throw error;
+  }
+}
+
+// Extract owner and repo from GitHub URL
+function parseGitHubUrl(url) {
+  const match = url.match(/github\.com\/([^\/]+)\/([^\/\?#]+)/);
+  if (!match) return null;
+  return { owner: match[1], repo: match[2].replace(/\.git$/, '') };
+}
+
+// Get repository information
+async function getRepoInfo(repoUrl) {
+  const parsed = parseGitHubUrl(repoUrl);
+  if (!parsed) return null;
+
+  try {
+    const response = await rateLimitedRequest(
+      `https://api.github.com/repos/${parsed.owner}/${parsed.repo}`
+    );
+
+    return {
+      name: response.data.name,
+      fullName: response.data.full_name,
+      description: response.data.description,
+      stars: response.data.stargazers_count,
+      forks: response.data.forks_count,
+      watchers: response.data.watchers_count,
+      language: response.data.language,
+      topics: response.data.topics || [],
+      createdAt: response.data.created_at,
+      updatedAt: response.data.updated_at,
+      pushedAt: response.data.pushed_at,
+      homepage: response.data.homepage,
+      license: response.data.license?.name
+    };
+  } catch (error) {
+    // Silently skip 404s (deleted/moved repos) - don't clutter output
+    if (error.response?.status === 404) {
+      return null;
+    }
+    // Only log non-404 errors
+    console.error(chalk.red(`Failed to fetch ${parsed.owner}/${parsed.repo}:`), error.message);
+    return null;
+  }
+}
+
+// Get README content
+async function getReadme(repoUrl) {
+  const parsed = parseGitHubUrl(repoUrl);
+  if (!parsed) return null;
+
+  // Try different README URLs
+  const urls = [
+    `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/main/README.md`,
+    `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/master/README.md`,
+    `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/main/readme.md`,
+    `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/master/readme.md`,
+    `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/main/Readme.md`,
+    `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/master/Readme.md`
+  ];
+
+  for (const url of urls) {
+    try {
+      const response = await rateLimitedRequest(url);
+      if (response.data) {
+        return {
+          content: response.data,
+          url: url
+        };
+      }
+    } catch (error) {
+      continue;
+    }
+  }
+
+  return null;
+}
+
+// Get commits information
+async function getLatestCommit(repoUrl) {
+  const parsed = parseGitHubUrl(repoUrl);
+  if (!parsed) return null;
+
+  try {
+    const response = await rateLimitedRequest(
+      `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/commits?per_page=1`
+    );
+
+    if (response.data && response.data.length > 0) {
+      const commit = response.data[0];
+      return {
+        sha: commit.sha,
+        message: commit.commit.message,
+        author: commit.commit.author.name,
+        date: commit.commit.author.date
+      };
+    }
+  } catch (error) {
+    return null;
+  }
+
+  return null;
+}
+
+// Get list of awesome lists from main awesome repo
+async function getAwesomeListsIndex() {
+  try {
+    const response = await rateLimitedRequest(
+      'https://raw.githubusercontent.com/sindresorhus/awesome/main/readme.md'
+    );
+    return response.data;
+  } catch (error) {
+    throw new Error('Failed to fetch awesome lists index: ' + error.message);
+  }
+}
+
+module.exports = {
+  getRepoInfo,
+  getReadme,
+  getLatestCommit,
+  getAwesomeListsIndex,
+  parseGitHubUrl,
+  rateLimitedRequest
+};