fix: github workflow
This commit is contained in:
valknarness
60
.github/workflows/build-database.yml
vendored
60
.github/workflows/build-database.yml
vendored
@@ -59,6 +59,8 @@ jobs:
|
||||
|
||||
- name: Build awesome database
|
||||
id: build
|
||||
env:
|
||||
CI: true
|
||||
run: |
|
||||
# Capture start time
|
||||
START_TIME=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
|
||||
@@ -68,43 +70,27 @@ jobs:
|
||||
INDEX_MODE="${{ github.event.inputs.index_mode || 'full' }}"
|
||||
echo "Index mode: $INDEX_MODE"
|
||||
|
||||
# Build the index with automated selection
|
||||
if [ "$INDEX_MODE" = "sample" ]; then
|
||||
# For sample mode, we'll need to modify the script to accept input
|
||||
echo "Building sample index (10 lists)..."
|
||||
timeout 150m node -e "
|
||||
const indexer = require('./lib/indexer');
|
||||
(async () => {
|
||||
try {
|
||||
// Simulate user choosing 'sample' option
|
||||
process.stdin.push('sample\n');
|
||||
await indexer.buildIndex(false);
|
||||
console.log('Sample index built successfully');
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Failed to build index:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
" || echo "Index building completed with timeout"
|
||||
else
|
||||
echo "Building full index..."
|
||||
timeout 150m node -e "
|
||||
const indexer = require('./lib/indexer');
|
||||
(async () => {
|
||||
try {
|
||||
// Simulate user choosing 'full' option
|
||||
process.stdin.push('full\n');
|
||||
await indexer.buildIndex(false);
|
||||
console.log('Full index built successfully');
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Failed to build index:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
" || echo "Index building completed with timeout"
|
||||
fi
|
||||
# Build the index in non-interactive mode
|
||||
timeout 150m node -e "
|
||||
const indexer = require('./lib/indexer');
|
||||
(async () => {
|
||||
try {
|
||||
await indexer.buildIndex(false, '${INDEX_MODE}');
|
||||
console.log('Index built successfully');
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Failed to build index:', error.message);
|
||||
console.error(error.stack);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
" || {
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -eq 124 ]; then
|
||||
echo "Index building timed out after 150 minutes"
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
}
|
||||
|
||||
# Capture end time
|
||||
END_TIME=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
|
||||
|
||||
120
lib/indexer.js
120
lib/indexer.js
@@ -89,29 +89,42 @@ function isAwesomeList(url, name, description) {
|
||||
}
|
||||
|
||||
// Build the complete index
|
||||
async function buildIndex(force = false) {
|
||||
async function buildIndex(force = false, mode = null) {
|
||||
console.clear();
|
||||
console.log(purpleGold('\n🚀 AWESOME INDEX BUILDER 🚀\n'));
|
||||
|
||||
// Check if running in CI/non-interactive mode
|
||||
const isNonInteractive = process.env.CI === 'true' || mode !== null;
|
||||
|
||||
if (force) {
|
||||
const { confirm } = await inquirer.prompt([
|
||||
{
|
||||
type: 'confirm',
|
||||
name: 'confirm',
|
||||
message: chalk.yellow('⚠️ Force rebuild will clear all indexed data (bookmarks will be preserved). Continue?'),
|
||||
default: false
|
||||
}
|
||||
]);
|
||||
if (isNonInteractive) {
|
||||
// Clear index data without confirmation in CI
|
||||
console.log(chalk.gray('\nClearing existing index...'));
|
||||
const dbInstance = require('./database').getDb();
|
||||
dbInstance.exec('DELETE FROM readmes');
|
||||
dbInstance.exec('DELETE FROM repositories');
|
||||
dbInstance.exec('DELETE FROM awesome_lists');
|
||||
console.log(chalk.green('✓ Index cleared\n'));
|
||||
} else {
|
||||
const { confirm } = await inquirer.prompt([
|
||||
{
|
||||
type: 'confirm',
|
||||
name: 'confirm',
|
||||
message: chalk.yellow('⚠️ Force rebuild will clear all indexed data (bookmarks will be preserved). Continue?'),
|
||||
default: false
|
||||
}
|
||||
]);
|
||||
|
||||
if (!confirm) return;
|
||||
if (!confirm) return;
|
||||
|
||||
// Clear index data (keep bookmarks)
|
||||
console.log(chalk.gray('\nClearing existing index...'));
|
||||
const dbInstance = require('./database').getDb();
|
||||
dbInstance.exec('DELETE FROM readmes');
|
||||
dbInstance.exec('DELETE FROM repositories');
|
||||
dbInstance.exec('DELETE FROM awesome_lists');
|
||||
console.log(chalk.green('✓ Index cleared\n'));
|
||||
// Clear index data (keep bookmarks)
|
||||
console.log(chalk.gray('\nClearing existing index...'));
|
||||
const dbInstance = require('./database').getDb();
|
||||
dbInstance.exec('DELETE FROM readmes');
|
||||
dbInstance.exec('DELETE FROM repositories');
|
||||
dbInstance.exec('DELETE FROM awesome_lists');
|
||||
console.log(chalk.green('✓ Index cleared\n'));
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch main awesome list
|
||||
@@ -132,46 +145,61 @@ async function buildIndex(force = false) {
|
||||
const awesomeLists = parseMarkdownLinks(mainReadme);
|
||||
console.log(chalk.green(`✓ Found ${awesomeLists.length} awesome lists!\n`));
|
||||
|
||||
// Ask user what to index
|
||||
const { indexChoice } = await inquirer.prompt([
|
||||
{
|
||||
type: 'list',
|
||||
name: 'indexChoice',
|
||||
message: 'What would you like to index?',
|
||||
choices: [
|
||||
{ name: '🎯 Index everything (recommended for first run)', value: 'full' },
|
||||
{ name: '📋 Index lists only (metadata, no READMEs)', value: 'lists' },
|
||||
{ name: '🎲 Index a random sample (10 lists)', value: 'sample' },
|
||||
{ name: '🔍 Select specific categories', value: 'select' },
|
||||
{ name: '← Back', value: 'cancel' }
|
||||
]
|
||||
}
|
||||
]);
|
||||
let indexChoice = mode;
|
||||
|
||||
// Ask user what to index (only if interactive)
|
||||
if (!isNonInteractive) {
|
||||
const result = await inquirer.prompt([
|
||||
{
|
||||
type: 'list',
|
||||
name: 'indexChoice',
|
||||
message: 'What would you like to index?',
|
||||
choices: [
|
||||
{ name: '🎯 Index everything (recommended for first run)', value: 'full' },
|
||||
{ name: '📋 Index lists only (metadata, no READMEs)', value: 'lists' },
|
||||
{ name: '🎲 Index a random sample (10 lists)', value: 'sample' },
|
||||
{ name: '🔍 Select specific categories', value: 'select' },
|
||||
{ name: '← Back', value: 'cancel' }
|
||||
]
|
||||
}
|
||||
]);
|
||||
indexChoice = result.indexChoice;
|
||||
}
|
||||
|
||||
if (indexChoice === 'cancel') return;
|
||||
|
||||
// Default to 'full' if no mode specified
|
||||
if (!indexChoice) indexChoice = 'full';
|
||||
|
||||
console.log(chalk.cyan(`Index mode: ${indexChoice}\n`));
|
||||
|
||||
let listsToIndex = awesomeLists;
|
||||
|
||||
if (indexChoice === 'sample') {
|
||||
listsToIndex = awesomeLists.sort(() => 0.5 - Math.random()).slice(0, 10);
|
||||
} else if (indexChoice === 'select') {
|
||||
const categories = [...new Set(awesomeLists.map(l => l.category).filter(Boolean))];
|
||||
const { selectedCategories } = await inquirer.prompt([
|
||||
{
|
||||
type: 'checkbox',
|
||||
name: 'selectedCategories',
|
||||
message: 'Select categories to index:',
|
||||
choices: categories,
|
||||
pageSize: 15
|
||||
if (isNonInteractive) {
|
||||
console.log(chalk.yellow('Select mode not available in non-interactive mode, using full'));
|
||||
indexChoice = 'full';
|
||||
} else {
|
||||
const categories = [...new Set(awesomeLists.map(l => l.category).filter(Boolean))];
|
||||
const { selectedCategories } = await inquirer.prompt([
|
||||
{
|
||||
type: 'checkbox',
|
||||
name: 'selectedCategories',
|
||||
message: 'Select categories to index:',
|
||||
choices: categories,
|
||||
pageSize: 15
|
||||
}
|
||||
]);
|
||||
|
||||
if (selectedCategories.length === 0) {
|
||||
console.log(chalk.yellow('No categories selected'));
|
||||
return;
|
||||
}
|
||||
]);
|
||||
|
||||
if (selectedCategories.length === 0) {
|
||||
console.log(chalk.yellow('No categories selected'));
|
||||
return;
|
||||
listsToIndex = awesomeLists.filter(l => selectedCategories.includes(l.category));
|
||||
}
|
||||
|
||||
listsToIndex = awesomeLists.filter(l => selectedCategories.includes(l.category));
|
||||
}
|
||||
|
||||
console.log(pinkPurple(`\n✨ Starting index of ${listsToIndex.length} awesome lists ✨\n`));
|
||||
|
||||
Reference in New Issue
Block a user