import { DaemonStatusSchema, ListProjectsSchema, ListVersionsSchema, ListSpidersSchema, ListJobsSchema, ScheduleJobSchema, CancelJobSchema, DeleteVersionSchema, DeleteProjectSchema, AddVersionSchema, type ScheduleJobParams, type CancelJobParams, type ListVersionsParams, type ListSpidersParams, type ListJobsParams, type DeleteVersionParams, type DeleteProjectParams, } from "./types"; // Get credentials from environment variables (server-side only) const SCRAPYD_URL = process.env.SCRAPYD_URL || "https://scrapy.pivoine.art"; const SCRAPYD_USERNAME = process.env.SCRAPYD_USERNAME || ""; const SCRAPYD_PASSWORD = process.env.SCRAPYD_PASSWORD || ""; /** * Create Basic Auth header */ function getAuthHeader(): string { const credentials = Buffer.from(`${SCRAPYD_USERNAME}:${SCRAPYD_PASSWORD}`).toString("base64"); return `Basic ${credentials}`; } /** * Base fetch wrapper with auth */ async function fetchScrapyd(endpoint: string, options: RequestInit = {}) { const url = `${SCRAPYD_URL}/${endpoint}`; const response = await fetch(url, { ...options, headers: { Authorization: getAuthHeader(), ...options.headers, }, }); if (!response.ok) { throw new Error(`Scrapyd API error: ${response.status} ${response.statusText}`); } return response.json(); } /** * ScrapydClient - Server-side API client for Scrapyd * All methods use environment variables for authentication */ export const ScrapydClient = { /** * Get daemon status */ async getDaemonStatus() { const data = await fetchScrapyd("daemonstatus.json"); return DaemonStatusSchema.parse(data); }, /** * List all projects */ async listProjects() { const data = await fetchScrapyd("listprojects.json"); return ListProjectsSchema.parse(data); }, /** * List versions for a project */ async listVersions(params: ListVersionsParams) { const url = new URLSearchParams({ project: params.project }); const data = await fetchScrapyd(`listversions.json?${url}`); return ListVersionsSchema.parse(data); }, /** * List spiders for a project */ async listSpiders(params: ListSpidersParams) { const url = new URLSearchParams({ project: params.project, ...(params.version && { _version: params.version }), }); const data = await fetchScrapyd(`listspiders.json?${url}`); return ListSpidersSchema.parse(data); }, /** * List jobs (pending, running, finished) for a project */ async listJobs(params: ListJobsParams) { const url = new URLSearchParams({ project: params.project }); const data = await fetchScrapyd(`listjobs.json?${url}`); return ListJobsSchema.parse(data); }, /** * Schedule a spider job */ async scheduleJob(params: ScheduleJobParams) { const formData = new URLSearchParams({ project: params.project, spider: params.spider, ...(params.jobid && { jobid: params.jobid }), }); // Add custom settings if (params.settings) { Object.entries(params.settings).forEach(([key, value]) => { formData.append(`setting`, `${key}=${value}`); }); } // Add spider arguments if (params.args) { Object.entries(params.args).forEach(([key, value]) => { formData.append(key, value); }); } const data = await fetchScrapyd("schedule.json", { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded", }, body: formData.toString(), }); return ScheduleJobSchema.parse(data); }, /** * Cancel a job */ async cancelJob(params: CancelJobParams) { const formData = new URLSearchParams({ project: params.project, job: params.job, }); const data = await fetchScrapyd("cancel.json", { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded", }, body: formData.toString(), }); return CancelJobSchema.parse(data); }, /** * Delete a project version */ async deleteVersion(params: DeleteVersionParams) { const formData = new URLSearchParams({ project: params.project, version: params.version, }); const data = await fetchScrapyd("delversion.json", { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded", }, body: formData.toString(), }); return DeleteVersionSchema.parse(data); }, /** * Delete a project */ async deleteProject(params: DeleteProjectParams) { const formData = new URLSearchParams({ project: params.project, }); const data = await fetchScrapyd("delproject.json", { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded", }, body: formData.toString(), }); return DeleteProjectSchema.parse(data); }, /** * Add/upload a project version (egg file) */ async addVersion(project: string, version: string, eggFile: Buffer) { const formData = new FormData(); formData.append("project", project); formData.append("version", version); formData.append("egg", new Blob([new Uint8Array(eggFile)]), "project.egg"); const data = await fetchScrapyd("addversion.json", { method: "POST", body: formData, }); return AddVersionSchema.parse(data); }, };