scrapy-ui/lib/scrapyd-client.ts

import {
  DaemonStatusSchema,
  ListProjectsSchema,
  ListVersionsSchema,
  ListSpidersSchema,
  ListJobsSchema,
  ScheduleJobSchema,
  CancelJobSchema,
  DeleteVersionSchema,
  DeleteProjectSchema,
  AddVersionSchema,
  type ScheduleJobParams,
  type CancelJobParams,
  type ListVersionsParams,
  type ListSpidersParams,
  type ListJobsParams,
  type DeleteVersionParams,
  type DeleteProjectParams,
} from "./types";

// Get credentials from environment variables (server-side only)
const SCRAPYD_URL = process.env.SCRAPYD_URL || "https://scrapy.pivoine.art";
const SCRAPYD_USERNAME = process.env.SCRAPYD_USERNAME || "";
const SCRAPYD_PASSWORD = process.env.SCRAPYD_PASSWORD || "";

/**
 * Create Basic Auth header
 */
function getAuthHeader(): string {
  const credentials = Buffer.from(`${SCRAPYD_USERNAME}:${SCRAPYD_PASSWORD}`).toString("base64");
  return `Basic ${credentials}`;
}

/**
 * Base fetch wrapper with auth
 */
async function fetchScrapyd(endpoint: string, options: RequestInit = {}) {
  const url = `${SCRAPYD_URL}/${endpoint}`;

  const response = await fetch(url, {
    ...options,
    headers: {
      Authorization: getAuthHeader(),
      ...options.headers,
    },
  });

  if (!response.ok) {
    throw new Error(`Scrapyd API error: ${response.status} ${response.statusText}`);
  }

  return response.json();
}

/**
 * ScrapydClient - Server-side API client for Scrapyd
 * All methods use environment variables for authentication
 */
export const ScrapydClient = {
  /**
   * Get daemon status
   */
  async getDaemonStatus() {
    const data = await fetchScrapyd("daemonstatus.json");
    return DaemonStatusSchema.parse(data);
  },

  /**
   * List all projects
   */
  async listProjects() {
    const data = await fetchScrapyd("listprojects.json");
    return ListProjectsSchema.parse(data);
  },

  /**
   * List versions for a project
   */
  async listVersions(params: ListVersionsParams) {
    const url = new URLSearchParams({ project: params.project });
    const data = await fetchScrapyd(`listversions.json?${url}`);
    return ListVersionsSchema.parse(data);
  },

  /**
   * List spiders for a project
   */
  async listSpiders(params: ListSpidersParams) {
    const url = new URLSearchParams({
      project: params.project,
      ...(params.version && { _version: params.version }),
    });
    const data = await fetchScrapyd(`listspiders.json?${url}`);
    return ListSpidersSchema.parse(data);
  },

  /**
   * List jobs (pending, running, finished) for a project
   */
  async listJobs(params: ListJobsParams) {
    const url = new URLSearchParams({ project: params.project });
    const data = await fetchScrapyd(`listjobs.json?${url}`);
    return ListJobsSchema.parse(data);
  },

  /**
   * Schedule a spider job
   */
  async scheduleJob(params: ScheduleJobParams) {
    const formData = new URLSearchParams({
      project: params.project,
      spider: params.spider,
      ...(params.jobid && { jobid: params.jobid }),
    });

    // Add custom settings
    if (params.settings) {
      Object.entries(params.settings).forEach(([key, value]) => {
        formData.append(`setting`, `${key}=${value}`);
      });
    }

    // Add spider arguments
    if (params.args) {
      Object.entries(params.args).forEach(([key, value]) => {
        formData.append(key, value);
      });
    }

    const data = await fetchScrapyd("schedule.json", {
      method: "POST",
      headers: {
        "Content-Type": "application/x-www-form-urlencoded",
      },
      body: formData.toString(),
    });

    return ScheduleJobSchema.parse(data);
  },

  /**
   * Cancel a job
   */
  async cancelJob(params: CancelJobParams) {
    const formData = new URLSearchParams({
      project: params.project,
      job: params.job,
    });

    const data = await fetchScrapyd("cancel.json", {
      method: "POST",
      headers: {
        "Content-Type": "application/x-www-form-urlencoded",
      },
      body: formData.toString(),
    });

    return CancelJobSchema.parse(data);
  },

  /**
   * Delete a project version
   */
  async deleteVersion(params: DeleteVersionParams) {
    const formData = new URLSearchParams({
      project: params.project,
      version: params.version,
    });

    const data = await fetchScrapyd("delversion.json", {
      method: "POST",
      headers: {
        "Content-Type": "application/x-www-form-urlencoded",
      },
      body: formData.toString(),
    });

    return DeleteVersionSchema.parse(data);
  },

  /**
   * Delete a project
   */
  async deleteProject(params: DeleteProjectParams) {
    const formData = new URLSearchParams({
      project: params.project,
    });

    const data = await fetchScrapyd("delproject.json", {
      method: "POST",
      headers: {
        "Content-Type": "application/x-www-form-urlencoded",
      },
      body: formData.toString(),
    });

    return DeleteProjectSchema.parse(data);
  },

  /**
   * Add/upload a project version (egg file)
   */
  async addVersion(project: string, version: string, eggFile: Buffer) {
    const formData = new FormData();
    formData.append("project", project);
    formData.append("version", version);
    formData.append("egg", new Blob([new Uint8Array(eggFile)]), "project.egg");

    const data = await fetchScrapyd("addversion.json", {
      method: "POST",
      body: formData,
    });

    return AddVersionSchema.parse(data);
  },
};