feat: initial commit - Scrapyd UI web interface
- Next.js 16.0.1 + React 19.2.0 + Tailwind CSS 4.1.16 - Complete Scrapyd API integration (all 12 endpoints) - Dashboard with real-time job monitoring - Projects management (upload, list, delete) - Spiders management with scheduling - Jobs monitoring with filtering and cancellation - System status monitoring - Dark/light theme toggle with next-themes - Server-side authentication via environment variables - Docker deployment with multi-stage builds - GitHub Actions CI/CD workflow 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
282
app/(dashboard)/spiders/page.tsx
Normal file
282
app/(dashboard)/spiders/page.tsx
Normal file
@@ -0,0 +1,282 @@
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { Header } from "@/components/header";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
DialogTrigger,
|
||||
} from "@/components/ui/dialog";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import { Bug, PlayCircle, AlertCircle } from "lucide-react";
|
||||
import { ListProjects, ListSpiders, ScheduleJob } from "@/lib/types";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
|
||||
export default function SpidersPage() {
|
||||
const queryClient = useQueryClient();
|
||||
const [selectedProject, setSelectedProject] = useState<string>("");
|
||||
const [scheduleDialogOpen, setScheduleDialogOpen] = useState(false);
|
||||
const [selectedSpider, setSelectedSpider] = useState<string>("");
|
||||
|
||||
// Fetch projects
|
||||
const { data: projects, isLoading: isProjectsLoading } = useQuery({
|
||||
queryKey: ["projects"],
|
||||
queryFn: async (): Promise<ListProjects> => {
|
||||
const res = await fetch("/ui/api/scrapyd/projects");
|
||||
if (!res.ok) throw new Error("Failed to fetch projects");
|
||||
return res.json();
|
||||
},
|
||||
});
|
||||
|
||||
// Fetch spiders for selected project
|
||||
const { data: spiders, isLoading: isSpidersLoading } = useQuery({
|
||||
queryKey: ["spiders", selectedProject],
|
||||
queryFn: async (): Promise<ListSpiders> => {
|
||||
const res = await fetch(
|
||||
`/ui/api/scrapyd/spiders?project=${selectedProject}`
|
||||
);
|
||||
if (!res.ok) throw new Error("Failed to fetch spiders");
|
||||
return res.json();
|
||||
},
|
||||
enabled: !!selectedProject,
|
||||
});
|
||||
|
||||
// Schedule job mutation
|
||||
const scheduleJobMutation = useMutation({
|
||||
mutationFn: async (data: {
|
||||
project: string;
|
||||
spider: string;
|
||||
args?: Record<string, string>;
|
||||
}) => {
|
||||
const res = await fetch("/ui/api/scrapyd/jobs", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(data),
|
||||
});
|
||||
if (!res.ok) throw new Error("Failed to schedule job");
|
||||
return res.json() as Promise<ScheduleJob>;
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["jobs"] });
|
||||
setScheduleDialogOpen(false);
|
||||
setSelectedSpider("");
|
||||
},
|
||||
});
|
||||
|
||||
const handleSchedule = (e: React.FormEvent<HTMLFormElement>) => {
|
||||
e.preventDefault();
|
||||
const formData = new FormData(e.currentTarget);
|
||||
const argsStr = formData.get("args") as string;
|
||||
|
||||
let args: Record<string, string> | undefined;
|
||||
if (argsStr.trim()) {
|
||||
try {
|
||||
args = JSON.parse(argsStr);
|
||||
} catch (error) {
|
||||
alert("Invalid JSON format for arguments");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
scheduleJobMutation.mutate({
|
||||
project: selectedProject,
|
||||
spider: selectedSpider,
|
||||
args,
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<Header
|
||||
title="Spiders"
|
||||
description="Browse and schedule spider jobs"
|
||||
/>
|
||||
|
||||
{/* Project Selector */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Select Project</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{isProjectsLoading ? (
|
||||
<Skeleton className="h-10 w-full" />
|
||||
) : (
|
||||
<Select value={selectedProject} onValueChange={setSelectedProject}>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder="Select a project" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{projects?.projects.map((project) => (
|
||||
<SelectItem key={project} value={project}>
|
||||
{project}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Spiders List */}
|
||||
{selectedProject && (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>
|
||||
Spiders in "{selectedProject}"
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{isSpidersLoading ? (
|
||||
<div className="space-y-2">
|
||||
{[1, 2, 3].map((i) => (
|
||||
<Skeleton key={i} className="h-16 w-full" />
|
||||
))}
|
||||
</div>
|
||||
) : spiders?.spiders && spiders.spiders.length > 0 ? (
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Spider Name</TableHead>
|
||||
<TableHead>Status</TableHead>
|
||||
<TableHead className="text-right">Actions</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{spiders.spiders.map((spider) => (
|
||||
<TableRow key={spider}>
|
||||
<TableCell>
|
||||
<div className="flex items-center gap-2">
|
||||
<Bug className="h-4 w-4 text-muted-foreground" />
|
||||
<span className="font-medium">{spider}</span>
|
||||
</div>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Badge variant="secondary">Available</Badge>
|
||||
</TableCell>
|
||||
<TableCell className="text-right">
|
||||
<Dialog
|
||||
open={scheduleDialogOpen && selectedSpider === spider}
|
||||
onOpenChange={(open) => {
|
||||
setScheduleDialogOpen(open);
|
||||
if (open) setSelectedSpider(spider);
|
||||
}}
|
||||
>
|
||||
<DialogTrigger asChild>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => setSelectedSpider(spider)}
|
||||
>
|
||||
<PlayCircle className="mr-2 h-4 w-4" />
|
||||
Schedule
|
||||
</Button>
|
||||
</DialogTrigger>
|
||||
<DialogContent>
|
||||
<form onSubmit={handleSchedule}>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Schedule Spider Job</DialogTitle>
|
||||
<DialogDescription>
|
||||
Schedule "{spider}" to run on "{selectedProject}"
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<div className="gap-4 py-4">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="project-name">Project</Label>
|
||||
<Input
|
||||
id="project-name"
|
||||
value={selectedProject}
|
||||
disabled
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="spider-name">Spider</Label>
|
||||
<Input
|
||||
id="spider-name"
|
||||
value={spider}
|
||||
disabled
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="args">
|
||||
Arguments (JSON)
|
||||
</Label>
|
||||
<Textarea
|
||||
id="args"
|
||||
name="args"
|
||||
placeholder='{"url": "https://example.com", "pages": 10}'
|
||||
className="font-mono text-sm"
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Optional: Provide spider arguments in JSON format
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<DialogFooter>
|
||||
<Button
|
||||
type="submit"
|
||||
disabled={scheduleJobMutation.isPending}
|
||||
>
|
||||
{scheduleJobMutation.isPending
|
||||
? "Scheduling..."
|
||||
: "Schedule Job"}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</form>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
) : (
|
||||
<div className="flex flex-col items-center justify-center py-12 text-center">
|
||||
<AlertCircle className="mb-4 h-12 w-12 text-muted-foreground" />
|
||||
<h3 className="mb-2 text-lg font-semibold">No spiders found</h3>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
This project doesn't have any spiders yet
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{!selectedProject && !isProjectsLoading && (
|
||||
<Card>
|
||||
<CardContent className="flex flex-col items-center justify-center py-12">
|
||||
<Bug className="mb-4 h-12 w-12 text-muted-foreground" />
|
||||
<h3 className="mb-2 text-lg font-semibold">Select a project</h3>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Choose a project to view its spiders
|
||||
</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user