feat: add BullMQ job queue with admin monitoring UI

- Add BullMQ to backend; mail jobs (verification, password reset) now enqueued instead of sent inline - Mail worker processes jobs with 3-attempt exponential backoff retry - Admin GraphQL resolvers: adminQueues, adminQueueJobs, adminRetryJob, adminRemoveJob, adminPauseQueue, adminResumeQueue - Admin frontend page at /admin/queues: queue cards with counts, job table with status filter, retry/remove/pause actions Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-08 18:25:09 +01:00
parent 6dcdc0130b
commit ea23233645
16 changed files with 927 additions and 6 deletions
--- a/packages/frontend/src/lib/i18n/locales/en.ts
+++ b/packages/frontend/src/lib/i18n/locales/en.ts
@@ -913,6 +913,7 @@ export default {
      articles: "Articles",
      comments: "Comments",
      recordings: "Recordings",
+      queues: "Queues",
    },
    common: {
      save_changes: "Save changes",
@@ -1058,6 +1059,36 @@ export default {
      delete_success: "Recording deleted",
      delete_error: "Failed to delete recording",
    },
+    queues: {
+      title: "Job Queues",
+      pause: "Pause",
+      resume: "Resume",
+      paused_badge: "Paused",
+      retry: "Retry",
+      remove: "Remove",
+      retry_success: "Job retried",
+      retry_error: "Failed to retry job",
+      remove_success: "Job removed",
+      remove_error: "Failed to remove job",
+      pause_success: "Queue paused",
+      pause_error: "Failed to pause queue",
+      resume_success: "Queue resumed",
+      resume_error: "Failed to resume queue",
+      col_id: "ID",
+      col_name: "Name",
+      col_status: "Status",
+      col_attempts: "Attempts",
+      col_created: "Created",
+      col_actions: "Actions",
+      no_jobs: "No jobs found",
+      status_all: "All",
+      status_waiting: "Waiting",
+      status_active: "Active",
+      status_completed: "Completed",
+      status_failed: "Failed",
+      status_delayed: "Delayed",
+      failed_reason: "Reason: {reason}",
+    },
    article_form: {
      new_title: "New article",
      edit_title: "Edit article",
--- a/packages/frontend/src/lib/services.ts
+++ b/packages/frontend/src/lib/services.ts
@@ -1876,3 +1876,145 @@ export async function adminDeleteRecording(id: string): Promise<void> {
    await getGraphQLClient().request(ADMIN_DELETE_RECORDING_MUTATION, { id });
  });
 }
+
+// --- Queues ---
+
+export type JobCounts = {
+  waiting: number;
+  active: number;
+  completed: number;
+  failed: number;
+  delayed: number;
+  paused: number;
+};
+
+export type QueueInfo = {
+  name: string;
+  counts: JobCounts;
+  isPaused: boolean;
+};
+
+export type Job = {
+  id: string;
+  name: string;
+  queue: string;
+  status: string;
+  data: Record<string, unknown>;
+  result: unknown;
+  failedReason: string | null;
+  attemptsMade: number;
+  createdAt: string;
+  processedAt: string | null;
+  finishedAt: string | null;
+  progress: number | null;
+};
+
+const ADMIN_QUEUES_QUERY = gql`
+  query AdminQueues {
+    adminQueues {
+      name
+      isPaused
+      counts {
+        waiting
+        active
+        completed
+        failed
+        delayed
+        paused
+      }
+    }
+  }
+`;
+
+export async function getAdminQueues(
+  fetchFn?: typeof globalThis.fetch,
+  token?: string,
+): Promise<QueueInfo[]> {
+  return loggedApiCall("getAdminQueues", async () => {
+    const client = token ? getAuthClient(token, fetchFn) : getGraphQLClient(fetchFn);
+    const data = await client.request<{ adminQueues: QueueInfo[] }>(ADMIN_QUEUES_QUERY);
+    return data.adminQueues;
+  });
+}
+
+const ADMIN_QUEUE_JOBS_QUERY = gql`
+  query AdminQueueJobs($queue: String!, $status: String, $limit: Int, $offset: Int) {
+    adminQueueJobs(queue: $queue, status: $status, limit: $limit, offset: $offset) {
+      id
+      name
+      queue
+      status
+      data
+      result
+      failedReason
+      attemptsMade
+      createdAt
+      processedAt
+      finishedAt
+      progress
+    }
+  }
+`;
+
+export async function getAdminQueueJobs(
+  queue: string,
+  status?: string,
+  limit?: number,
+  offset?: number,
+): Promise<Job[]> {
+  return loggedApiCall("getAdminQueueJobs", async () => {
+    const data = await getGraphQLClient().request<{ adminQueueJobs: Job[] }>(
+      ADMIN_QUEUE_JOBS_QUERY,
+      { queue, status, limit, offset },
+    );
+    return data.adminQueueJobs;
+  });
+}
+
+const ADMIN_RETRY_JOB_MUTATION = gql`
+  mutation AdminRetryJob($queue: String!, $jobId: String!) {
+    adminRetryJob(queue: $queue, jobId: $jobId)
+  }
+`;
+
+export async function adminRetryJob(queue: string, jobId: string): Promise<void> {
+  return loggedApiCall("adminRetryJob", async () => {
+    await getGraphQLClient().request(ADMIN_RETRY_JOB_MUTATION, { queue, jobId });
+  });
+}
+
+const ADMIN_REMOVE_JOB_MUTATION = gql`
+  mutation AdminRemoveJob($queue: String!, $jobId: String!) {
+    adminRemoveJob(queue: $queue, jobId: $jobId)
+  }
+`;
+
+export async function adminRemoveJob(queue: string, jobId: string): Promise<void> {
+  return loggedApiCall("adminRemoveJob", async () => {
+    await getGraphQLClient().request(ADMIN_REMOVE_JOB_MUTATION, { queue, jobId });
+  });
+}
+
+const ADMIN_PAUSE_QUEUE_MUTATION = gql`
+  mutation AdminPauseQueue($queue: String!) {
+    adminPauseQueue(queue: $queue)
+  }
+`;
+
+const ADMIN_RESUME_QUEUE_MUTATION = gql`
+  mutation AdminResumeQueue($queue: String!) {
+    adminResumeQueue(queue: $queue)
+  }
+`;
+
+export async function adminPauseQueue(queue: string): Promise<void> {
+  return loggedApiCall("adminPauseQueue", async () => {
+    await getGraphQLClient().request(ADMIN_PAUSE_QUEUE_MUTATION, { queue });
+  });
+}
+
+export async function adminResumeQueue(queue: string): Promise<void> {
+  return loggedApiCall("adminResumeQueue", async () => {
+    await getGraphQLClient().request(ADMIN_RESUME_QUEUE_MUTATION, { queue });
+  });
+}
--- a/packages/frontend/src/routes/admin/+layout.svelte
+++ b/packages/frontend/src/routes/admin/+layout.svelte
@@ -14,6 +14,11 @@
      href: "/admin/recordings",
      icon: "icon-[ri--record-circle-line]",
    },
+    {
+      name: $_("admin.nav.queues"),
+      href: "/admin/queues",
+      icon: "icon-[ri--stack-line]",
+    },
  ]);

  function isActive(href: string) {
--- a/packages/frontend/src/routes/admin/queues/+page.server.ts
+++ b/packages/frontend/src/routes/admin/queues/+page.server.ts
@@ -0,0 +1,7 @@
+import { getAdminQueues } from "$lib/services";
+
+export async function load({ fetch, cookies }) {
+  const token = cookies.get("session_token") || "";
+  const queues = await getAdminQueues(fetch, token).catch(() => []);
+  return { queues };
+}
--- a/packages/frontend/src/routes/admin/queues/+page.svelte
+++ b/packages/frontend/src/routes/admin/queues/+page.svelte
@@ -0,0 +1,298 @@
+<script lang="ts">
+  import { invalidateAll } from "$app/navigation";
+  import { toast } from "svelte-sonner";
+  import { _ } from "svelte-i18n";
+  import {
+    getAdminQueueJobs,
+    adminRetryJob,
+    adminRemoveJob,
+    adminPauseQueue,
+    adminResumeQueue,
+  } from "$lib/services";
+  import { Button } from "$lib/components/ui/button";
+  import { Badge } from "$lib/components/ui/badge";
+  import type { Job, QueueInfo } from "$lib/services";
+
+  const { data } = $props();
+
+  const queues = $derived(data.queues);
+  let selectedQueue = $state<string | null>(data.queues[0]?.name ?? null);
+  let selectedStatus = $state<string | null>(null);
+  let jobs = $state<Job[]>([]);
+  let loadingJobs = $state(false);
+  let togglingQueue = $state<string | null>(null);
+
+  const STATUS_FILTERS = [
+    { value: null, label: $_("admin.queues.status_all") },
+    { value: "waiting", label: $_("admin.queues.status_waiting") },
+    { value: "active", label: $_("admin.queues.status_active") },
+    { value: "completed", label: $_("admin.queues.status_completed") },
+    { value: "failed", label: $_("admin.queues.status_failed") },
+    { value: "delayed", label: $_("admin.queues.status_delayed") },
+  ];
+
+  async function loadJobs() {
+    if (!selectedQueue) return;
+    loadingJobs = true;
+    try {
+      jobs = await getAdminQueueJobs(selectedQueue, selectedStatus ?? undefined, 50, 0);
+    } finally {
+      loadingJobs = false;
+    }
+  }
+
+  async function selectQueue(name: string) {
+    selectedQueue = name;
+    selectedStatus = null;
+    await loadJobs();
+  }
+
+  async function selectStatus(status: string | null) {
+    selectedStatus = status;
+    await loadJobs();
+  }
+
+  async function retryJob(job: Job) {
+    try {
+      await adminRetryJob(job.queue, job.id);
+      toast.success($_("admin.queues.retry_success"));
+      await loadJobs();
+      await refreshCounts();
+    } catch {
+      toast.error($_("admin.queues.retry_error"));
+    }
+  }
+
+  async function removeJob(job: Job) {
+    try {
+      await adminRemoveJob(job.queue, job.id);
+      toast.success($_("admin.queues.remove_success"));
+      jobs = jobs.filter((j) => j.id !== job.id);
+      await refreshCounts();
+    } catch {
+      toast.error($_("admin.queues.remove_error"));
+    }
+  }
+
+  async function toggleQueue(queueName: string, isPaused: boolean) {
+    togglingQueue = queueName;
+    try {
+      if (isPaused) {
+        await adminResumeQueue(queueName);
+        toast.success($_("admin.queues.resume_success"));
+      } else {
+        await adminPauseQueue(queueName);
+        toast.success($_("admin.queues.pause_success"));
+      }
+      await refreshCounts();
+    } catch {
+      toast.error(isPaused ? $_("admin.queues.resume_error") : $_("admin.queues.pause_error"));
+    } finally {
+      togglingQueue = null;
+    }
+  }
+
+  async function refreshCounts() {
+    await invalidateAll();
+  }
+
+  $effect(() => {
+    if (selectedQueue) loadJobs();
+  });
+
+  function statusColor(status: string): string {
+    switch (status) {
+      case "active":
+        return "text-blue-500 border-blue-500/30 bg-blue-500/10";
+      case "completed":
+        return "text-green-500 border-green-500/30 bg-green-500/10";
+      case "failed":
+        return "text-destructive border-destructive/30 bg-destructive/10";
+      case "delayed":
+        return "text-yellow-500 border-yellow-500/30 bg-yellow-500/10";
+      default:
+        return "text-muted-foreground border-border/40 bg-muted/20";
+    }
+  }
+
+  function formatDate(iso: string | null): string {
+    if (!iso) return "—";
+    return new Date(iso).toLocaleString();
+  }
+</script>
+
+<div class="py-3 sm:py-6 sm:pl-6">
+  <div class="flex items-center justify-between mb-6 px-3 sm:px-0">
+    <h1 class="text-2xl font-bold">{$_("admin.queues.title")}</h1>
+  </div>
+
+  <!-- Queue cards -->
+  <div class="flex flex-wrap gap-3 mb-6 px-3 sm:px-0">
+    {#each queues as queue (queue.name)}
+      {@const isSelected = selectedQueue === queue.name}
+      <button
+        class={`flex-1 min-w-48 rounded-lg border p-4 text-left transition-colors cursor-pointer ${
+          isSelected
+            ? "border-primary/50 bg-primary/5"
+            : "border-border/40 bg-card hover:border-border/70"
+        }`}
+        onclick={() => selectQueue(queue.name)}
+        aria-pressed={isSelected}
+      >
+        <div class="flex items-center justify-between mb-3">
+          <span class="font-semibold capitalize">{queue.name}</span>
+          <div class="flex items-center gap-1.5">
+            {#if queue.isPaused}
+              <Badge variant="outline" class="text-yellow-600 border-yellow-500/40 bg-yellow-500/10"
+                >{$_("admin.queues.paused_badge")}</Badge
+              >
+            {/if}
+            <Button
+              size="sm"
+              variant="ghost"
+              class="h-6 px-2 text-xs"
+              disabled={togglingQueue === queue.name}
+              onclick={(e) => {
+                e.stopPropagation();
+                toggleQueue(queue.name, queue.isPaused);
+              }}
+            >
+              {queue.isPaused ? $_("admin.queues.resume") : $_("admin.queues.pause")}
+            </Button>
+          </div>
+        </div>
+        <div class="flex flex-wrap gap-2 text-xs">
+          {#if queue.counts.waiting > 0}
+            <span class="text-muted-foreground">{queue.counts.waiting} waiting</span>
+          {/if}
+          {#if queue.counts.active > 0}
+            <span class="text-blue-500">{queue.counts.active} active</span>
+          {/if}
+          {#if queue.counts.completed > 0}
+            <span class="text-green-500">{queue.counts.completed} completed</span>
+          {/if}
+          {#if queue.counts.failed > 0}
+            <span class="text-destructive font-medium">{queue.counts.failed} failed</span>
+          {/if}
+          {#if queue.counts.delayed > 0}
+            <span class="text-yellow-500">{queue.counts.delayed} delayed</span>
+          {/if}
+          {#if Object.values(queue.counts).every((v) => v === 0)}
+            <span class="text-muted-foreground">empty</span>
+          {/if}
+        </div>
+      </button>
+    {/each}
+  </div>
+
+  {#if selectedQueue}
+    <!-- Status filter tabs -->
+    <div class="flex gap-1 mb-4 px-3 sm:px-0 flex-wrap">
+      {#each STATUS_FILTERS as f (f.value ?? "all")}
+        <Button
+          size="sm"
+          variant={selectedStatus === f.value ? "default" : "outline"}
+          onclick={() => selectStatus(f.value)}
+        >
+          {f.label}
+        </Button>
+      {/each}
+    </div>
+
+    <!-- Jobs table -->
+    <div class="sm:rounded-lg border-y sm:border border-border/40 overflow-x-auto">
+      <table class="w-full text-sm">
+        <thead class="bg-muted/30">
+          <tr>
+            <th class="px-4 py-3 text-left font-medium text-muted-foreground"
+              >{$_("admin.queues.col_id")}</th
+            >
+            <th class="px-4 py-3 text-left font-medium text-muted-foreground"
+              >{$_("admin.queues.col_name")}</th
+            >
+            <th class="px-4 py-3 text-left font-medium text-muted-foreground"
+              >{$_("admin.queues.col_status")}</th
+            >
+            <th
+              class="px-4 py-3 text-left font-medium text-muted-foreground hidden md:table-cell"
+            >
+              {$_("admin.queues.col_attempts")}
+            </th>
+            <th
+              class="px-4 py-3 text-left font-medium text-muted-foreground hidden lg:table-cell"
+            >
+              {$_("admin.queues.col_created")}
+            </th>
+            <th class="px-4 py-3 text-right font-medium text-muted-foreground"
+              >{$_("admin.queues.col_actions")}</th
+            >
+          </tr>
+        </thead>
+        <tbody class="divide-y divide-border/30">
+          {#if loadingJobs}
+            <tr>
+              <td colspan="6" class="px-4 py-8 text-center text-muted-foreground"
+                >{$_("common.loading")}</td
+              >
+            </tr>
+          {:else}
+            {#each jobs as job (job.id)}
+              <tr class="hover:bg-muted/10 transition-colors">
+                <td class="px-4 py-3 font-mono text-xs text-muted-foreground">{job.id}</td>
+                <td class="px-4 py-3">
+                  <div>
+                    <p class="font-medium">{job.name}</p>
+                    {#if job.failedReason}
+                      <p class="text-xs text-destructive mt-0.5 max-w-xs truncate">
+                        {$_("admin.queues.failed_reason", { values: { reason: job.failedReason } })}
+                      </p>
+                    {/if}
+                  </div>
+                </td>
+                <td class="px-4 py-3">
+                  <Badge variant="outline" class={statusColor(job.status)}>{job.status}</Badge>
+                </td>
+                <td class="px-4 py-3 text-muted-foreground hidden md:table-cell"
+                  >{job.attemptsMade}</td
+                >
+                <td class="px-4 py-3 text-muted-foreground hidden lg:table-cell text-xs"
+                  >{formatDate(job.createdAt)}</td
+                >
+                <td class="px-4 py-3 text-right">
+                  <div class="flex items-center justify-end gap-1">
+                    {#if job.status === "failed"}
+                      <Button
+                        size="sm"
+                        variant="ghost"
+                        aria-label={$_("admin.queues.retry")}
+                        onclick={() => retryJob(job)}
+                      >
+                        <span class="icon-[ri--restart-line] h-4 w-4"></span>
+                      </Button>
+                    {/if}
+                    <Button
+                      size="sm"
+                      variant="ghost"
+                      aria-label={$_("admin.queues.remove")}
+                      class="text-destructive hover:text-destructive hover:bg-destructive/10"
+                      onclick={() => removeJob(job)}
+                    >
+                      <span class="icon-[ri--delete-bin-line] h-4 w-4"></span>
+                    </Button>
+                  </div>
+                </td>
+              </tr>
+            {/each}
+            {#if jobs.length === 0}
+              <tr>
+                <td colspan="6" class="px-4 py-8 text-center text-muted-foreground"
+                  >{$_("admin.queues.no_jobs")}</td
+                >
+              </tr>
+            {/if}
+          {/if}
+        </tbody>
+      </table>
+    </div>
+  {/if}
+</div>