Create background job runner and memory refresh jobs

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/background-jobs-routes.test.ts

@@ -0,0 +1,175 @@
+import express from "express";
+import request from "supertest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { errorHandler } from "../middleware/index.js";
+import { backgroundJobRoutes } from "../routes/background-jobs.js";
+
+const companyA = "11111111-1111-4111-8111-111111111111";
+const companyB = "22222222-2222-4222-8222-222222222222";
+const jobId = "33333333-3333-4333-8333-333333333333";
+const runId = "44444444-4444-4444-8444-444444444444";
+
+const mockBackgroundJobService = vi.hoisted(() => ({
+  createOrUpdateJob: vi.fn(),
+  listJobs: vi.fn(),
+  getJob: vi.fn(),
+  createRun: vi.fn(),
+  listRuns: vi.fn(),
+  getRun: vi.fn(),
+  listRunEvents: vi.fn(),
+  requestCancelRun: vi.fn(),
+}));
+
+const mockLogActivity = vi.hoisted(() => vi.fn());
+
+vi.mock("../services/index.js", () => ({
+  backgroundJobService: () => mockBackgroundJobService,
+  logActivity: mockLogActivity,
+}));
+
+function createApp(actor: Record<string, unknown>) {
+  const app = express();
+  app.use(express.json());
+  app.use((req, _res, next) => {
+    (req as any).actor = actor;
+    next();
+  });
+  app.use("/api", backgroundJobRoutes({} as any));
+  app.use(errorHandler);
+  return app;
+}
+
+describe("background job routes", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockLogActivity.mockResolvedValue(undefined);
+    mockBackgroundJobService.createOrUpdateJob.mockResolvedValue({
+      id: jobId,
+      companyId: companyA,
+      key: "memory.refresh",
+      jobType: "memory_refresh",
+      displayName: "Memory refresh",
+      backendKind: "server_worker",
+      status: "active",
+      config: {},
+    });
+    mockBackgroundJobService.createRun.mockResolvedValue({
+      id: runId,
+      companyId: companyA,
+      jobId,
+      jobKey: "memory.refresh",
+      jobType: "memory_refresh",
+      trigger: "manual",
+      status: "queued",
+    });
+    mockBackgroundJobService.requestCancelRun.mockResolvedValue({
+      id: runId,
+      companyId: companyA,
+      jobId,
+      jobKey: "memory.refresh",
+      jobType: "memory_refresh",
+      trigger: "manual",
+      status: "running",
+      cancellationRequestedAt: new Date("2026-04-20T00:00:00.000Z"),
+    });
+  });
+
+  it("blocks board users outside the company from listing jobs", async () => {
+    const app = createApp({
+      type: "board",
+      userId: "board-user",
+      source: "session",
+      companyIds: [companyB],
+      isInstanceAdmin: false,
+    });
+
+    const res = await request(app)
+      .get(`/api/companies/${companyA}/background-jobs`)
+      .set("Origin", "http://localhost:3100");
+
+    expect(res.status).toBe(403);
+    expect(mockBackgroundJobService.listJobs).not.toHaveBeenCalled();
+  });
+
+  it("creates a generic background job definition and logs activity", async () => {
+    const app = createApp({
+      type: "board",
+      userId: "board-user",
+      source: "session",
+      companyIds: [companyA],
+      isInstanceAdmin: false,
+    });
+
+    const res = await request(app)
+      .post(`/api/companies/${companyA}/background-jobs`)
+      .set("Origin", "http://localhost:3100")
+      .send({
+        key: "memory.refresh",
+        jobType: "memory_refresh",
+        displayName: "Memory refresh",
+        backendKind: "server_worker",
+      });
+
+    expect(res.status).toBe(201);
+    expect(mockBackgroundJobService.createOrUpdateJob).toHaveBeenCalledWith(
+      companyA,
+      expect.objectContaining({ key: "memory.refresh", jobType: "memory_refresh" }),
+      expect.objectContaining({ actorType: "user", userId: "board-user" }),
+    );
+    expect(mockLogActivity).toHaveBeenCalledWith(
+      expect.anything(),
+      expect.objectContaining({
+        companyId: companyA,
+        action: "background_job.upserted",
+        entityId: jobId,
+      }),
+    );
+  });
+
+  it("creates a generic background job run", async () => {
+    const app = createApp({
+      type: "board",
+      userId: "board-user",
+      source: "session",
+      companyIds: [companyA],
+      isInstanceAdmin: false,
+    });
+
+    const res = await request(app)
+      .post(`/api/companies/${companyA}/background-job-runs`)
+      .set("Origin", "http://localhost:3100")
+      .send({ jobId, trigger: "manual", totalItems: 3 });
+
+    expect(res.status).toBe(201);
+    expect(mockBackgroundJobService.createRun).toHaveBeenCalledWith(
+      companyA,
+      { jobId, trigger: "manual", totalItems: 3 },
+      expect.objectContaining({ actorType: "user", userId: "board-user" }),
+    );
+  });
+
+  it("requests cancellation for a run", async () => {
+    const app = createApp({
+      type: "board",
+      userId: "board-user",
+      source: "session",
+      companyIds: [companyA],
+      isInstanceAdmin: false,
+    });
+
+    const res = await request(app)
+      .post(`/api/companies/${companyA}/background-job-runs/${runId}/cancel`)
+      .set("Origin", "http://localhost:3100")
+      .send({});
+
+    expect(res.status).toBe(200);
+    expect(mockBackgroundJobService.requestCancelRun).toHaveBeenCalledWith(companyA, runId);
+    expect(mockLogActivity).toHaveBeenCalledWith(
+      expect.anything(),
+      expect.objectContaining({
+        action: "background_job_run.cancel_requested",
+        entityId: runId,
+      }),
+    );
+  });
+});

server/src/__tests__/memory-routes.test.ts

@@ -30,6 +30,7 @@ const mockMemoryService = vi.hoisted(() => ({
   getRecord: vi.fn(),
   listOperations: vi.fn(),
   listExtractionJobs: vi.fn(),
+  startRefreshJob: vi.fn(),
 }));
 
 const mockAgentService = vi.hoisted(() => ({
@@ -300,4 +301,66 @@ describe("memory routes", () => {
     expect(res.status).toBe(403);
     expect(mockMemoryService.setProjectOverride).not.toHaveBeenCalled();
   });
+
+  it("starts memory refresh jobs through the memory service and logs activity", async () => {
+    mockMemoryService.startRefreshJob.mockResolvedValue({
+      job: {
+        id: "99999999-9999-4999-8999-999999999999",
+        companyId: companyA,
+        key: "memory.refresh",
+        jobType: "memory_refresh",
+      },
+      run: {
+        id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+        companyId: companyA,
+        jobKey: "memory.refresh",
+        jobType: "memory_refresh",
+        status: "queued",
+      },
+      dryRun: false,
+      sourceCounts: {
+        issue: 1,
+        issue_comment: 2,
+        issue_document: 1,
+        run: 0,
+      },
+      recordCount: 0,
+    });
+    const app = createApp({
+      type: "board",
+      userId: "board-user",
+      source: "session",
+      companyIds: [companyA],
+      isInstanceAdmin: false,
+    });
+
+    const res = await request(app)
+      .post(`/api/companies/${companyA}/memory/refresh-jobs`)
+      .set("Origin", "http://localhost:3100")
+      .send({
+        sourceKinds: ["issue", "issue_comment", "issue_document"],
+        issueIds: ["bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"],
+        dryRun: false,
+      });
+
+    expect(res.status).toBe(202);
+    expect(mockMemoryService.startRefreshJob).toHaveBeenCalledWith(
+      companyA,
+      expect.objectContaining({
+        sourceKinds: ["issue", "issue_comment", "issue_document"],
+        issueIds: ["bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"],
+        dryRun: false,
+      }),
+      expect.objectContaining({ actorType: "user", userId: "board-user" }),
+    );
+    expect(mockLogActivity).toHaveBeenCalledWith(
+      expect.anything(),
+      expect.objectContaining({
+        companyId: companyA,
+        action: "memory.refresh_job_started",
+        entityType: "background_job_run",
+        entityId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+      }),
+    );
+  });
 });

server/src/__tests__/memory-service.test.ts

@@ -38,6 +38,137 @@ function makeBinding(id: string, key: string, providerKey = "local_basic") {
   };
 }
 
+function makeMemoryRecordRow(overrides: Record<string, unknown> = {}) {
+  const now = new Date("2026-04-01T00:00:00.000Z");
+  return {
+    id: "44444444-4444-4444-8444-444444444444",
+    companyId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+    bindingId: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
+    providerKey: "local_basic",
+    scopeAgentId: null,
+    scopeProjectId: null,
+    scopeIssueId: "55555555-5555-4555-8555-555555555555",
+    scopeRunId: null,
+    scopeSubjectId: null,
+    scopeType: "org",
+    scopeId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+    scopeWorkspaceId: null,
+    scopeTeamId: null,
+    sourceKind: "issue_comment",
+    sourceIssueId: "55555555-5555-4555-8555-555555555555",
+    sourceCommentId: "66666666-6666-4666-8666-666666666666",
+    sourceDocumentKey: null,
+    sourceRunId: null,
+    sourceActivityId: null,
+    sourceExternalRef: null,
+    ownerType: "user",
+    ownerId: "board-user",
+    createdByActorType: "user",
+    createdByActorId: "board-user",
+    sensitivityLabel: "internal",
+    retentionPolicy: null,
+    expiresAt: null,
+    retentionState: "active",
+    reviewState: "accepted",
+    reviewedAt: null,
+    reviewedByActorType: null,
+    reviewedByActorId: null,
+    reviewNote: null,
+    citationJson: null,
+    supersedesRecordId: null,
+    supersededByRecordId: null,
+    revokedAt: null,
+    revokedByActorType: null,
+    revokedByActorId: null,
+    revocationReason: null,
+    title: "Issue comment",
+    content: "Remember that the launch checklist lives in the issue document.",
+    summary: "Remember that the launch checklist lives in the issue document.",
+    metadata: {},
+    createdByOperationId: "77777777-7777-4777-8777-777777777777",
+    deletedAt: null,
+    createdAt: now,
+    updatedAt: now,
+    ...overrides,
+  };
+}
+
+function makeOperationRow(overrides: Record<string, unknown> = {}) {
+  const now = new Date("2026-04-01T00:00:00.000Z");
+  return {
+    id: "77777777-7777-4777-8777-777777777777",
+    companyId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+    bindingId: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
+    providerKey: "local_basic",
+    operationType: "upsert",
+    triggerKind: "hook",
+    hookKind: "issue_comment_capture",
+    status: "succeeded",
+    actorType: "user",
+    actorId: "board-user",
+    agentId: null,
+    userId: "board-user",
+    scopeAgentId: null,
+    scopeProjectId: null,
+    scopeIssueId: "55555555-5555-4555-8555-555555555555",
+    scopeRunId: null,
+    scopeSubjectId: null,
+    scopeType: "org",
+    scopeId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+    scopeWorkspaceId: null,
+    scopeTeamId: null,
+    maxSensitivityLabel: null,
+    sourceKind: "issue_comment",
+    sourceIssueId: "55555555-5555-4555-8555-555555555555",
+    sourceCommentId: "66666666-6666-4666-8666-666666666666",
+    sourceDocumentKey: null,
+    sourceRunId: null,
+    sourceActivityId: null,
+    sourceExternalRef: null,
+    queryText: null,
+    recordCount: 1,
+    requestJson: null,
+    resultJson: null,
+    policyDecisionJson: null,
+    revocationSelectorJson: null,
+    retentionActionJson: null,
+    usageJson: [],
+    error: null,
+    costEventId: null,
+    financeEventId: null,
+    occurredAt: now,
+    createdAt: now,
+    ...overrides,
+  };
+}
+
+function makeExtractionJobRow(overrides: Record<string, unknown> = {}) {
+  const now = new Date("2026-04-01T00:00:00.000Z");
+  return {
+    id: "88888888-8888-4888-8888-888888888888",
+    companyId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
+    bindingId: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
+    providerKey: "local_basic",
+    operationId: null,
+    status: "running",
+    providerJobId: null,
+    sourceKind: "issue_comment",
+    sourceIssueId: "55555555-5555-4555-8555-555555555555",
+    sourceCommentId: "66666666-6666-4666-8666-666666666666",
+    sourceDocumentKey: null,
+    sourceRunId: null,
+    sourceActivityId: null,
+    sourceExternalRef: null,
+    resultJson: null,
+    error: null,
+    startedAt: now,
+    completedAt: null,
+    createdAt: now,
+    updatedAt: now,
+    ...overrides,
+  };
+}
+
 describe("memoryService.forget", () => {
   it("rejects record sets that span multiple bindings", async () => {
     const rows = [
@@ -180,3 +311,130 @@ describe("memoryService.resolveBinding", () => {
     ).rejects.toThrow("Memory scope project does not belong to company");
   });
 });
+
+describe("memoryService hook policies", () => {
+  it("runs issue comment capture through an extraction job when policy requests Paperclip-managed extraction", async () => {
+    const binding = {
+      ...makeBinding("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", "company"),
+      config: {
+        enableIssueCommentCapture: true,
+        hookPolicies: {
+          issue_comment_capture: {
+            enabled: true,
+            extractionMode: "paperclip_managed",
+            runMode: "sync",
+            harness: "server_worker",
+            sensitivityLabel: "internal",
+            reviewState: "accepted",
+          },
+        },
+      },
+    };
+    const target = {
+      id: "target-company",
+      companyId: binding.companyId,
+      bindingId: binding.id,
+      targetType: "company",
+      targetId: binding.companyId,
+      createdAt: binding.createdAt,
+      updatedAt: binding.updatedAt,
+    };
+    const selectResults = [
+      [{ target, binding }],
+      [{ target, binding }],
+    ];
+    const insertResults = [
+      [makeExtractionJobRow()],
+      [makeMemoryRecordRow()],
+      [makeOperationRow()],
+    ];
+    const updateResults = [
+      [
+        makeExtractionJobRow({
+          status: "succeeded",
+          operationId: "77777777-7777-4777-8777-777777777777",
+          completedAt: new Date("2026-04-01T00:00:01.000Z"),
+        }),
+      ],
+    ];
+    const insertedValues: unknown[] = [];
+    const updatedValues: unknown[] = [];
+    const selectThenable = () => ({
+      from: vi.fn().mockReturnThis(),
+      innerJoin: vi.fn().mockReturnThis(),
+      where: vi.fn().mockReturnThis(),
+      orderBy: vi.fn().mockReturnThis(),
+      limit: vi.fn().mockReturnThis(),
+      then: vi.fn((resolve: (rows: unknown[]) => unknown) => Promise.resolve(resolve(selectResults.shift() ?? []))),
+    });
+    const db = {
+      select: vi.fn(() => selectThenable()),
+      insert: vi.fn(() => ({
+        values: vi.fn((value: unknown) => {
+          insertedValues.push(value);
+          return {
+            returning: vi.fn().mockResolvedValue(insertResults.shift() ?? []),
+          };
+        }),
+      })),
+      update: vi.fn(() => ({
+        set: vi.fn((value: unknown) => {
+          updatedValues.push(value);
+          return {
+            where: vi.fn().mockReturnValue({
+              returning: vi.fn().mockResolvedValue(updateResults.shift() ?? []),
+            }),
+          };
+        }),
+      })),
+    } as any;
+
+    const result = await memoryService(db).captureIssueComment({
+      companyId: binding.companyId,
+      issueId: "55555555-5555-4555-8555-555555555555",
+      commentId: "66666666-6666-4666-8666-666666666666",
+      body: "Remember that the launch checklist lives in the issue document.",
+      actor: {
+        actorType: "user",
+        actorId: "board-user",
+        agentId: null,
+        userId: "board-user",
+        runId: null,
+      },
+    });
+
+    expect(result?.operation.hookKind).toBe("issue_comment_capture");
+    expect(db.insert).toHaveBeenCalledTimes(3);
+    expect(insertedValues[0]).toMatchObject({
+      companyId: binding.companyId,
+      bindingId: binding.id,
+      status: "running",
+      sourceKind: "issue_comment",
+      resultJson: {
+        hookKind: "issue_comment_capture",
+        policy: {
+          extractionMode: "paperclip_managed",
+          harness: "server_worker",
+        },
+      },
+    });
+    expect(insertedValues[1]).toMatchObject({
+      reviewState: "accepted",
+      metadata: {
+        extraction: {
+          mode: "paperclip_managed",
+          harness: "server_worker",
+          jobId: "88888888-8888-4888-8888-888888888888",
+        },
+      },
+    });
+    expect(updatedValues[0]).toMatchObject({
+      status: "succeeded",
+      operationId: "77777777-7777-4777-8777-777777777777",
+      resultJson: {
+        hookKind: "issue_comment_capture",
+        recordIds: ["44444444-4444-4444-8444-444444444444"],
+      },
+    });
+  });
+});

server/src/app.ts

@@ -27,6 +27,7 @@ import { sidebarBadgeRoutes } from "./routes/sidebar-badges.js";
 import { inboxDismissalRoutes } from "./routes/inbox-dismissals.js";
 import { instanceSettingsRoutes } from "./routes/instance-settings.js";
 import { memoryRoutes } from "./routes/memory.js";
+import { backgroundJobRoutes } from "./routes/background-jobs.js";
 import { llmRoutes } from "./routes/llms.js";
 import { assetRoutes } from "./routes/assets.js";
 import { accessRoutes } from "./routes/access.js";
@@ -226,6 +227,7 @@ export async function createApp(
   api.use(approvalRoutes(db));
   api.use(secretRoutes(db));
   api.use(costRoutes(db));
+  api.use(backgroundJobRoutes(db));
   api.use(memoryRoutes(db, { pluginMemoryProviders: memoryProviderDispatcher }));
   api.use(activityRoutes(db));
   api.use(dashboardRoutes(db));

server/src/routes/authz.ts

@@ -39,6 +39,7 @@ export function getActorInfo(req: Request) {
       actorType: "agent" as const,
       actorId: req.actor.agentId ?? "unknown-agent",
       agentId: req.actor.agentId ?? null,
+      userId: null,
       runId: req.actor.runId ?? null,
     };
   }
@@ -47,6 +48,7 @@ export function getActorInfo(req: Request) {
     actorType: "user" as const,
     actorId: req.actor.userId ?? "board",
     agentId: null,
+    userId: req.actor.userId ?? null,
     runId: req.actor.runId ?? null,
   };
 }

server/src/routes/background-jobs.ts

@@ -0,0 +1,127 @@
+import { Router } from "express";
+import type { Db } from "@paperclipai/db";
+import {
+  createBackgroundJobRunSchema,
+  createBackgroundJobSchema,
+  listBackgroundJobRunsQuerySchema,
+  listBackgroundJobsQuerySchema,
+} from "@paperclipai/shared";
+import { validate } from "../middleware/validate.js";
+import { backgroundJobService, logActivity } from "../services/index.js";
+import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js";
+
+export function backgroundJobRoutes(db: Db) {
+  const router = Router();
+  const jobs = backgroundJobService(db);
+
+  router.get("/companies/:companyId/background-jobs", async (req, res) => {
+    const companyId = req.params.companyId as string;
+    assertCompanyAccess(req, companyId);
+    assertBoard(req);
+    const parsed = listBackgroundJobsQuerySchema.parse(req.query);
+    res.json(await jobs.listJobs(companyId, parsed));
+  });
+
+  router.post("/companies/:companyId/background-jobs", validate(createBackgroundJobSchema), async (req, res) => {
+    const companyId = req.params.companyId as string;
+    assertCompanyAccess(req, companyId);
+    assertBoard(req);
+    const actor = getActorInfo(req);
+    const job = await jobs.createOrUpdateJob(companyId, req.body, {
+      actorType: actor.actorType,
+      actorId: actor.actorId,
+      agentId: actor.agentId,
+      userId: actor.userId,
+    });
+    await logActivity(db, {
+      companyId,
+      actorType: actor.actorType,
+      actorId: actor.actorId,
+      agentId: actor.agentId,
+      action: "background_job.upserted",
+      entityType: "background_job",
+      entityId: job.id,
+      details: {
+        key: job.key,
+        jobType: job.jobType,
+        backendKind: job.backendKind,
+      },
+    });
+    res.status(201).json(job);
+  });
+
+  router.get("/companies/:companyId/background-job-runs", async (req, res) => {
+    const companyId = req.params.companyId as string;
+    assertCompanyAccess(req, companyId);
+    assertBoard(req);
+    const parsed = listBackgroundJobRunsQuerySchema.parse(req.query);
+    res.json(await jobs.listRuns(companyId, parsed));
+  });
+
+  router.post("/companies/:companyId/background-job-runs", validate(createBackgroundJobRunSchema), async (req, res) => {
+    const companyId = req.params.companyId as string;
+    assertCompanyAccess(req, companyId);
+    assertBoard(req);
+    const actor = getActorInfo(req);
+    const run = await jobs.createRun(companyId, req.body, {
+      actorType: actor.actorType,
+      actorId: actor.actorId,
+      agentId: actor.agentId,
+      userId: actor.userId,
+    });
+    await logActivity(db, {
+      companyId,
+      actorType: actor.actorType,
+      actorId: actor.actorId,
+      agentId: actor.agentId,
+      action: "background_job_run.created",
+      entityType: "background_job_run",
+      entityId: run.id,
+      details: {
+        jobKey: run.jobKey,
+        jobType: run.jobType,
+        trigger: run.trigger,
+      },
+    });
+    res.status(201).json(run);
+  });
+
+  router.get("/companies/:companyId/background-job-runs/:runId", async (req, res) => {
+    const companyId = req.params.companyId as string;
+    assertCompanyAccess(req, companyId);
+    assertBoard(req);
+    res.json(await jobs.getRun(companyId, req.params.runId as string));
+  });
+
+  router.get("/companies/:companyId/background-job-runs/:runId/events", async (req, res) => {
+    const companyId = req.params.companyId as string;
+    assertCompanyAccess(req, companyId);
+    assertBoard(req);
+    const limit = Number(req.query.limit ?? 200);
+    res.json(await jobs.listRunEvents(companyId, req.params.runId as string, Number.isFinite(limit) ? limit : 200));
+  });
+
+  router.post("/companies/:companyId/background-job-runs/:runId/cancel", async (req, res) => {
+    const companyId = req.params.companyId as string;
+    assertCompanyAccess(req, companyId);
+    assertBoard(req);
+    const actor = getActorInfo(req);
+    const run = await jobs.requestCancelRun(companyId, req.params.runId as string);
+    await logActivity(db, {
+      companyId,
+      actorType: actor.actorType,
+      actorId: actor.actorId,
+      agentId: actor.agentId,
+      action: "background_job_run.cancel_requested",
+      entityType: "background_job_run",
+      entityId: run.id,
+      details: {
+        jobKey: run.jobKey,
+        jobType: run.jobType,
+      },
+    });
+    res.json(run);
+  });
+
+  return router;
+}

server/src/routes/memory.ts

@@ -8,6 +8,7 @@ import {
   memoryListOperationsQuerySchema,
   memoryListRecordsQuerySchema,
   memoryQuerySchema,
+  memoryRefreshJobSchema,
   memoryRetentionSweepSchema,
   memoryReviewSchema,
   memoryRevokeSchema,
@@ -383,5 +384,28 @@ export function memoryRoutes(
     res.json(await memory.listExtractionJobs(companyId, parsed));
   });
 
+  router.post("/companies/:companyId/memory/refresh-jobs", validate(memoryRefreshJobSchema), async (req, res) => {
+    const companyId = req.params.companyId as string;
+    assertCompanyAccess(req, companyId);
+    assertBoard(req);
+    const result = await memory.startRefreshJob(companyId, req.body, actorInfoFromReq(req));
+    const actor = getActorInfo(req);
+    await logActivity(db, {
+      companyId,
+      actorType: actor.actorType,
+      actorId: actor.actorId,
+      agentId: actor.agentId,
+      action: "memory.refresh_job_started",
+      entityType: "background_job_run",
+      entityId: result.run.id,
+      details: {
+        jobId: result.job.id,
+        dryRun: result.dryRun,
+        sourceCounts: result.sourceCounts,
+      },
+    });
+    res.status(202).json(result);
+  });
+
   return router;
 }

server/src/services/background-jobs.ts

@@ -0,0 +1,494 @@
+import { and, desc, eq } from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+import {
+  backgroundJobCostEvents,
+  backgroundJobEvents,
+  backgroundJobRuns,
+  backgroundJobs,
+} from "@paperclipai/db";
+import type {
+  BackgroundJob,
+  BackgroundJobCostEvent,
+  BackgroundJobEvent,
+  BackgroundJobRun,
+} from "@paperclipai/shared";
+import {
+  completeBackgroundJobRunSchema,
+  createBackgroundJobEventSchema,
+  createBackgroundJobRunSchema,
+  createBackgroundJobSchema,
+  listBackgroundJobRunsQuerySchema,
+  listBackgroundJobsQuerySchema,
+  updateBackgroundJobRunProgressSchema,
+  type CompleteBackgroundJobRun,
+  type CreateBackgroundJob,
+  type CreateBackgroundJobEvent,
+  type CreateBackgroundJobRun,
+  type ListBackgroundJobRunsQuery,
+  type ListBackgroundJobsQuery,
+  type UpdateBackgroundJobRunProgress,
+} from "@paperclipai/shared";
+import { conflict, notFound, unprocessable } from "../errors.js";
+
+export interface BackgroundJobActor {
+  actorType: "agent" | "user" | "system";
+  actorId: string;
+  agentId?: string | null;
+  userId?: string | null;
+}
+
+type JobRow = typeof backgroundJobs.$inferSelect;
+type RunRow = typeof backgroundJobRuns.$inferSelect;
+type EventRow = typeof backgroundJobEvents.$inferSelect;
+type CostEventRow = typeof backgroundJobCostEvents.$inferSelect;
+
+function mapJob(row: JobRow): BackgroundJob {
+  return {
+    id: row.id,
+    companyId: row.companyId,
+    key: row.key,
+    jobType: row.jobType,
+    displayName: row.displayName,
+    description: row.description ?? null,
+    backendKind: row.backendKind,
+    status: row.status,
+    config: row.config ?? {},
+    createdByAgentId: row.createdByAgentId ?? null,
+    createdByUserId: row.createdByUserId ?? null,
+    sourceIssueId: row.sourceIssueId ?? null,
+    sourceProjectId: row.sourceProjectId ?? null,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+  };
+}
+
+function mapRun(row: RunRow): BackgroundJobRun {
+  return {
+    id: row.id,
+    companyId: row.companyId,
+    jobId: row.jobId ?? null,
+    jobKey: row.jobKey,
+    jobType: row.jobType,
+    trigger: row.trigger,
+    status: row.status,
+    requestedByActorType: row.requestedByActorType as BackgroundJobRun["requestedByActorType"],
+    requestedByActorId: row.requestedByActorId,
+    requestedByAgentId: row.requestedByAgentId ?? null,
+    requestedByUserId: row.requestedByUserId ?? null,
+    sourceIssueId: row.sourceIssueId ?? null,
+    sourceProjectId: row.sourceProjectId ?? null,
+    sourceAgentId: row.sourceAgentId ?? null,
+    heartbeatRunId: row.heartbeatRunId ?? null,
+    totalItems: row.totalItems ?? null,
+    processedItems: row.processedItems,
+    succeededItems: row.succeededItems,
+    failedItems: row.failedItems,
+    skippedItems: row.skippedItems,
+    progressPercent: row.progressPercent ?? null,
+    currentItem: row.currentItem ?? null,
+    cancellationRequestedAt: row.cancellationRequestedAt ?? null,
+    cancelledAt: row.cancelledAt ?? null,
+    error: row.error ?? null,
+    result:
+      typeof row.result === "object" && row.result !== null && !Array.isArray(row.result)
+        ? (row.result as Record<string, unknown>)
+        : null,
+    startedAt: row.startedAt ?? null,
+    finishedAt: row.finishedAt ?? null,
+    durationMs: row.durationMs ?? null,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+  };
+}
+
+function mapEvent(row: EventRow): BackgroundJobEvent {
+  return {
+    id: row.id,
+    companyId: row.companyId,
+    runId: row.runId,
+    eventType: row.eventType,
+    level: row.level,
+    message: row.message ?? null,
+    progressPercent: row.progressPercent ?? null,
+    totalItems: row.totalItems ?? null,
+    processedItems: row.processedItems ?? null,
+    succeededItems: row.succeededItems ?? null,
+    failedItems: row.failedItems ?? null,
+    skippedItems: row.skippedItems ?? null,
+    currentItem: row.currentItem ?? null,
+    details:
+      typeof row.details === "object" && row.details !== null && !Array.isArray(row.details)
+        ? (row.details as Record<string, unknown>)
+        : null,
+    createdAt: row.createdAt,
+  };
+}
+
+function mapCostEvent(row: CostEventRow): BackgroundJobCostEvent {
+  return {
+    id: row.id,
+    companyId: row.companyId,
+    runId: row.runId,
+    costEventId: row.costEventId,
+    createdAt: row.createdAt,
+  };
+}
+
+function calculateProgressPercent(processedItems: number, totalItems: number | null | undefined) {
+  if (!totalItems || totalItems <= 0) return null;
+  return Math.max(0, Math.min(100, Math.round((processedItems / totalItems) * 100)));
+}
+
+export function backgroundJobService(db: Db) {
+  async function getRunRow(runId: string): Promise<RunRow> {
+    const run = await db
+      .select()
+      .from(backgroundJobRuns)
+      .where(eq(backgroundJobRuns.id, runId))
+      .then((rows) => rows[0] ?? null);
+    if (!run) throw notFound("Background job run not found");
+    return run;
+  }
+
+  return {
+    createOrUpdateJob: async (
+      companyId: string,
+      data: CreateBackgroundJob,
+      actor: BackgroundJobActor,
+    ): Promise<BackgroundJob> => {
+      const parsed = createBackgroundJobSchema.parse(data);
+      const existing = await db
+        .select()
+        .from(backgroundJobs)
+        .where(and(eq(backgroundJobs.companyId, companyId), eq(backgroundJobs.key, parsed.key)))
+        .then((rows) => rows[0] ?? null);
+
+      if (existing) {
+        const [row] = await db
+          .update(backgroundJobs)
+          .set({
+            jobType: parsed.jobType,
+            displayName: parsed.displayName,
+            description: parsed.description ?? null,
+            backendKind: parsed.backendKind,
+            status: parsed.status,
+            config: parsed.config,
+            sourceIssueId: parsed.sourceIssueId ?? null,
+            sourceProjectId: parsed.sourceProjectId ?? null,
+            updatedAt: new Date(),
+          })
+          .where(eq(backgroundJobs.id, existing.id))
+          .returning();
+        return mapJob(row);
+      }
+
+      const [row] = await db
+        .insert(backgroundJobs)
+        .values({
+          companyId,
+          key: parsed.key,
+          jobType: parsed.jobType,
+          displayName: parsed.displayName,
+          description: parsed.description ?? null,
+          backendKind: parsed.backendKind,
+          status: parsed.status,
+          config: parsed.config,
+          createdByAgentId: actor.agentId ?? null,
+          createdByUserId: actor.userId ?? null,
+          sourceIssueId: parsed.sourceIssueId ?? null,
+          sourceProjectId: parsed.sourceProjectId ?? null,
+        })
+        .returning();
+      return mapJob(row);
+    },
+
+    listJobs: async (companyId: string, query: ListBackgroundJobsQuery): Promise<BackgroundJob[]> => {
+      const parsed = listBackgroundJobsQuerySchema.parse(query);
+      const conditions = [eq(backgroundJobs.companyId, companyId)];
+      if (parsed.jobType) conditions.push(eq(backgroundJobs.jobType, parsed.jobType));
+      if (parsed.status) conditions.push(eq(backgroundJobs.status, parsed.status));
+      const rows = await db
+        .select()
+        .from(backgroundJobs)
+        .where(and(...conditions))
+        .orderBy(desc(backgroundJobs.createdAt))
+        .limit(parsed.limit);
+      return rows.map(mapJob);
+    },
+
+    getJob: async (companyId: string, jobId: string): Promise<BackgroundJob> => {
+      const row = await db
+        .select()
+        .from(backgroundJobs)
+        .where(and(eq(backgroundJobs.companyId, companyId), eq(backgroundJobs.id, jobId)))
+        .then((rows) => rows[0] ?? null);
+      if (!row) throw notFound("Background job not found");
+      return mapJob(row);
+    },
+
+    createRun: async (
+      companyId: string,
+      data: CreateBackgroundJobRun,
+      actor: BackgroundJobActor,
+    ): Promise<BackgroundJobRun> => {
+      const parsed = createBackgroundJobRunSchema.parse(data);
+      let job: JobRow | null = null;
+      if (parsed.jobId) {
+        job = await db
+          .select()
+          .from(backgroundJobs)
+          .where(and(eq(backgroundJobs.companyId, companyId), eq(backgroundJobs.id, parsed.jobId)))
+          .then((rows) => rows[0] ?? null);
+        if (!job) throw notFound("Background job not found");
+      }
+      if (job?.status === "archived") throw conflict("Archived background jobs cannot be run");
+      if (job?.status === "paused") throw conflict("Paused background jobs cannot be run");
+
+      const jobKey = job?.key ?? parsed.jobKey;
+      const jobType = job?.jobType ?? parsed.jobType;
+      if (!jobKey || !jobType) {
+        throw unprocessable("Background job run requires job key and type");
+      }
+
+      const [row] = await db
+        .insert(backgroundJobRuns)
+        .values({
+          companyId,
+          jobId: job?.id ?? null,
+          jobKey,
+          jobType,
+          trigger: parsed.trigger,
+          requestedByActorType: actor.actorType,
+          requestedByActorId: actor.actorId,
+          requestedByAgentId: actor.agentId ?? null,
+          requestedByUserId: actor.userId ?? null,
+          sourceIssueId: parsed.sourceIssueId ?? job?.sourceIssueId ?? null,
+          sourceProjectId: parsed.sourceProjectId ?? job?.sourceProjectId ?? null,
+          sourceAgentId: parsed.sourceAgentId ?? null,
+          heartbeatRunId: parsed.heartbeatRunId ?? null,
+          totalItems: parsed.totalItems ?? null,
+          progressPercent: calculateProgressPercent(0, parsed.totalItems ?? null),
+        })
+        .returning();
+      await db.insert(backgroundJobEvents).values({
+        companyId,
+        runId: row.id,
+        eventType: "queued",
+        level: "info",
+        message: "Job run queued",
+        totalItems: parsed.totalItems ?? null,
+        processedItems: 0,
+        progressPercent: calculateProgressPercent(0, parsed.totalItems ?? null),
+      });
+      return mapRun(row);
+    },
+
+    startRun: async (runId: string): Promise<BackgroundJobRun> => {
+      const current = await getRunRow(runId);
+      if (current.status !== "queued") throw conflict("Only queued background job runs can be started");
+      const now = new Date();
+      const [row] = await db
+        .update(backgroundJobRuns)
+        .set({ status: "running", startedAt: now, updatedAt: now })
+        .where(eq(backgroundJobRuns.id, runId))
+        .returning();
+      await db.insert(backgroundJobEvents).values({
+        companyId: row.companyId,
+        runId,
+        eventType: "started",
+        level: "info",
+        message: "Job run started",
+        totalItems: row.totalItems ?? null,
+        processedItems: row.processedItems,
+        progressPercent: row.progressPercent ?? null,
+      });
+      return mapRun(row);
+    },
+
+    updateRunProgress: async (
+      runId: string,
+      data: UpdateBackgroundJobRunProgress,
+    ): Promise<BackgroundJobRun> => {
+      const parsed = updateBackgroundJobRunProgressSchema.parse(data);
+      const current = await getRunRow(runId);
+      if (current.status !== "running") throw conflict("Only running background job runs can report progress");
+      const nextProcessed = parsed.processedItems ?? current.processedItems;
+      const nextTotal = parsed.totalItems === undefined ? current.totalItems : parsed.totalItems;
+      const progressPercent =
+        parsed.progressPercent === undefined
+          ? calculateProgressPercent(nextProcessed, nextTotal)
+          : parsed.progressPercent;
+      const [row] = await db
+        .update(backgroundJobRuns)
+        .set({
+          totalItems: nextTotal ?? null,
+          processedItems: nextProcessed,
+          succeededItems: parsed.succeededItems ?? current.succeededItems,
+          failedItems: parsed.failedItems ?? current.failedItems,
+          skippedItems: parsed.skippedItems ?? current.skippedItems,
+          progressPercent,
+          currentItem: parsed.currentItem === undefined ? current.currentItem : parsed.currentItem,
+          updatedAt: new Date(),
+        })
+        .where(eq(backgroundJobRuns.id, runId))
+        .returning();
+      await db.insert(backgroundJobEvents).values({
+        companyId: row.companyId,
+        runId,
+        eventType: "progress",
+        level: "info",
+        message: row.currentItem,
+        progressPercent: row.progressPercent ?? null,
+        totalItems: row.totalItems ?? null,
+        processedItems: row.processedItems,
+        succeededItems: row.succeededItems,
+        failedItems: row.failedItems,
+        skippedItems: row.skippedItems,
+        currentItem: row.currentItem ?? null,
+      });
+      return mapRun(row);
+    },
+
+    appendEvent: async (runId: string, data: CreateBackgroundJobEvent): Promise<BackgroundJobEvent> => {
+      const run = await getRunRow(runId);
+      const parsed = createBackgroundJobEventSchema.parse(data);
+      const [row] = await db
+        .insert(backgroundJobEvents)
+        .values({
+          companyId: run.companyId,
+          runId,
+          eventType: parsed.eventType,
+          level: parsed.level,
+          message: parsed.message ?? null,
+          progressPercent: parsed.progressPercent ?? null,
+          totalItems: parsed.totalItems ?? null,
+          processedItems: parsed.processedItems ?? null,
+          succeededItems: parsed.succeededItems ?? null,
+          failedItems: parsed.failedItems ?? null,
+          skippedItems: parsed.skippedItems ?? null,
+          currentItem: parsed.currentItem ?? null,
+          details: parsed.details ?? null,
+        })
+        .returning();
+      return mapEvent(row);
+    },
+
+    completeRun: async (runId: string, data: CompleteBackgroundJobRun): Promise<BackgroundJobRun> => {
+      const parsed = completeBackgroundJobRunSchema.parse(data);
+      const current = await getRunRow(runId);
+      if (!["queued", "running"].includes(current.status)) {
+        throw conflict("Background job run is already terminal");
+      }
+      const finishedAt = new Date();
+      const durationMs = current.startedAt ? finishedAt.getTime() - current.startedAt.getTime() : null;
+      const [row] = await db
+        .update(backgroundJobRuns)
+        .set({
+          status: parsed.status,
+          error: parsed.error ?? null,
+          result: parsed.result ?? null,
+          progressPercent: parsed.status === "succeeded" ? 100 : current.progressPercent,
+          cancelledAt: parsed.status === "cancelled" ? finishedAt : current.cancelledAt,
+          finishedAt,
+          durationMs,
+          updatedAt: finishedAt,
+        })
+        .where(eq(backgroundJobRuns.id, runId))
+        .returning();
+      await db.insert(backgroundJobEvents).values({
+        companyId: row.companyId,
+        runId,
+        eventType: parsed.status === "succeeded" ? "completed" : parsed.status,
+        level: parsed.status === "failed" ? "error" : "info",
+        message: parsed.error ?? `Job run ${parsed.status}`,
+        progressPercent: row.progressPercent ?? null,
+        totalItems: row.totalItems ?? null,
+        processedItems: row.processedItems,
+        succeededItems: row.succeededItems,
+        failedItems: row.failedItems,
+        skippedItems: row.skippedItems,
+        details: parsed.result ?? null,
+      });
+      return mapRun(row);
+    },
+
+    requestCancelRun: async (companyId: string, runId: string): Promise<BackgroundJobRun> => {
+      const current = await getRunRow(runId);
+      if (current.companyId !== companyId) throw notFound("Background job run not found");
+      if (!["queued", "running"].includes(current.status)) return mapRun(current);
+      const now = new Date();
+      const [row] = await db
+        .update(backgroundJobRuns)
+        .set({ cancellationRequestedAt: now, updatedAt: now })
+        .where(eq(backgroundJobRuns.id, runId))
+        .returning();
+      await db.insert(backgroundJobEvents).values({
+        companyId,
+        runId,
+        eventType: "log",
+        level: "warn",
+        message: "Cancellation requested",
+      });
+      return mapRun(row);
+    },
+
+    isCancellationRequested: async (runId: string): Promise<boolean> => {
+      const run = await getRunRow(runId);
+      return Boolean(run.cancellationRequestedAt);
+    },
+
+    linkCostEvent: async (companyId: string, runId: string, costEventId: string): Promise<BackgroundJobCostEvent> => {
+      const run = await getRunRow(runId);
+      if (run.companyId !== companyId) throw notFound("Background job run not found");
+      const [row] = await db
+        .insert(backgroundJobCostEvents)
+        .values({ companyId, runId, costEventId })
+        .onConflictDoNothing()
+        .returning();
+      return row ? mapCostEvent(row) : {
+        id: "",
+        companyId,
+        runId,
+        costEventId,
+        createdAt: new Date(),
+      };
+    },
+
+    listRuns: async (companyId: string, query: ListBackgroundJobRunsQuery): Promise<BackgroundJobRun[]> => {
+      const parsed = listBackgroundJobRunsQuerySchema.parse(query);
+      const conditions = [eq(backgroundJobRuns.companyId, companyId)];
+      if (parsed.jobId) conditions.push(eq(backgroundJobRuns.jobId, parsed.jobId));
+      if (parsed.jobType) conditions.push(eq(backgroundJobRuns.jobType, parsed.jobType));
+      if (parsed.status) conditions.push(eq(backgroundJobRuns.status, parsed.status));
+      if (parsed.sourceIssueId) conditions.push(eq(backgroundJobRuns.sourceIssueId, parsed.sourceIssueId));
+      if (parsed.sourceProjectId) conditions.push(eq(backgroundJobRuns.sourceProjectId, parsed.sourceProjectId));
+      const rows = await db
+        .select()
+        .from(backgroundJobRuns)
+        .where(and(...conditions))
+        .orderBy(desc(backgroundJobRuns.createdAt))
+        .limit(parsed.limit);
+      return rows.map(mapRun);
+    },
+
+    getRun: async (companyId: string, runId: string): Promise<BackgroundJobRun> => {
+      const row = await getRunRow(runId);
+      if (row.companyId !== companyId) throw notFound("Background job run not found");
+      return mapRun(row);
+    },
+
+    listRunEvents: async (companyId: string, runId: string, limit = 200): Promise<BackgroundJobEvent[]> => {
+      const run = await getRunRow(runId);
+      if (run.companyId !== companyId) throw notFound("Background job run not found");
+      const rows = await db
+        .select()
+        .from(backgroundJobEvents)
+        .where(eq(backgroundJobEvents.runId, runId))
+        .orderBy(desc(backgroundJobEvents.createdAt))
+        .limit(Math.min(Math.max(limit, 1), 500));
+      return rows.reverse().map(mapEvent);
+    },
+  };
+}
+
+export type BackgroundJobService = ReturnType<typeof backgroundJobService>;

server/src/services/index.ts

@@ -21,6 +21,7 @@ export { dashboardService } from "./dashboard.js";
 export { sidebarBadgeService } from "./sidebar-badges.js";
 export { inboxDismissalService } from "./inbox-dismissals.js";
 export { memoryService } from "./memory.js";
+export { backgroundJobService } from "./background-jobs.js";
 export { accessService } from "./access.js";
 export { boardAuthService } from "./board-auth.js";
 export { instanceSettingsService } from "./instance-settings.js";