fix: preserve status-card deadline ownership through restart

Use a dedicated advisory-lock connection so a one-slot write pool can finish evaluation and result writeback. Record internal deadline provenance so startup recovers lost membership hints without advancing saved policy or retry debt.

Co-Authored-By: Paperclip <paperclip@paperclip.ing>

server/src/__tests__/status-card-evaluation-work.test.ts

@@ -3,7 +3,7 @@ import { documentService } from "../services/documents.js";
 import { randomUUID } from "node:crypto";
 import { eq, sql } from "drizzle-orm";
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
-import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents,
+import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, withDedicatedDbConnection,
   heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db";
 import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared";
 import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
@@ -188,6 +188,86 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
       .toHaveLength(1);
   });
 
+  it("finishes generation and writeback with a one-connection pool and releases dedicated locks", async () => {
+    vi.useRealTimers();
+    const previousDb = db;
+    db = createDb(external ?? temporary!.connectionString, { maxConnections: 1, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
+    try {
+      const created = await card("reactive", { ...policy(), debounceSeconds: 1 });
+      const countLocks = async () => Number((await db.execute<{ count: string }>(sql`
+        select count(*) from pg_stat_activity where datname = current_database()
+        and application_name = 'paperclip-workspace-finalization-lock'`))[0]!.count);
+      const baseline = await countLocks();
+      await issueService(db).create(companyId, { title: "Pool-one blocker", status: "blocked", priority: "medium" });
+      const service = statusCardEvaluationService(db, { wakeup });
+      const acquired = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
+      const holder = withDedicatedDbConnection(db, connection => connection.transaction(async tx => {
+        await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`);
+        acquired.resolve(); await release.promise;
+      }));
+      await acquired.promise;
+      try {
+        await service.run(new AbortController().signal);
+        expect(wakeup).not.toHaveBeenCalled();
+        expect(await service.next()).toBeGreaterThan(Date.now());
+      } finally { release.resolve(); await holder; }
+      await service.run(new AbortController().signal);
+      expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0);
+      await vi.waitFor(async () => {
+        await service.run(new AbortController().signal);
+        expect(wakeup).toHaveBeenCalledOnce();
+      }, { timeout: 5000, interval: 100 });
+      await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!);
+      await service.run(new AbortController().signal);
+      expect(await service.next()).toBeNull();
+      expect(idleWorkSnapshot().active).toBe(0);
+      await vi.waitFor(async () => expect(await countLocks()).toBe(baseline));
+      // Failure exits also release the physical connection and advisory lock.
+      await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id));
+      await service.run(new AbortController().signal);
+      expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("retry");
+      await vi.waitFor(async () => expect(await countLocks()).toBe(baseline));
+      await withDedicatedDbConnection(db, connection => connection.transaction(async tx => {
+        expect((await tx.execute<{ acquired: boolean }>(sql`select pg_try_advisory_xact_lock(hashtextextended(${created.id}, 0)) as acquired`))[0]!.acquired).toBe(true);
+      }));
+    } finally { await db.$client.end({ timeout: 1 }); db = previousDb; }
+  }, 15_000);
+
+  it.each([false, true])("recovers a lost committed hint before a future membership deadline (legacy=%s)", async legacy => {
+    await issueService(db).create(companyId, { title: "Original blocker", status: "blocked", priority: "medium" });
+    compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1d" });
+    const created = await card();
+    expect(created.nextEvalReason).toBe("membership");
+    const oldExpiry = created.nextEvalAt!.getTime();
+    // No coordinator exists to receive this committed source hint. A fresh
+    // service has no in-memory hint and must reconstruct from durable truth.
+    await issueService(db).create(companyId, { title: "Committed before crash", status: "blocked", priority: "medium" });
+    if (legacy) await db.update(statusCards).set({ nextEvalReason: null }).where(eq(statusCards.id, created.id));
+    const restarted = statusCardEvaluationService(db, { wakeup });
+    await restarted.run(new AbortController().signal);
+    const pending = (await statusCardService(db).getById(created.id))!;
+    expect(pending.pendingChangeCount).toBe(1);
+    expect(pending.nextEvalReason).toBe("policy");
+    expect(pending.nextEvalAt!.getTime()).toBeLessThan(oldExpiry);
+    expect(wakeup).not.toHaveBeenCalled();
+    vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
+    expect(wakeup).toHaveBeenCalledOnce();
+    await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!);
+    expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("membership");
+  });
+
+  it("preserves a legacy ambiguous five-minute claim without extending it on restart", async () => {
+    const created = await card();
+    const retry = new Date(Date.now() + 240_000);
+    await db.update(statusCards).set({ nextEvalAt: retry, nextEvalReason: null }).where(eq(statusCards.id, created.id));
+    await issueService(db).create(companyId, { title: "Unconsumed source", status: "blocked", priority: "medium" });
+    const restarted = statusCardEvaluationService(db, { wakeup });
+    await restarted.run(new AbortController().signal);
+    expect(await restarted.next()).toBe(retry.getTime());
+    vi.setSystemTime(retry); await restarted.run(new AbortController().signal);
+    expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0);
+  });
+
   it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => {
     vi.setSystemTime(new Date("2026-11-02T18:00:00Z"));
     const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}),
@@ -335,8 +415,12 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" });
     await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
     await vi.advanceTimersByTimeAsync(60_000); await settled();
+    await vi.waitFor(async () => {
+      await vi.advanceTimersByTimeAsync(1000);
+      expect((await statusCardService(db).getById(created.id))!.state).toBe("error");
+    });
     const failed = (await statusCardService(db).getById(created.id))!;
-    expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null });
+    expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null, nextEvalReason: null });
     expect(errors.length).toBeGreaterThan(0);
     const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id));
     const ledger = history.find(row => row.status === "failed")!;

server/src/__tests__/status-cards.test.ts

@@ -192,6 +192,7 @@ describeEmbeddedPostgres("status card routes", () => {
       .send({ title: "Launch health", titlePinned: true });
     expect(patched.status).toBe(200);
     expect(patched.body).toMatchObject({ title: "Launch health", titlePinned: true });
+    expect(patched.body).not.toHaveProperty("nextEvalReason");
 
     const scheduled = await request(app)
       .patch(`/api/status-cards/${created.body.id}`)
@@ -1152,6 +1153,7 @@ describeEmbeddedPostgres("status card routes", () => {
     });
     expect(summaryWrite.status).toBe(200);
     expect(summaryWrite.body.card).toMatchObject({ state: "active", queryVersion: 1, generatingIssueId: null });
+    expect(summaryWrite.body.card).not.toHaveProperty("nextEvalReason");
     expect(summaryWrite.body.document.latestBody).toContain("**Decide:**");
     expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "full"))).toMatchObject({ status: "ok", finishedAt: expect.any(Date), inputTokens: 5200, outputTokens: 980 });
 

server/src/routes/status-cards.ts

@@ -16,6 +16,12 @@ import { accessService, heartbeatService, instanceSettingsService, issueService,
 import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "../services/issue-assignment-wakeup.js";
 import { assertCompanyAccess, getAccessibleResource, getActorInfo, hasCompanyAccess } from "./authz.js";
 
+// Deadline provenance belongs to scheduling, not the public card contract.
+function publicCard<T extends { nextEvalReason?: unknown }>(card: T) {
+  const { nextEvalReason: _reason, ...result } = card;
+  return result;
+}
+
 export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWakeupDeps } = {}) {
   const router = Router();
   const access = accessService(db);
@@ -144,9 +150,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
     assertCompanyAccess(req, companyId);
     await assertStatusCardsEnabled();
     const query = listStatusCardsQuerySchema.parse(req.query);
-    res.json(await service.list(companyId, query.archived, {
+    res.json((await service.list(companyId, query.archived, {
       issueReadCondition: await issueReadSqlCondition(db, req.actor),
-    }));
+    })).map(publicCard));
   });
 
   router.post("/companies/:companyId/status-cards", validate(createStatusCardSchema), async (req, res) => {
@@ -162,7 +168,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
     try {
       const compile = await enqueueCompile(req, card.id);
       await logMutation(req, companyId, "status_card.created", card.id, { state: card.state });
-      res.status(201).json(compile.card);
+      res.status(201).json(publicCard(compile.card));
     } catch (error) {
       await service.remove(card.id);
       throw error;
@@ -173,9 +179,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
     await assertStatusCardsEnabled();
     const card = await getAccessibleResource(req, res, service.getById(req.params.id as string), "Status card not found");
     if (!card) return;
-    res.json(await service.hydrate(card, {
+    res.json(publicCard(await service.hydrate(card, {
       issueReadCondition: await issueReadSqlCondition(db, req.actor),
-    }));
+    })));
   });
 
   router.patch("/status-cards/:id", validate(patchStatusCardSchema), async (req, res) => {
@@ -197,7 +203,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
       fields: Object.keys(req.body),
       archived: Boolean(updated.archivedAt),
     });
-    res.json(compile?.card ?? restore?.card ?? updated);
+    res.json(publicCard(compile?.card ?? restore?.card ?? updated));
   });
 
   router.delete("/status-cards/:id", async (req, res) => {
@@ -234,7 +240,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
       generatingIssueId: result.generatingIssue.id,
       alreadyGenerating: result.alreadyGenerating,
     });
-    res.status(result.alreadyGenerating ? 200 : 202).json(result);
+    res.status(result.alreadyGenerating ? 200 : 202).json({ ...result, card: publicCard(result.card) });
   });
 
   router.post("/status-cards/:id/refresh", validate(refreshStatusCardSchema), async (req, res) => {
@@ -249,7 +255,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
       alreadyGenerating: result.alreadyGenerating,
       enqueued: result.enqueued,
     });
-    res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json(result);
+    res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json({ ...result, card: publicCard(result.card) });
   });
 
   router.get("/status-cards/:id/dry-run", async (req, res) => {
@@ -289,7 +295,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
       generationIssueId: req.body.generationIssueId,
       changeSummary: req.body.changeSummary,
     });
-    res.json(updated);
+    res.json(publicCard(updated));
   });
 
   router.put("/status-cards/:id/summary", validate(writeStatusCardSummarySchema), async (req, res) => {
@@ -309,7 +315,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
       documentId: result.document.id,
       changeSummary: req.body.changeSummary,
     });
-    res.json(result);
+    res.json({ ...result, card: publicCard(result.card) });
   });
 
   return router;

server/src/services/status-card-finalization.ts

@@ -47,6 +47,7 @@ export async function finalizeStatusCardsForStalledGeneration(
       failureReason,
       generatingIssueId: null,
       nextEvalAt: null,
+      nextEvalReason: null,
       updatedAt: now,
     })
     .where(

server/src/services/status-cards.ts

@@ -1,6 +1,6 @@
 import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
 import { createHash } from "node:crypto";
-import { and, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm";
+import { and, desc, eq, gt, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm";
 import {
   agents,
   companies,
@@ -11,6 +11,7 @@ import {
   issueComments,
   statusCards,
   statusCardUpdates,
+  withDedicatedDbConnection,
   type Db,
 } from "@paperclipai/db";
 import type {
@@ -301,11 +302,12 @@ export function statusCardService(
       ...(input.refreshPolicy !== undefined
         ? {
             refreshPolicy: input.refreshPolicy,
+            nextEvalReason: "refresh",
             nextEvalAt: card.archivedAt ? null : input.refreshPolicy.mode === "reactive" && card.queries.length > 0 ? now : nextStatusCardEvaluationAt(input.refreshPolicy, now),
           }
         : {}),
       ...(archiveChanged && input.archived
-        ? { archivedAt: now, archivedByAgentId: actor.agentId, archivedByUserId: actor.userId, nextEvalAt: null }
+        ? { archivedAt: now, archivedByAgentId: actor.agentId, archivedByUserId: actor.userId, nextEvalAt: null, nextEvalReason: null }
         : {}),
       ...(archiveChanged && !input.archived
         ? {
@@ -315,6 +317,7 @@ export function statusCardService(
             lastChangeAt: now,
             lastUpdateRunKind: null,
             nextEvalAt: card.queries.length > 0 ? now : null,
+            nextEvalReason: "refresh",
           }
         : {}),
     };
@@ -537,9 +540,10 @@ export function statusCardService(
     companyId: string,
     issueIds: string[],
     options: StatusCardReadOptions = {},
+    queryDb: Db = db,
   ): Promise<CompanySearchIssueSummary[]> {
     if (issueIds.length === 0) return [];
-    const rows = await db
+    const rows = await queryDb
       .select({
         id: issues.id,
         identifier: issues.identifier,
@@ -589,11 +593,11 @@ export function statusCardService(
     return loadIssueSummaries(card.companyId, card.mentionedIssueIds ?? [], options);
   }
 
-  async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void) {
+  async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void, queryDb: Db = db) {
     const issueMap = new Map<string, CompanySearchIssueSummary>();
     for (const storedQuery of card.queries) {
       const query = companySearchQuerySchema.parse(storedQuery);
-      const response = await searchSvc.search(card.companyId, query, {
+      const response = await companySearchService(queryDb).search(card.companyId, query, {
         issueReadCondition: options.issueReadCondition, onNextMembershipChange,
       });
       for (const result of response.results) {
@@ -605,12 +609,12 @@ export function statusCardService(
     const mentioned = await loadIssueSummaries(
       card.companyId,
       (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)),
-      options,
+      options, queryDb,
     );
     for (const issue of mentioned) issueMap.set(issue.id, issue);
     const snapshot = [...issueMap.values()];
     if (snapshot.length === 0) return snapshot;
-    const latestHumanComments = await db
+    const latestHumanComments = await queryDb
       .select({
         issueId: issueComments.issueId,
         // The postgres-js driver returns the `max()` aggregate over a timestamp
@@ -672,6 +676,7 @@ export function statusCardService(
         lastChangeAt: null,
         state: "active",
         nextEvalAt,
+        nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null,
       }).where(eq(statusCards.id, card.id)).returning());
       return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false };
     }
@@ -699,6 +704,7 @@ export function statusCardService(
         pendingChangeHash,
         lastChangeAt,
         state: decision.action === "pause_budget" ? "paused_budget" : decision.action === "pause_hours" ? "paused_hours" : "active",
+        nextEvalReason: "policy",
         nextEvalAt: nextStatusCardPolicyEligibility({
           policy: card.refreshPolicy, now, lastChangeAt,
           completedUpdateTimes: recent.filter(row => row.kind !== "compile" && row.finishedAt).map(row => row.startedAt),
@@ -755,6 +761,7 @@ export function statusCardService(
       lastChangeAt,
       state: "active",
       nextEvalAt,
+      nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null,
       failureReason: null,
     }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning());
     if (!next) {
@@ -788,9 +795,16 @@ export function statusCardService(
   async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet<string> | "changed", reconstruct = false) {
     const changedScope = reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size
       ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`;
-    // Startup restores every disarmed reactive scope. A concurrent committed
-    // source hint can additionally preempt its company's saved future deadline.
-    const reactiveScope = or(reconstruct ? isNull(statusCards.nextEvalAt) : undefined, changedScope);
+    // Startup recovers lost source hints for membership deadlines as well as
+    // disarmed cards. Saved policy gates and retry leases keep their owner.
+    // Legacy untagged leases are at most five minutes from their claim time:
+    // preserve that bounded ambiguity, never a days-away membership deadline.
+    const startupScope = reconstruct ? or(
+      isNull(statusCards.nextEvalAt), eq(statusCards.nextEvalReason, "membership"),
+      and(isNull(statusCards.nextEvalReason), isNull(statusCards.pendingChangeHash),
+        eq(statusCards.state, "active"), gt(statusCards.nextEvalAt, new Date(now.getTime() + 5 * 60_000))),
+    ) : undefined;
+    const reactiveScope = or(startupScope, changedScope);
     const due = await db.select({ card: statusCards }).from(statusCards)
       .innerJoin(companies, eq(companies.id, statusCards.companyId))
       .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId),
@@ -805,13 +819,13 @@ export function statusCardService(
       // event may preempt a genuine future deadline, so serialize scheduler
       // evaluation across connections without holding a row lock while the
       // existing issue/card services commit through their own transactions.
-      await db.transaction(async lockTx => {
+      await withDedicatedDbConnection(db, connection => connection.transaction(async lockTx => {
         const [lock] = await lockTx.execute<{ acquired: boolean }>(
           sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`,
         );
         if (!lock?.acquired) { deferredCompanyIds.add(candidate.companyId); return; }
         const claimUntil = new Date(now.getTime() + 5 * 60 * 1000);
-        const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil })
+        const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil, nextEvalReason: "retry" })
           .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId),
             candidate.nextEvalAt ? eq(statusCards.nextEvalAt, candidate.nextEvalAt) : isNull(statusCards.nextEvalAt)))
           .returning());
@@ -826,7 +840,7 @@ export function statusCardService(
             "status card scheduled refresh failed",
           );
         }
-      });
+      }));
     }
     return { evaluated, enqueued, deferredCompanyIds };
   }
@@ -859,7 +873,9 @@ export function statusCardService(
       // Current watched membership: compiled-query matches plus the issues this
       // summary mentions.
       let membershipAt: Date | null = null;
-      const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; });
+      // All watched-source reads use this transaction. Borrowing the root
+      // pool here would deadlock a one-connection installation.
+      const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }, tx as unknown as Db);
       const watchedNow = buildStatusCardFingerprint(watchedSnapshot);
       let snapshot: StatusCardFingerprint;
       if (payloadFingerprint) {
@@ -931,6 +947,9 @@ export function statusCardService(
         pendingChangeCount: 0,
         pendingChangeHash: null,
         lastChangeAt: null,
+        nextEvalReason: current.refreshPolicy.mode === "reactive" &&
+          filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0
+          ? "refresh" : membershipAt ? "membership" : null,
         nextEvalAt: current.refreshPolicy.mode === "reactive" &&
           filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0
           ? now : nextEvaluationAt(current, membershipAt, now),