fix(server): drive secret-proposal expiry from committed changes

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/secret-proposals-routes.test.ts

@@ -1,3 +1,6 @@
+import { registerSecretProposalExpiryWork } from "../services/secret-proposal-expiry-work.js";
+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
+import { idleWorkSnapshot } from "../services/task-admission.js";
 import { randomUUID } from "node:crypto";
 import { mkdirSync, rmSync } from "node:fs";
 import os from "node:os";
@@ -5,7 +8,7 @@ import path from "node:path";
 import express from "express";
 import request from "supertest";
 import { and, eq, sql } from "drizzle-orm";
-import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
 import {
   activityLog,
   agents,
@@ -38,7 +41,8 @@ import {
   startEmbeddedPostgresTestDatabase,
 } from "./helpers/embedded-postgres.js";
 
-const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const externalTestDatabase = process.env.PAPERCLIP_SECRET_PROPOSAL_TEST_DATABASE_URL;
+const embeddedPostgresSupport = externalTestDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
 const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
 
 describeEmbeddedPostgres("secret proposal routes", () => {
@@ -50,9 +54,10 @@ describeEmbeddedPostgres("secret proposal routes", () => {
   beforeAll(async () => {
     mkdirSync(secretsTmpDir, { recursive: true });
     process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = path.join(secretsTmpDir, "master.key");
+    if (externalTestDatabase) { db = createDb(externalTestDatabase, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); return; }
     const started = await startEmbeddedPostgresTestDatabase("secret-proposal-routes");
     stopDb = started.cleanup;
-    db = createDb(started.connectionString);
+    db = createDb(started.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
   });
 
   afterEach(async () => {
@@ -74,6 +79,7 @@ describeEmbeddedPostgres("secret proposal routes", () => {
   });
 
   afterAll(async () => {
+    await db?.$client.end({ timeout: 1 });
     await stopDb?.();
     if (previousKeyFile === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE;
     else process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = previousKeyFile;
@@ -221,6 +227,151 @@ describeEmbeddedPostgres("secret proposal routes", () => {
     return app;
   }
 
+  describe("expiry scheduling", () => {
+    let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
+    let proposals: ReturnType<typeof createSecretProposalsService>;
+    let fixture: Awaited<ReturnType<typeof seedRun>>;
+    let draining: boolean;
+    const errors: unknown[] = [];
+    const expiryMs = 14 * 24 * 60 * 60_000;
+
+    beforeEach(async () => {
+      await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
+      fixture = await seedRun();
+      proposals = createSecretProposalsService(db);
+      draining = false;
+      errors.length = 0;
+      vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true });
+    });
+    afterEach(async () => {
+      await coordinator?.stop();
+      coordinator = undefined;
+      vi.useRealTimers();
+      expect(errors).toEqual([]);
+    });
+    async function start() {
+      coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining,
+        canReconcile: () => true, onError: error => errors.push(error) });
+      await registerSecretProposalExpiryWork(coordinator, proposals, () => !draining).ready;
+    }
+    async function settle(ms = 60_000) {
+      await vi.advanceTimersByTimeAsync(ms);
+      if (!draining) await vi.waitFor(() => expect(idleWorkSnapshot().active === 0).toBe(true), { timeout: 5000 });
+    }
+    async function quiet() {
+      await settle();
+      expect(coordinator!.nextWakeAt()).toBeNull();
+      const reads = vi.spyOn(db, "select");
+      await settle(60 * 60_000);
+      expect(reads).not.toHaveBeenCalled();
+      reads.mockRestore();
+    }
+    const create = (service = proposals) => service.createSecret({ ...fixture, registerForRedaction: () => {} }, {
+      name: "dev/scheduled/secret", value: "expiry-test-only", justification: "Test durable expiry",
+    });
+    const bind = (secretProposalId: string) => proposals.createBinding(fixture, {
+      secretProposalId, configPath: "access.SCHEDULED", justification: "Test dependent expiry",
+      bindingTargetPolicy: "self_and_reports",
+    });
+
+    it("stays quiet until the real proposal deadline, scrubs ciphertext, and leaves terminal history quiet", async () => {
+      await start(); await quiet();
+      const secret = await create();
+      const binding = await bind(secret.id);
+      expect(secret.valueCiphertext).not.toBeNull();
+      await settle();
+      expect(coordinator!.nextWakeAt()).toBe(secret.expiresAt.getTime());
+      const reads = vi.spyOn(db, "select");
+      await settle(24 * 60 * 60_000);
+      expect(reads).not.toHaveBeenCalled(); reads.mockRestore();
+      await settle(expiryMs);
+      expect(await proposals.getById(fixture.companyId, secret.id)).toMatchObject({
+        status: "expired", valueCiphertext: null, ciphertextScrubbedAt: expect.any(Date),
+      });
+      expect(await proposals.getById(fixture.companyId, binding.id)).toMatchObject({ status: "rejected" });
+      expect(await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, binding.interactionId!)))
+        .toEqual([expect.objectContaining({ status: "rejected" })]);
+      await quiet();
+    });
+
+    it.each([false, true])("only observes the producer after outer settlement (rollback %s)", async rollback => {
+      await start(); await quiet();
+      let release!: () => void, written!: () => void;
+      const gate = new Promise<void>(resolve => { release = resolve; });
+      const staged = new Promise<void>(resolve => { written = resolve; });
+      const rollbackError = new Error("rollback fixture");
+      const writing = db.transaction(async tx => {
+        await create(createSecretProposalsService(tx as unknown as typeof db));
+        written(); await gate;
+        if (rollback) throw rollbackError;
+      }).catch(error => { if (error !== rollbackError) throw error; });
+      await staged;
+      try {
+        await settle();
+        expect(await proposals.nextExpiryAt()).toBeNull();
+      } finally { release(); }
+      await writing; await settle();
+      expect(await proposals.nextExpiryAt()).toEqual(rollback ? null : expect.any(Number));
+      expect(coordinator!.nextWakeAt()).toBe(await proposals.nextExpiryAt());
+    });
+
+    it("reconstructs a deadline after restart and disarms when the proposer withdraws", async () => {
+      const secret = await create();
+      await start();
+      expect(coordinator!.nextWakeAt()).toBe(secret.expiresAt.getTime());
+      await proposals.transition(fixture.companyId, secret.id, "withdrawn", { proposerAgentId: fixture.agentId });
+      await quiet();
+      expect(await proposals.getById(fixture.companyId, secret.id)).toMatchObject({ valueCiphertext: null });
+    });
+
+    it.each(["startup", "first_write", "failed_pass"])("retains %s work through drain and still rejects expired approval on use", async mode => {
+      if (mode === "startup") draining = true;
+      await start();
+      if (mode === "first_write") { await quiet(); draining = true; }
+      const secret = await create();
+      if (mode === "failed_pass") {
+        vi.spyOn(proposals, "sweepExpired").mockRejectedValueOnce(new Error("database unavailable"));
+        await vi.advanceTimersByTimeAsync(30_000);
+        await vi.waitFor(() => expect(errors).toHaveLength(1)); errors.length = 0;
+        draining = true;
+      }
+      await settle();
+      vi.setSystemTime(Date.now() + expiryMs);
+      const reads = vi.spyOn(db, "select");
+      await settle(2 * 60 * 60_000);
+      expect(reads).not.toHaveBeenCalled(); reads.mockRestore();
+      expect(await proposals.getById(fixture.companyId, secret.id)).toMatchObject({ status: "pending" });
+      await expect(proposals.approve(fixture.companyId, secret.id, { resolvedByUserId: "board-user" }))
+        .rejects.toThrow("Expired proposals cannot be approved");
+      draining = false; await settle();
+      expect(await proposals.getById(fixture.companyId, secret.id)).toMatchObject({ status: "expired", valueCiphertext: null });
+      await quiet();
+    });
+
+    it.each(["rejected", "execution_failed"])("reschedules when the interaction owner resolves a binding: %s", async mode => {
+      const secret = await create();
+      await start();
+      const approved = await proposals.approve(fixture.companyId, secret.id, { resolvedByUserId: "board-user" });
+      await quiet();
+      const remaining = await proposals.createBinding(fixture, { secretId: approved.createdSecretId,
+        configPath: "access.SECOND", justification: "Independent binding", bindingTargetPolicy: "self_and_reports" });
+      await settle();
+      expect(coordinator!.nextWakeAt()).toBe(remaining.expiresAt.getTime());
+      const interactions = issueThreadInteractionService(db);
+      if (mode === "rejected") {
+        await interactions.rejectInteraction({ id: fixture.issueId, companyId: fixture.companyId, status: "in_progress" },
+          remaining.interactionId!, { reason: "No longer required" }, { userId: "board-user" });
+      } else {
+        await db.update(issueThreadInteractions).set({ status: "accepted", result: { version: 1, outcome: "accepted" },
+          resolvedByUserId: "board-user", resolvedAt: new Date() }).where(eq(issueThreadInteractions.id, remaining.interactionId!));
+        await interactions.recordSecretProposalExecutionResult({ id: fixture.issueId, companyId: fixture.companyId },
+          remaining.interactionId!, remaining.id, { status: "failed", errorCode: "permission_denied" });
+      }
+      expect(await proposals.getById(fixture.companyId, remaining.id)).toMatchObject({ status: "rejected" });
+      await quiet();
+    });
+  });
+
   it("requires company admin access to reject secret proposals", async () => {
     const fixture = await seedRun();
     const proposed = await request(createAgentApp(fixture))

server/src/index.ts

@@ -1,3 +1,4 @@
+import { registerSecretProposalExpiryWork } from "./services/secret-proposal-expiry-work.js";
 import { registerCustomImageCleanupWork } from "./services/custom-image-cleanup-work.js";
 import { registerEnvironmentCleanupWork } from "./services/environment-cleanup-work.js";
 import { loginCleanupSchedule } from "./services/login-cleanup-state.js";
@@ -1757,6 +1758,8 @@ async function startServerWithDatabaseTeardown(
         work().catch((err: unknown) => {
           logger.error({ err, startupStep: name }, "startup housekeeping step failed");
         });
+      await step("secret_proposal_expiry", () => registerSecretProposalExpiryWork(deliveryWork, secretProposals,
+        () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready);
       await step("custom_image_setup_cleanup", () => registerCustomImageCleanupWork(deliveryWork, environmentCustomImages,
         () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped,
         config.heartbeatSchedulerIntervalMs).ready);
@@ -1853,14 +1856,6 @@ async function startServerWithDatabaseTeardown(
             logger.error({ err }, "periodic tool connection health sweep failed");
           }));
 
-        trackHeartbeatSchedulerWork(secretProposals.sweepExpired()
-          .then((expired) => {
-            if (expired > 0) logger.warn({ expired }, "periodic secret proposal expiry scrubbed proposals");
-          })
-          .catch((err) => {
-            logger.error({ err }, "periodic secret proposal expiry sweep failed");
-          }));
-
         trackHeartbeatSchedulerWork(app.locals.toolGateway.sweepActionReviews().catch((err: unknown) => logger.error({ err }, "tool review recovery failed")));
         trackHeartbeatSchedulerWork(app.locals.toolGateway.cleanupExpiredSessions().catch((err: unknown) => logger.error({ err }, "gateway token cleanup failed")));
 

server/src/services/custom-image-cleanup-work.ts

@@ -1,34 +1,22 @@
 import { logger } from "../middleware/logger.js";
 import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
+import { registerDeadlineWork } from "./deadline-work.js";
 import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
 import type { environmentCustomImageService } from "./environment-custom-images.js";
 
-/** Existing setup rows own expiry; provider operations remain in their service. */
 export function registerCustomImageCleanupWork(
   coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
   service: Pick<ReturnType<typeof environmentCustomImageService>, "cleanupExpiredSetupSessions" | "nextSetupCleanupAt">,
   canRun: () => boolean,
   retryMs = 30_000,
 ) {
-  let requested = 1, reconciled = 0;
-  let next: number | null = null;
-  let earliest = 0;
-  return coordinator.register(DELIVERY_QUEUES.customImageExpiry, {
+  return registerDeadlineWork(coordinator, DELIVERY_QUEUES.customImageExpiry, {
     retryMs,
-    pausePassiveMaintenance: true,
-    onSettled: () => { requested++; },
+    canRun,
     run: async () => {
-      if (!canRun()) { requested++; return; }
-      if (Date.now() < earliest) { next = earliest; return; }
-      const generation = ++requested;
-      earliest = Date.now() + retryMs;
       const result = await service.cleanupExpiredSetupSessions();
       if (result.timedOut > 0 || result.failed > 0) logger.warn(result, "environment customImage setup cleanup changed sessions");
-      const deadline = await service.nextSetupCleanupAt();
-      next = deadline === null ? null : Math.max(earliest, deadline);
-      reconciled = generation;
     },
-    hasPending: async () => requested !== reconciled,
-    nextRunAt: () => next,
+    next: () => service.nextSetupCleanupAt(),
   });
 }

server/src/services/deadline-work.ts

@@ -0,0 +1,34 @@
+import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
+import type { DeliveryQueue } from "./delivery-work-notifications.js";
+
+export function registerDeadlineWork(
+  coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
+  queue: DeliveryQueue,
+  task: {
+    run: () => Promise<unknown>;
+    next: () => Promise<number | null>;
+    canRun: () => boolean;
+    retryMs: number;
+  },
+) {
+  let requested = 1, reconciled = 0;
+  let next: number | null = null;
+  let earliest = 0;
+  return coordinator.register(queue, {
+    retryMs: task.retryMs,
+    pausePassiveMaintenance: true,
+    onSettled: () => { requested++; },
+    run: async () => {
+      if (!task.canRun()) { requested++; return; }
+      if (Date.now() < earliest) { next = earliest; return; }
+      const generation = ++requested;
+      earliest = Date.now() + task.retryMs;
+      await task.run();
+      const deadline = await task.next();
+      next = deadline === null ? null : Math.max(earliest, deadline);
+      reconciled = generation;
+    },
+    hasPending: async () => requested !== reconciled,
+    nextRunAt: () => next,
+  });
+}

server/src/services/delivery-work-notifications.ts

@@ -8,6 +8,7 @@ export const DELIVERY_QUEUES = {
   executionStatus: "execution-status",
   executionDisposition: "execution-disposition",
   environmentCleanup: "environment-cleanup",
+  secretProposalExpiry: "secret-proposal-expiry",
   customImageExpiry: "custom-image-expiry",
   historicalCleanup: "historical-cleanup",
   adapterLoginCleanup: "adapter-login-cleanup",

server/src/services/issue-thread-interactions.ts

@@ -1159,6 +1159,7 @@ async function resolveLinkedSecretProposal(
 ) {
   const proposalId = linkedSecretProposalId(interaction);
   if (!proposalId) return;
+  await notifyDeliveryWork(db, DELIVERY_QUEUES.secretProposalExpiry);
   const [proposal] = await db
     .update(companySecretProposals)
     .set({
@@ -3154,6 +3155,7 @@ export function issueThreadInteractionService(
           : execution.status;
         if (executionStatus === "failed" && proposal.status === "pending") {
           const resolutionReason = `Interaction acceptance failed: ${execution.errorCode ?? "secret_proposal_execution_failed"}`;
+          await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry);
           await tx
             .update(companySecretProposals)
             .set({

server/src/services/secret-proposal-expiry-work.ts

@@ -0,0 +1,21 @@
+import { logger } from "../middleware/logger.js";
+import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
+import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
+import { registerDeadlineWork } from "./deadline-work.js";
+import type { createSecretProposalsService } from "./secret-proposals.js";
+
+export function registerSecretProposalExpiryWork(
+  coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
+  service: Pick<ReturnType<typeof createSecretProposalsService>, "sweepExpired" | "nextExpiryAt">,
+  canRun: () => boolean,
+) {
+  return registerDeadlineWork(coordinator, DELIVERY_QUEUES.secretProposalExpiry, {
+    retryMs: 30_000,
+    canRun,
+    run: async () => {
+      const expired = await service.sweepExpired();
+      if (expired > 0) logger.warn({ expired }, "secret proposal expiry scrubbed proposals");
+    },
+    next: () => service.nextExpiryAt(),
+  });
+}

server/src/services/secret-proposals.ts

@@ -1,3 +1,4 @@
+import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js";
 import { withAgentAppearance } from "@paperclipai/shared";
 import { and, count, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm";
@@ -144,6 +145,7 @@ export function createSecretProposalsService(db: Db) {
         });
         return { denial, value: null };
       }
+      await notifyDeliveryWork(txDb, DELIVERY_QUEUES.secretProposalExpiry);
       return { denial: null, value: await create(txDb) };
     });
     if (result.denial) throw unprocessable(result.denial.message);
@@ -619,6 +621,7 @@ export function createSecretProposalsService(db: Db) {
     appliedBindingConfigPath?: string | null;
   }) {
     const now = new Date();
+    await notifyDeliveryWork(txDb, DELIVERY_QUEUES.secretProposalExpiry);
     const updated = await txDb.update(companySecretProposals).set({
       status: "approved",
       resolvedByUserId: input.resolvedByUserId,
@@ -794,6 +797,7 @@ export function createSecretProposalsService(db: Db) {
     const now = new Date();
     return db.transaction(async (tx) => {
       const txDb = tx as unknown as Db;
+      await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry);
       const updated = await tx.update(companySecretProposals).set({
         status,
         resolvedByUserId: input.resolvedByUserId ?? null,
@@ -887,5 +891,11 @@ export function createSecretProposalsService(db: Db) {
     return expiredCount;
   }
 
-  return { getById, view: enrich, createSecret, createBinding, listForAgent, listForBoard, assertBindingSnapshotCurrent, approve, transition, sweepExpired };
+  async function nextExpiryAt(): Promise<number | null> {
+    const [row] = await db.select({ at: sql<string | Date | null>`min(${companySecretProposals.expiresAt})` })
+      .from(companySecretProposals).where(eq(companySecretProposals.status, "pending"));
+    return row?.at == null ? null : new Date(row.at).getTime();
+  }
+
+  return { getById, view: enrich, createSecret, createBinding, listForAgent, listForBoard, assertBindingSnapshotCurrent, approve, transition, sweepExpired, nextExpiryAt };
 }