fix(server): prove completed history before idle sleep (#18646)

Reviewed and verified at 592bce69c6.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/chat-channels.integration.test.ts

@@ -1,3 +1,5 @@
+import { chatHistoryActionBlocks } from "./helpers/chat-history-proof.js";
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
 import { resolveCommittedNativeRecoveryNotices, pendingNativeRecoveryNotices } from "../services/native-runtime/native-recovery-notices.js";
 import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
 import { projectCommittedRun } from "../services/native-runtime/native-run-projection.js";
@@ -287,7 +289,8 @@ import {
   startEmbeddedPostgresTestDatabase,
 } from "./helpers/embedded-postgres.js";
 
-const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL;
+const historyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL");
+const externalTestDatabaseUrl = historyTestDatabaseUrl ?? process.env.PAPERCLIP_TEST_DATABASE_URL;
 const embeddedPostgresSupport = externalTestDatabaseUrl
   ? { supported: true }
   : await getEmbeddedPostgresTestSupport();
@@ -1125,6 +1128,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
     // Do not expire or time out its connections on those virtual clock jumps.
     db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, {
       idleTimeoutSeconds: 0, connectTimeoutSeconds: 0, maxLifetimeSeconds: 0,
+      ...(historyTestDatabaseUrl ? { maxConnections: 4 } : {}),
     });
   }, 90_000);
 
@@ -3009,6 +3013,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       const [review] = await readChatGitHubReviewsByEndpointId(f.endpoint.id);
       expect(review.assessment?.findings[0].category).toBe(categories[0]);
       expect((await f.action()).status).toBe("processed");
+      expect(await chatHistoryActionBlocks(db, (await f.action()).id)).toBe(false);
       expect(f.responseComments.size).toBe(1);
     });
     it("publishes one score heading when a newly configured bot submits a formatted summary", async () => {
@@ -3072,6 +3077,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       const [response] = await db.select().from(chatActions).where(and(eq(chatActions.deliveryId, delivery.id),
         eq(chatActions.kind, "github_response_comment")));
       expect(response).toMatchObject({ status: "cancelled", result: { final: true, code: "passive_response_wait" } });
+      expect(await chatHistoryActionBlocks(db, response.id)).toBe(false);
       const writes = f.responseWrites.length;
       const updatedAt = response.updatedAt;
       await responses.processPending();
@@ -3317,6 +3323,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       const body = [...f.responseComments.values()][0].body;
       expect(body).not.toMatch(/Review queued|Working on this/);
       expect(body).toContain("paperclip-response-state:final");
+      const responses = await db.select().from(chatActions).where(and(eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "github_response_comment")));
+      expect(responses.length).toBeGreaterThan(0);
+      for (const receipt of responses) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(false);
     });
     it("retires a legacy coalesced GitHub placeholder before replacing its receipt", async () => {
       const f = await publicationRetryFixture();
@@ -3428,6 +3437,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       expect([...f.responseComments.values()].some(comment => comment.body.includes("Newer progress"))).toBe(true);
       const [settled] = await db.select().from(chatActions).where(eq(chatActions.id, first.id));
       expect(settled.result?.final).toBe(false);
+      expect(await chatHistoryActionBlocks(db, settled.id)).toBe(true);
     });
     it.each([false, true])("recovers an approved queued GitHub review without another approval (revoked: %s)", async revoked => {
       const f = await publicationRetryFixture();
@@ -3463,6 +3473,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       await db.update(chatActions).set({ result: { ...action.result, retryAt: new Date(0).toISOString() } }).where(eq(chatActions.id, action.id));
       await f.service.processPending();
       expect((await f.action()).status).toBe(revoked ? "cancelled" : "processed");
+      expect(await chatHistoryActionBlocks(db, (await f.action()).id)).toBe(false);
       if (revoked) await expect(gateway.executeTool({ sessionToken: session.token, tool: tool.name, parameters: f.assessment })).rejects.toThrow();
       else expect(await gateway.executeTool({ sessionToken: session.token, tool: tool.name, parameters: f.assessment }))
         .toMatchObject({ status: "replayed", result: { data: { status: "processed", score: 3 } } });
@@ -4833,6 +4844,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       expect(f.responseWrites.filter(write => write.method === "POST")).toHaveLength(2);
       const [deleted] = await db.select().from(chatActions).where(and(eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "github_response_comment"), sql`${chatActions.result}->>'id' = ${String(firstId)}`));
       expect(deleted).toMatchObject({ status: "cancelled", result: { final: true, code: "response_deleted" } });
+      expect(await chatHistoryActionBlocks(db, deleted.id)).toBe(false);
       await githubResponseCommentService(db, f.providerFetch).processPending();
       expect(f.responseComments.size).toBe(1);
       await expect(githubRunReplyState(db, { ...f.session, endpointId: f.endpoint.id })).resolves.toBe("confirmed");
@@ -9849,6 +9861,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
         expect(f.provider.mock.calls.filter(([url]) => String(url).endsWith("apps.manifest.create"))).toHaveLength(0);
         expect((await f.service.get(f.endpoint.id)).status).not.toBe("revoked");
         expect((await f.service.get(f.endpoint.id)).setup.slackRegistration).toMatchObject({ status: "configured", appId: f.appId });
+        const [reconnectReceipt] = await db.select().from(chatActions).where(and(eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "slack_managed_reconnect")));
+        expect(reconnectReceipt).toMatchObject({ status: "processed", result: null });
+        expect(await chatHistoryActionBlocks(db, reconnectReceipt.id)).toBe(false);
         await f.service.slackRegistration.managed.provision(f.endpoint.id, actor, reconnectInput);
         await f.provision();
         expect(f.provider.mock.calls.filter(([url]) => String(url).endsWith("apps.managedInstall"))).toHaveLength(1);
@@ -17125,12 +17140,13 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
           context.endpoint.id,
         );
         expect(context.posts).toEqual([]);
-        if (change === "edited comment")
+        if (change === "edited comment") {
           expect(await context.receipt()).toMatchObject({
             status: "cancelled",
             result: { attempts: 0, code: "source_changed_or_unavailable" },
           });
-        else expect(await context.receipt()).toBeUndefined();
+          expect(await chatHistoryActionBlocks(db, (await context.receipt())!.id)).toBe(false);
+        } else expect(await context.receipt()).toBeUndefined();
         expect(context.wakeup).not.toHaveBeenCalled();
       } finally {
         await retirePublicationFixture(context.service, context.endpoint.id);
@@ -17811,6 +17827,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
         result: { code: "github_webhook_ingress_runtime_superseded" },
       });
       expect((await context.ingress())!.payload).not.toHaveProperty("body");
+      expect(await chatHistoryActionBlocks(db, (await context.ingress())!.id)).toBe(false);
       expect(sdkDispatch).not.toHaveBeenCalled();
       expect(
         await db
@@ -17862,6 +17879,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
         result: { code: "github_webhook_recovery_source_changed" },
       });
       expect(cancelled!.payload).not.toHaveProperty("body");
+      expect(await chatHistoryActionBlocks(db, cancelled!.id)).toBe(false);
       expect(cancelled!.result?.retryable).not.toBe(true);
       expect(context.requests).toEqual([]);
       expect(
@@ -43636,6 +43654,11 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       expect(original?.state).toBe("published");
       expect(original?.payload.card?.actions).toEqual(expect.arrayContaining([expect.objectContaining({ type: "callback" })]));
       if (kind === "connection_intent") expect(original?.payload.card?.actions).toContainEqual({ type: "link", label: "Review connection", url: expect.any(String) });
+      const controls = (await db.select().from(chatActions).where(and(eq(chatActions.endpointId, endpoint.id),
+        eq(sql<string>`${chatActions.payload}->>'interactionId'`, interaction.id))))
+        .filter(row => ["confirmation_response", "question_form_open", "question_form_submit", "question_answer"].includes(row.kind));
+      expect(controls.length).toBeGreaterThan(0);
+      for (const control of controls) expect(await chatHistoryActionBlocks(db, control.id)).toBe(true);
       if (kind === "connection_intent") await interactions.resolveConnectionIntent(issue, interaction.id, { version: 1, outcome: "declined" }, { userId: "owner-user" });
       else if (kind === "request_item_verdicts") await interactions.submitItemVerdicts(issue, interaction.id, { verdicts: [{ id: "docs", verdict: "approve" }] }, { userId: "owner-user" });
       else await interactions.rejectInteraction(issue, interaction.id, {}, { userId: "owner-user" });
@@ -43645,6 +43668,11 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       expect(terminal?.state).toBe("published");
       expect(terminal?.providerMessageId).toBe(original.providerMessageId);
       expect(terminal?.payload.card?.actions).toBeUndefined();
+      for (const control of controls) expect(await chatHistoryActionBlocks(db, control.id)).toBe(false);
+      const control = controls[0]!;
+      await db.update(chatActions).set({ payload: { ...control.payload, interactionId: randomUUID() } }).where(eq(chatActions.id, control.id));
+      expect(await chatHistoryActionBlocks(db, control.id)).toBe(true);
+      await db.update(chatActions).set({ payload: control.payload }).where(eq(chatActions.id, control.id));
     } finally {
       await service.shutdown();
       if (previousPublicUrl === undefined) delete process.env.PAPERCLIP_PUBLIC_URL;
@@ -81351,6 +81379,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
         expect(context.consentCards).toHaveLength(1);
         expect(context.fileCards).toEqual([]);
         expect(context.uploadRequest).not.toHaveBeenCalled();
+        expect(await chatHistoryActionBlocks(db, intent!.id)).toBe(true);
         const before = await context.service.getPublicationBatchStatus(
           context.endpoint.id,
           context.conversation.id,
@@ -81379,6 +81408,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
           body: context.bytes,
         });
         expect(context.fileCards).toHaveLength(1);
+        expect(await chatHistoryActionBlocks(db, intent!.id)).toBe(false);
         const status = await context.service.getPublicationBatchStatus(
           context.endpoint.id,
           context.conversation.id,

server/src/__tests__/chat-run-milestone-recovery-work.test.ts

@@ -22,7 +22,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
   const errors: unknown[] = [];
   beforeAll(async () => {
     if (!external) temporary = await startEmbeddedPostgresTestDatabase("chat-recovery-milestones-");
-    db = createDb(external ?? temporary!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
+    db = createDb(external ?? temporary!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
     fixtures = createChatMilestoneFixtures(db);
   }, 60_000);
   beforeEach(async () => {
@@ -77,6 +77,9 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
   });
 
   it("wakes an older failed run when recovery refreshes its quota classification", async () => {
+    // Keep both idempotence checks before the provider's midnight reset even
+    // when this file runs near midnight and quiet() advances the fake clock.
+    vi.setSystemTime(new Date("2026-11-02T12:00:00.000Z"));
     const runId = randomUUID(), old = new Date(Date.now() - 48 * 60 * 60_000);
     await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId,
       status: "failed", runtimeMode: "legacy", responsibleUserId: f.userId,
@@ -96,6 +99,9 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     await flush();
     expect(await publications()).toEqual([expect.objectContaining({ idempotencyKey: `run:${runId}:failed:${f.endpointId}` })]);
     await quiet();
+    const [monitored] = await db.select({ nextCheckAt: issues.monitorNextCheckAt }).from(issues).where(eq(issues.id, f.issueId));
+    expect(monitored.nextCheckAt).toEqual(new Date("2026-11-03T00:00:00.000Z"));
+    expect(monitored.nextCheckAt!.getTime()).toBeGreaterThan(Date.now());
     expect(await recovery.reconcileStrandedAssignedIssues()).toMatchObject({ providerQuotaMonitored: 0, skipped: 1 });
     await flush(); await quiet(); expect(await publications()).toHaveLength(1);
   });

server/src/__tests__/environment-cleanup-work.test.ts

@@ -28,12 +28,13 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
   let companyId: string, environmentId: string, draining: boolean, failing: boolean;
   let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
   let worker: ReturnType<typeof registerEnvironmentCleanupWork> | undefined;
-  let releases: ReturnType<typeof vi.fn>, inspections: ReturnType<typeof vi.fn>, reads: ReturnType<typeof vi.spyOn>;
+  let releases: ReturnType<typeof vi.fn>, reads: ReturnType<typeof vi.spyOn>;
+  let inspections: ReturnType<typeof vi.fn<(row: { providerLeaseId: string | null }) => Promise<{ providerLeaseId: string | null; state: string }>>>;
   const errors: unknown[] = [], disposers: Array<() => void> = [];
   beforeAll(async () => {
     if (!external) database = await startEmbeddedPostgresTestDatabase("environment-cleanup-work-");
-    db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
-    await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
+    db = createDb(external ?? database!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
+    await Promise.all(Array.from({ length: 4 }, () => db.execute(sql`select pg_sleep(0.01)`)));
   }, 90_000);
   beforeEach(async () => {
     await db.execute(sql`TRUNCATE sandbox_creation_operations, companies, environments CASCADE`);
@@ -48,6 +49,9 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
   afterEach(async () => {
     worker?.stop(); await coordinator?.stop(); coordinator = undefined; worker = undefined;
     for (const stop of disposers.splice(0)) stop();
+    // Creation ownership intentionally survives company deletion; company
+    // cascades therefore cannot isolate this case from the next test suite.
+    await db.delete(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId, companyId));
     vi.useRealTimers(); vi.restoreAllMocks();
     expect(errors).toEqual([]);
   });
@@ -61,12 +65,13 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
     let historySettled = !history;
     worker = registerEnvironmentCleanupWork(db, coordinator, {
       sweepPending: sweepOverride ?? (backoffMs => heartbeat.sweepPendingCleanupLeases({ backoffMs })),
-      historical: history ? { hasPending: history.hasPending,
+      historical: history ? { hasPending: () => history.hasPending(),
         sweep: async () => { const result = await history.sweep(); historySettled = true; return result; } } : undefined,
       canRun: () => !draining,
     });
     await worker.ready;
     await vi.waitFor(() => expect(historySettled).toBe(true));
+    return history || undefined;
   }
   async function settle(ms = 30_000, wait = true) {
     await vi.advanceTimersByTimeAsync(ms);
@@ -116,7 +121,7 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
     ]));
     await quiet();
   });
-  it.each([false, true])("settles a deleted-company creation as manual history across restart without idle polling (contextual=%s)", async contextual => {
+  it.each([false, true])("retains deleted-company creation uncertainty across a quiet worker restart (contextual=%s)", async contextual => {
     await start(); await quiet();
     const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {},
       metadata: contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {} },
@@ -126,8 +131,11 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
     await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([
       expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }),
     ]));
-    if (contextual) await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId,
-      providerLeaseId: created.providerLeaseId });
+    if (contextual) {
+      if (!created.providerLeaseId) throw new Error("Fixture creation returned no provider lease identity");
+      await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId,
+        providerLeaseId: created.providerLeaseId });
+    }
     await quiet();
     worker!.stop(); await coordinator!.stop();
     await start(); await quiet();
@@ -135,9 +143,84 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
     const ownerId = randomUUID();
     const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 };
     expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none"))
-      .toEqual({ version: 1, backgroundWork: "none" });
+      .toEqual({ version: 1, backgroundWork: "present" });
     await quiet();
   });
+  it("discharges an empty historical metadata notification during provider spacing", async () => {
+    const lease = await pending();
+    await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" });
+    const history = await start(true);
+    await settle(30_000);
+    expect(await history!.hasPending()).toBe(false);
+    const checked = inspections.mock.calls.length;
+    await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", presentation: "changed" });
+    await vi.waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); });
+    expect(inspections).toHaveBeenCalledTimes(checked);
+    expect(await history!.hasPending()).toBe(false);
+    const count = reads.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(300_000);
+    expect(reads).toHaveBeenCalledTimes(count); expect(inspections).toHaveBeenCalledTimes(checked);
+  });
+  it.each([false, true])("preserves historical debt arriving during an empty fast-path read (unresolved transaction=%s)", async unresolved => {
+    const passive = await pending(), debt = await pending();
+    await environmentService(db).releaseLease(passive.id, "released", { cleanupStatus: "success" });
+    await environmentService(db).releaseLease(debt.id, "failed", { cleanupStatus: "success" });
+    const history = (await start(true))!;
+    await settle(30_000); expect(inspections).toHaveBeenCalledTimes(1);
+    let readStarted = false, releaseRead!: () => void, writeReady!: () => void, releaseWrite!: () => void;
+    const readGate = new Promise<void>(resolve => { releaseRead = resolve; });
+    const written = new Promise<void>(resolve => { writeReady = resolve; });
+    const writeGate = new Promise<void>(resolve => { releaseWrite = resolve; });
+    const original = history.hasPending;
+    const check = vi.spyOn(history, "hasPending").mockImplementationOnce(async () => {
+      const pending = await original(); expect(pending).toBe(false); readStarted = true; await readGate; return pending;
+    });
+    let writing: Promise<unknown> | undefined;
+    try {
+      await environmentService(db).updateLeaseMetadata(passive.id, { presentation: "changed" });
+      await vi.waitFor(() => expect(readStarted).toBe(true));
+      writing = db.transaction(async tx => {
+        await environmentService(tx as unknown as Db).updateLeaseMetadata(debt.id, { driver: "sandbox", changed: true });
+        writeReady(); if (unresolved) await writeGate;
+      });
+      await written;
+      if (!unresolved) await writing;
+      releaseRead();
+      await vi.advanceTimersByTimeAsync(1);
+      expect(idleWorkSnapshot().active).toBeGreaterThan(0);
+      expect(inspections).toHaveBeenCalledTimes(1);
+      releaseWrite(); await writing;
+      await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1); expect(await original()).toBe(true); });
+      expect(idleWorkSnapshot().active).toBeGreaterThan(0);
+      const ownerId = randomUUID();
+      // One drain hold must retain its identity while the safety read runs.
+      const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000),
+        expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 };
+      expect((await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")).backgroundWork).toBe("present");
+      await settle(300_000);
+      await vi.waitFor(() => { expect(inspections).toHaveBeenCalledTimes(2); expect(idleWorkSnapshot().active).toBe(0); });
+      expect(await original()).toBe(false); await quiet();
+    } finally { releaseRead(); releaseWrite(); await writing; check.mockRestore(); }
+  });
+  it("retains a failed historical fast-path read through drain and clears it after resume", async () => {
+    const lease = await pending();
+    await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" });
+    const history = (await start(true))!;
+    await settle(30_000);
+    vi.spyOn(history, "hasPending").mockRejectedValueOnce(new Error("historical debt read unavailable"));
+    await environmentService(db).updateLeaseMetadata(lease.id, { presentation: "changed" });
+    await vi.waitFor(() => expect(errors).toHaveLength(1));
+    expect(idleWorkSnapshot().active).toBeGreaterThan(0); expect(coordinator!.nextWakeAt()).not.toBeNull();
+    errors.length = 0; draining = true;
+    const count = reads.mock.calls.length;
+    await settle(60_000, false);
+    expect(reads).toHaveBeenCalledTimes(count); expect(idleWorkSnapshot().active).toBeGreaterThan(0);
+    draining = false; await settle();
+    await vi.waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); });
+    expect(inspections).not.toHaveBeenCalled(); await quiet();
+    worker!.stop(); await coordinator!.stop();
+    expect(idleWorkSnapshot().active).toBe(0);
+  });
   it("verifies ordinary cleanup carrying creation provenance and stops provider polling", async () => {
     const lease = await pending();
     await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", creationOperationId: randomUUID() });

server/src/__tests__/helpers/chat-history-proof.ts

@@ -0,0 +1,11 @@
+import { sql } from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+import { idleChatHistoryChecks } from "../../services/chat-channels/idle-history-proof.js";
+
+/** Scope the actual shared predicate to a real producer's receipt. Other pending
+ * fixture work remains independently observable through its owning gates. */
+export async function chatHistoryActionBlocks(db: Db, actionId: string): Promise<boolean> {
+  const check = idleChatHistoryChecks.find(check => check.reason === "chat_actions")!;
+  const [row] = await db.execute<{ blocked: boolean }>(sql`SELECT EXISTS (${sql.raw(check.query)} AND a.id = ${actionId}::uuid) AS blocked`);
+  return row.blocked;
+}

server/src/__tests__/sandbox-creation-operations.test.ts

@@ -10,6 +10,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi }
 import { eq, sql } from "drizzle-orm";
 import { agents, builtInManagedResources, companies, companySecretVersions, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db";
 import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
 import { environmentRuntimeService } from "../services/environment-runtime.js";
 import { environmentService } from "../services/environments.js";
 import { HttpError } from "../errors.js";
@@ -18,8 +19,9 @@ import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
 import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js";
 import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
 
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL");
 describe("durable sandbox creation through the runtime entry point", () => {
-  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
   let db: ReturnType<typeof createDb>;
   let companyId: string, environmentId: string, pluginId: string, runId: string, spool: string;
   let creates: number, destroys: number, loseReply: boolean, failDestroy: boolean;
@@ -27,8 +29,8 @@ describe("durable sandbox creation through the runtime entry point", () => {
   const resources = new Map<string, string>();
   let workers: PluginWorkerManager;
   beforeAll(async () => {
-    database = await startEmbeddedPostgresTestDatabase("sandbox-creation-operations-");
-    db = createDb(database.connectionString);
+    if (!external) database = await startEmbeddedPostgresTestDatabase("sandbox-creation-operations-");
+    db = createDb(external ?? database!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
   }, 90_000);
   beforeEach(async () => {
     await db.execute(sql`truncate sandbox_creation_operations, companies, environments, plugins cascade`);
@@ -267,7 +269,7 @@ describe("durable sandbox creation through the runtime entry point", () => {
     return secret;
   }
   it.each(["deleted", "disabled", "archived", "version-disabled", "vault-disabled", "vault-coming-soon"] as const)(
-    "retains a %s creation credential as quiet manual reconciliation across restart", async state => {
+    "retains a %s creation credential with a quiet worker while sleep remains blocked across restart", async state => {
     const secrets = secretOwner.secretService(db);
     const vault = state.startsWith("vault-") ? await secrets.createProviderConfig(companyId, {
       provider: "local_encrypted", displayName: "Creation vault", status: "ready", config: {},
@@ -281,8 +283,10 @@ describe("durable sandbox creation through the runtime entry point", () => {
     const inspectCalls = () => vi.mocked(workers.call).mock.calls.filter(value => value[1] === "environmentInspectLease").length;
     await due(); await runtime().flushDeferredOrphanCleanups();
     expect(await db.select().from(sandboxCreationOperations)).toEqual([
-      expect.objectContaining({ state: "needs_reconciliation" }),
+      expect.objectContaining({ companyId, state: "needs_reconciliation",
+        metadata: expect.objectContaining({ creationReconciliationReason: "creation_identity_unavailable" }) }),
     ]);
+    expect(resources.size).toBe(1);
     expect(await db.select().from(environmentLeases)).toEqual([]);
     expect(inspectCalls()).toBe(0);
     expect(await nextEnvironmentCleanupAt(db)).toBeNull();
@@ -293,12 +297,12 @@ describe("durable sandbox creation through the runtime entry point", () => {
     await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId));
     const ownerId = randomUUID();
     const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 };
-    // An installed plugin is a separate blocker until its worker proves quiet.
+    // A quiet plugin worker cannot discharge the retained creation uncertainty.
     expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none"))
       .toEqual({ version: 1, backgroundWork: "present" });
     expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none",
       async () => ({ backgroundWork: "none", pluginIds: [pluginId] })))
-      .toEqual({ version: 1, backgroundWork: "none" });
+      .toEqual({ version: 1, backgroundWork: "present" });
     expect(await readdir(spool)).toEqual([]);
   });
   it("retries a temporary creation credential resolution failure without losing ownership", async () => {

server/src/__tests__/storage-local-provider.test.ts

@@ -119,6 +119,32 @@ describe("local disk storage provider", () => {
     await expect(service.getObject("company-1", stored.objectKey)).rejects.toMatchObject({ status: 404 });
   });
 
+  it.each(["EACCES", "EIO"])("retains storage uncertainty for %s instead of reporting absence", async code => {
+    const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-storage-"));
+    tempRoots.push(root);
+    const service = createStorageService(createLocalDiskStorageProvider(root));
+    const stored = await service.putFile({ companyId: "company-1", namespace: "issues/issue-1", originalFilename: "retained.txt",
+      contentType: "text/plain", body: Buffer.from("must not disappear") });
+    const failure = Object.assign(new Error("controlled filesystem failure"), { code });
+    const unlink = vi.spyOn(fs, "unlink").mockRejectedValueOnce(failure);
+    try { await expect(service.deleteObject("company-1", stored.objectKey)).rejects.toBe(failure); }
+    finally { unlink.mockRestore(); }
+    expect(await service.headObject("company-1", stored.objectKey)).toMatchObject({ exists: true });
+    const stat = vi.spyOn(fs, "stat").mockRejectedValueOnce(failure);
+    try { await expect(service.headObject("company-1", stored.objectKey)).rejects.toBe(failure); }
+    finally { stat.mockRestore(); }
+    expect(await readStreamToBuffer((await service.getObject("company-1", stored.objectKey)).stream)).toEqual(Buffer.from("must not disappear"));
+  });
+
+  it("treats a non-directory object path as absent", async () => {
+    const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-storage-"));
+    tempRoots.push(root);
+    await fs.writeFile(path.join(root, "company-1"), "not a directory");
+    const service = createStorageService(createLocalDiskStorageProvider(root));
+    expect(await service.headObject("company-1", "company-1/object")).toEqual({ exists: false });
+    await expect(service.deleteObject("company-1", "company-1/object")).resolves.toBeUndefined();
+  });
+
   it.each(["headObject", "deleteObject"] as const)("cancels %s even if a provider stops responding", async method => {
     const provider = createLocalDiskStorageProvider(os.tmpdir());
     const request = vi.spyOn(provider, method).mockImplementation(async () => new Promise<never>(() => {}));

server/src/__tests__/voice-session-store.integration.test.ts

@@ -1,3 +1,5 @@
+import { chatHistoryActionBlocks } from "./helpers/chat-history-proof.js";
+import { voiceCallReports } from "../services/voice/voice-call-reports.js";
 import { subscribeDatabaseWork } from "@paperclipai/db";
 import * as voiceReplyNotifications from "../services/voice/voice-reply-notifications.js";
 import { voiceReplySchedule } from "../services/voice/voice-reply-schedule.js";
@@ -40,15 +42,16 @@ import { buildPaperclipRuntimeMcpServers } from "../services/heartbeat.js";
 import { createToolGatewayService } from "../services/tool-gateway.js";
 import { initializeRunIdentity } from "../services/run-identity.js";
 import { syncSpekoVoiceTools, spekoToolsForSession, executeSpekoVoiceTool } from "../services/voice/speko-agent-tools.js";
-const external = resolveExternalTestDatabaseUrl("PAPERCLIP_VOICE_TEST_DATABASE_URL");
+const historyExternal = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL");
+const external = historyExternal ?? resolveExternalTestDatabaseUrl("PAPERCLIP_VOICE_TEST_DATABASE_URL");
 const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
 (support.supported ? describe : describe.skip)("durable Speko voice sessions", () => {
   let db: ReturnType<typeof createDb>;
   let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
   beforeAll(async () => {
     if (!external) database = await startEmbeddedPostgresTestDatabase("paperclip-speko-store-");
-    if (external) await applyPendingMigrations(external);
-    db = createDb(external ?? database!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
+    if (external && !historyExternal) await applyPendingMigrations(external);
+    db = createDb(external ?? database!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0, ...(historyExternal ? { maxConnections: 4 } : {}) });
   }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
   afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
 
@@ -75,7 +78,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
       return { companyId, endpointId, sessionId, token: reserved.token, envelope, webhookId: `msg_${toolId}`, fingerprint: createHash("sha256").update(JSON.stringify(envelope)).digest("hex"), accept };
     };
     async function publication(state: "streaming" | "published" = "published") {
-      const [row] = await db.insert(chatPublications).values({ companyId, endpointId, conversationId, issueId, idempotencyKey: randomUUID(), state, payload: { text: "Approved result" } }).returning();
+      const [row] = await db.insert(chatPublications).values({ companyId, endpointId, conversationId, issueId, idempotencyKey: randomUUID(), state, publishedAt: state === "published" ? new Date() : null, payload: { text: "Approved result" } }).returning();
       return row;
     }
     return { companyId, endpointId, agentId, connectionId, issueId, conversationId, callerId, sessionId, store, request, reserved, accept, toolInput, publication };
@@ -164,6 +167,13 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
       const report = f.sign({ type: "call.report", session_id: f.call.providerSessionId, organization_id: f.event.organization_id });
       await f.service.inbound.lifecycle(f.endpoint.publicId, report.body, report.headers);
       await settle(); expect(f.transport.callReport).toHaveBeenCalledTimes(2); expect(onError).not.toHaveBeenCalled();
+      const receipts = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId)))
+        .filter(row => row.kind === "speko_lifecycle_receipt");
+      expect(receipts.length).toBeGreaterThan(0);
+      for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true);
+      await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId));
+      expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]);
+      for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(false);
     } finally {
       await coordinator.stop(); vi.useRealTimers();
       const ended = f.sign({ type: "call.status", call_id: f.call.providerSessionId, status: "ended" });
@@ -1381,7 +1391,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
   it("sends long approved phone answers as durable ordered parts and resumes after a rate limit", async () => {
     const f = await pushFixture(), text = "😀 answer ".repeat(4000) + "FINAL";
     await db.update(chatPublications).set({payload: {text}}).where(eq(chatPublications.id, f.publication.id));
-    const [later] = await db.insert(chatPublications).values({companyId: f.companyId, endpointId: f.endpointId, conversationId: f.conversationId, issueId: f.issueId, idempotencyKey: randomUUID(), state: "published", payload: {text: "Later answer B"}}).returning();
+    const [later] = await db.insert(chatPublications).values({companyId: f.companyId, endpointId: f.endpointId, conversationId: f.conversationId, issueId: f.issueId, idempotencyKey: randomUUID(), state: "published", publishedAt: new Date(), payload: {text: "Later answer B"}}).returning();
     await f.store.enqueuePublication(f.companyId, later.id);
     f.transport.sendCallMessage.mockResolvedValueOnce({messageId: "part-0"}).mockRejectedValueOnce(new SpekoProviderError("provider_unavailable", false, 429));
     await f.push();
@@ -1405,6 +1415,13 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     for (const call of calls) {expect(call[1].length).toBeLessThanOrEqual(16_000); expect(/^[\uDC00-\uDFFF]|[\uD800-\uDBFF]$/.test(call[1])).toBe(false);}
     const [delivered] = await db.select().from(chatVoiceReplies).where(eq(chatVoiceReplies.id, reply.id));
     expect(delivered.deliveredAt).toBeInstanceOf(Date); expect(delivered.spokenAt).toBeNull();
+    const accepted = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId)))
+      .filter(row => row.kind === "speko_voice_reply_push");
+    for (const receipt of accepted) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true);
+    await f.service.end(f.companyId, f.session.id, f.input.caller);
+    await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId));
+    expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]);
+    for (const receipt of accepted) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(false);
   });
   it("does not retry an uncertain long-answer part or send its suffix", async () => {
     const f = await pushFixture();
@@ -1416,6 +1433,13 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     expect(f.transport.sendCallMessage).toHaveBeenCalledTimes(2);
     const [reply] = await db.select().from(chatVoiceReplies).where(eq(chatVoiceReplies.publicationId, f.publication.id));
     expect(reply.deliveredAt).toBeNull();
+    await f.service.end(f.companyId, f.session.id, f.input.caller);
+    await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId));
+    expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]);
+    const uncertain = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId)))
+      .filter(row => row.kind === "speko_voice_reply_push" && row.status === "unknown");
+    expect(uncertain).toHaveLength(1);
+    expect(await chatHistoryActionBlocks(db, uncertain[0]!.id)).toBe(true);
   });
   it("pushes a published phone answer exactly once across concurrent dispatch and restart", async () => {
     const f = await pushFixture();
@@ -2027,6 +2051,15 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     expect(tools).toHaveLength(3);
     expect(f.transport.configureTool).toHaveBeenCalledTimes(7);
     expect(f.transport.configureTool.mock.calls.slice(-3).every((call) => Boolean(call[3]))).toBe(true);
+    const receipts = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId)))
+      .filter(row => ["speko_tool_setup", "speko_lifecycle_setup"].includes(row.kind));
+    expect(receipts).toHaveLength(4);
+    for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true); // real24h refresh obligation
+    await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId));
+    expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]);
+    // The resource helper alone has no post-configuration commit receipt;
+    // defaults/runtime completion belongs to the ordinary configure owner.
+    for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true);
   });
 
   it.each(["current", "legacy"])("moves only receipt-owned %s tools and hooks when callback origin and signing secret change", async proofVersion => {