diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts index 8d14bb2f9da..87548d4b507 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts @@ -1986,6 +1986,7 @@ describe("ACPX runtime host", () => { it("retains a settled turn when shutdown cleanup must be retried", async () => { const fixture = await hostFixture(); + const credential = credentialLeaseFixture(fixture.root); let settleTurn!: (value: { stopReason: string }) => void; const turn = { ...runtimeTurn(), @@ -2000,6 +2001,8 @@ describe("ACPX runtime host", () => { if (failRuntimeClose) throw new Error("runtime cleanup failed"); }, }); + const dependencies = fixture.dependencies({ openRuntime: async () => runtime }); + dependencies.stageCredential = vi.fn(async () => credential); const host = await AcpxRuntimeHost.open( { ...fixture.options, @@ -2008,7 +2011,7 @@ describe("ACPX runtime host", () => { permissionMode: "approve-reads", environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, }, - fixture.dependencies({ openRuntime: async () => runtime }), + dependencies, ); host.startTurn({ text: "Complete the task.", requestId: "turn-1" }); @@ -2016,6 +2019,8 @@ describe("ACPX runtime host", () => { settleTurn({ stopReason: "end_turn" }); await expect(firstClose).rejects.toThrow(/cleanup failed/); expect(turn.cancel).toHaveBeenCalledOnce(); + expect(credential.close).not.toHaveBeenCalled(); + expect(host.isClosed()).toBe(false); failRuntimeClose = false; await expect( @@ -2023,6 +2028,8 @@ describe("ACPX runtime host", () => { ).resolves.toBeUndefined(); expect(turn.cancel).toHaveBeenCalledTimes(2); expect(runtime.close).toHaveBeenCalledTimes(2); + expect(credential.close).toHaveBeenCalledOnce(); + expect(host.isClosed()).toBe(true); }); it("closes a command lease that resolves after admission is aborted", async () => { @@ -2307,7 +2314,7 @@ describe("ACPX runtime host", () => { }, ACPX_LONG_WAIT_TEST_TIMEOUT_MS); }); -// Deadline tests exercise runtime admission and cleanup, not kernel lease +// Runtime lifecycle tests exercise admission and cleanup, not kernel lease // acquisition. Keep their credential owner deterministic: real lease staging // uses hashed loopback ports that unrelated parallel tests can occupy. The // credential and retained-owner integration tests still use real leases. diff --git a/server/src/__tests__/sandbox-creation-operations.test.ts b/server/src/__tests__/sandbox-creation-operations.test.ts index 7c5c1924ecb..920b5e65679 100644 --- a/server/src/__tests__/sandbox-creation-operations.test.ts +++ b/server/src/__tests__/sandbox-creation-operations.test.ts @@ -10,6 +10,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } import { eq, sql } from "drizzle-orm"; import { agents, builtInManagedResources, companies, companySecretVersions, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { environmentRuntimeService } from "../services/environment-runtime.js"; import { environmentService } from "../services/environments.js"; import { HttpError } from "../errors.js"; @@ -18,8 +19,9 @@ import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); describe("durable sandbox creation through the runtime entry point", () => { - let database: Awaited>; + let database: Awaited> | undefined; let db: ReturnType; let companyId: string, environmentId: string, pluginId: string, runId: string, spool: string; let creates: number, destroys: number, loseReply: boolean, failDestroy: boolean; @@ -27,8 +29,8 @@ describe("durable sandbox creation through the runtime entry point", () => { const resources = new Map(); let workers: PluginWorkerManager; beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("sandbox-creation-operations-"); - db = createDb(database.connectionString); + if (!external) database = await startEmbeddedPostgresTestDatabase("sandbox-creation-operations-"); + db = createDb(external ?? database!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); }, 90_000); beforeEach(async () => { await db.execute(sql`truncate sandbox_creation_operations, companies, environments, plugins cascade`); @@ -267,7 +269,7 @@ describe("durable sandbox creation through the runtime entry point", () => { return secret; } it.each(["deleted", "disabled", "archived", "version-disabled", "vault-disabled", "vault-coming-soon"] as const)( - "retains a %s creation credential as quiet manual reconciliation across restart", async state => { + "retains a %s creation credential with a quiet worker while sleep remains blocked across restart", async state => { const secrets = secretOwner.secretService(db); const vault = state.startsWith("vault-") ? await secrets.createProviderConfig(companyId, { provider: "local_encrypted", displayName: "Creation vault", status: "ready", config: {}, @@ -281,8 +283,10 @@ describe("durable sandbox creation through the runtime entry point", () => { const inspectCalls = () => vi.mocked(workers.call).mock.calls.filter(value => value[1] === "environmentInspectLease").length; await due(); await runtime().flushDeferredOrphanCleanups(); expect(await db.select().from(sandboxCreationOperations)).toEqual([ - expect.objectContaining({ state: "needs_reconciliation" }), + expect.objectContaining({ companyId, state: "needs_reconciliation", + metadata: expect.objectContaining({ creationReconciliationReason: "creation_identity_unavailable" }) }), ]); + expect(resources.size).toBe(1); expect(await db.select().from(environmentLeases)).toEqual([]); expect(inspectCalls()).toBe(0); expect(await nextEnvironmentCleanupAt(db)).toBeNull(); @@ -293,12 +297,12 @@ describe("durable sandbox creation through the runtime entry point", () => { await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); const ownerId = randomUUID(); const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 }; - // An installed plugin is a separate blocker until its worker proves quiet. + // A quiet plugin worker cannot discharge the retained creation uncertainty. expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")) .toEqual({ version: 1, backgroundWork: "present" }); expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none", async () => ({ backgroundWork: "none", pluginIds: [pluginId] }))) - .toEqual({ version: 1, backgroundWork: "none" }); + .toEqual({ version: 1, backgroundWork: "present" }); expect(await readdir(spool)).toEqual([]); }); it("retries a temporary creation credential resolution failure without losing ownership", async () => {