Merge origin/master and retain both Cloud skill fixture contracts

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/agent-harness-verification-remote.test.ts

@@ -0,0 +1,67 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
+
+// Keep the real directory, command and adapter checks. Runtime staging is a
+// separate operation and must not copy files or contact a provider in this test.
+vi.mock("@paperclipai/adapter-utils/execution-target", async importOriginal => ({
+  ...await importOriginal<typeof import("@paperclipai/adapter-utils/execution-target")>(),
+  prepareAdapterExecutionTargetRuntime: async () => ({
+    workspaceRemoteDir: "/workspace", runtimeRootDir: "/workspace/.paperclip-runtime",
+    assetDirs: {}, restoreWorkspace: async () => {},
+  }),
+}));
+
+import { testEnvironment } from "@paperclipai/adapter-opencode-local/server";
+import { assertHarnessTestPassed } from "../services/agent-harness-verification.js";
+import { lifecycleFailureMessage, lifecycleVerificationNeedsRetry } from "../modules/agent-lifecycle/application/failure.js";
+
+describe("remote harness verification recovery", () => {
+  let configHome: string;
+  beforeAll(async () => { configHome = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-verification-")); });
+  afterAll(async () => { if (configHome) await rm(configHome, { recursive: true, force: true }); });
+  const context = { stage: "host" as const, phase: "verifying" as const, verificationFailures: 100 };
+
+  it.each([
+    ["directory", "timeout"], ["directory", "transport"],
+    ["command", "timeout"], ["command", "transport"],
+    ["directory", "missing"], ["command", "missing"],
+  ] as const)("retains %s %s through the real adapter check and recovers when repaired", async (check, fault) => {
+    let failing = true;
+    const target: AdapterExecutionTarget = {
+      kind: "remote", transport: "sandbox", remoteCwd: "/workspace", runner: {
+        execute: async input => {
+          const script = input.args?.join(" ") ?? "";
+          const selected = check === "directory" ? script.includes("[ -d ") : script.includes("command -v ");
+          if (failing && selected && fault === "transport") {
+            throw Object.assign(new Error("private provider output"), { code: "ECONNRESET" });
+          }
+          return {
+            exitCode: failing && selected ? fault === "timeout" ? null : 1 : 0,
+            timedOut: failing && selected && fault === "timeout", signal: null, stderr: "", pid: null,
+            startedAt: new Date().toISOString(),
+            stdout: input.command === "opencode" ? JSON.stringify({ type: "text", part: { text: "hello" } }) : "",
+          };
+        },
+      },
+    };
+    const run = () => testEnvironment({ companyId: "company", adapterType: "opencode_local", executionTarget: target,
+      config: { command: "opencode", model: "provider/model", env: { XDG_CONFIG_HOME: configHome } } });
+    const result = await run();
+    const failed = result.checks.find(entry => entry.code === (check === "directory" ? "opencode_cwd_invalid" : "opencode_command_unresolvable"));
+    expect(failed?.level).toBe("error");
+    expect(failed?.retryable === true).toBe(fault !== "missing");
+    let error: unknown;
+    try { assertHarnessTestPassed(result); } catch (caught) { error = caught; }
+    expect(error).toMatchObject({ code: fault !== "missing" ? "test_environment_unavailable"
+      : check === "directory" ? "harness_cwd_invalid" : "harness_command_unavailable" });
+    expect(lifecycleVerificationNeedsRetry(context, error)).toBe(fault === "missing");
+    expect(lifecycleFailureMessage(context, error)).not.toContain("private provider output");
+    if (fault !== "missing") expect(lifecycleFailureMessage(context, error)).toContain("retry automatically");
+    failing = false;
+    const recovered = await run();
+    expect(() => assertHarnessTestPassed(recovered)).not.toThrow();
+  });
+});

server/src/__tests__/agent-harness-verification.test.ts

@@ -24,6 +24,13 @@ describe("saved harness verification", () => {
       expect(error).toMatchObject({ code, message: "The harness test failed" });
     }
   });
+  it("preserves a retryable failed check before classifying configuration failures", () => {
+    const temporary = result("fail", ["opencode_cwd_invalid", "opencode_command_unresolvable"]);
+    temporary.checks[0] = { ...temporary.checks[0], level: "error", retryable: true };
+    expect(() => assertHarnessTestPassed(temporary)).toThrow(expect.objectContaining({ code: "test_environment_unavailable" }));
+    temporary.checks[0].level = "info";
+    expect(() => assertHarnessTestPassed(temporary)).toThrow(expect.objectContaining({ code: "harness_command_unavailable" }));
+  });
   it.each([
     ["fail", []], ["warn", ["claude_hello_probe_auth_required"]],
     ["warn", ["gemini_hello_probe_timed_out"]], ["pass", ["claude_hello_probe_skipped_custom_command"]],

server/src/__tests__/environment-cleanup-work.test.ts

@@ -1,3 +1,5 @@
+import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
+import { createOwnedSandbox } from "../services/sandbox-creation-operations.js";
 import { mkdtemp, writeFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -7,7 +9,7 @@ import { idleWorkSnapshot } from "../services/task-admission.js";
 import { randomUUID } from "node:crypto";
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
 import { eq, sql } from "drizzle-orm";
-import { companies, createDb, environmentLeases, environments, type Db } from "@paperclipai/db";
+import { companies, createDb, sandboxCreationOperations, environmentLeases, environments, type Db } from "@paperclipai/db";
 import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
 import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
 import { registerEnvironmentCleanupWork } from "../services/environment-cleanup-work.js";
@@ -34,7 +36,7 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
     await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
   }, 90_000);
   beforeEach(async () => {
-    await db.execute(sql`TRUNCATE companies, environments CASCADE`);
+    await db.execute(sql`TRUNCATE sandbox_creation_operations, companies, environments CASCADE`);
     companyId = randomUUID(); environmentId = randomUUID(); draining = false; failing = false; errors.length = 0;
     await db.insert(companies).values({ id: companyId, name: "Cleanup worker", issuePrefix: "CLN" });
     await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" });
@@ -99,6 +101,64 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
     await settle(180_000); expect(inspections).toHaveBeenCalledTimes(3);
     await quiet();
   });
+  it("reconstructs a creation deadline after restart and leaves manual reconciliation quiet", async () => {
+    await start(); await quiet();
+    await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} },
+      async () => ({ providerLeaseId: "fixture-unattached" }));
+    await settle();
+    expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now());
+    worker!.stop(); await coordinator!.stop();
+    await start();
+    expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now());
+    await settle(31 * 60_000);
+    await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([
+      expect.objectContaining({ state: "needs_reconciliation" }),
+    ]));
+    await quiet();
+  });
+  it("settles a deleted-company creation as manual history across restart without idle polling", async () => {
+    await start(); await quiet();
+    await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} },
+      async () => ({ providerLeaseId: "fixture-deleted-company" }));
+    await db.delete(companies).where(eq(companies.id, companyId));
+    await settle(31 * 60_000);
+    await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([
+      expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }),
+    ]));
+    await quiet();
+    worker!.stop(); await coordinator!.stop();
+    await start(); await quiet();
+    expect(releases).not.toHaveBeenCalled(); expect(inspections).not.toHaveBeenCalled();
+    const ownerId = randomUUID();
+    const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 };
+    expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none"))
+      .toEqual({ version: 1, backgroundWork: "none" });
+    await quiet();
+  });
+  it("verifies ordinary cleanup carrying creation provenance and stops provider polling", async () => {
+    const lease = await pending();
+    await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", creationOperationId: randomUUID() });
+    await environmentService(db).releaseLease(lease.id, "failed", { cleanupStatus: "success" });
+    await start(true);
+    expect(inspections).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ providerLeaseId: lease.providerLeaseId,
+      metadata: expect.objectContaining({ creationOperationId: expect.any(String) }) }));
+    await quiet();
+    worker!.stop(); await coordinator!.stop();
+    await start(true); expect(inspections).toHaveBeenCalledTimes(1); await quiet();
+  });
+  it("preserves a first creation signal during drain and disarms after atomic attachment", async () => {
+    await start(); await quiet(); draining = true;
+    const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} },
+      async () => ({ providerLeaseId: "fixture-attached" }));
+    const count = reads.mock.calls.length;
+    await settle(60_000, false); expect(reads).toHaveBeenCalledTimes(count);
+    draining = false;
+    await environmentService(db).acquireLease({ companyId, environmentId, provider: "fixture", providerLeaseId: created.providerLeaseId,
+      creationOperationId: String(created.metadata!.creationOperationId), metadata: created.metadata });
+    await settle();
+    expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ state: "attached" })]);
+    await quiet();
+  });
   it("wakes on a real orphan insert, waits for its backoff, then disarms after deletion", async () => {
     await start(); await quiet();
     const lease = await pending(); await settle();

server/src/__tests__/fixtures/sandbox-creation-receiver.ts

@@ -0,0 +1,34 @@
+import { createDb } from "@paperclipai/db";
+import { environmentRuntimeService } from "../../services/environment-runtime.js";
+import { environmentService } from "../../services/environments.js";
+import type { PluginWorkerManager } from "../../services/plugin-worker-manager.js";
+
+const db = createDb(process.env.CREATION_FIXTURE_DATABASE_URL!);
+const workers = {
+  isRunning: () => true,
+  getWorker: () => ({ supportedMethods: ["environmentAcquireLease", "environmentInspectLease", "environmentDestroyLease"] }),
+  call: async (_pluginId: string, method: string, input: unknown) => {
+    const response = await fetch(process.env.CREATION_FIXTURE_PROVIDER_URL!, {
+      method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ method, input }),
+    });
+    if (!response.ok) throw new Error("Controlled provider unavailable");
+    return response.json();
+  },
+} as unknown as PluginWorkerManager;
+try {
+  const runtime = environmentRuntimeService(db, {
+    pluginWorkerManager: workers, orphanCleanupSpoolDir: process.env.CREATION_FIXTURE_EMPTY_DISK,
+  });
+  if (process.env.CREATION_FIXTURE_ACTION === "acquire") {
+    await runtime.acquireRunLease({ companyId: process.env.CREATION_FIXTURE_COMPANY_ID!,
+      environment: (await environmentService(db).getById(process.env.CREATION_FIXTURE_ENVIRONMENT_ID!))!,
+      heartbeatRunId: process.env.CREATION_FIXTURE_RUN_ID!, issueId: null, persistedExecutionWorkspace: null });
+  } else {
+    await runtime.flushDeferredOrphanCleanups();
+  }
+  process.stdout.write(JSON.stringify({ ok: true }));
+} catch {
+  process.stdout.write(JSON.stringify({ ok: false }));
+} finally {
+  await db.$client.end({ timeout: 1 });
+}

server/src/__tests__/heartbeat-plugin-environment.test.ts

@@ -278,6 +278,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => {
     }, { timeout: 5_000 });
 
     expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentAcquireLease", {
+      creationOperationId: expect.any(String),
       driverKey: "sandbox",
       companyId,
       environmentId,
@@ -745,6 +746,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => {
     }, { timeout: 5_000 });
 
     expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentAcquireLease", {
+      creationOperationId: expect.any(String),
       driverKey: "sandbox",
       companyId,
       environmentId: newEnvironmentId,

server/src/__tests__/managed-skill-content.test.ts

@@ -166,7 +166,12 @@ describe.skipIf(!support.supported)("managed skills use committed versions", ()
     }
   }, 180_000);
 
-  it("reports a provider mismatch for local versions under persisted Cloud configuration", async () => {
+  it.each([
+    { storage: "legacy local", env: { PAPERCLIP_STORAGE_PROVIDER: "local_disk" }, expectedStatus: "local_storage" },
+    { storage: "shared S3", env: { PAPERCLIP_STORAGE_PROVIDER: "s3",
+      PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-cli-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1",
+      PAPERCLIP_SECRETS_MASTER_KEY: "12345678901234567890123456789012" }, expectedStatus: "provider_mismatch" },
+  ])("loads persisted Cloud configuration for read-only $storage maintenance without seeding it", async ({ env, expectedStatus }) => {
     const companyId = await company();
     const skill = await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" });
     const local = await skillInstallationMaintenance(db).verifyVersions({ companyId, limit: 50 });
@@ -178,8 +183,7 @@ describe.skipIf(!support.supported)("managed skills use committed versions", ()
       `--phase=${phase}`, `--company=${companyId}`, "--limit=50",
     ], { env: { ...process.env, DATABASE_URL: database.connectionString,
       PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture",
-      PAPERCLIP_STORAGE_PROVIDER: "s3", PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-cli-fixture",
-      PAPERCLIP_STORAGE_S3_REGION: "us-east-1", PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64) },
+      PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects"), ...env },
       timeout: 60_000, maxBuffer: 1024 * 1024 });
     await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration");
     expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0);
@@ -188,7 +192,7 @@ describe.skipIf(!support.supported)("managed skills use committed versions", ()
       const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!);
       expect(eligibility.results).toEqual([expect.objectContaining({ versionId: skill.currentVersionId, status: "version_backed" })]);
       const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
-      expect(verification.results).toEqual([expect.objectContaining({ versionId: skill.currentVersionId, status: "provider_mismatch" })]);
+      expect(verification.results).toEqual([expect.objectContaining({ versionId: skill.currentVersionId, status: expectedStatus })]);
       expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toEqual([persisted]);
     } finally {
       await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key));

server/src/__tests__/plugin-warm-tasks.integration.test.ts

@@ -1,11 +1,13 @@
-import { flushDeferredEnvironmentCleanup, hasDeferredEnvironmentCleanup, registerDeferredEnvironmentCleanup, subscribeDeferredEnvironmentCleanup } from "../services/environment-cleanup-notifications.js";
 import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
+import { mkdtemp, readdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
 import { randomUUID } from "node:crypto";
 import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
-import { eq } from "drizzle-orm";
-import { agents, companies, createDb, applyPendingMigrations, environmentLeases, environments, heartbeatRuns, issues, plugins, projects } from "@paperclipai/db";
+import { eq, sql } from "drizzle-orm";
+import { agents, companies, createDb, applyPendingMigrations, environmentLeases, environments, heartbeatRuns, issues, plugins, projects, sandboxCreationOperations } from "@paperclipai/db";
 import { environmentRuntimeService } from "../services/environment-runtime.js";
-import * as orphanCleanup from "../services/sandbox-orphan-cleanup-spool.js";
+import { recordSandboxCreationResource } from "../services/sandbox-creation-operations.js";
 import { resolvePluginTaskResume } from "../services/environment-task-runtime.js";
 import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
 import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -51,7 +53,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport
     if(p.operation.kind==="describe") return {kind:"description",taskId:p.taskId,runner:{version:"fixture",digest:`sha256:${"a".repeat(64)}`},warmTaskVersion:1};
     return {kind:"accepted",taskId:p.taskId};
    }} as unknown as PluginWorkerManager;
-  const runtime=()=>environmentRuntimeService(db,{pluginWorkerManager:workers});
+  const runtime=(orphanCleanupSpoolDir?:string)=>environmentRuntimeService(db,{pluginWorkerManager:workers,orphanCleanupSpoolDir});
   const acquire=()=>runtime().acquireRunLease({companyId,environment:environment as never,issueId,agentId,heartbeatRunId:runId,persistedExecutionWorkspace:null,adapterType:"paperclip_runner",runnerTaskProjectIds:projectIds,requestedExpiresAt:deadline});
   const first=await acquire();
   const identity={runnerInstanceId:"runner",environmentLeaseId:first.lease.id,runId,normalizedSessionId:"session",turnId:"turn",itemId:"item"};
@@ -126,58 +128,64 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport
   await expect(resolvePluginTaskResume(db,{companyId:f.companyId,leaseId:next.lease.id,priorIdentity:f.identity})).resolves.toBe(f.first.lease.providerLeaseId);
  });
 
- it.each(["ownership-lost","database-error","cleanup-error","spool"] as const)("compensates an unowned replacement task: %s",async mode=>{
+ it.each(["ownership-lost","database-error"] as const)("retains a durable operation for an unattached replacement task: %s",async mode=>{
   const f=await fixture();await f.nextRun();const next=await f.acquire();f.stopped();
-  if(mode==="cleanup-error" || mode==="spool") f.failRelease();
-  const append=vi.fn(async()=>true);
-  let delayFlush=false,flushCalls=0,releaseFlush!:()=>void;
-  const flushGate=new Promise<void>(resolve=>{releaseFlush=resolve;});
-  const stopGate=mode==="spool" ? registerDeferredEnvironmentCleanup(db,async()=>{
-   if(delayFlush && ++flushCalls===1) await flushGate;
-   return {recovered:0,pending:0};
-  }) : undefined;
-  const wake=vi.fn();
-  const unsubscribe=subscribeDeferredEnvironmentCleanup(db,wake);
-  const spool=mode==="spool" ? vi.spyOn(orphanCleanup,"createSandboxOrphanCleanupSpool").mockReturnValue({append,load:async()=>[],remove:async()=>{}}) : undefined;
-  let replacementFailed=false;
-  const transact=db.transaction.bind(db);
-  const transaction=mode==="spool" ? vi.spyOn(db,"transaction").mockImplementation((...args)=>replacementFailed
-   ? Promise.reject(Error("database unavailable")) : transact(...args)) : undefined;
-  const original=db.update.bind(db);
-  const update=vi.spyOn(db,"update").mockImplementation(((table:any)=>{
-   const query=original(table),set=query.set.bind(query);
-   query.set=((values:any)=>values.providerLeaseId === "task-2"
-    ? {where:()=>({returning:async()=>{replacementFailed=true;if(mode!=="ownership-lost") throw Error("replacement write failed");return [];}})}
-    : set(values)) as never;
-   return query;
-  }) as never);
+  const disk=await mkdtemp(join(tmpdir(),"replacement-intent-"));
+  const runtime=()=>f.runtime(disk);
+  await db.execute(sql.raw(`create or replace function reject_task_replacement() returns trigger language plpgsql as $$ begin
+    if new.provider_lease_id = 'task-2' then ${mode === "ownership-lost" ? "return null;" : "raise exception 'replacement write failed';"} end if; return new; end $$`));
+  await db.execute(sql`create trigger reject_task_replacement before update on environment_leases for each row execute function reject_task_replacement()`);
   try {
-   await expect(f.runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}}))
-    .rejects.toThrow("reconcile the same task");
-   if(mode==="spool") {
-    expect(wake).toHaveBeenCalledOnce();
-    expect(hasDeferredEnvironmentCleanup(db)).toBe(true);
-    await expect(flushDeferredEnvironmentCleanup(db)).rejects.toThrow("Deferred orphan cleanup failed");
-    expect(hasDeferredEnvironmentCleanup(db)).toBe(true);
-   }
-  } finally {update.mockRestore();transaction?.mockRestore();spool?.mockRestore();unsubscribe();}
-  expect(f.calls).toContain("environmentReleaseLease");
-  if(mode==="spool") expect(append).toHaveBeenCalledWith(expect.objectContaining({providerLeaseId:"task-2",metadata:expect.objectContaining({driver:"plugin",runnerTaskDispatched:false})}));
-  const rows=await db.select().from(environmentLeases).where(eq(environmentLeases.companyId,f.companyId));
-  expect(rows.filter(row=>row.providerLeaseId === "task-2")).toEqual(mode==="cleanup-error"
-   ? [expect.objectContaining({status:"pending_cleanup",failureReason:"plugin_task_replacement_unowned",metadata:expect.objectContaining({runnerTaskDispatched:false})})] : []);
-  if(mode==="spool") {
-   delayFlush=true;
-   const delayed=flushDeferredEnvironmentCleanup(db);
-   try { expect(await flushDeferredEnvironmentCleanup(db)).toEqual({recovered:1,pending:0}); }
-   finally { releaseFlush(); stopGate!(); }
-   expect(await delayed).toEqual({recovered:1,pending:0});
-   expect(hasDeferredEnvironmentCleanup(db)).toBe(false);
-   const recovered=await db.select().from(environmentLeases).where(eq(environmentLeases.companyId,f.companyId));
-   expect(recovered.filter(row=>row.providerLeaseId==="task-2")).toEqual([
-    expect.objectContaining({status:"pending_cleanup",failureReason:"plugin_task_replacement_unowned"}),
-   ]);
-  }
+   await expect(runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}})).rejects.toThrow("reconcile the same task");
+  } finally { await db.execute(sql`drop trigger reject_task_replacement on environment_leases`); }
+  expect(f.calls).not.toContain("environmentReleaseLease");
+  const [lease]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,next.lease.id));
+  expect(lease.providerLeaseId).toBe(next.lease.providerLeaseId);
+  const operations=await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId,f.companyId));
+  const pending=operations.filter(row=>row.state === "pending");
+  expect(pending).toHaveLength(1);expect(pending[0].requestKey).toBe(next.lease.providerLeaseId);
+  expect(pending[0].metadata.creationProviderLeaseId).toBe("task-2");expect(pending[0].config).toEqual({});
+  expect(f.acquireRequests.at(-1)?.creationOperationId).toBe(pending[0].id);
+  const creates=f.acquireRequests.length;
+  await expect(runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}})).rejects.toThrow("reconcile the same task");
+  expect(f.acquireRequests).toHaveLength(creates);
+  await db.update(sandboxCreationOperations).set({nextAttemptAt:new Date(0)}).where(eq(sandboxCreationOperations.id,pending[0].id));
+  await runtime().flushDeferredOrphanCleanups();
+  const [manual]=await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id,pending[0].id));
+  expect(manual.state).toBe("needs_reconciliation");expect(manual.metadata.creationProviderLeaseId).toBe("task-2");
+  await runtime().flushDeferredOrphanCleanups();expect(f.acquireRequests).toHaveLength(creates);
+  expect(f.calls).not.toContain("environmentReleaseLease");
+  expect(await readdir(disk)).toEqual([]);
+  await rm(disk,{recursive:true,force:true});
+ });
+
+ it("retains the known replacement identity after a committed attachment response is lost",async()=>{
+  const f=await fixture();await f.nextRun();const next=await f.acquire();f.stopped();
+  const transaction=db.transaction.bind(db);let lost=false;
+  const response=vi.spyOn(db,"transaction").mockImplementation((async(...args:any[])=>{
+   const result=await (transaction as any)(...args);
+   if(!lost && result?.providerLeaseId === "task-2") {lost=true;throw new Error("Attachment commit response lost");}
+   return result;
+  }) as typeof db.transaction);
+  try {
+   await expect(f.runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}})).rejects.toThrow("reconcile the same task");
+  } finally {response.mockRestore();}
+  expect(lost).toBe(true);
+  const operations=await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId,f.companyId));
+  const replacement=operations.find(operation=>operation.requestKey === next.lease.providerLeaseId)!;
+  expect(replacement).toMatchObject({state:"attached",leaseId:next.lease.id,metadata:{creationProviderLeaseId:"task-2"}});
+  const identity={id:replacement.id,companyId:f.companyId,environmentId:replacement.originalEnvironmentId,providerLeaseId:"task-2"};
+  await recordSandboxCreationResource(db,identity);
+  await expect(recordSandboxCreationResource(db,{...identity,providerLeaseId:"different-resource"})).rejects.toThrow("identity changed");
+  await expect(recordSandboxCreationResource(db,{...identity,companyId:randomUUID()})).rejects.toThrow("identity changed");
+  expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id,replacement.id)))[0])
+   .toMatchObject({state:"attached",metadata:{creationProviderLeaseId:"task-2"}});
+  const [saved]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,next.lease.id));
+  expect(saved).toMatchObject({providerLeaseId:"task-2",status:"active"});
+  await db.update(sandboxCreationOperations).set({nextAttemptAt:new Date(0)}).where(eq(sandboxCreationOperations.id,replacement.id));
+  await f.runtime().flushDeferredOrphanCleanups();
+  expect((await f.runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}})).taskId).toBe("task-2");
+  expect(f.acquireRequests).toHaveLength(2);expect(f.calls).not.toContain("environmentReleaseLease");
  });
 
  it.each(["added","removed"] as const)("retires retained mounts when an authorized project is %s after controller restart",async mode=>{
@@ -203,7 +211,9 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport
   const replacing=f.runtime().task({companyId:f.companyId,leaseId:next.lease.id,operation:{kind:"describe"}});
   if(["violating","missing-bounded","invalid","expired"].includes(mode)) {
    await expect(replacing).rejects.toThrow("reconcile the same task");
-   expect(f.calls).toContain("environmentReleaseLease");
+   expect(f.calls).not.toContain("environmentReleaseLease");
+   const operations=await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId,f.companyId));
+   expect(operations.filter(row=>row.state === "pending")).toHaveLength(1);
   } else {
    const description=await replacing;
    const [saved]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,next.lease.id));

server/src/__tests__/sandbox-creation-operations.test.ts

@@ -0,0 +1,333 @@
+import { execFile } from "node:child_process";
+import { createServer } from "node:http";
+import { promisify } from "node:util";
+import { fileURLToPath } from "node:url";
+import { randomUUID } from "node:crypto";
+import { mkdtemp, readFile, readdir, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { eq, sql } from "drizzle-orm";
+import { agents, builtInManagedResources, companies, companySecretVersions, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db";
+import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { environmentRuntimeService } from "../services/environment-runtime.js";
+import { environmentService } from "../services/environments.js";
+import { HttpError } from "../errors.js";
+import * as secretOwner from "../services/secrets.js";
+import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
+import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js";
+import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
+
+describe("durable sandbox creation through the runtime entry point", () => {
+  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  let db: ReturnType<typeof createDb>;
+  let companyId: string, environmentId: string, pluginId: string, runId: string, spool: string;
+  let creates: number, destroys: number, loseReply: boolean, failDestroy: boolean;
+  let beforeReply: (() => Promise<void>) | undefined;
+  const resources = new Map<string, string>();
+  let workers: PluginWorkerManager;
+  beforeAll(async () => {
+    database = await startEmbeddedPostgresTestDatabase("sandbox-creation-operations-");
+    db = createDb(database.connectionString);
+  }, 90_000);
+  beforeEach(async () => {
+    await db.execute(sql`truncate sandbox_creation_operations, companies, environments, plugins cascade`);
+    companyId = randomUUID(); environmentId = randomUUID(); pluginId = randomUUID(); runId = randomUUID();
+    const agentId = randomUUID(); creates = 0; destroys = 0; loseReply = false; failDestroy = false; beforeReply = undefined; resources.clear();
+    spool = await mkdtemp(join(tmpdir(), "empty-sandbox-web-"));
+    await db.insert(companies).values({ id: companyId, name: "Creation fixture", issuePrefix: "CRT" });
+    await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", role: "engineer", adapterType: "codex_local" });
+    await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", invocationSource: "manual" });
+    await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox", config: { provider: "creation-fixture", image: "fixture", reuseLease: false } });
+    await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.creation-fixture", packageName: "creation-fixture", version: "1.0.0", apiVersion: 1,
+      status: "ready", categories: ["automation"], manifestJson: { id: "paperclip.creation-fixture", version: "1.0.0", apiVersion: 1,
+      displayName: "Creation fixture", description: "Controlled remote provider", author: "Paperclip", categories: ["automation"],
+      capabilities: ["environment.drivers.register"], entrypoints: { worker: "fixture.js" }, environmentDrivers: [{
+        driverKey: "creation-fixture", kind: "sandbox_provider", displayName: "Fixture", creationOperationVersion: 1, configSchema: { type: "object" },
+      }] } });
+    workers = {
+      isRunning: () => true,
+      getWorker: () => ({ supportedMethods: ["environmentAcquireLease", "environmentInspectLease", "environmentDestroyLease"] }),
+      call: vi.fn(async (_pluginId: string, method: string, input: any) => {
+        if (method === "environmentAcquireLease") {
+          expect(await db.select().from(sandboxCreationOperations)).toContainEqual(expect.objectContaining({ id: input.creationOperationId, state: "pending" }));
+          creates++;
+          const resource = `provider-${input.creationOperationId}`;
+          resources.set(input.creationOperationId, resource);
+          await beforeReply?.();
+          if (loseReply) throw new Error("Lost provider reply");
+          return { providerLeaseId: resource, metadata: { remoteCwd: "/workspace" } };
+        }
+        if (method === "environmentInspectLease") {
+          const id = input.creationOperation.id;
+          const resource = resources.get(id);
+          if (!resource) return { providerLeaseId: id, state: "unknown" };
+          return { providerLeaseId: resource, state: "active", creationCleanup: { providerLeaseId: resource,
+            observedProviderLeaseId: resource, companyId, environmentId, runId: input.creationOperation.runId, attemptId: id,
+            accountFingerprint: "a".repeat(64), labels: { "paperclip-create-attempt": id } } };
+        }
+        if (method === "environmentDestroyLease") {
+          destroys++;
+          if (failDestroy) throw new Error("Fixture teardown failed");
+          resources.delete(input.leaseMetadata.creationOperationId);
+          return { providerLeaseId: input.providerLeaseId, state: "destroyed" };
+        }
+        throw new Error(`Unexpected fixture method ${method}`);
+      }),
+    } as unknown as PluginWorkerManager;
+  });
+  afterEach(async () => {
+    await db.execute(sql`drop trigger if exists reject_fixture_lease on environment_leases`);
+    await db.execute(sql`drop trigger if exists reject_fixture_creation on sandbox_creation_operations`);
+    await rm(spool, { recursive: true, force: true });
+  });
+  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
+  function runtime() { return environmentRuntimeService(db, { pluginWorkerManager: workers, orphanCleanupSpoolDir: spool }); }
+  async function acquire() {
+    return runtime().acquireRunLease({ companyId, environment: (await environmentService(db).getById(environmentId))!,
+      heartbeatRunId: runId, issueId: null, persistedExecutionWorkspace: null, assertCompanyBinding: true });
+  }
+  async function due() { await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) }); }
+  async function rejectWrites(table: "environment_leases" | "sandbox_creation_operations", trigger: string) {
+    await db.execute(sql`create or replace function reject_fixture_write() returns trigger language plpgsql as $$ begin raise exception 'Fixture database write unavailable'; end $$`);
+    await db.execute(sql.raw(`create trigger ${trigger} before insert on ${table} for each row execute function reject_fixture_write()`));
+  }
+  it("can reapply the creation migration without losing a committed intent", async () => {
+    loseReply = true; await expect(acquire()).rejects.toThrow("Lost provider reply");
+    const [before] = await db.select().from(sandboxCreationOperations);
+    const migration = await readFile(new URL("../../../packages/db/src/migrations/0418_curved_lady_mastermind.sql", import.meta.url), "utf8");
+    for (let attempt = 0; attempt < 2; attempt++) {
+      for (const statement of migration.split("--> statement-breakpoint").map(value => value.trim()).filter(Boolean)) {
+        await db.execute(sql.raw(statement));
+      }
+    }
+    expect(await db.select().from(sandboxCreationOperations)).toEqual([before]);
+    expect(creates).toBe(1);
+  });
+  it("fails before the external action when the durable admission write fails", async () => {
+    await rejectWrites("sandbox_creation_operations", "reject_fixture_creation");
+    await expect(acquire()).rejects.toThrow();
+    expect(creates).toBe(0); expect(resources.size).toBe(0); expect(await readdir(spool)).toEqual([]);
+  });
+  it("attaches once and reuses the committed transfer after a lost response", async () => {
+    const first = await acquire();
+    const retry = await acquire();
+    expect(retry.lease.id).toBe(first.lease.id); expect(creates).toBe(1); expect(destroys).toBe(0);
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    expect(await db.select().from(environmentLeases)).toHaveLength(1);
+    expect(await nextEnvironmentCleanupAt(db)).toBeNull();
+  });
+  it("recovers a failed lease write with a fresh receiver and empty web disk", async () => {
+    await rejectWrites("environment_leases", "reject_fixture_lease");
+    await expect(acquire()).rejects.toThrow();
+    expect(creates).toBe(1); expect(await readdir(spool)).toEqual([]);
+    expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ state: "pending" })]);
+    await db.execute(sql`drop trigger reject_fixture_lease on environment_leases`);
+    await due();
+    await Promise.all([runtime().flushDeferredOrphanCleanups(), runtime().flushDeferredOrphanCleanups()]);
+    const leases = await db.select().from(environmentLeases);
+    expect(leases).toEqual([expect.objectContaining({ status: "pending_cleanup", providerLeaseId: [...resources.values()][0] })]);
+    expect(creates).toBe(1);
+    failDestroy = true;
+    await expect(runtime().retryPendingSandboxTeardown({ environment: null, lease: (await environmentService(db).getLeaseById(leases[0]!.id))! })).rejects.toThrow("Fixture teardown failed");
+    expect(resources.size).toBe(1);
+    failDestroy = false;
+    await runtime().retryPendingSandboxTeardown({ environment: null, lease: (await environmentService(db).getLeaseById(leases[0]!.id))! });
+    expect(resources.size).toBe(0); expect(destroys).toBe(2);
+  });
+  it("reconstructs ownership in a replacement receiver process with a different empty web disk", async () => {
+    const provider = createServer(async (request, response) => {
+      try {
+        let body = ""; for await (const chunk of request) body += chunk.toString();
+        const { method, input } = JSON.parse(body);
+        response.setHeader("content-type", "application/json");
+        response.end(JSON.stringify(await (workers.call as any)(pluginId, method, input)));
+      } catch { response.statusCode = 500; response.end("{}"); }
+    });
+    await new Promise<void>(resolve => provider.listen(0, "127.0.0.1", resolve));
+    const address = provider.address() as { port: number };
+    const replacementDisk = await mkdtemp(join(tmpdir(), "replacement-sandbox-web-"));
+    const fixture = fileURLToPath(new URL("./fixtures/sandbox-creation-receiver.ts", import.meta.url));
+    async function receiver(action: string, disk: string) {
+      const result = await promisify(execFile)(process.execPath, ["--import", fileURLToPath(import.meta.resolve("tsx")), fixture], {
+        cwd: fileURLToPath(new URL("../../..", import.meta.url)), timeout: 60_000,
+        env: { ...process.env, CREATION_FIXTURE_ACTION: action, CREATION_FIXTURE_EMPTY_DISK: disk,
+          CREATION_FIXTURE_DATABASE_URL: database.connectionString, CREATION_FIXTURE_PROVIDER_URL: `http://127.0.0.1:${address.port}`,
+          CREATION_FIXTURE_COMPANY_ID: companyId, CREATION_FIXTURE_ENVIRONMENT_ID: environmentId, CREATION_FIXTURE_RUN_ID: runId },
+      });
+      return result.stdout;
+    }
+    try {
+      await rejectWrites("environment_leases", "reject_fixture_lease");
+      expect(await receiver("acquire", spool)).toContain('"ok":false');
+      expect(creates).toBe(1); expect(resources.size).toBe(1); expect(await readdir(spool)).toEqual([]);
+      await db.execute(sql`drop trigger reject_fixture_lease on environment_leases`);
+      await due();
+      expect(await receiver("recover", replacementDisk)).toContain('"ok":true');
+      expect(await receiver("recover", replacementDisk)).toContain('"ok":true');
+      expect(await db.select().from(environmentLeases)).toEqual([
+        expect.objectContaining({ status: "pending_cleanup", providerLeaseId: [...resources.values()][0] }),
+      ]);
+      expect(creates).toBe(1); expect(await readdir(replacementDisk)).toEqual([]);
+    } finally {
+      await new Promise<void>((resolve, reject) => provider.close(error => error ? reject(error) : resolve()));
+      await rm(replacementDisk, { recursive: true, force: true });
+    }
+  }, 180_000);
+  it("never repeats creation after a lost provider reply", async () => {
+    loseReply = true;
+    await expect(acquire()).rejects.toThrow("Lost provider reply");
+    await expect(acquire()).rejects.toThrow("requires reconciliation");
+    expect(creates).toBe(1);
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    expect(await db.select().from(environmentLeases)).toEqual([expect.objectContaining({ status: "pending_cleanup" })]);
+  });
+  it("retains cleanup ownership when the environment is deleted during creation", async () => {
+    beforeReply = async () => { await environmentService(db).remove(environmentId); };
+    await expect(acquire()).rejects.toThrow();
+    expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ environmentId: null, originalEnvironmentId: environmentId, state: "pending" })]);
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    expect(await db.select().from(environmentLeases)).toEqual([expect.objectContaining({ environmentId: null, status: "pending_cleanup" })]);
+  });
+  it("rejects attachment after a company rebind and retains cleanup", async () => {
+    beforeReply = async () => {
+      const foreign = randomUUID(); await db.insert(companies).values({ id: foreign, name: "Foreign", issuePrefix: "FOR" });
+      await db.insert(builtInManagedResources).values({ companyId: foreign, bundleKey: "managed-environment", resourceKind: "environment",
+        resourceKey: "sandbox", resourceId: environmentId, stockVersion: "1", stockHash: "fixture" });
+    };
+    await expect(acquire()).rejects.toMatchObject({ status: 403 });
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    expect(await db.select().from(environmentLeases)).toEqual([expect.objectContaining({ companyId, status: "pending_cleanup" })]);
+  });
+  it("backs off a failed lookup and recovers unrelated creations in the same pass", async () => {
+    loseReply = true; await expect(acquire()).rejects.toThrow("Lost provider reply");
+    const [first] = await db.select().from(sandboxCreationOperations);
+    const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
+    runId = randomUUID(); await db.insert(heartbeatRuns).values({ ...run!, id: runId });
+    await expect(acquire()).rejects.toThrow("Lost provider reply");
+    const originalCall = workers.call;
+    (workers as any).call = vi.fn(async (plugin: string, method: string, params: any, timeout: any) => {
+      if (method === "environmentInspectLease" && params.creationOperation.id === first!.id) throw new Error("Transient fixture lookup");
+      return originalCall(plugin, method as any, params, timeout);
+    });
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    const rows = await db.select().from(sandboxCreationOperations);
+    expect(rows.find(value => value.id === first!.id)).toMatchObject({ state: "pending" });
+    expect(rows.find(value => value.id === first!.id)!.nextAttemptAt.getTime()).toBeGreaterThan(Date.now());
+    expect(rows.filter(value => value.state === "cleanup_pending")).toHaveLength(1);
+  });
+  it("does not attach a sandbox to a run stopped during creation", async () => {
+    beforeReply = async () => { await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, runId)); };
+    await expect(acquire()).rejects.toThrow("run authority changed");
+    expect(await db.select().from(environmentLeases)).toHaveLength(0);
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    expect(await db.select().from(environmentLeases)).toEqual([expect.objectContaining({ status: "pending_cleanup" })]);
+  });
+  it.each(["before lookup", "during lookup", "lookup fails after deletion"])("records manual reconciliation when the company is deleted %s", async (deletion) => {
+    loseReply = true; await expect(acquire()).rejects.toThrow("Lost provider reply");
+    async function deleteCompany() {
+      await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
+      await db.delete(agents).where(eq(agents.companyId, companyId));
+      await db.delete(companies).where(eq(companies.id, companyId));
+    }
+    if (deletion === "before lookup") await deleteCompany();
+    const originalCall = workers.call;
+    const calls = vi.fn(async (plugin: string, method: string, params: any, timeout: any) => {
+      if (method === "environmentInspectLease" && deletion !== "before lookup") {
+        await deleteCompany();
+        if (deletion === "lookup fails after deletion") throw new Error("Company credential no longer exists");
+      }
+      return originalCall(plugin, method as any, params, timeout);
+    });
+    (workers as any).call = calls;
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    expect(await db.select().from(sandboxCreationOperations)).toEqual([
+      expect.objectContaining({ companyId, state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }),
+    ]);
+    expect(await db.select().from(environmentLeases)).toEqual([]);
+    expect(calls.mock.calls.filter(value => value[1] === "environmentInspectLease")).toHaveLength(deletion === "before lookup" ? 0 : 1);
+    const count = calls.mock.calls.length;
+    await runtime().flushDeferredOrphanCleanups(); await runtime().flushDeferredOrphanCleanups();
+    expect(calls).toHaveBeenCalledTimes(count);
+    expect(await nextEnvironmentCleanupAt(db)).toBeNull(); expect(creates).toBe(1);
+    expect(await readdir(spool)).toEqual([]);
+  });
+  async function savedCreationCredential(providerConfigId?: string) {
+    const secrets = secretOwner.secretService(db);
+    const secret = await secrets.create(companyId, {
+      name: `creation-key-${randomUUID()}`, provider: "local_encrypted", providerConfigId, value: "fixture-private-key",
+    });
+    await secrets.createBinding({ companyId, secretId: secret.id, targetType: "environment", targetId: environmentId, configPath: "apiKey" });
+    await db.update(plugins).set({ manifestJson: sql`jsonb_set(${plugins.manifestJson}, '{environmentDrivers,0,configSchema}',
+      '{"type":"object","properties":{"apiKey":{"type":"string","format":"secret-ref"}}}'::jsonb)` }).where(eq(plugins.id, pluginId));
+    await db.update(environments).set({ config: { provider: "creation-fixture", image: "fixture", reuseLease: false, apiKey: secret.id } })
+      .where(eq(environments.id, environmentId));
+    loseReply = true;
+    await expect(acquire()).rejects.toThrow("Lost provider reply");
+    return secret;
+  }
+  it.each(["deleted", "disabled", "archived", "version-disabled", "vault-disabled", "vault-coming-soon"] as const)(
+    "retains a %s creation credential as quiet manual reconciliation across restart", async state => {
+    const secrets = secretOwner.secretService(db);
+    const vault = state.startsWith("vault-") ? await secrets.createProviderConfig(companyId, {
+      provider: "local_encrypted", displayName: "Creation vault", status: "ready", config: {},
+    }) : undefined;
+    const secret = await savedCreationCredential(vault?.id);
+    if (state === "deleted") await secrets.remove(secret.id);
+    else if (state === "disabled" || state === "archived") await secrets.update(secret.id, { status: state });
+    else if (state === "version-disabled") await db.update(companySecretVersions).set({ status: "disabled" })
+      .where(eq(companySecretVersions.secretId, secret.id));
+    else await secrets.updateProviderConfig(vault!.id, { status: state === "vault-disabled" ? "disabled" : "coming_soon" });
+    const inspectCalls = () => vi.mocked(workers.call).mock.calls.filter(value => value[1] === "environmentInspectLease").length;
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    expect(await db.select().from(sandboxCreationOperations)).toEqual([
+      expect.objectContaining({ state: "needs_reconciliation" }),
+    ]);
+    expect(await db.select().from(environmentLeases)).toEqual([]);
+    expect(inspectCalls()).toBe(0);
+    expect(await nextEnvironmentCleanupAt(db)).toBeNull();
+    const restarted = runtime();
+    await restarted.flushDeferredOrphanCleanups(); await restarted.flushDeferredOrphanCleanups();
+    expect(inspectCalls()).toBe(0); expect(creates).toBe(1); expect(destroys).toBe(0);
+    expect(await nextEnvironmentCleanupAt(db)).toBeNull();
+    await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId));
+    const ownerId = randomUUID();
+    const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 };
+    // An installed plugin is a separate blocker until its worker proves quiet.
+    expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none"))
+      .toEqual({ version: 1, backgroundWork: "present" });
+    expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none",
+      async () => ({ backgroundWork: "none", pluginIds: [pluginId] })))
+      .toEqual({ version: 1, backgroundWork: "none" });
+    expect(await readdir(spool)).toEqual([]);
+  });
+  it("retries a temporary creation credential resolution failure without losing ownership", async () => {
+    await savedCreationCredential();
+    const original = secretOwner.secretService;
+    const resolution = vi.fn(async () => { throw new HttpError(503, "Temporary credential provider failure"); });
+    const owner = vi.spyOn(secretOwner, "secretService").mockImplementation(client => ({
+      ...original(client), resolveSecretValueForSandboxCleanup: resolution,
+    }));
+    try {
+      await due(); await runtime().flushDeferredOrphanCleanups();
+      expect(resolution).toHaveBeenCalledTimes(1);
+      expect(await db.select().from(sandboxCreationOperations)).toEqual([
+        expect.objectContaining({ state: "pending", nextAttemptAt: expect.any(Date) }),
+      ]);
+      expect(await nextEnvironmentCleanupAt(db)).toBeGreaterThan(Date.now());
+      expect(vi.mocked(workers.call).mock.calls.filter(value => value[1] === "environmentInspectLease")).toHaveLength(0);
+    } finally { owner.mockRestore(); }
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ state: "cleanup_pending" })]);
+    expect(await db.select().from(environmentLeases)).toHaveLength(1);
+    expect(creates).toBe(1); expect(destroys).toBe(0); expect(await readdir(spool)).toEqual([]);
+  });
+  it("makes unsupported uncertain outcomes explicit without another creation", async () => {
+    loseReply = true; await expect(acquire()).rejects.toThrow();
+    await db.update(plugins).set({ manifestJson: sql`${plugins.manifestJson} #- '{environmentDrivers,0,creationOperationVersion}'` }).where(eq(plugins.id, pluginId));
+    await due(); await runtime().flushDeferredOrphanCleanups();
+    expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ state: "needs_reconciliation" })]);
+    expect(await nextEnvironmentCleanupAt(db)).toBeNull();
+    await expect(acquire()).rejects.toThrow("requires reconciliation"); expect(creates).toBe(1);
+  });
+});