diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index e535edcd569..7b8ef115430 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -101,6 +101,17 @@ The manager permits at most two outstanding inspection RPCs across workers. Caller timeouts retain their slots until the actual worker reply or exit; timeouts cannot accumulate additional provider reads in later sweeps. +If a failed ephemeral lease has no usable local inspection path, a managed +Daytona instance can ask Cloud for a cleanup-only absence receipt. Cloud checks +the stack's historical credential organization and reads the exact allocation +with its manager credential. Only confirmed absence can produce a signed receipt +bound to the stack, company, environment, lease, provider handle and nullable run. +The separate cleanup type and audience cannot authorize run recovery. Older +Cloud servers, unsupported providers, invalid signatures, changed organizations +and failed reads leave the lease unresolved. Existing bounded cleanup batches +and retry deadlines own these reads; no new idle polling is added. An explicit +local provider observation is retained rather than replaced by this fallback. + Host-owned verification metadata binds the company, environment, resource, provider handle, optional run, and exact row version. Concurrent resume, replacement, cleanup, or configuration changes invalidate verification. diff --git a/maps/agent-access-map/instance-operations.md b/maps/agent-access-map/instance-operations.md index 3e21ef72901..f2d4250a4cb 100644 --- a/maps/agent-access-map/instance-operations.md +++ b/maps/agent-access-map/instance-operations.md @@ -62,6 +62,14 @@ authority as company-agent API access. Gaps: Instance start/stop and task workspace start/stop are different capabilities. Local service control is not agent company authority. +Historical Cloud sandbox cleanup remains server-owned. Cleanup-only absence +reads authenticate the tenant host with its server capability; they expose no +new board or agent endpoint. Their signed scope cannot authorize task recovery. +[Runtime and idle-gate regressions](../../server/src/__tests__/historical-cleanup-verification.test.ts) +cover reconciliation, while [receipt checks](../../server/src/services/sandbox-absence-proof.test.ts) +cover signature, identity, freshness and separation from recovery authority. +These checks do not establish a deployed sleep/wake result. + ## `configuration` Capability ID: `instance-operations/configuration`. diff --git a/maps/feature-map/instance-operations.md b/maps/feature-map/instance-operations.md index 7a01c24ebf6..dd43901323e 100644 --- a/maps/feature-map/instance-operations.md +++ b/maps/feature-map/instance-operations.md @@ -61,6 +61,15 @@ served by the admitted application before it finishes. Repeat with a disconnect before headers arrive. Confirm that neither leaves a permanent request hold. An unfinished application write must still prevent sleep after its client leaves. +Automated: [historical cleanup](../../server/src/__tests__/historical-cleanup-verification.test.ts) +exercises the runtime inspection path for failed Cloud sandbox leases, including +verification leases without a run. A signed cleanup-only absence observation +can remove this blocker without changing failed outcomes or granting run +continuation. An unresolved provider resource still prevents idle sleep. +Manual: on an owned Cloud staging workspace, confirm the exact published build, +reconcile failed test allocations, then verify actual provider suspension and +normal hosted entry after wake. Local tests do not prove that hosted journey. + ### `settings-backup` Cloud startup reads nonsecret application choices from a private, versioned diff --git a/server/src/__tests__/historical-cleanup-verification.test.ts b/server/src/__tests__/historical-cleanup-verification.test.ts index 37bc2dd6692..1269c02f1a2 100644 --- a/server/src/__tests__/historical-cleanup-verification.test.ts +++ b/server/src/__tests__/historical-cleanup-verification.test.ts @@ -1,8 +1,9 @@ -import { randomUUID } from "node:crypto"; +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db"; import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js"; @@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests (support.supported ? describe : describe.skip)("historical cleanup verification", () => { let db: ReturnType, database: Awaited>; beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000); - afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); }); + afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); }); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true); const state = async () => (await read()).backgroundWork; @@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }), }); } + it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => { + const f = await seed(adHoc); + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", + PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture", + PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret", + PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value); + let absent = false; + const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => { + if (!absent) return Response.json({ absent: false }); + const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000); + expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id, + leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId }); + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url"); + const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1", + sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url"); + return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` }); + }); + vi.stubGlobal("fetch", fetchImpl); + const runtime = environmentRuntimeService(db); + const service = historicalCleanupVerificationService(db, { + inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined, + }); + expect(await state()).toBe("present"); + expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 }); + expect(await state()).toBe("present"); + absent = true; + expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 }); + expect(await state()).toBe("none"); + const [current] = await db.select().from(environmentLeases); + expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId }); + expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination"); + expect(await service.sweep()).toMatchObject({ checked: 0 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => { + await seed(true); + for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", + PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture", + PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value); + const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", + acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null, + inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }), + }] }); + const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined }); + expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 }); + expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled(); + }); it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => { const f = await seed(adHoc); expect(await state()).toBe("present"); const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 }); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 277a5a0254c..33f161457f9 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js"; import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js"; import { createHash, randomUUID } from "node:crypto"; import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; @@ -4021,7 +4022,12 @@ export function environmentRuntimeService( const lease = await environmentsSvc.getLeaseById(leaseId); if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined; const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null; - return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease }); + // Legacy Cloud leases can lack a credential version that permits a + // provider-local observation. Cloud's manager verifies the historical + // organization and signs only exact absence; no replacement key or + // cleanup label is treated as proof. Explicit provider answers win. + return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease }) + ?? await inspectCloudSandboxCleanupAbsence(lease); }, async retryPendingSandboxTeardown(input: { diff --git a/server/src/services/sandbox-absence-proof.test.ts b/server/src/services/sandbox-absence-proof.test.ts index d145425137e..82a7a3cfbba 100644 --- a/server/src/services/sandbox-absence-proof.test.ts +++ b/server/src/services/sandbox-absence-proof.test.ts @@ -1,6 +1,7 @@ import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; -import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js"; +import { inspectCloudSandboxCleanupAbsence, requestSandboxAbsenceProof, verifySandboxAbsenceReceipt, + verifySandboxCleanupAbsenceReceipt } from "./sandbox-absence-proof.js"; const { privateKey, publicKey } = generateKeyPairSync("ed25519"); const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(), @@ -9,10 +10,10 @@ const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORI PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) }; const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000; -function receipt(overrides: Record = {}) { - const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url"); - const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack", - ...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url"); +function receipt(overrides: Record = {}, purpose = "sandbox-absence", source: Record = binding) { + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: `paperclip-${purpose}+jwt` })).toString("base64url"); + const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: `paperclip-${purpose}/v1`, sub: "fixture-stack", + ...source, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url"); return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`; } describe("sandbox absence proof", () => { @@ -45,3 +46,44 @@ describe("sandbox absence proof", () => { expect(fetchImpl).not.toHaveBeenCalled(); }); }); + +describe("cleanup-only sandbox absence", () => { + const cleanup = { purpose: "cleanup" as const, companyId: binding.companyId, environmentId: randomUUID(), + leaseId: binding.leaseId, heartbeatRunId: null as string | null, provider: binding.provider, providerLeaseId: binding.providerLeaseId }; + const lease = { id: cleanup.leaseId, ...cleanup }; + const signed = (overrides: Record = {}) => receipt(overrides, "sandbox-cleanup-absence", cleanup); + it.each([null, binding.runId])("accepts exact absence with nullable run %s without issuing a run receipt", async heartbeatRunId => { + const source = { ...cleanup, heartbeatRunId }; + const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt({}, "sandbox-cleanup-absence", source) })); + expect(await inspectCloudSandboxCleanupAbsence({ ...lease, heartbeatRunId }, { env, now, fetchImpl })) + .toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence"); + expect(JSON.parse(init.body)).toEqual(source); + expect(init.headers.authorization).not.toContain(env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN); + expect(init.redirect).toBe("error"); + }); + it("requires every binding, current stack, signature and freshness, and rejects recovery receipts", () => { + expect(verifySandboxCleanupAbsenceReceipt(signed(), cleanup, env, now)).toMatchObject(cleanup); + for (const overrides of [ + ...Object.keys(cleanup).map(key => ({ [key]: randomUUID() })), + { sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-sandbox-absence/v1" }, { iss: "other" }, + { iat: seconds + 31 }, { exp: seconds }, { exp: seconds + 61 }, + ]) expect(verifySandboxCleanupAbsenceReceipt(signed(overrides), cleanup, env, now)).toBeNull(); + expect(verifySandboxCleanupAbsenceReceipt(signed().slice(0, -8) + "tampered", cleanup, env, now)).toBeNull(); + expect(verifySandboxCleanupAbsenceReceipt(receipt(), cleanup, env, now)).toBeNull(); + expect(verifySandboxAbsenceReceipt(signed(), binding, env, now)).toBeNull(); + }); + it("keeps old Clouds, outages, invalid replies and unconfigured instances inconclusive", async () => { + for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: signed({ heartbeatRunId: binding.runId }) }]) + expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeUndefined(); + for (const status of [400, 401, 403, 503]) expect(await inspectCloudSandboxCleanupAbsence(lease, + { env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status })) })).toBeUndefined(); + expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeUndefined(); + const fetchImpl = vi.fn(); + expect(await inspectCloudSandboxCleanupAbsence(lease, { env: {}, now, fetchImpl })).toBeUndefined(); + for (const change of [{ provider: "other" }, { environmentId: null }, { providerLeaseId: "invalid" }]) + expect(await inspectCloudSandboxCleanupAbsence({ ...lease, ...change }, { env, now, fetchImpl })).toBeUndefined(); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/services/sandbox-absence-proof.ts b/server/src/services/sandbox-absence-proof.ts index 8ae852688ed..ae6044dd0af 100644 --- a/server/src/services/sandbox-absence-proof.ts +++ b/server/src/services/sandbox-absence-proof.ts @@ -7,8 +7,18 @@ const bindingSchema = z.object({ companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(), leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"), providerLeaseId: z.string().uuid(), -}); +}).strict(); +const cleanupBindingSchema = z.object({ + purpose: z.literal("cleanup"), companyId: z.string().uuid(), environmentId: z.string().uuid(), + leaseId: z.string().uuid(), heartbeatRunId: z.string().uuid().nullable(), + provider: z.literal("daytona"), providerLeaseId: z.string().uuid(), +}).strict(); export type SandboxAbsenceBinding = z.infer; +export type SandboxCleanupAbsenceBinding = z.infer; +export type SandboxCleanupAbsenceProof = SandboxCleanupAbsenceBinding & { + v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-cleanup-absence/v1"; sub: string; + absent: true; iat: number; exp: number; +}; export type SandboxAbsenceProof = SandboxAbsenceBinding & { v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string; absent: true; iat: number; exp: number; @@ -16,19 +26,30 @@ export type SandboxAbsenceProof = SandboxAbsenceBinding & { export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding, env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null { + const parsed = bindingSchema.safeParse(binding); + return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-absence", env, now) as SandboxAbsenceProof | null : null; +} + +export function verifySandboxCleanupAbsenceReceipt(receipt: string, binding: SandboxCleanupAbsenceBinding, + env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxCleanupAbsenceProof | null { + const parsed = cleanupBindingSchema.safeParse(binding); + return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-cleanup-absence", env, now) as SandboxCleanupAbsenceProof | null : null; +} + +function verifyReceipt(receipt: string, binding: Record, purpose: string, + env: NodeJS.ProcessEnv, now: Date): Record | null { try { - const parsed = bindingSchema.safeParse(binding); - if (receipt.length > 16_000 || !parsed.success) return null; + if (receipt.length > 16_000) return null; const parts = receipt.split("."); if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null; const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8")); - if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null; + if (header.alg !== "EdDSA" || header.typ !== `paperclip-${purpose}+jwt` || typeof header.kid !== "string") return null; if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null; - const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof; + const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")); const seconds = Math.floor(now.getTime() / 1000); - if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1" + if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== `paperclip-${purpose}/v1` || !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true - || Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding]) + || Object.keys(binding).some(key => proof[key] !== binding[key]) || !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30 || proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null; return proof; @@ -38,11 +59,35 @@ export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbs export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date; } = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> { + const parsed = bindingSchema.safeParse(binding); + if (!parsed.success) return null; + const receipt = await requestReceipt(parsed.data, options); + if (!receipt) return null; + const proof = verifySandboxAbsenceReceipt(receipt, parsed.data, options.env, options.now); + return proof ? { proof, receipt } : null; +} + +/** Cleanup-only observation: never grants continuation authority to a run. */ +export async function inspectCloudSandboxCleanupAbsence(lease: { + id: string; companyId: string; environmentId: string | null; heartbeatRunId: string | null; + provider: string | null; providerLeaseId: string | null; +}, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date } = {}) { + const parsed = cleanupBindingSchema.safeParse({ purpose: "cleanup", companyId: lease.companyId, + environmentId: lease.environmentId, leaseId: lease.id, heartbeatRunId: lease.heartbeatRunId, + provider: lease.provider, providerLeaseId: lease.providerLeaseId }); + if (!parsed.success) return undefined; + const receipt = await requestReceipt(parsed.data, options); + if (!receipt || !verifySandboxCleanupAbsenceReceipt(receipt, parsed.data, options.env, options.now)) return undefined; + return { providerLeaseId: parsed.data.providerLeaseId, state: "destroyed" as const }; +} + +async function requestReceipt(binding: Record, options: { + env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; +}): Promise { const env = options.env ?? process.env; const context = getCloudStackContext(env); const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET; - const parsed = bindingSchema.safeParse(binding); - if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null; + if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim()) return null; try { const origin = new URL(context.cloudOrigin); const host = new URL(`https://${context.primaryHost}`); @@ -51,12 +96,11 @@ export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, { method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000), headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" }, - body: JSON.stringify(parsed.data), + body: JSON.stringify(binding), }); if (!response.ok) return null; const payload = await response.json() as { receipt?: unknown }; if (typeof payload.receipt !== "string") return null; - const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now); - return proof ? { proof, receipt: payload.receipt } : null; + return payload.receipt; } catch { return null; } }