mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-31 22:24:16 +01:00
fix(server): reconcile failed sandbox cleanup with signed absence (#18407)
Reconcile failed ephemeral Cloud leases through a signed cleanup-only absence observation when local historical inspection is unavailable. Preserve failure outcomes, exact resource bindings, retention and row-version fences; grant no run-continuation authority. All 54 current-head checks/statuses passed. Greptile is 5/5, Medium, with no findings or unresolved threads. The 38 focused tests pass; both new reconciliation cases fail without the fallback. Isolated baseline and branch idle suites each pass all 225 cases. The original broader local teardown timeout is retained in the PR, and the full local run remains in progress. Cloud protocol support is already serving in staging. Exact candidate composition and hosted provider sleep/wake qualification follow this merge. Production has not been changed. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
105a67b52c
commit
87ae20ae65
7 files changed
+190
-20
No files matched your search
@@ -101,6 +101,17 @@ The manager permits at most two outstanding inspection RPCs across workers.
|
||||
Caller timeouts retain their slots until the actual worker reply or exit;
|
||||
timeouts cannot accumulate additional provider reads in later sweeps.
|
||||
|
||||
If a failed ephemeral lease has no usable local inspection path, a managed
|
||||
Daytona instance can ask Cloud for a cleanup-only absence receipt. Cloud checks
|
||||
the stack's historical credential organization and reads the exact allocation
|
||||
with its manager credential. Only confirmed absence can produce a signed receipt
|
||||
bound to the stack, company, environment, lease, provider handle and nullable run.
|
||||
The separate cleanup type and audience cannot authorize run recovery. Older
|
||||
Cloud servers, unsupported providers, invalid signatures, changed organizations
|
||||
and failed reads leave the lease unresolved. Existing bounded cleanup batches
|
||||
and retry deadlines own these reads; no new idle polling is added. An explicit
|
||||
local provider observation is retained rather than replaced by this fallback.
|
||||
|
||||
Host-owned verification metadata binds the company, environment, resource,
|
||||
provider handle, optional run, and exact row version. Concurrent resume,
|
||||
replacement, cleanup, or configuration changes invalidate verification.
|
||||
|
||||
@@ -62,6 +62,14 @@ authority as company-agent API access.
|
||||
Gaps: Instance start/stop and task workspace start/stop are different capabilities.
|
||||
Local service control is not agent company authority.
|
||||
|
||||
Historical Cloud sandbox cleanup remains server-owned. Cleanup-only absence
|
||||
reads authenticate the tenant host with its server capability; they expose no
|
||||
new board or agent endpoint. Their signed scope cannot authorize task recovery.
|
||||
[Runtime and idle-gate regressions](../../server/src/__tests__/historical-cleanup-verification.test.ts)
|
||||
cover reconciliation, while [receipt checks](../../server/src/services/sandbox-absence-proof.test.ts)
|
||||
cover signature, identity, freshness and separation from recovery authority.
|
||||
These checks do not establish a deployed sleep/wake result.
|
||||
|
||||
## `configuration`
|
||||
|
||||
Capability ID: `instance-operations/configuration`.
|
||||
|
||||
@@ -61,6 +61,15 @@ served by the admitted application before it finishes. Repeat with a disconnect
|
||||
before headers arrive. Confirm that neither leaves a permanent request hold. An
|
||||
unfinished application write must still prevent sleep after its client leaves.
|
||||
|
||||
Automated: [historical cleanup](../../server/src/__tests__/historical-cleanup-verification.test.ts)
|
||||
exercises the runtime inspection path for failed Cloud sandbox leases, including
|
||||
verification leases without a run. A signed cleanup-only absence observation
|
||||
can remove this blocker without changing failed outcomes or granting run
|
||||
continuation. An unresolved provider resource still prevents idle sleep.
|
||||
Manual: on an owned Cloud staging workspace, confirm the exact published build,
|
||||
reconcile failed test allocations, then verify actual provider suspension and
|
||||
normal hosted entry after wake. Local tests do not prove that hosted journey.
|
||||
|
||||
### `settings-backup`
|
||||
|
||||
Cloud startup reads nonsecret application choices from a private, versioned
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db";
|
||||
import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
|
||||
@@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
|
||||
(support.supported ? describe : describe.skip)("historical cleanup verification", () => {
|
||||
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000);
|
||||
afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true);
|
||||
const state = async () => (await read()).backgroundWork;
|
||||
@@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
|
||||
inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }),
|
||||
});
|
||||
}
|
||||
it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => {
|
||||
const f = await seed(adHoc);
|
||||
const { privateKey, publicKey } = generateKeyPairSync("ed25519");
|
||||
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
|
||||
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
|
||||
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value);
|
||||
let absent = false;
|
||||
const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => {
|
||||
if (!absent) return Response.json({ absent: false });
|
||||
const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000);
|
||||
expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id,
|
||||
leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId });
|
||||
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url");
|
||||
const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1",
|
||||
sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url");
|
||||
return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` });
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchImpl);
|
||||
const runtime = environmentRuntimeService(db);
|
||||
const service = historicalCleanupVerificationService(db, {
|
||||
inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined,
|
||||
});
|
||||
expect(await state()).toBe("present");
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 });
|
||||
expect(await state()).toBe("present");
|
||||
absent = true;
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
|
||||
expect(await state()).toBe("none");
|
||||
const [current] = await db.select().from(environmentLeases);
|
||||
expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId });
|
||||
expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination");
|
||||
expect(await service.sweep()).toMatchObject({ checked: 0 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => {
|
||||
await seed(true);
|
||||
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
|
||||
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value);
|
||||
const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl);
|
||||
const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox",
|
||||
acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null,
|
||||
inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }),
|
||||
}] });
|
||||
const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined });
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 });
|
||||
expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => {
|
||||
const f = await seed(adHoc); expect(await state()).toBe("present");
|
||||
const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
|
||||
|
||||
@@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s
|
||||
import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
|
||||
import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js";
|
||||
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js";
|
||||
import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
|
||||
@@ -4021,7 +4022,12 @@ export function environmentRuntimeService(
|
||||
const lease = await environmentsSvc.getLeaseById(leaseId);
|
||||
if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined;
|
||||
const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null;
|
||||
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease });
|
||||
// Legacy Cloud leases can lack a credential version that permits a
|
||||
// provider-local observation. Cloud's manager verifies the historical
|
||||
// organization and signs only exact absence; no replacement key or
|
||||
// cleanup label is treated as proof. Explicit provider answers win.
|
||||
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease })
|
||||
?? await inspectCloudSandboxCleanupAbsence(lease);
|
||||
},
|
||||
|
||||
async retryPendingSandboxTeardown(input: {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js";
|
||||
import { inspectCloudSandboxCleanupAbsence, requestSandboxAbsenceProof, verifySandboxAbsenceReceipt,
|
||||
verifySandboxCleanupAbsenceReceipt } from "./sandbox-absence-proof.js";
|
||||
|
||||
const { privateKey, publicKey } = generateKeyPairSync("ed25519");
|
||||
const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(),
|
||||
@@ -9,10 +10,10 @@ const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORI
|
||||
PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
|
||||
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) };
|
||||
const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000;
|
||||
function receipt(overrides: Record<string, unknown> = {}) {
|
||||
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url");
|
||||
const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack",
|
||||
...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
|
||||
function receipt(overrides: Record<string, unknown> = {}, purpose = "sandbox-absence", source: Record<string, unknown> = binding) {
|
||||
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: `paperclip-${purpose}+jwt` })).toString("base64url");
|
||||
const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: `paperclip-${purpose}/v1`, sub: "fixture-stack",
|
||||
...source, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
|
||||
return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`;
|
||||
}
|
||||
describe("sandbox absence proof", () => {
|
||||
@@ -45,3 +46,44 @@ describe("sandbox absence proof", () => {
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("cleanup-only sandbox absence", () => {
|
||||
const cleanup = { purpose: "cleanup" as const, companyId: binding.companyId, environmentId: randomUUID(),
|
||||
leaseId: binding.leaseId, heartbeatRunId: null as string | null, provider: binding.provider, providerLeaseId: binding.providerLeaseId };
|
||||
const lease = { id: cleanup.leaseId, ...cleanup };
|
||||
const signed = (overrides: Record<string, unknown> = {}) => receipt(overrides, "sandbox-cleanup-absence", cleanup);
|
||||
it.each([null, binding.runId])("accepts exact absence with nullable run %s without issuing a run receipt", async heartbeatRunId => {
|
||||
const source = { ...cleanup, heartbeatRunId };
|
||||
const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt({}, "sandbox-cleanup-absence", source) }));
|
||||
expect(await inspectCloudSandboxCleanupAbsence({ ...lease, heartbeatRunId }, { env, now, fetchImpl }))
|
||||
.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" });
|
||||
const [url, init] = fetchImpl.mock.calls[0];
|
||||
expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence");
|
||||
expect(JSON.parse(init.body)).toEqual(source);
|
||||
expect(init.headers.authorization).not.toContain(env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN);
|
||||
expect(init.redirect).toBe("error");
|
||||
});
|
||||
it("requires every binding, current stack, signature and freshness, and rejects recovery receipts", () => {
|
||||
expect(verifySandboxCleanupAbsenceReceipt(signed(), cleanup, env, now)).toMatchObject(cleanup);
|
||||
for (const overrides of [
|
||||
...Object.keys(cleanup).map(key => ({ [key]: randomUUID() })),
|
||||
{ sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-sandbox-absence/v1" }, { iss: "other" },
|
||||
{ iat: seconds + 31 }, { exp: seconds }, { exp: seconds + 61 },
|
||||
]) expect(verifySandboxCleanupAbsenceReceipt(signed(overrides), cleanup, env, now)).toBeNull();
|
||||
expect(verifySandboxCleanupAbsenceReceipt(signed().slice(0, -8) + "tampered", cleanup, env, now)).toBeNull();
|
||||
expect(verifySandboxCleanupAbsenceReceipt(receipt(), cleanup, env, now)).toBeNull();
|
||||
expect(verifySandboxAbsenceReceipt(signed(), binding, env, now)).toBeNull();
|
||||
});
|
||||
it("keeps old Clouds, outages, invalid replies and unconfigured instances inconclusive", async () => {
|
||||
for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: signed({ heartbeatRunId: binding.runId }) }])
|
||||
expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeUndefined();
|
||||
for (const status of [400, 401, 403, 503]) expect(await inspectCloudSandboxCleanupAbsence(lease,
|
||||
{ env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status })) })).toBeUndefined();
|
||||
expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeUndefined();
|
||||
const fetchImpl = vi.fn();
|
||||
expect(await inspectCloudSandboxCleanupAbsence(lease, { env: {}, now, fetchImpl })).toBeUndefined();
|
||||
for (const change of [{ provider: "other" }, { environmentId: null }, { providerLeaseId: "invalid" }])
|
||||
expect(await inspectCloudSandboxCleanupAbsence({ ...lease, ...change }, { env, now, fetchImpl })).toBeUndefined();
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -7,8 +7,18 @@ const bindingSchema = z.object({
|
||||
companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(),
|
||||
leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"),
|
||||
providerLeaseId: z.string().uuid(),
|
||||
});
|
||||
}).strict();
|
||||
const cleanupBindingSchema = z.object({
|
||||
purpose: z.literal("cleanup"), companyId: z.string().uuid(), environmentId: z.string().uuid(),
|
||||
leaseId: z.string().uuid(), heartbeatRunId: z.string().uuid().nullable(),
|
||||
provider: z.literal("daytona"), providerLeaseId: z.string().uuid(),
|
||||
}).strict();
|
||||
export type SandboxAbsenceBinding = z.infer<typeof bindingSchema>;
|
||||
export type SandboxCleanupAbsenceBinding = z.infer<typeof cleanupBindingSchema>;
|
||||
export type SandboxCleanupAbsenceProof = SandboxCleanupAbsenceBinding & {
|
||||
v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-cleanup-absence/v1"; sub: string;
|
||||
absent: true; iat: number; exp: number;
|
||||
};
|
||||
export type SandboxAbsenceProof = SandboxAbsenceBinding & {
|
||||
v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string;
|
||||
absent: true; iat: number; exp: number;
|
||||
@@ -16,19 +26,30 @@ export type SandboxAbsenceProof = SandboxAbsenceBinding & {
|
||||
|
||||
export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding,
|
||||
env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null {
|
||||
const parsed = bindingSchema.safeParse(binding);
|
||||
return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-absence", env, now) as SandboxAbsenceProof | null : null;
|
||||
}
|
||||
|
||||
export function verifySandboxCleanupAbsenceReceipt(receipt: string, binding: SandboxCleanupAbsenceBinding,
|
||||
env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxCleanupAbsenceProof | null {
|
||||
const parsed = cleanupBindingSchema.safeParse(binding);
|
||||
return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-cleanup-absence", env, now) as SandboxCleanupAbsenceProof | null : null;
|
||||
}
|
||||
|
||||
function verifyReceipt(receipt: string, binding: Record<string, unknown>, purpose: string,
|
||||
env: NodeJS.ProcessEnv, now: Date): Record<string, unknown> | null {
|
||||
try {
|
||||
const parsed = bindingSchema.safeParse(binding);
|
||||
if (receipt.length > 16_000 || !parsed.success) return null;
|
||||
if (receipt.length > 16_000) return null;
|
||||
const parts = receipt.split(".");
|
||||
if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null;
|
||||
const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8"));
|
||||
if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null;
|
||||
if (header.alg !== "EdDSA" || header.typ !== `paperclip-${purpose}+jwt` || typeof header.kid !== "string") return null;
|
||||
if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null;
|
||||
const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof;
|
||||
const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8"));
|
||||
const seconds = Math.floor(now.getTime() / 1000);
|
||||
if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1"
|
||||
if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== `paperclip-${purpose}/v1`
|
||||
|| !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true
|
||||
|| Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding])
|
||||
|| Object.keys(binding).some(key => proof[key] !== binding[key])
|
||||
|| !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30
|
||||
|| proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null;
|
||||
return proof;
|
||||
@@ -38,11 +59,35 @@ export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbs
|
||||
export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: {
|
||||
env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date;
|
||||
} = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> {
|
||||
const parsed = bindingSchema.safeParse(binding);
|
||||
if (!parsed.success) return null;
|
||||
const receipt = await requestReceipt(parsed.data, options);
|
||||
if (!receipt) return null;
|
||||
const proof = verifySandboxAbsenceReceipt(receipt, parsed.data, options.env, options.now);
|
||||
return proof ? { proof, receipt } : null;
|
||||
}
|
||||
|
||||
/** Cleanup-only observation: never grants continuation authority to a run. */
|
||||
export async function inspectCloudSandboxCleanupAbsence(lease: {
|
||||
id: string; companyId: string; environmentId: string | null; heartbeatRunId: string | null;
|
||||
provider: string | null; providerLeaseId: string | null;
|
||||
}, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date } = {}) {
|
||||
const parsed = cleanupBindingSchema.safeParse({ purpose: "cleanup", companyId: lease.companyId,
|
||||
environmentId: lease.environmentId, leaseId: lease.id, heartbeatRunId: lease.heartbeatRunId,
|
||||
provider: lease.provider, providerLeaseId: lease.providerLeaseId });
|
||||
if (!parsed.success) return undefined;
|
||||
const receipt = await requestReceipt(parsed.data, options);
|
||||
if (!receipt || !verifySandboxCleanupAbsenceReceipt(receipt, parsed.data, options.env, options.now)) return undefined;
|
||||
return { providerLeaseId: parsed.data.providerLeaseId, state: "destroyed" as const };
|
||||
}
|
||||
|
||||
async function requestReceipt(binding: Record<string, unknown>, options: {
|
||||
env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch;
|
||||
}): Promise<string | null> {
|
||||
const env = options.env ?? process.env;
|
||||
const context = getCloudStackContext(env);
|
||||
const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET;
|
||||
const parsed = bindingSchema.safeParse(binding);
|
||||
if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null;
|
||||
if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim()) return null;
|
||||
try {
|
||||
const origin = new URL(context.cloudOrigin);
|
||||
const host = new URL(`https://${context.primaryHost}`);
|
||||
@@ -51,12 +96,11 @@ export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding,
|
||||
const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, {
|
||||
method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000),
|
||||
headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" },
|
||||
body: JSON.stringify(parsed.data),
|
||||
body: JSON.stringify(binding),
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
const payload = await response.json() as { receipt?: unknown };
|
||||
if (typeof payload.receipt !== "string") return null;
|
||||
const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now);
|
||||
return proof ? { proof, receipt: payload.receipt } : null;
|
||||
return payload.receipt;
|
||||
} catch { return null; }
|
||||
}
|
||||
Reference in new issue
Block a user