diff --git a/doc/task-recovery.md b/doc/task-recovery.md index 9198e53c9e5..6773c02b4be 100644 --- a/doc/task-recovery.md +++ b/doc/task-recovery.md @@ -47,6 +47,19 @@ Execution containment and file preservation are independent facts: `isolated-workspace-attempts` namespace. It does not modify the original directory or satisfy that source's preservation/cleanup obligation. +A missing managed sandbox cannot answer another stop request. Core can request +a signed absence observation from Cloud under a host-bound server capability. +Cloud reads the exact allocation in the historical credential's organization. +Only a fresh signature for the exact stack, company, issue, run, lease, +allocation and physical recovery owner can settle execution absence. Permission +failures, a changed organization, unsigned answers and provider timeouts remain +inconclusive. This protocol requires Cloud support before the Core consumer. + +An absence receipt does not recover files or complete the task. The original run +stays failed and its source remains protected. Existing authorized input may use +normal fresh workspace admission only after every executor is contained. Known +publications and uncertain external effects retain their normal receipt fences. + Fresh attempts currently require Core-managed storage. Plugin-owned storage does not yet support importing these preserved snapshots. Preparation stops before calling that provider or acquiring an environment. It retains the diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md index 77ea407fd97..0cee692325b 100644 --- a/maps/agent-access-map/recovery.md +++ b/maps/agent-access-map/recovery.md @@ -218,6 +218,14 @@ held physical workspace owner, and its current stop-only cleanup claim. The ordinary resume and destruction fences still apply to protected sources. The assembled-runtime regression checks this separation before any provider call. +Managed missing-source reconciliation uses the server-only Cloud +`POST /v1/tenant/sandbox-absence` protocol. A host-bound capability authenticates +the stack; board and agent credentials cannot call it. Core requires a fresh +Cloud signature for the exact company, issue, run, lease, allocation and physical +owner. This can establish execution absence, never recovered files or successful +task completion. Normal Retry admission and external-effect fences still apply. +Signature and PostgreSQL regressions are local evidence, not deployed acceptance. + Review: `operation-reviewed`. Each workspace repair/export/reconstruction/physical-owner operation has its own proof and authority. diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 4fb2ff30d4d..66be99d9dd4 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -230,6 +230,14 @@ provider call. Verify saved and newer local files, one preservation receipt, and no additional agent turn. The PostgreSQL regression covers this boundary; the opt-in provider probe also uses this runtime path with the protection marker. +For a missing managed allocation, verify a fresh Cloud-signed absence observation +under the exact physical recovery owner. The existing Retry may then select an +independent workspace through normal admission. Verify that the original run +stays failed and missing files are not called recovered. Refuse a foreign stack, +source, allocation or owner, expired signatures, changed provider organizations, +permission failures and provider outages. Local PostgreSQL and signed-protocol +regressions cover these boundaries; deployed recovery requires separate proof. + Repeat source-only repair with a `succeeded` legacy run that still has a retained file obligation. Its result must remain successful, its current task owner and status must stay unchanged, and later saved input must become eligible for normal diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index af91a95ebb1..855f6ee5714 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -72368,7 +72368,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { status: "running", contextSnapshot: {}, }); - const runsBefore = await readHeartbeatRunsByCompanyId(lane.fixture.companyId); + const runsBefore = (await readHeartbeatRunsByCompanyId(lane.fixture.companyId)) + .toSorted((a, b) => a.id.localeCompare(b.id)); let stoppedId = 0; lane.setHook(async (method, body) => { if (method.endsWith("Draft") && !stoppedId) { @@ -72397,7 +72398,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { .where(eq(issueComments.id, publication!.commentId!)); expect(comment!.body).toContain("END-first-stopped"); expect( - await readHeartbeatRunsByCompanyId(lane.fixture.companyId), + (await readHeartbeatRunsByCompanyId(lane.fixture.companyId)) + .toSorted((a, b) => a.id.localeCompare(b.id)), ).toEqual(runsBefore); expect(lane.context.cancelRun).not.toHaveBeenCalled(); expect( diff --git a/server/src/services/sandbox-absence-proof.test.ts b/server/src/services/sandbox-absence-proof.test.ts new file mode 100644 index 00000000000..d145425137e --- /dev/null +++ b/server/src/services/sandbox-absence-proof.test.ts @@ -0,0 +1,47 @@ +import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js"; + +const { privateKey, publicKey } = generateKeyPairSync("ed25519"); +const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(), + ownerToken: randomUUID(), provider: "daytona" as const, providerLeaseId: randomUUID() }; +const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret", + PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) }; +const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000; +function receipt(overrides: Record = {}) { + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url"); + const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack", + ...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url"); + return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`; +} +describe("sandbox absence proof", () => { + it("requires signed, fresh proof for every exact source and physical owner field", () => { + expect(verifySandboxAbsenceReceipt(receipt(), binding, env, now)).toMatchObject({ ...binding, absent: true }); + for (const overrides of [ + ...Object.keys(binding).map(key => ({ [key]: key === "provider" ? "other" : randomUUID() })), + { sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-workspace-worker-stop/v1" }, { iss: "other" }, + { iat: seconds + 31, exp: seconds + 60 }, { exp: seconds }, { exp: seconds + 61 }, + ]) expect(verifySandboxAbsenceReceipt(receipt(overrides), binding, env, now)).toBeNull(); + expect(verifySandboxAbsenceReceipt(receipt().slice(0, -8) + "tampered", binding, env, now)).toBeNull(); + }); + it("uses a host-bound purpose capability, never the fleet token or a browser credential", async () => { + const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt() })); + expect(await requestSandboxAbsenceProof(binding, { env, now, fetchImpl })).toMatchObject({ proof: { absent: true } }); + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence"); + expect(init.headers).toEqual({ authorization: `Bearer ${createHmac("sha256", "fixture-secret").update("paperclip-cloud-sandbox-absence-v1\nfixture.paperclip.test").digest("hex")}`, + "content-type": "application/json", "x-paperclip-cloud-tenant-host": "fixture.paperclip.test" }); + expect(JSON.parse(init.body)).toEqual(binding); + expect(init.redirect).toBe("error"); + }); + it("keeps outages, authentication errors, missing configuration and unsigned answers inconclusive", async () => { + for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: receipt({ ownerToken: randomUUID() }) }]) + expect(await requestSandboxAbsenceProof(binding, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeNull(); + expect(await requestSandboxAbsenceProof(binding, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeNull(); + expect(await requestSandboxAbsenceProof(binding, { env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status: 403 })) })).toBeNull(); + const fetchImpl = vi.fn(); + expect(await requestSandboxAbsenceProof(binding, { env: {}, now, fetchImpl })).toBeNull(); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/services/sandbox-absence-proof.ts b/server/src/services/sandbox-absence-proof.ts new file mode 100644 index 00000000000..8ae852688ed --- /dev/null +++ b/server/src/services/sandbox-absence-proof.ts @@ -0,0 +1,62 @@ +import { createHmac, verify } from "node:crypto"; +import { z } from "zod"; +import { cloudRuntimePublicKeyForKid } from "./cloud-runtime-identity.js"; +import { getCloudStackContext } from "./cloud-instance.js"; + +const bindingSchema = z.object({ + companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(), + leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"), + providerLeaseId: z.string().uuid(), +}); +export type SandboxAbsenceBinding = z.infer; +export type SandboxAbsenceProof = SandboxAbsenceBinding & { + v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string; + absent: true; iat: number; exp: number; +}; + +export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding, + env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null { + try { + const parsed = bindingSchema.safeParse(binding); + if (receipt.length > 16_000 || !parsed.success) return null; + const parts = receipt.split("."); + if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null; + const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8")); + if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null; + if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null; + const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof; + const seconds = Math.floor(now.getTime() / 1000); + if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1" + || !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true + || Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding]) + || !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30 + || proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null; + return proof; + } catch { return null; } +} + +export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: { + env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date; +} = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> { + const env = options.env ?? process.env; + const context = getCloudStackContext(env); + const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET; + const parsed = bindingSchema.safeParse(binding); + if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null; + try { + const origin = new URL(context.cloudOrigin); + const host = new URL(`https://${context.primaryHost}`); + if (origin.protocol !== "https:" || origin.username || origin.password || host.hostname !== context.primaryHost.toLowerCase()) return null; + const capability = createHmac("sha256", secret).update(`paperclip-cloud-sandbox-absence-v1\n${host.hostname}`).digest("hex"); + const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, { + method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000), + headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" }, + body: JSON.stringify(parsed.data), + }); + if (!response.ok) return null; + const payload = await response.json() as { receipt?: unknown }; + if (typeof payload.receipt !== "string") return null; + const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now); + return proof ? { proof, receipt: payload.receipt } : null; + } catch { return null; } +} diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts index 86fabf4a849..01f96262f19 100644 --- a/server/src/services/sandbox-stop-and-retain.ts +++ b/server/src/services/sandbox-stop-and-retain.ts @@ -1,9 +1,10 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql, type SQL } from "drizzle-orm"; -import { environmentLeases, type Db } from "@paperclipai/db"; +import { environmentLeases, heartbeatRuns, type Db } from "@paperclipai/db"; import { isBuiltinSandboxProvider } from "./sandbox-provider-runtime.js"; import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "./remote-execution-termination.js"; import { hasNativeWorkspaceExportResume, readNativeWorkspaceExportResume, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js"; +import type { SandboxAbsenceProof } from "./sandbox-absence-proof.js"; type Lease = Pick; @@ -148,3 +149,28 @@ export async function settleStopOnlyCleanup(db: Db, lease: Lease, options: { att )).returning(); return updated ?? null; } + +export async function settleAbsentSandbox(db: Db, lease: typeof environmentLeases.$inferSelect, + evidence: { proof: SandboxAbsenceProof; receipt: string }) { + const { proof } = evidence; + if (proof.companyId !== lease.companyId || proof.runId !== lease.heartbeatRunId || proof.issueId !== lease.issueId + || proof.leaseId !== lease.id || proof.provider !== lease.provider || proof.providerLeaseId !== lease.providerLeaseId) return false; + const termination = remoteTerminationReceipt(lease, { providerLeaseId: proof.providerLeaseId, state: "destroyed" }); + if (!termination) return false; + const now = new Date(); + const changed = await db.update(environmentLeases).set({ + status: "released", cleanupStatus: "success", failureReason: "workspace_source_unavailable", releasedAt: now, updatedAt: now, + metadata: sql`(${environmentLeases.metadata} - 'sandboxStopAndRetain' - 'nativeWorkspaceExportResume' - 'sandboxStopAndRetainReceipt') || ${JSON.stringify({ + remoteExecutionTermination: termination, sandboxAbsence: evidence, + pendingCleanupInFlight: false, pendingCleanupLeaseExpiresAtMs: 0, + })}::jsonb`, + }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), eq(environmentLeases.providerLeaseId, proof.providerLeaseId), + eq(environmentLeases.status, lease.status), + sql`exists (select 1 from ${heartbeatRuns} where ${heartbeatRuns.id} = ${lease.heartbeatRunId} + and ${heartbeatRuns.companyId} = ${lease.companyId} + and ${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceFinalizationOwner,token}' = ${proof.ownerToken})`, + sql`${environmentLeases.metadata} is not distinct from ${lease.metadata === null ? null : JSON.stringify(lease.metadata)}::jsonb`, + )).returning({ id: environmentLeases.id }); + return changed.length === 1; +} diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 6b10de5b617..9edfdf1d391 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -16,7 +16,7 @@ import { createSystemRecovery, failSystemRecovery, SystemRecoveryFailure } from import { requestTaskRetry } from "./task-retry.js"; import { readFreshWorkspaceReadiness } from "./fresh-workspace-continuation.js"; import { execFile } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -29,6 +29,7 @@ import type { EnvironmentLease } from "@paperclipai/shared"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { prepareNativeWorkspaceSync } from "./native-runtime/native-workspace-sync.js"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { prepareStoppedSandboxForRecovery } from "./sandbox-stop-and-retain.js"; import { recoverRetainedWorkspace, requestWorkspaceRecovery, scheduleRecoveredWorkspaceCleanup, reconcileRetainedWorkspaces, retainedWorkspaceCandidates } from "./workspace-recovery.js"; import * as activity from "./activity-log.js"; import { heartbeatService } from "./heartbeat.js"; @@ -579,6 +580,82 @@ describe("automatic retained workspace recovery", () => { expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n"); }, 30_000); + it.each(["single", "multiple", "interrupted"])("continues an existing Retry after signed absence evidence for every retained allocation: %s", async mode => { + const count = mode === "single" ? 1 : 2; + const f = await seed(); + const leaseIds = [f.lease.id]; + if (count === 2) { + const [additional] = await db.insert(environmentLeases).values({ ...f.lease, id: randomUUID(), + providerLeaseId: randomUUID(), status: "pending_cleanup", cleanupStatus: "failed", releasedAt: null }).returning(); + leaseIds.push(additional.id); + const recovery = { schema: "paperclip.workspace-restore-recovery.v1", leaseIds }; + await db.update(heartbeatRuns).set({ resultJson: sql`${heartbeatRuns.resultJson} || ${JSON.stringify({ workspaceRestoreRecovery: recovery })}::jsonb` }) + .where(eq(heartbeatRuns.id, f.runId)); + await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ workspaceRestoreRecovery: recovery })}::jsonb` }) + .where(eq(issueRecoveryActions.id, f.actionId)); + } + const source = await run(f.runId); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" }); + const [stoppedLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(await prepareStoppedSandboxForRecovery(db, stoppedLease)).not.toBeNull(); + f.runtime.resumeRunLease.mockRejectedValue(new Error("retained allocation unavailable")); + f.runtime.retryPendingSandboxTeardown.mockRejectedValue(new Error("retained allocation unavailable")); + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board", + actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() }); + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "fixture-stack"); + vi.stubEnv("PAPERCLIP_CLOUD_API_ORIGIN", "https://cloud.example"); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "fixture-token"); + vi.stubEnv("PAPERCLIP_PRIMARY_HOST", "fixture.paperclip.test"); + vi.stubEnv("PAPERCLIP_AUTH_SECRET", "fixture-auth-secret"); + vi.stubEnv("PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS", JSON.stringify({ keys: [{ + ...publicKey.export({ format: "jwk" }), kid: "absence-test", alg: "EdDSA", use: "sig", + }] })); + const originalFetch = globalThis.fetch; + const observed: string[] = []; + let interrupted = mode === "interrupted"; + const fetchProof = vi.spyOn(globalThis, "fetch").mockImplementation(async (url, init) => { + if (!String(url).endsWith("/v1/tenant/sandbox-absence")) return originalFetch(url, init); + const binding = JSON.parse(String(init?.body)); + observed.push(binding.leaseId); + if (mode === "interrupted" && binding.leaseId === f.lease.id && observed.length > 2) return new Response(null, { status: 503 }); + if (interrupted && binding.leaseId !== f.lease.id) return new Response(null, { status: 503 }); + const seconds = Math.floor(Date.now() / 1000); + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", typ: "paperclip-sandbox-absence+jwt", kid: "absence-test" })).toString("base64url"); + const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", + sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url"); + return Response.json({ receipt: `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}` }); + }); + try { + await recoverRetainedWorkspace(f.input); + if (interrupted) { + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).not.toBe("exhausted"); + interrupted = false; + await makeRetryDue(f.actionId); + await recoverRetainedWorkspace(f.input); + } + const leases = await db.select().from(environmentLeases).where(inArray(environmentLeases.id, leaseIds)); + expect(leases).toHaveLength(count); + expect(observed).toEqual(mode === "interrupted" ? [leaseIds[0], leaseIds[1], leaseIds[1]] : leaseIds); + for (const lease of leases) expect(lease).toMatchObject({ status: "released", cleanupStatus: "success", + metadata: { remoteExecutionTermination: { state: "destroyed" } } }); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ + state: "exhausted", failure: { code: "source_unavailable", retryClass: "use_alternative_strategy" }, + }); + expect(f.runtime.resumeRunLease).not.toHaveBeenCalled(); + expect(f.runtime.retryPendingSandboxTeardown).not.toHaveBeenCalled(); + expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.companyId, f.companyId))).toHaveLength(0); + const heartbeat = heartbeatService(db); + await Promise.all([heartbeat.resumeTaskRetry(f.companyId, request.requestId!), heartbeat.resumeTaskRetry(f.companyId, request.requestId!)]); + const turns = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued"))); + expect(turns).toHaveLength(1); + expect(await readFreshWorkspaceReadiness(db, turns[0])).toMatchObject({ mode: "fresh_isolated_workspace", retryIntentId: request.requestId }); + expect((await run(f.runId)).status).toBe("failed"); + expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n"); + } finally { fetchProof.mockRestore(); vi.unstubAllEnvs(); } + }, 30_000); + it.each(["pause", "cancelled", "approval", "budget", "reassigned"])("keeps the %s gate after files recover", async gate => { const f = await seed(), commentId = randomUUID(), queueId = randomUUID(); const source = await run(f.runId); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index ec050a5b519..984adf7fb63 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -7,7 +7,8 @@ import { logger } from "../middleware/logger.js"; import type { EnvironmentRuntimeService } from "./environment-runtime.js"; import { resolveEnvironmentExecutionTarget } from "./environment-execution-target.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; -import { hasStopOnlyCleanup, prepareHistoricalSandboxStopVerification, prepareSandboxStopAndRetain, prepareStoppedSandboxForRecovery, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; +import { hasStopOnlyCleanup, prepareHistoricalSandboxStopVerification, prepareSandboxStopAndRetain, prepareStoppedSandboxForRecovery, settleAbsentSandbox, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; +import { requestSandboxAbsenceProof } from "./sandbox-absence-proof.js"; import { persistActivity, publishActivity } from "./activity-log.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; import { hasVerifiedWorkspaceRepair, readWorkspaceRecovery, type WorkspaceRecovery } from "./workspace-recovery-receipt.js"; @@ -101,8 +102,24 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ try { await observeActive(); const inspected = await inspectWorkspaceRecoverySource(db, input); + let sourceUnavailable = false; for (const source of inspected.sources) { const { lease, environment, remoteCwd } = source; + if (hasRemoteTerminationReceipt(lease) + && (lease.metadata?.remoteExecutionTermination as Record | undefined)?.state === "destroyed") { + sourceUnavailable = true; + continue; + } + if (lease.provider === "daytona" && lease.providerLeaseId) { + const absence = await requestSandboxAbsenceProof({ companyId: input.companyId, issueId: input.issueId, runId: input.runId, leaseId: lease.id, + provider: "daytona", providerLeaseId: lease.providerLeaseId, ownerToken: ownership.token }); + if (absence) { + await ownership.assertHeld(); + if (!await settleAbsentSandbox(db, lease, absence)) throw new SystemRecoveryFailure("agent_stop_unconfirmed", "recheck_evidence"); + sourceUnavailable = true; + continue; + } + } let reference = source.reference; if (reference) { try { await readNativeWorkspaceSyncLocalCwd({ runId: input.runId, companyId: input.companyId, @@ -221,6 +238,7 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ } } await ownership.assertHeld(); + if (sourceUnavailable) throw new SystemRecoveryFailure("source_unavailable", "use_alternative_strategy"); const next = { ...current!, revision: current!.revision + 1, phase: "reconcile_effects" as const, state: "complete" as const, nextAttemptAt: null }; const activity = await db.transaction(async tx => {