diff --git a/packages/shared/src/crash-point-registry.ts b/packages/shared/src/crash-point-registry.ts index 708b1e43103..8d6f60e8ad8 100644 --- a/packages/shared/src/crash-point-registry.ts +++ b/packages/shared/src/crash-point-registry.ts @@ -28,6 +28,13 @@ export type CrashPointDefinition = { export const CRASH_POINT_NAME_PATTERN = /^[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*\.(before|after)_[a-z0-9_]+$/; export const CRASH_POINTS = { + "remote_workspace.prepare.after_publish_before_provision": { + description: "A run restored its remote workspace's saved version and published the root as its dirty epoch, with the unchanged baseline; no provisioning, runner command or agent process has started. Recovery must hand the workspace on at its saved version and keep the root, not abandon the epoch.", + environments: ["daytona"], + kinds: ["remote_workspace_unsaved_files"], + invariants: ["I2", "I7"], + prerequisite: "a previously saved remote workspace: the point counts only a preparation that restores a saved version (a follow-up or continuation run, for example workload S1), never a workspace's first preparation", + }, "sandbox.create.after_intent_before_provider": { description: "The sandbox_creation_operations intent is committed; the provider create call has not been made. Recovery must settle the intent without a sandbox to find.", environments: ["daytona"], diff --git a/server/src/__tests__/ownership-reconciliation-report.test.ts b/server/src/__tests__/ownership-reconciliation-report.test.ts index ea11b0b60eb..fa8573c50d2 100644 --- a/server/src/__tests__/ownership-reconciliation-report.test.ts +++ b/server/src/__tests__/ownership-reconciliation-report.test.ts @@ -527,6 +527,66 @@ describe("report-only ownership reconciliation", () => { .toEqual({ kind: "free" }); }); + it("hands on a preparation-only epoch at its saved version only when the unchanged baseline is proven", async () => { + // Each writer restored saved version `snapshotId` and was killed in + // preparation. Only `clean` has nothing after the restore; every other + // writer lacks the proof and keeps today's abandonment. + const cases = ["clean", "noBaseline", "otherEpochBaseline", "releasedBaseline", "provisioned", "processStarted"] as const; + const snapshotId = randomUUID(); + const setup = Object.fromEntries(cases.map(name => [name, { + runId: randomUUID(), workspaceId: randomUUID(), leaseId: randomUUID(), providerLeaseId: `sandbox-${randomUUID()}`, + }])) as Record<(typeof cases)[number], { runId: string; workspaceId: string; leaseId: string; providerLeaseId: string }>; + const remoteCwd = (name: string) => `/home/daytona/paperclip-workspace/.paperclip-workspaces/${setup[name as keyof typeof setup].workspaceId}`; + await db.insert(heartbeatRuns).values(cases.map(name => { + const s = setup[name]; + return { id: s.runId, companyId, agentId, status: "failed", invocationSource: "manual", executionStage: "preparing", + finishedAt: minutesAgo(2), errorCode: "process_lost", + ...(name === "processStarted" ? { processPid: 4_194_303 } : {}), + runnerProfileJson: { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId: s.workspaceId, + leaseId: s.leaseId, providerLeaseId: s.providerLeaseId, remoteCwd: remoteCwd(name), epoch: 2, state: "prepared", snapshotId } } }; + })); + await db.insert(executionWorkspaces).values(cases.map(name => { + const s = setup[name]; + const baseline = name === "noBaseline" ? {} + : { remoteWorkspaceUnchangedBaseline: { schema: "paperclip.remote-workspace-unchanged-baseline.v1", + epoch: name === "otherEpochBaseline" ? 1 : 2, runId: s.runId, snapshotId, + ...(name === "releasedBaseline" ? { releasedAt: minutesAgo(5).toISOString() } : {}) } }; + return { id: s.workspaceId, companyId, mode: "isolated_workspace", strategyType: "test", name, + metadata: { ...baseline, remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 2, snapshotId, + source: { kind: "git", repoUrl: "https://example.invalid/repo.git", ref: null }, dirty: true, writerRunId: s.runId, + location: { environmentId: randomUUID(), leaseId: s.leaseId, providerLeaseId: s.providerLeaseId, remoteCwd: remoteCwd(name) } } } }; + })); + // The restore itself is file-neutral; a provisioning command is not. + await db.insert(workspaceOperations).values(cases.map(name => ({ companyId, executionWorkspaceId: setup[name].workspaceId, + heartbeatRunId: setup[name].runId, phase: "remote_workspace_restore", status: "succeeded", metadata: { epoch: 2, snapshotId } }))); + await db.insert(workspaceOperations).values({ companyId, executionWorkspaceId: setup.provisioned.workspaceId, + heartbeatRunId: setup.provisioned.runId, phase: "remote_workspace_provision", status: "failed", metadata: { epoch: 2 } }); + const read = async (name: (typeof cases)[number]) => (await db.select().from(executionWorkspaces) + .where(eq(executionWorkspaces.id, setup[name].workspaceId)))[0]!; + + await runOwnershipReconciliationPass(db, { state: createOwnershipReportState() }); + + const clean = await read("clean"); + expect(readRemoteWorkspaceAuthority(clean.metadata)).toMatchObject({ epoch: 2, dirty: false, writerRunId: null, snapshotId, + location: { leaseId: setup.clean.leaseId, providerLeaseId: setup.clean.providerLeaseId, remoteCwd: remoteCwd("clean") } }); + expect(clean.metadata?.remoteWorkspaceAbandonedEpochs).toBeUndefined(); + expect(clean.metadata?.remoteWorkspaceUnchangedBaseline).toMatchObject({ releasedAt: expect.any(String) }); + const [cleanRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, setup.clean.runId)); + expect(cleanRun.runnerProfileJson?.remoteWorkspace).toMatchObject({ state: "saved", epoch: 2 }); + // The retry is admitted at once, and the killed run's sandbox is no longer a prepared source holding it. + expect(await readRemoteWorkspaceWriterHold(db, { companyId, authority: readRemoteWorkspaceAuthority(clean.metadata)!, + runId: randomUUID() })).toEqual({ kind: "free" }); + // A recorded process is runtime work: neither handed on nor abandoned here. + expect(readRemoteWorkspaceAuthority((await read("processStarted")).metadata)) + .toMatchObject({ epoch: 2, dirty: true, writerRunId: setup.processStarted.runId }); + for (const name of cases.filter(name => name !== "clean" && name !== "processStarted")) { + const row = await read(name); + expect(readRemoteWorkspaceAuthority(row.metadata), name) + .toMatchObject({ epoch: 3, dirty: false, writerRunId: null, location: null }); + expect(row.metadata?.remoteWorkspaceAbandonedEpochs, name).toEqual([expect.objectContaining({ epoch: 2, runId: setup[name].runId })]); + } + }); + it("settles a workspace command whose process died mid-command, and keeps live or unprovable ones", async () => { const run = (id: string, status: string) => ({ id, companyId, agentId, status, invocationSource: "manual", ...(status === "running" ? {} : { finishedAt: minutesAgo(2) }) }); diff --git a/server/src/services/recovery/remote-workspace-unsaved-files-settlement.ts b/server/src/services/recovery/remote-workspace-unsaved-files-settlement.ts index 1dbdf5c02cf..5931f763b7e 100644 --- a/server/src/services/recovery/remote-workspace-unsaved-files-settlement.ts +++ b/server/src/services/recovery/remote-workspace-unsaved-files-settlement.ts @@ -3,6 +3,7 @@ import { executionWorkspaces, heartbeatRunEvents, heartbeatRuns, nativeRunFinali import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import type { OwnerProof } from "../owner-proof.js"; import { abandonUnverifiedRemoteWorkspaceEpoch } from "../remote-workspace-epoch-abandonment.js"; +import { releaseUnchangedRemoteWorkspaceWriterById } from "../remote-workspaces.js"; import { notifyTerminalWorkspaceWork } from "../terminal-workspace-work.js"; import type { InProgressRecord, SettleOutcome } from "./ownership-registry.js"; @@ -43,6 +44,16 @@ export async function settlePreparationOnlyRemoteWorkspaceEpoch(db: Db, record: eq(workspaceOperations.phase, "remote_workspace_snapshot"), inArray(workspaceOperations.status, ["running", "pending"]), sql`${workspaceOperations.metadata}->>'epoch' = ${String(authority.epoch)}`)).limit(1); if (save) return { settled: false, reason: "save_in_flight" }; + // A preparation that restored the saved version and then did nothing else + // left exactly that version in its root. When the durable unchanged-baseline + // proof holds, hand the workspace on at its saved version and keep the root, + // so the retry reuses the same directory and the saved session still names + // it. Anything unproven (no baseline from an older release, a provisioning + // command, a runner command, a process, a kill before the restore published) + // is abandoned exactly as before. + if (await releaseUnchangedRemoteWorkspaceWriterById(transaction, { companyId, workspaceId: record.recordId, runId })) { + return { settled: true, reason: context.proof.reason }; + } await notifyTerminalWorkspaceWork(tx, `workspace:${record.recordId}`); try { await abandonUnverifiedRemoteWorkspaceEpoch(transaction, { diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 3c4c3c69376..2920a4f6447 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -8,6 +8,7 @@ import { mkdtemp, readFile, rm, statfs } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { and, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { crashPoint } from "@paperclipai/shared/crash-points"; import { companies, environmentLeases, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, workspaceOperations, type Db, } from "@paperclipai/db"; @@ -618,6 +619,12 @@ export async function prepareRemoteWorkspace(input: { return { workspace: savedWorkspace!, lease: savedLease! as EnvironmentLease, remoteRoot, reference: runReference }; }); target.remoteCwd = remoteRoot; + // Counted only for a preparation that published an unchanged baseline (a + // restore of a saved version), so the point exercises the saved-root handoff. + const publishedBaseline = readUnchangedBaseline(prepared.workspace.metadata); + if (publishedBaseline?.runId === input.runId && publishedBaseline.epoch === prepared.reference.epoch && !publishedBaseline.releasedAt) { + crashPoint("remote_workspace.prepare.after_publish_before_provision"); + } if (restoredExport?.cleanupPending) await cleanupSavedExports({ db: input.db, companyId: input.workspace.companyId, workspaceId: input.workspace.id, location: readRemoteWorkspaceAuthority(prepared.workspace.metadata)!.location!, target, assertOwned: input.assertOwned ?? (async () => undefined) }); diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index b5fa7d9b65c..6eccbfd0a56 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -226,7 +226,7 @@ "packages/shared/src/connector-identity.test.ts": "d58357707faefa7c98bb906cdaeb4db39d31081a", "packages/shared/src/connector-identity.ts": "5ede83c9c5225dbe8af9cf3a49c74e477ef5b955", "packages/shared/src/constants.ts": "3149ee1cc5ffa34b50ecd493276b3a70fc46adc2", - "packages/shared/src/crash-point-registry.ts": "708b1e431032ecf370fd259ab180d11e4c0b5cf3", + "packages/shared/src/crash-point-registry.ts": "8d6f60e8ad8f90338fb3047955bb32160f50b9ae", "packages/shared/src/crash-points.test.ts": "804a3acfd7a51010f31dbc3fb9a4dca771562c99", "packages/shared/src/crash-points.ts": "d9f31d95c78f32290190d7728f42cd9573681fc8", "packages/shared/src/customer-success.ts": "a2d02aaa8223164c0eae4267b2a6da82242eaa6f",