diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index bd2c6804dc9..09598fe618e 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3406,7 +3406,23 @@ have no recovery receipt. When its saved reference names the exact snapshot that its own succeeded `remote_workspace_restore` loaded, at the same epoch, it left no unsaved files. A later continuation into another workspace skips that source instead of failing with `remote_workspace_recovery_receipt_invalid`. A -source that saved any other snapshot still needs the canonical receipt. +source that saved any other snapshot still needs the canonical receipt to be +attached. + +A predecessor whose recovery receipt cannot be verified never stops a +continuation or a Retry. Examples are a receipt on a run reference that was +never saved, a receipt from another epoch or for another run, a saved +reference without a receipt, a reference that the current release cannot read, +and a receipt whose snapshot is gone. Preparation then attaches nothing from +that source. The run starts from its workspace's newest verified save, as a run +without a predecessor does. Preparation logs the skip with the code +`remote_workspace_recovery_receipt_invalid` and a reason. The source keeps its +saved snapshot, restore hold, receipts and recovery action, so workspace +recovery can still settle it. A recovered copy that is already in the workspace +but cannot be verified is not published again. Its files stay where they are. +Before this rule, the refusal could not change the saved state, so every Retry +failed in the same way. Rows that earlier releases saved follow the same rule +with no migration. A foreign writer and a failed snapshot read still refuse. A queued turn for the same task that never started is not a workspace writer for automatic recovery. The hold gates its dispatch until the recovery receipt is diff --git a/server/src/services/remote-workspace-recovery-attachments.ts b/server/src/services/remote-workspace-recovery-attachments.ts index 046f11dea3b..687a77b5671 100644 --- a/server/src/services/remote-workspace-recovery-attachments.ts +++ b/server/src/services/remote-workspace-recovery-attachments.ts @@ -16,6 +16,21 @@ import { assertExecutionWorkspaceRecordAccess } from "./task-workspace-access.js import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js"; import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; import { getStorageService, type StorageService } from "../storage/index.js"; +import { logger } from "../middleware/logger.js"; + +/** A source whose recovery receipt cannot be verified is skipped, never fatal. + * The run then starts from its workspace's newest verified save, as a run with + * no predecessor does. Nothing unverified is imported, and the source keeps its + * saved snapshot, restore hold, receipts and recovery action, so workspace + * recovery can still settle it. A refusal here could not change that saved + * state, so every Retry failed the same way, including on rows that an earlier + * release saved. */ +function skipUnverifiedRecoverySource(input: { companyId: string; issueId?: string | null; runId: string }, + sourceRunId: string, reason: string) { + logger.warn({ companyId: input.companyId, issueId: input.issueId, runId: input.runId, sourceRunId, + code: "remote_workspace_recovery_receipt_invalid", reason }, + "Saved work of a previous run was not attached because its recovery receipt could not be verified"); +} /** Import verified canonical recovery as a sidecar. The selected workspace and * its own files stay intact; this does not grant execution or replay authority. */ @@ -43,23 +58,26 @@ export async function attachPreservedRemoteWorkspaceSnapshots(input: { const result: Array<{ operationId: string; path: string; sourceRunId: string }> = []; for (const source of sources) { if (source.runnerProfileJson?.remoteWorkspace == null) { - if (canonicalSources.has(source.id)) throw new Error("remote_workspace_recovery_receipt_invalid"); + if (canonicalSources.has(source.id)) skipUnverifiedRecoverySource(input, source.id, "receipt_without_reference"); continue; } const parsed = remoteWorkspaceRunReferenceSchema.safeParse(source.runnerProfileJson.remoteWorkspace); if (!parsed.success) { - if (hasRequiredWorkspaceRecovery(source.resultJson)) throw new Error("remote_workspace_recovery_receipt_invalid"); + if (hasRequiredWorkspaceRecovery(source.resultJson)) skipUnverifiedRecoverySource(input, source.id, "reference_unreadable"); continue; } const remote = parsed.data; - if (canonicalSources.has(source.id) && (remote.state !== "saved" || !remote.snapshotId)) - throw new Error("remote_workspace_recovery_receipt_invalid"); + if (canonicalSources.has(source.id) && (remote.state !== "saved" || !remote.snapshotId)) { + skipUnverifiedRecoverySource(input, source.id, "receipt_on_unsaved_reference"); + continue; + } if (remote.workspaceId === input.workspaceId || remote.state !== "saved" || !remote.snapshotId) continue; const proof = await immutableRemoteWorkspaceRecoveryAttachment(input.db, source); if (!proof) { - if (hasRequiredWorkspaceRecovery(source.resultJson) && !await savedOnlyItsRestoredSnapshot(input.db, source, remote)) { - throw new Error("remote_workspace_recovery_receipt_invalid"); - } + // A source that saved only the snapshot its own restore loaded left no + // unsaved files; any other unverified source is reported and skipped. + if (hasRequiredWorkspaceRecovery(source.resultJson) && !await savedOnlyItsRestoredSnapshot(input.db, source, remote)) + skipUnverifiedRecoverySource(input, source.id, "receipt_unverified"); continue; } const operationId = proof.operationId; @@ -70,8 +88,10 @@ export async function attachPreservedRemoteWorkspaceSnapshots(input: { input.companyId, remote.workspaceId); const saved = await getWorkspaceSnapshot(input.db, { companyId: input.companyId, workspaceId: remote.workspaceId, snapshotId: remote.snapshotId }); - if (!saved || saved.sha256 !== proof.sha256 || saved.generation !== proof.generation) - throw new Error("remote_workspace_recovery_snapshot_missing"); + if (!saved || saved.sha256 !== proof.sha256 || saved.generation !== proof.generation) { + skipUnverifiedRecoverySource(input, source.id, "receipt_snapshot_missing"); + continue; + } const provenance = JSON.stringify({ schema: "paperclip.remote-workspace-recovery-attachment.v1", operationId, sourceRunId: source.id, workspaceId: remote.workspaceId, snapshotId: remote.snapshotId, sha256: saved.sha256 }); @@ -253,7 +273,13 @@ export async function retainRemoteRecoveryAttachments(input: { provenance = await retainedRecoveryAttachmentProvenance({ db: input.db, companyId: input.workspace.companyId, workspace: input.workspace, issueIds: [issueId, input.workspace.sourceIssueId].filter((id): id is string => typeof id === "string"), sourceRunId: item.sourceRunId, operationId: item.operationId, verifySelectedSnapshot }); - if (!provenance) throw new Error("remote_workspace_recovery_receipt_invalid"); + // An unverifiable record is not published again. Its files stay where + // they are in the workspace; only the recovered-copy claim is dropped. + if (!provenance) { + skipUnverifiedRecoverySource({ companyId: input.workspace.companyId, + issueId: input.workspace.sourceIssueId, runId: input.runId }, item.sourceRunId, "retained_copy_unverified"); + continue; + } } await input.assertOwned?.(); const location = { target: input.target, remoteCwd: input.remoteRoot, operationId: item.operationId, provenance }; diff --git a/server/src/services/remote-workspace-recovery-receipt-fallback.test.ts b/server/src/services/remote-workspace-recovery-receipt-fallback.test.ts new file mode 100644 index 00000000000..7d87535f65a --- /dev/null +++ b/server/src/services/remote-workspace-recovery-receipt-fallback.test.ts @@ -0,0 +1,127 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readdir } from "node:fs/promises"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { describe, expect, it, vi } from "vitest"; +import { executionWorkspaceSnapshots, executionWorkspaces, heartbeatRuns, issueRecoveryActions, workspaceOperations } from "@paperclipai/db"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { attachPreservedRemoteWorkspaceSnapshots } from "./remote-workspace-recovery-attachments.js"; +import { useWorkspaceRecoveryFixture, db, failedRemotePreparation, root, run } from "./__tests__/workspace-recovery-fixture.js"; + +const targets = vi.hoisted(() => new Map()); +vi.mock("./environment-execution-target.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, resolveEnvironmentExecutionTarget: async (input: Parameters[0]) => + (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; +}); + +// A predecessor whose recovery receipt cannot be verified is skipped, never +// fatal: preparation attaches nothing unverified and leaves its saved state. +describe("remote workspace recovery receipt fallback", () => { + useWorkspaceRecoveryFixture(targets); + + // Nov 10 staging: a continuation into a fresh workspace failed + // with remote_workspace_recovery_receipt_invalid. Its predecessor restored the + // committed save, failed before provider work, and kept a restore hold with + // no recovery receipt. It saved the very snapshot it restored, so it left no + // unsaved files to attach. A source that saved anything else is not attached + // either: without a receipt it is skipped, and its saved state stays as it was. + it.each(["restored_unchanged", "saved_other_snapshot", "restore_not_succeeded", "restored_other_epoch"] as const)( + "decides a receipt-less restore hold of a source that %s", async variant => { + const f = await failedRemotePreparation(); + const template = await run(f.runId); + const restoredSnapshotId = randomUUID(), sourceId = randomUUID(), leaseId = randomUUID(); + await db.insert(executionWorkspaceSnapshots).values({ id: restoredSnapshotId, companyId: f.companyId, + executionWorkspaceId: f.workspaceId, sourceOperationId: randomUUID(), sourceRunId: randomUUID(), + generation: 4, epoch: 3, status: "committed", sha256: "a".repeat(64) }); + await db.insert(heartbeatRuns).values({ id: sourceId, companyId: f.companyId, agentId: template.agentId, + nativeIssueId: f.issueId, issueId: f.issueId, status: "failed", runtimeMode: "native", finishedAt: new Date(), + contextSnapshot: { issueId: f.issueId }, + errorCode: "native_continuation_requires_reconciliation", + error: "native_controller_recovery_missing_evidence:retained_sandbox_binding", + runnerProfileJson: { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", epoch: 6, state: "saved", + leaseId, providerLeaseId: randomUUID(), remoteCwd: `/home/daytona/paperclip-workspace/.paperclip-workspaces/${f.workspaceId}-6-${leaseId}`, + workspaceId: f.workspaceId, snapshotId: variant === "saved_other_snapshot" ? randomUUID() : restoredSnapshotId } }, + resultJson: { stopReason: "adapter_failed", executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, + workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] } } }); + await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId, heartbeatRunId: sourceId, + executionWorkspaceId: f.workspaceId, phase: "remote_workspace_restore", + status: variant === "restore_not_succeeded" ? "failed" : "succeeded", + metadata: { epoch: variant === "restored_other_epoch" ? 5 : 6, snapshotId: restoredSnapshotId } }); + const [writer] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: template.agentId, + issueId: f.issueId, status: "running", responsibleUserId: "local-board", contextSnapshot: { issueId: f.issueId } }).returning(); + const destination = path.join(root, writer.id); await mkdir(destination); + const attach = attachPreservedRemoteWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId, + runId: writer.id, workspaceId: randomUUID(), sourceRunIds: [sourceId], cwd: destination }); + await expect(attach).resolves.toEqual([]); + expect(await readdir(destination)).toEqual([]); + expect((await run(sourceId)).resultJson).toMatchObject({ workspaceRestoreRecovery: { leaseIds: [leaseId] } }); + }, 30_000); + + // Nov 10 staging, after a deploy: Retry of a task whose previous run failed + // with native_controller_recovery_missing_evidence:runner_not_settled failed in + // setup with remote_workspace_recovery_receipt_invalid, and every later Retry + // would fail the same way. A predecessor whose receipt cannot be verified must + // not dead-end the task: the Retry starts from the workspace's newest verified + // save, attaches nothing unverified, and leaves the source's saved snapshot, + // restore hold and receipts exactly as they were. + it.each([ + "receipt_without_reference", "unreadable_reference_with_hold", "receipt_on_prepared_reference", + "superseded_snapshot_with_hold", "receipt_from_other_epoch", "receipt_for_other_run", + ] as const)("starts a Retry from the newest verified save when a predecessor has %s", async variant => { + const f = await failedRemotePreparation(); + const template = await run(f.runId); + const sourceId = randomUUID(), leaseId = randomUUID(), providerLeaseId = randomUUID(), savedSnapshotId = randomUUID(); + const sourceWorkspaceId = variant === "receipt_on_prepared_reference" ? f.workspaceId : randomUUID(); + if (sourceWorkspaceId !== f.workspaceId) await db.insert(executionWorkspaces).values({ id: sourceWorkspaceId, + companyId: f.companyId, sourceIssueId: f.issueId, name: "Previous run workspace", providerType: "cloud_sandbox", + mode: "isolated_workspace", strategyType: "project_primary", cwd: `/workspaces/${sourceWorkspaceId}` }); + await db.insert(executionWorkspaceSnapshots).values({ id: savedSnapshotId, companyId: f.companyId, + executionWorkspaceId: sourceWorkspaceId, sourceOperationId: randomUUID(), sourceRunId: sourceId, + generation: 7, epoch: 6, status: variant === "superseded_snapshot_with_hold" ? "superseded" : "committed", + sha256: "b".repeat(64) }); + const reference = { schema: "paperclip.remote-workspace-run.v1", epoch: 6, + state: variant === "receipt_on_prepared_reference" ? "prepared" : "saved", leaseId, providerLeaseId, + remoteCwd: `/home/daytona/paperclip-workspace/.paperclip-workspaces/${sourceWorkspaceId}-6-${leaseId}`, + workspaceId: sourceWorkspaceId, snapshotId: variant === "receipt_on_prepared_reference" ? null : savedSnapshotId, + ...(variant === "unreadable_reference_with_hold" ? { savedBy: "previous-release" } : {}) }; + const hold = { workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] } }; + await db.insert(heartbeatRuns).values({ id: sourceId, companyId: f.companyId, agentId: template.agentId, + nativeIssueId: f.issueId, issueId: f.issueId, status: "failed", runtimeMode: "native", + finishedAt: new Date(Date.now() - 60_000), createdAt: new Date(Date.now() - 120_000), contextSnapshot: { issueId: f.issueId }, + errorCode: "native_continuation_requires_reconciliation", + error: "native_controller_recovery_missing_evidence:runner_not_settled", + runnerProfileJson: variant === "receipt_without_reference" ? {} : { remoteWorkspace: reference }, + resultJson: { stopReason: "adapter_failed", ...hold } }); + const [recoveryAction] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", status: "resolved", ownerType: "board", cause: "native_continuation_requires_reconciliation", + fingerprint: `native-workspace:${sourceId}`, nextAction: "Recover saved work", + evidence: { runId: sourceId, ...hold, workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", + runId: variant === "receipt_for_other_run" ? randomUUID() : sourceId, status: "recovered", stage: "saving_receipts", + attempt: 1, requestIds: [] } } }).returning(); + if (variant !== "unreadable_reference_with_hold" && variant !== "superseded_snapshot_with_hold") + await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId, heartbeatRunId: sourceId, + executionWorkspaceId: sourceWorkspaceId, phase: "workspace_recovery", status: "succeeded", finishedAt: new Date(), + metadata: { recoveryActionId: recoveryAction.id, leaseId, providerLeaseId, snapshotId: savedSnapshotId, + generation: 7, sha256: "b".repeat(64), storage: "remote_workspace", + epoch: variant === "receipt_from_other_epoch" ? 5 : 6 } }); + const [writer] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: template.agentId, + nativeIssueId: f.issueId, issueId: f.issueId, status: "running", runtimeMode: "native", retryOfRunId: sourceId, + responsibleUserId: "local-board", contextSnapshot: { issueId: f.issueId, taskRetryRequestId: randomUUID() } }).returning(); + const destination = path.join(root, writer.id); await mkdir(destination); + const before = { source: await run(sourceId), operations: await db.select().from(workspaceOperations) + .where(eq(workspaceOperations.heartbeatRunId, sourceId)), snapshots: await db.select().from(executionWorkspaceSnapshots) + .where(eq(executionWorkspaceSnapshots.id, savedSnapshotId)) }; + const attach = attachPreservedRemoteWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId, + runId: writer.id, workspaceId: f.workspaceId, sourceRunIds: [writer.id], cwd: destination }); + await expect(attach).resolves.toEqual([]); + expect(await readdir(destination)).toEqual([]); + // The fallback attaches nothing unverified and never touches the source's saved state. + expect(await run(sourceId)).toEqual(before.source); + expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, sourceId))).toEqual(before.operations); + expect(await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, savedSnapshotId))) + .toEqual(before.snapshots); + }, 30_000); +}); diff --git a/server/src/services/workspace-recovery-preparation.test.ts b/server/src/services/workspace-recovery-preparation.test.ts index 44924d4ba27..2f7047f37a8 100644 --- a/server/src/services/workspace-recovery-preparation.test.ts +++ b/server/src/services/workspace-recovery-preparation.test.ts @@ -46,6 +46,9 @@ vi.mock("./environment-execution-target.js", async importOriginal => { (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; }); +const publishedSources = (workspace: { metadata?: Record | null }) => + ((workspace.metadata?.remoteWorkspaceRecoveredFiles ?? []) as Array<{ sourceRunId: string }>).map(item => item.sourceRunId); + describe("automatic retained workspace recovery", () => { useWorkspaceRecoveryFixture(targets); @@ -339,9 +342,11 @@ describe("automatic retained workspace recovery", () => { await db.update(heartbeatRuns).set({ runnerProfileJson: { legacyWorkspaceRecoveryMigration: { ...marker, workspaceId: fault === "source_marker_workspace" ? randomUUID() : placeholder.id } } }) .where(eq(heartbeatRuns.id, related.id)); - await expect(prepareRemoteWorkspace({ db, storage, workspace: withoutProof, runId: childRun.id, - lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: cold.remoteRoot } })) - .rejects.toThrow("remote_workspace_recovery_receipt_invalid"); + // An unverifiable retained copy is not published again; the run still starts. + const unverified = await prepareRemoteWorkspace({ db, storage, workspace: withoutProof, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: cold.remoteRoot } }); + expect(publishedSources(unverified.workspace)).not.toContain(related.id); + expect(publishedSources(unverified.workspace)).toContain(source.id); } await db.update(workspaceOperations).set({ status: "succeeded", metadata: { legacyRecoverySources: [{ runId: related.id, issueId: earlierChild.id }] } }).where(eq(workspaceOperations.id, migration.id)); @@ -349,9 +354,9 @@ describe("automatic retained workspace recovery", () => { { ...marker, workspaceId: placeholder.id } } }).where(eq(heartbeatRuns.id, related.id)); await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.id, (olderPaths.find(item => item.sourceRunId === source.id))!.operationId)); - await expect(prepareRemoteWorkspace({ db, storage, workspace: withoutProof, runId: childRun.id, - lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: cold.remoteRoot } })) - .rejects.toThrow("remote_workspace_recovery_receipt_invalid"); + const withoutReceipt = await prepareRemoteWorkspace({ db, storage, workspace: withoutProof, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: cold.remoteRoot } }); + expect(publishedSources(withoutReceipt.workspace)).not.toContain(source.id); expect(await readFile(path.join(inherited.remoteRoot, newest!.path, "files", "ticks.log"), "utf8")).toBe(editedLog); await db.update(workspaceOperations).set({ status: "succeeded" }).where(eq(workspaceOperations.id, (olderPaths.find(item => item.sourceRunId === source.id))!.operationId)); @@ -379,14 +384,14 @@ describe("automatic retained workspace recovery", () => { remoteWorkspaceRecoveredFiles: olderPaths.map(({ operationId, sourceRunId, path }) => ({ operationId, sourceRunId, path })) } }; const authority = readRemoteWorkspaceAuthority(legacySnapshot.metadata)!; await db.update(executionWorkspaceSnapshots).set({ status: "uploaded" }).where(eq(executionWorkspaceSnapshots.id, authority.snapshotId!)); - await expect(prepareRemoteWorkspace({ db, storage, workspace: legacySnapshot, runId: childRun.id, - lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot } })) - .rejects.toThrow("remote_workspace_recovery_receipt_invalid"); + const uncommitted = await prepareRemoteWorkspace({ db, storage, workspace: legacySnapshot, runId: childRun.id, + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot } }); + expect(publishedSources(uncommitted.workspace)).not.toContain(related.id); await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, authority.snapshotId!)); - await expect(prepareRemoteWorkspace({ db, storage, workspace: { ...legacySnapshot, metadata: { ...legacySnapshot.metadata, + const otherSnapshot = await prepareRemoteWorkspace({ db, storage, workspace: { ...legacySnapshot, metadata: { ...legacySnapshot.metadata, remoteWorkspace: { ...authority, snapshotId: sourceReference.snapshotId } } }, runId: childRun.id, - lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot } })) - .rejects.toThrow("remote_workspace_recovery_receipt_invalid"); + lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot } }); + expect(publishedSources(otherSnapshot.workspace)).not.toContain(related.id); const unreadable = { ...storage, getObject: vi.fn().mockRejectedValue(new Error("fixture: selected snapshot unavailable")) }; await expect(prepareRemoteWorkspace({ db, storage: unreadable, workspace: legacySnapshot, runId: childRun.id, lease: childLease as EnvironmentLease, target: { ...target, leaseId: childLease.id, remoteCwd: inherited.remoteRoot } })) @@ -504,7 +509,7 @@ describe("automatic retained workspace recovery", () => { }, 30_000); it.each(["missing_receipt", "missing_reference", "prepared_reference", "missing_snapshot_id", "uncommitted", "foreign_snapshot", "wrong_lease", "missing_object", "corrupt_object", "wrong_company", "wrong_issue"])( - "refuses canonical recovery attachments with %s evidence", async fault => { + "never attaches canonical recovery attachments with %s evidence", async fault => { const f = await failedRemotePreparation(); await recoverRetainedWorkspace(f.input); const source = await run(f.runId), remote = source.runnerProfileJson!.remoteWorkspace as { snapshotId: string }; @@ -532,53 +537,19 @@ describe("automatic retained workspace recovery", () => { return { ...object, stream: Readable.from([Buffer.from("corrupt")]) }; }) : undefined; try { - await expect(attachPreservedRemoteWorkspaceSnapshots({ db, + const attach = attachPreservedRemoteWorkspaceSnapshots({ db, companyId: fault === "wrong_company" ? randomUUID() : f.companyId, issueId: fault === "wrong_issue" ? randomUUID() : f.issueId, - runId: writer.id, workspaceId: randomUUID(), sourceRunIds: [source.id], cwd: destination })).rejects.toThrow(); + runId: writer.id, workspaceId: randomUUID(), sourceRunIds: [source.id], cwd: destination }); + // An unverifiable receipt falls back to the newest verified save and + // attaches nothing. A failed read and a foreign writer still refuse. + if (["missing_object", "corrupt_object", "wrong_company", "wrong_issue"].includes(fault)) await expect(attach).rejects.toThrow(); + else await expect(attach).resolves.toEqual([]); expect(await readdir(destination)).toEqual(fault === "missing_object" || fault === "corrupt_object" ? [".paperclip-recovered"] : []); expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n"); } finally { failingRead?.mockRestore(); } }, 30_000); - // Nov 10 staging (PAP-3881): a continuation into a fresh workspace failed - // with remote_workspace_recovery_receipt_invalid. Its predecessor restored the - // committed save, failed before provider work, and kept a restore hold with - // no recovery receipt. It saved the very snapshot it restored, so it left no - // unsaved files to attach. A source that saved anything else stays strict. - it.each(["restored_unchanged", "saved_other_snapshot", "restore_not_succeeded", "restored_other_epoch"] as const)( - "decides a receipt-less restore hold of a source that %s", async variant => { - const f = await failedRemotePreparation(); - const template = await run(f.runId); - const restoredSnapshotId = randomUUID(), sourceId = randomUUID(), leaseId = randomUUID(); - await db.insert(executionWorkspaceSnapshots).values({ id: restoredSnapshotId, companyId: f.companyId, - executionWorkspaceId: f.workspaceId, sourceOperationId: randomUUID(), sourceRunId: randomUUID(), - generation: 4, epoch: 3, status: "committed", sha256: "a".repeat(64) }); - await db.insert(heartbeatRuns).values({ id: sourceId, companyId: f.companyId, agentId: template.agentId, - nativeIssueId: f.issueId, issueId: f.issueId, status: "failed", runtimeMode: "native", finishedAt: new Date(), - contextSnapshot: { issueId: f.issueId }, - errorCode: "native_continuation_requires_reconciliation", - error: "native_controller_recovery_missing_evidence:retained_sandbox_binding", - runnerProfileJson: { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", epoch: 6, state: "saved", - leaseId, providerLeaseId: randomUUID(), remoteCwd: `/home/daytona/paperclip-workspace/.paperclip-workspaces/${f.workspaceId}-6-${leaseId}`, - workspaceId: f.workspaceId, snapshotId: variant === "saved_other_snapshot" ? randomUUID() : restoredSnapshotId } }, - resultJson: { stopReason: "adapter_failed", executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, - workspaceRestoreFailure: "restore_failed", - workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] } } }); - await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId, heartbeatRunId: sourceId, - executionWorkspaceId: f.workspaceId, phase: "remote_workspace_restore", - status: variant === "restore_not_succeeded" ? "failed" : "succeeded", - metadata: { epoch: variant === "restored_other_epoch" ? 5 : 6, snapshotId: restoredSnapshotId } }); - const [writer] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: template.agentId, - issueId: f.issueId, status: "running", responsibleUserId: "local-board", contextSnapshot: { issueId: f.issueId } }).returning(); - const destination = path.join(root, writer.id); await mkdir(destination); - const attach = attachPreservedRemoteWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId, - runId: writer.id, workspaceId: randomUUID(), sourceRunIds: [sourceId], cwd: destination }); - if (variant === "restored_unchanged") await expect(attach).resolves.toEqual([]); - else await expect(attach).rejects.toThrow("remote_workspace_recovery_receipt_invalid"); - expect(await readdir(destination)).toEqual([]); - }, 30_000); - it.each(["unchanged", "advanced"])("deduplicates immutable canonical sidecars when the source workspace is %s", async state => { const f = await failedRemotePreparation(); await recoverRetainedWorkspace(f.input); const source = await run(f.runId), destination = path.join(root, randomUUID()); await mkdir(destination);