diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 887c4b3056b..0cb9258466d 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -24,6 +24,16 @@ branch-deletion receipts. PostgreSQL and real-file tests prove cold restoration of saved bytes, index state, and unpushed commits, including after pruning the old Git object store. Invalid or unavailable retained data still blocks import. +Administrative migration can recover a missing empty task-directory payload only +when storage confirms that its object is absent. The existing encrypted key +envelope must authenticate the exact empty payload hash and length, company, +workspace, restore-point ID, and generation. Metadata counters alone never +establish emptiness. This path leaves the original reference and missing object +unchanged and publishes the recovered bytes through the normal verified remote +snapshot owner. Ordinary restoration remains unchanged. Expired checkpoints, +nonempty or forged payloads, unreadable objects, and unknown object absence still +block migration. + An archived isolated checkout can be gone while its recorded deleted commit survives in the company/project Git cache. After validating the exact successful branch-deletion receipt, the importer can read immutable objects from that diff --git a/server/src/__tests__/missing-empty-task-directory.test.ts b/server/src/__tests__/missing-empty-task-directory.test.ts new file mode 100644 index 00000000000..a545ae4b526 --- /dev/null +++ b/server/src/__tests__/missing-empty-task-directory.test.ts @@ -0,0 +1,117 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; +import { createStorageService } from "../storage/service.js"; +import { sealRestorePoint, type RestorePointIdentity } from "../services/git-restore-point.js"; +import { captureTaskDirectorySnapshot, materializeTaskDirectorySnapshot } from "../services/isolated-task-directory-snapshot.js"; +import { loadTaskDirectoryRestorePoint, TASK_DIRECTORY_RESTORE_POINT_SCHEMA, type TaskDirectoryRestorePointRef } from "../services/isolated-task-directory-retention.js"; + +let root: string; + +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-empty-retention-"))); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "home")); + vi.stubEnv("PAPERCLIP_INSTANCE_ID", "empty-retention-test"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "31".repeat(32)); +}); + +afterEach(async () => { + vi.restoreAllMocks(); vi.unstubAllEnvs(); + await fs.rm(root, { recursive: true, force: true }); +}); + +async function fixture(kind = "empty", scope?: keyof RestorePointIdentity) { + const directory = path.join(root, "source"); await fs.mkdir(directory); + const empty = await captureTaskDirectorySnapshot(directory); + if (kind === "file") await fs.writeFile(path.join(directory, "saved.txt"), "must never disappear\n"); + if (kind === "directory") await fs.mkdir(path.join(directory, "saved-directory")); + const snapshot = await captureTaskDirectorySnapshot(directory); + const identity = { companyId: randomUUID(), workspaceId: randomUUID(), restorePointId: randomUUID(), generation: 1 }; + const foreign = { ...identity }; + if (scope === "generation") foreign.generation = 2; + else if (scope) foreign[scope] = randomUUID(); + const sealed = await sealRestorePoint(snapshot.payload, foreign); + const provider = createLocalDiskStorageProvider(path.join(root, "objects")); + const storage = createStorageService(provider); + const objectKey = `${identity.companyId}/git-restore-points/${identity.workspaceId}/1/task-directory/${identity.restorePointId}.pcrp`; + const put = (body: Buffer) => storage.putFile({ companyId: identity.companyId, namespace: "git-restore-points", objectKey, + originalFilename: null, contentType: "application/octet-stream", body }); + await put(sealed.ciphertext); + const ref: TaskDirectoryRestorePointRef = { schema: TASK_DIRECTORY_RESTORE_POINT_SCHEMA, id: identity.restorePointId, objectKey, generation: 1, + sha256: sealed.sha256, byteSize: sealed.ciphertext.length, key: sealed.key, digest: snapshot.digest, files: snapshot.files, excluded: snapshot.excluded }; + const load = (recoverMissingEmpty = true) => loadTaskDirectoryRestorePoint({ storage, workspace: { companyId: identity.companyId, id: identity.workspaceId }, ref, recoverMissingEmpty }); + return { ref, storage, provider, put, load, empty, snapshot, directory, remove: () => storage.deleteObject(identity.companyId, objectKey) }; +} + +it("recovers only the exact empty payload authenticated by the original scoped key envelope", async () => { + const f = await fixture(); await f.remove(); await fs.rm(f.directory, { recursive: true }); + await expect(f.load(false)).rejects.toMatchObject({ reason: "restore_point_unavailable" }); + const payload = await f.load(); expect(payload).toEqual(f.snapshot.payload); + const restored = await materializeTaskDirectorySnapshot({ payload, target: path.join(root, "restored"), expectedDigest: f.ref.digest }); + expect(await fs.readdir(restored)).toEqual([]); + expect(await f.storage.headObject(f.ref.objectKey.split("/")[0]!, f.ref.objectKey)).toMatchObject({ exists: false }); +}); + +it.each(["file", "directory"])("refuses forged empty counters and digest for authenticated %s contents", async kind => { + const f = await fixture(kind); await f.remove(); + f.ref.files = 0; f.ref.excluded = 0; f.ref.digest = f.empty.digest; + await expect(f.load()).rejects.toMatchObject({ reason: "plaintext_hash_mismatch" }); +}); + +it.each(["companyId", "workspaceId", "restorePointId", "generation"] as const)("rejects a candidate from another %s", async scope => { + const f = await fixture("empty", scope); await f.remove(); + await expect(f.load()).rejects.toMatchObject({ reason: "key_identity_mismatch" }); +}); + +it("does not replace a present corrupt encrypted object with a reconstructed payload", async () => { + const f = await fixture(); await f.put(Buffer.from("corrupted")); + await expect(f.load()).rejects.toThrow(); +}); + +it("retains a missing nonempty checkpoint as an unresolved obligation", async () => { + const f = await fixture("file"); await f.remove(); + await expect(f.load()).rejects.toMatchObject({ reason: "restore_point_unavailable" }); + expect(await fs.readFile(path.join(f.directory, "saved.txt"), "utf8")).toBe("must never disappear\n"); +}); + +it("does not treat an unreadable present object as missing", async () => { + const f = await fixture(); + vi.spyOn(f.storage, "getObject").mockRejectedValue(new Error("read unavailable")); + await expect(f.load()).rejects.toMatchObject({ reason: "restore_point_unavailable" }); +}); + +it("does not reconstruct when object absence cannot be established", async () => { + const f = await fixture(); await f.remove(); + vi.spyOn(f.storage, "headObject").mockRejectedValue(new Error("head unavailable")); + await expect(f.load()).rejects.toThrow(); +}); + +it("refuses an expired checkpoint even when its exact contents are reconstructible", async () => { + const f = await fixture(); await f.remove(); + await expect(loadTaskDirectoryRestorePoint({ storage: f.storage, workspace: { companyId: f.ref.objectKey.split("/")[0]!, id: f.ref.objectKey.split("/")[2]! }, + ref: { ...f.ref, expiredAt: new Date().toISOString() }, recoverMissingEmpty: true })).rejects.toMatchObject({ reason: "restore_point_expired" }); +}); + +it("cancels a stalled absence check through the real storage service boundary", async () => { + const f = await fixture(); await f.remove(); + const controller = new AbortController(), reason = new Error("migration deadline"); + let entered!: () => void, release!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + vi.spyOn(f.provider, "headObject").mockImplementation(() => new Promise(resolve => { + release = () => resolve({ exists: false }); entered(); + })); + const result = loadTaskDirectoryRestorePoint({ storage: f.storage, + workspace: { companyId: f.ref.objectKey.split("/")[0]!, id: f.ref.objectKey.split("/")[2]! }, + ref: f.ref, recoverMissingEmpty: true, signal: controller.signal }).then(payload => ({ payload }), error => ({ error })); + await started; controller.abort(reason); + let timeout: ReturnType | undefined; + try { + const observed = await Promise.race([result, new Promise(resolve => { timeout = setTimeout(() => resolve("still waiting"), 1000); })]); + expect(observed).toEqual({ error: reason }); + } finally { + clearTimeout(timeout); release(); await result; + } +}); diff --git a/server/src/services/git-restore-point.ts b/server/src/services/git-restore-point.ts index 3af9997b693..dbdd6167393 100644 --- a/server/src/services/git-restore-point.ts +++ b/server/src/services/git-restore-point.ts @@ -554,6 +554,28 @@ export async function sealRestorePoint(payload: Buffer, identity: RestorePointId return { ciphertext, sha256: createHash("sha256").update(ciphertext).digest("hex"), key: material }; } +async function restorePointKey(input: { key: StoredSecretVersionMaterial; identity: RestorePointIdentity }) { + const unwrapped = JSON.parse(await localEncryptedProvider.resolveVersion({ material: input.key, externalRef: null } as Parameters[0])); + const { identity } = input; + if (unwrapped.schema !== KEY_SCHEMA || unwrapped.companyId !== identity.companyId || unwrapped.workspaceId !== identity.workspaceId + || unwrapped.restorePointId !== identity.restorePointId || unwrapped.generation !== identity.generation) { + throw new RestorePointRestoreFailure("key_identity_mismatch"); + } + return unwrapped as RestorePointIdentity & { dataKey: string; plaintextBytes: number; plaintextSha256: string }; +} + +function verifyRestorePointPayload(payload: Buffer, key: { plaintextBytes: number; plaintextSha256: string }) { + if (payload.length !== key.plaintextBytes || createHash("sha256").update(payload).digest("hex") !== key.plaintextSha256) { + throw new RestorePointRestoreFailure("plaintext_hash_mismatch"); + } +} + +export async function verifyRestorePointPayloadCandidate(input: { + payload: Buffer; key: StoredSecretVersionMaterial; identity: RestorePointIdentity; +}) { + verifyRestorePointPayload(input.payload, await restorePointKey(input)); +} + export async function openRestorePoint(input: { ciphertext: Buffer; sha256: string; @@ -567,12 +589,8 @@ export async function openRestorePoint(input: { if (input.ciphertext.length < minimum || !input.ciphertext.subarray(0, SEALED_MAGIC.length).equals(SEALED_MAGIC)) { throw new RestorePointRestoreFailure("invalid_ciphertext"); } - const unwrapped = JSON.parse(await localEncryptedProvider.resolveVersion({ material: input.key, externalRef: null } as Parameters[0])); + const unwrapped = await restorePointKey(input); const { identity } = input; - if (unwrapped.schema !== KEY_SCHEMA || unwrapped.companyId !== identity.companyId || unwrapped.workspaceId !== identity.workspaceId - || unwrapped.restorePointId !== identity.restorePointId || unwrapped.generation !== identity.generation) { - throw new RestorePointRestoreFailure("key_identity_mismatch"); - } const iv = input.ciphertext.subarray(SEALED_MAGIC.length, SEALED_MAGIC.length + 12); const tag = input.ciphertext.subarray(input.ciphertext.length - 16); const body = input.ciphertext.subarray(SEALED_MAGIC.length + 12, input.ciphertext.length - 16); @@ -585,9 +603,7 @@ export async function openRestorePoint(input: { } catch { throw new RestorePointRestoreFailure("ciphertext_authentication_failed"); } - if (payload.length !== unwrapped.plaintextBytes || createHash("sha256").update(payload).digest("hex") !== unwrapped.plaintextSha256) { - throw new RestorePointRestoreFailure("plaintext_hash_mismatch"); - } + verifyRestorePointPayload(payload, unwrapped); return payload; } diff --git a/server/src/services/isolated-task-directory-retention.ts b/server/src/services/isolated-task-directory-retention.ts index b4c83c70e66..102459580c8 100644 --- a/server/src/services/isolated-task-directory-retention.ts +++ b/server/src/services/isolated-task-directory-retention.ts @@ -7,7 +7,7 @@ import { environmentLeases, executionWorkspaceRestorePoints, executionWorkspaces import type { StoredSecretVersionMaterial } from "../secrets/types.js"; import type { StorageService } from "../storage/types.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; -import { openRestorePoint, sealRestorePoint } from "./git-restore-point.js"; +import { openRestorePoint, sealRestorePoint, verifyRestorePointPayloadCandidate } from "./git-restore-point.js"; import { notifyGitRestorePointWork } from "./git-restore-point-ledger.js"; import { abandonGitRestorePoint, activateGitRestorePoints, readRestorePointObject, storeSealedRestorePoint } from "./git-restore-point-storage.js"; import { allocatedBytes, describeGitExpirationFailure, expirationScope, isGitWorkspaceExpirationEligible, type GitExpirationFailure } from "./git-workspace-expiration.js"; @@ -15,6 +15,7 @@ import { IsolatedRepositorySourceBusy, withIsolatedRepositorySourceLock } from " import { logActivity } from "./activity-log.js"; import { captureTaskDirectorySnapshot, + createEmptyTaskDirectorySnapshot, inspectTaskDirectory, materializeTaskDirectorySnapshot, newestTaskDirectoryMtime, @@ -143,16 +144,26 @@ export async function loadTaskDirectoryRestorePoint(input: { storage: StorageService; workspace: { companyId: string; id: string }; ref: TaskDirectoryRestorePointRef; + recoverMissingEmpty?: boolean; signal?: AbortSignal; }) { const ref = validateTaskDirectoryRestorePointRef(input.ref, input.workspace); if (ref.expiredAt) throw new TaskDirectoryRestoreFailure("restore_point_expired"); - const ciphertext = await readRestorePointObject(input.storage, input.workspace.companyId, ref.objectKey, input.signal).catch(() => { - throw new TaskDirectoryRestoreFailure("restore_point_unavailable"); + const identity = { companyId: input.workspace.companyId, workspaceId: input.workspace.id, restorePointId: ref.id, generation: ref.generation }; + const ciphertext = await readRestorePointObject(input.storage, input.workspace.companyId, ref.objectKey, input.signal).catch(async () => { + input.signal?.throwIfAborted(); + if (!input.recoverMissingEmpty || ref.files !== 0 || ref.excluded !== 0) throw new TaskDirectoryRestoreFailure("restore_point_unavailable"); + const head = await input.storage.headObject(input.workspace.companyId, ref.objectKey, { signal: input.signal }).catch(() => null); + input.signal?.throwIfAborted(); + if (head?.exists !== false) throw new TaskDirectoryRestoreFailure("restore_point_unavailable"); + const candidate = createEmptyTaskDirectorySnapshot(); + if (candidate.digest !== ref.digest) throw new TaskDirectoryRestoreFailure("restored_contents_mismatch"); + await verifyRestorePointPayloadCandidate({ payload: candidate.payload, key: ref.key, identity }); + return null; }); + if (ciphertext === null) return createEmptyTaskDirectorySnapshot().payload; if (ciphertext.length !== ref.byteSize) throw new TaskDirectoryRestoreFailure("ciphertext_size_mismatch"); - return openRestorePoint({ ciphertext, sha256: ref.sha256, key: ref.key, - identity: { companyId: input.workspace.companyId, workspaceId: input.workspace.id, restorePointId: ref.id, generation: ref.generation } }); + return openRestorePoint({ ciphertext, sha256: ref.sha256, key: ref.key, identity }); } function skipReason(error: unknown) { diff --git a/server/src/services/isolated-task-directory-snapshot.ts b/server/src/services/isolated-task-directory-snapshot.ts index 1eafdfb126f..390e6596f63 100644 --- a/server/src/services/isolated-task-directory-snapshot.ts +++ b/server/src/services/isolated-task-directory-snapshot.ts @@ -148,7 +148,14 @@ export async function captureTaskDirectorySnapshot(root: string, options: { limi await handle.close(); } } }); - const manifest: TaskDirectorySnapshotManifest = { schema: TASK_DIRECTORY_SNAPSHOT_SCHEMA, entries: walked.entries, excluded: walked.excluded }; + return encodeTaskDirectorySnapshot({ schema: TASK_DIRECTORY_SNAPSHOT_SCHEMA, entries: walked.entries, excluded: walked.excluded }, bodies); +} + +export function createEmptyTaskDirectorySnapshot() { + return encodeTaskDirectorySnapshot({ schema: TASK_DIRECTORY_SNAPSHOT_SCHEMA, entries: [], excluded: [] }, []); +} + +function encodeTaskDirectorySnapshot(manifest: TaskDirectorySnapshotManifest, bodies: Buffer[]) { const encoded = Buffer.from(JSON.stringify(manifest)); const length = Buffer.alloc(4); length.writeUInt32BE(encoded.length); @@ -158,8 +165,8 @@ export async function captureTaskDirectorySnapshot(root: string, options: { limi return { payload, digest: taskDirectoryManifestDigest(manifest), - files: walked.entries.filter(entry => entry.type === "file").length, - excluded: walked.excluded.length, + files: manifest.entries.filter(entry => entry.type === "file").length, + excluded: manifest.excluded.length, }; } diff --git a/server/src/services/remote-workspace-migration.test.ts b/server/src/services/remote-workspace-migration.test.ts index a78367dd358..474bef43691 100644 --- a/server/src/services/remote-workspace-migration.test.ts +++ b/server/src/services/remote-workspace-migration.test.ts @@ -853,6 +853,26 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest }); expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("original saved work\n"); }); + it("migrates an exactly authenticated empty retained payload without rewriting its missing source", async () => { + const f = await seed(); await fs.rm(path.join(f.cwd, "saved.txt")); + const retained = await retainDirectory(f); await storage.deleteObject(f.companyId, retained.objectKey); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId)); + const before = await workspace(f.workspaceId); + expect(await migrateRemoteWorkspace({ db, storage, ...f, apply: true })).toMatchObject({ applied: true }); + const saved = await workspace(f.workspaceId), authority = readRemoteWorkspaceAuthority(saved.metadata)!; + expect(saved).toMatchObject({ status: "archived", closedAt: before.closedAt }); + expect(saved.metadata?.isolatedTaskDirectoryRetention).toEqual(before.metadata?.isolatedTaskDirectoryRetention); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]).toMatchObject({ status: "done", executionWorkspaceId: f.workspaceId }); + expect(await fs.stat(f.cwd).catch(() => null)).toBeNull(); + expect(await storage.headObject(f.companyId, retained.objectKey)).toMatchObject({ exists: false }); + const ref = await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! }); + const archivePath = path.join(root, "recovered-empty.pcws"), targetDir = path.join(root, "recovered-empty"); + await loadWorkspaceSnapshotArchive({ storage, reference: ref!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest }); + expect(await fs.readdir(targetDir)).toEqual([]); + expect(await migrateRemoteWorkspace({ db, storage, ...f, apply: true })).toMatchObject({ applied: true, replayed: true }); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)?.snapshotId).toBe(authority.snapshotId); + }); it.each(["missing", "expired", "corrupt"])("does not replace an unavailable retained directory with an empty workspace (%s)", async failure => { const f = await seed(), retained = await retainDirectory(f); if (failure === "missing") await storage.deleteObject(f.companyId, retained.objectKey); diff --git a/server/src/services/remote-workspace-migration.ts b/server/src/services/remote-workspace-migration.ts index 20a472b16c6..bfc28c4a809 100644 --- a/server/src/services/remote-workspace-migration.ts +++ b/server/src/services/remote-workspace-migration.ts @@ -321,7 +321,7 @@ export async function migrateRemoteWorkspace(input: { snapshotId: snapshot.id, targetPath: archivePath, signal: input.signal, }) : null; const sourceRoot = recovered ? null : retained ? await materializeTaskDirectorySnapshot({ - payload: await loadTaskDirectoryRestorePoint({ storage: input.storage, workspace: reserved.workspace, ref: retained }), + payload: await loadTaskDirectoryRestorePoint({ storage: input.storage, workspace: reserved.workspace, ref: retained, recoverMissingEmpty: true, signal: input.signal }), target: path.join(scratch, "retained-source"), expectedDigest: retained.digest, }) : reserved.archivedGitSource ? await materializeArchivedGitMigrationSource({ db: input.db, storage: input.storage, workspace: reserved.workspace, source: reserved.archivedGitSource, scratch }) : reserved.workspace.cwd ?? reserved.workspace.providerRef!;