diff --git a/doc/DATABASE.md b/doc/DATABASE.md index b96d485b48e..ab7d6cf7cb4 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -49,6 +49,11 @@ and leaves all existing cases passive. Source identities are not grants. ### Task query performance +The cancelled workspace-preservation partial index and bounded run/activity +projections share the restore-failure allowlist. Changing those failure codes +also changes the index predicate; use the normal migration-generation workflow. +A required metadata repair is excluded from ordinary cancelled preservation work. + Native session resume selects its newest progress barrier within the existing 32-run task/session window in one SQL snapshot. Only the chosen checkpoint source returns its saved native input, checkpoint, tool fingerprint, wake envelope, and diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 6ccaec0bfe4..8580413e81f 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -710,6 +710,22 @@ The directory merge still preserves host-only file changes under its existing rules. A changed host branch requires recovery. An intentional reset to an ancestor exports a full Git bundle so restore keeps the actual sandbox tip; an empty delta is reserved for an unchanged tip. +A full Git bundle from a shallow repository still omits history before its +shallow boundaries. When a delta import cannot satisfy its history, sandbox +restore exports those boundaries with the full bundle. The host unpacks it in +a temporary bare repository and uses Git's shallow fetch protocol to import it. +This preserves existing host ancestry and connectivity checks. File restore and +the normal concurrent-work merge must still finish before run finalization. +The boundary metadata has a 64 KiB transfer/read budget and at most 256 unique +commits. Duplicate entries do not repeat host Git probes. Boundary inspection +has a separate 15-second budget; cloning, unpacking and import share 60 seconds +without charging probe time. Oversized or malformed metadata fails before import +with a distinct non-retryable metadata classification. It requires file recovery +and never uses the unsafe-archive omission path to complete an accepted result. +The sandbox producer uses a dedicated exit status which the command client retains +and the export owner converts back to the typed metadata error. Bounded API +projections and cancelled-preservation selection use the shared failure allowlist; +task chat retains its recovery notice without offering another provider turn. Unrelated sandbox history keeps the existing history-preserving graft only when the recorded host has not advanced; it must not replace concurrent host work. Warm sandbox reuse must match the current host Git tip and branch as well as the @@ -1576,6 +1592,13 @@ and obtains a fresh stop receipt. An interrupted legacy run does not need an accepted native result. A successful `workspace_recovery` operation and matching recovery record prove persistence; adapter output or a user repair note cannot. The failed run and unknown external effects remain historical facts. +A reusable lease can also owe native copyback: the export-repair action must +select its exact immutable descriptor and the finalizer's accepted successful +result under the same company, task, run and completion contract. Lease reuse +policy alone does not discharge that persistence obligation. Ordinary reusable +sources without this proof remain ineligible. The preservation owner still +requires an independently stopped, exclusively bound allocation and no host writer; +it saves files and resumes the existing finalizer without another provider turn. Startup and the retained-workspace sweep reconstruct these obligations through the retained workspace index. Committed Retry, recovery admission, and recovery diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md index 2f9bd8214e7..e18533217d4 100644 --- a/maps/agent-access-map/recovery.md +++ b/maps/agent-access-map/recovery.md @@ -321,7 +321,7 @@ API: `present` — POST /api/issues/{id}/recovery-actions/recover-workspace with Agent access: `restricted` — board task-read and issue:mutate; not runtime:manage. Agents denied through REST/native fallback. -Policy: Exact terminal retained source/lease/reference; independent stop and physical ownership checks; no newer writer. Bounded queued repair preserves newer files and has retry state. Matching key/recovered state dedupes; permanent blocked source refuses. Recovery starts no provider turn. +Policy: Exact terminal retained source/lease/reference; independent stop and physical ownership checks; no newer writer. Bounded queued repair preserves newer files and has retry state. Matching key/recovered state dedupes; permanent blocked source refuses. Recovery starts no provider turn. For native export repair, a reusable lease is eligible only with its exact descriptor and the finalizer-selected accepted successful result under the same company/task/run/completion contract. This does not change actor access or permit another allocation owner or host writer. Invalid Git metadata is a non-retryable persistence failure requiring service repair; it cannot use unsafe-archive omission to complete the result. Run-detail/activity projections retain the allowlisted code, including bounded large-result retrieval, and cancelled-preservation discovery honors the same hold. Evidence: `source-only` — [recovery routes](../../server/src/routes/issues.ts), [retained recovery](../../server/src/services/workspace-recovery.ts); reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`. No live provider proof. diff --git a/maps/agent-access-map/runs-adapters.md b/maps/agent-access-map/runs-adapters.md index b1c907ebe4c..cf775968280 100644 --- a/maps/agent-access-map/runs-adapters.md +++ b/maps/agent-access-map/runs-adapters.md @@ -305,6 +305,13 @@ guard. This changes no actor permission or session-reset authority. Follow the [session recipe](../feature-map/runs-adapters.md#session) for recovery followed by ordinary turns; deterministic runner checks do not establish live provider access. +Sandbox export carries shallow Git boundaries through the full-bundle fallback +and uses normal Git fetch connectivity checks. Existing host ancestry and merge +guards remain effective. This grants no task retry or operator recovery authority; +the saved-result export API still requires board runtime management permission. +Follow the [session recipe](../feature-map/runs-adapters.md#session) for separately +fetched shallow history; filesystem controls do not establish live recovery. + Plugin suspension receipts remain required through recovery and admission, then are removed when the admitted session becomes live. A later warm run therefore does not inherit an obsolete stopped-run identity. Repository-free fresh plugin diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 2deba6248e4..00983d8aeb8 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -399,7 +399,18 @@ For a source with a retained-file protection marker, exercise recovery through the assembled environment runtime. Its exact preservation owner may reopen the stopped allocation to read saved files; ordinary resume stays blocked. Reject a changed claim, missing stop intent, wrong lease or wrong environment before a -provider call. Verify saved and newer local files, one preservation receipt, and +provider call. Repeat accepted-native-result copyback for both reusable lease policies. Require +the exact accepted result and completion contract; reject a rejected or unselected +result, another allocation owner, and an active host writer. Verify no provider +turn or wake is created, then finalize the original result once. Reject malformed +or oversized Git metadata through native finalization: the run must fail and +retain its file-recovery barrier, rather than successfully omitting the export. +Exercise both an oversized sender file and an oversized transfer report. Verify +one rejected export, no payload read after rejection, and unchanged host files. +Reload a large-result run and its activity: the failure code and task recovery +notice remain visible, and metadata repair offers no new provider turn. These database +and filesystem controls do not qualify a deployed provider. +Verify saved and newer local files, one preservation receipt, and no additional agent turn. The PostgreSQL regression covers this boundary; the opt-in provider probe also uses this runtime path with the protection marker. diff --git a/maps/feature-map/runs-adapters.md b/maps/feature-map/runs-adapters.md index 9acbc35d21e..21c853d8501 100644 --- a/maps/feature-map/runs-adapters.md +++ b/maps/feature-map/runs-adapters.md @@ -102,6 +102,13 @@ The [remote continuity regression](../../doc/testing-native-remote-continuity.md uses a real Linux runner and deterministic provider. Live provider acceptance is a separate check. +In a fresh sandbox workspace, fetch a different published branch with depth one, +then save a local result and finish. Verify export imports the shallow history, +retains host-only edits, and completes finalization without another provider turn. +Repeat when the host already has that branch history; its ancestry must remain +available. The filesystem sandbox regression covers transfer and merge behavior, +not live provider acceptance. + For a plugin computer, complete a task, let its warm session suspend, then send two follow-ups within the next warm window. Verify the first resumes the saved session and the second keeps the same provider history without an identity diff --git a/packages/adapter-utils/src/command-managed-runtime.test.ts b/packages/adapter-utils/src/command-managed-runtime.test.ts index f51855f2927..cc264f0cd35 100644 --- a/packages/adapter-utils/src/command-managed-runtime.test.ts +++ b/packages/adapter-utils/src/command-managed-runtime.test.ts @@ -1027,6 +1027,8 @@ describe("command managed runtime", () => { await expect(client.run("tar -cf workspace-download.tar .", { timeoutMs: 30_000 })).rejects.toThrow( /stdout: tar: workspace-download\.tar: Cannot open: Permission denied/, ); + await expect(client.run("tar -cf workspace-download.tar .", { timeoutMs: 30_000 })) + .rejects.toMatchObject({ exitCode: 2, timedOut: false }); }); it("test_channel_round_trips_all_byte_values", () => { diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts index ebe858153e6..501859e8eca 100644 --- a/packages/adapter-utils/src/command-managed-runtime.ts +++ b/packages/adapter-utils/src/command-managed-runtime.ts @@ -189,7 +189,9 @@ function formatFailedCommandOutput(result: RunProcessResult): string { function requireSuccessfulResult(result: RunProcessResult, action: string): void { if (result.exitCode === 0 && !result.timedOut) return; const detail = formatFailedCommandOutput(result); - throw new Error(`${action} failed with exit code ${result.exitCode ?? "null"}${detail}`); + throw Object.assign(new Error(`${action} failed with exit code ${result.exitCode ?? "null"}${detail}`), { + exitCode: result.exitCode, timedOut: result.timedOut, + }); } function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer { diff --git a/packages/adapter-utils/src/git-shallow-import-budget.test.ts b/packages/adapter-utils/src/git-shallow-import-budget.test.ts new file mode 100644 index 00000000000..6fdbdeae1ac --- /dev/null +++ b/packages/adapter-utils/src/git-shallow-import-budget.test.ts @@ -0,0 +1,66 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { expect, it, vi } from "vitest"; +import { fetchGitBundleIntoLocalRef, runLocalGit } from "./git-workspace-sync.js"; + +const timing = vi.hoisted(() => ({ enabled: false, elapsedProbeMs: 0, probeDurationMs: 14_000, fetchTimeout: 0 })); +vi.mock("node:child_process", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, execFile: (...args: Parameters) => { + const gitArgs = args[1] as string[]; + if (timing.enabled && gitArgs.includes("fetch")) timing.fetchTimeout = Number((args[2] as { timeout: number }).timeout); + // Keep real Git and its callbacks. Only its observed probe duration is + // controlled, so a fast fixture can exercise a nearly spent probe budget. + if (timing.enabled && gitArgs.some(arg => arg.endsWith("^@"))) { + const callback = args[3] as (...values: unknown[]) => void; + args[3] = ((...values: unknown[]) => { timing.elapsedProbeMs += timing.probeDurationMs; callback(...values); }) as typeof args[3]; + } + return actual.execFile(...args); + } }; +}); + +it.each([14_000, 16_000])("separates transfer and probe budgets (probe duration %s)", async probeDurationMs => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-git-probe-budget-")); + const git = async (cwd: string, args: string[]) => (await runLocalGit(cwd, args)).stdout.trim(); + const realNow = Date.now.bind(Date); + try { + const source = path.join(root, "source"), host = path.join(root, "host"), remote = path.join(root, "remote"); + await git(root, ["init", source]); + await git(source, ["config", "user.name", "Test"]); + await git(source, ["config", "user.email", "test@example.com"]); + await writeFile(path.join(source, "saved.txt"), "base\n"); + await git(source, ["add", "."]); await git(source, ["commit", "-m", "base"]); + await git(root, ["clone", source, host]); + await writeFile(path.join(source, "saved.txt"), "completed\n"); + await git(source, ["commit", "-am", "completed"]); + const head = await git(source, ["rev-parse", "HEAD"]); + await git(root, ["clone", "--depth=1", `file://${source}`, remote]); + const bundlePath = path.join(root, "work.bundle"), shallowPath = path.join(remote, ".git", "shallow"); + const exportRef = `refs/paperclip/export/${randomUUID()}`, importedRef = `refs/paperclip/import/${randomUUID()}`; + await git(remote, ["update-ref", exportRef, head]); + await git(remote, ["bundle", "create", bundlePath, exportRef]); + vi.spyOn(Date, "now").mockImplementation(() => realNow() + timing.elapsedProbeMs); + timing.enabled = true; timing.probeDurationMs = probeDurationMs; + if (probeDurationMs > 15_000) { + await expect(fetchGitBundleIntoLocalRef({ localDir: host, bundlePath, shallowPath, exportRef, importedRef, baseSha: head })) + .rejects.toThrow("boundary inspection timed out"); + expect(timing.fetchTimeout).toBe(0); + timing.enabled = false; + await expect(git(host, ["rev-parse", "--verify", importedRef])).rejects.toThrow(); + expect(await readFile(path.join(host, "saved.txt"), "utf8")).toBe("base\n"); + return; + } + expect(await fetchGitBundleIntoLocalRef({ localDir: host, bundlePath, shallowPath, exportRef, importedRef, baseSha: head })).toBe(head); + expect(timing.elapsedProbeMs).toBe(14_000); + expect(timing.fetchTimeout).toBeGreaterThan(55_000); + timing.enabled = false; + expect(await git(host, ["show", `${importedRef}:saved.txt`])).toBe("completed"); + expect(await git(host, ["fsck", "--connectivity-only"])).not.toContain("missing"); + expect(await readFile(path.join(host, "saved.txt"), "utf8")).toBe("base\n"); + } finally { + timing.enabled = false; timing.elapsedProbeMs = 0; timing.fetchTimeout = 0; + vi.restoreAllMocks(); await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/adapter-utils/src/git-workspace-sync.test.ts b/packages/adapter-utils/src/git-workspace-sync.test.ts index c1029239a18..72d8c515368 100644 --- a/packages/adapter-utils/src/git-workspace-sync.test.ts +++ b/packages/adapter-utils/src/git-workspace-sync.test.ts @@ -578,6 +578,84 @@ describe("git workspace sync", () => { } }); + it.each(["missing", "complete", "shallow"])("imports shallow full bundles without truncating existing host ancestry (host history: %s)", async (hostHistory) => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-shallow-bundle-")); + cleanupDirs.push(root); + const source = await createRepo(root); + await writeFile(path.join(source, "tracked.txt"), "host baseline\n"); + await git(source, ["commit", "-am", "host baseline"]); + await git(source, ["branch", "baseline"]); + await writeFile(path.join(source, "tracked.txt"), "published work\n"); + await git(source, ["commit", "-am", "published"]); + const published = await git(source, ["rev-parse", "HEAD"]); + const host = path.join(root, "host"); + await git(root, hostHistory === "shallow" + ? ["clone", "--depth=1", "--branch", "baseline", `file://${source}`, host] + : ["clone", source, host]); + const originalShallow = hostHistory === "shallow" ? await readFile(path.join(host, ".git", "shallow"), "utf8") : null; + if (hostHistory === "missing") { + // A host containing just the original base, never the separately fetched PR. + await git(host, ["reset", "--hard", "HEAD^"]); + await git(host, ["reflog", "expire", "--expire=now", "--all"]); + await git(host, ["update-ref", "-d", "refs/remotes/origin/main"]); + await git(host, ["symbolic-ref", "--delete", "refs/remotes/origin/HEAD"]); + await git(host, ["gc", "--prune=now"]); + await expect(git(host, ["cat-file", "-e", published])).rejects.toThrow(); + } + const before = await git(host, ["rev-list", "HEAD"]); + const sandbox = path.join(root, "sandbox"); + await git(root, ["clone", "--depth=1", `file://${source}`, sandbox]); + const bundlePath = path.join(root, "full.bundle"); + const shallowPath = path.join(root, "shallow"); + const exportRef = createRemoteGitExportRef("test"); + const importedRef = createImportedGitRef("test"); + await execFile("sh", ["-c", buildRemoteGitDeltaBundleScript({ + remoteDir: sandbox, baseSha: published, exportRef, bundlePath, shallowPath, forceFullBundle: true, + })]); + const tracePath = path.join(root, "git-trace.json"); + const previousTrace = process.env.GIT_TRACE2_EVENT; + process.env.GIT_TRACE2_EVENT = tracePath; + const gitStarts = async () => (await readFile(tracePath, "utf8")).trim().split("\n") + .filter((line) => JSON.parse(line).event === "start").length; + try { + expect(await fetchGitBundleIntoLocalRef({ localDir: host, bundlePath, exportRef, importedRef, + baseSha: published, shallowPath })).toBe(published); + const uniqueStarts = await gitStarts(); + await rm(tracePath); + await writeFile(shallowPath, `${published}\n`.repeat(12)); + expect(await fetchGitBundleIntoLocalRef({ localDir: host, bundlePath, exportRef, importedRef, + baseSha: published, shallowPath })).toBe(published); + // Duplicate sender metadata must not multiply host subprocess work. + expect(await gitStarts()).toBeLessThanOrEqual(uniqueStarts + 4); + await rm(tracePath); + await writeFile(shallowPath, `${published}\n`.repeat(2000)); + await expect(fetchGitBundleIntoLocalRef({ localDir: host, bundlePath, exportRef, importedRef, + baseSha: published, shallowPath })).rejects.toThrow("byte limit"); + await expect(stat(tracePath)).rejects.toMatchObject({ code: "ENOENT" }); + await writeFile(shallowPath, Array.from({ length: 257 }, (_, i) => + (i + 1).toString(16).padStart(40, "0")).join("\n")); + await expect(fetchGitBundleIntoLocalRef({ localDir: host, bundlePath, exportRef, importedRef, + baseSha: published, shallowPath })).rejects.toThrow("boundary limit"); + await expect(stat(tracePath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + if (previousTrace === undefined) delete process.env.GIT_TRACE2_EVENT; + else process.env.GIT_TRACE2_EVENT = previousTrace; + } + expect(await git(host, ["rev-list", "HEAD"])).toBe(before); + if (originalShallow) expect(await readFile(path.join(host, ".git", "shallow"), "utf8")).toBe(originalShallow); + expect(await git(host, ["fsck", "--connectivity-only"])).not.toContain("missing"); + + await writeFile(shallowPath, `${published}\n../../config\n`); + await expect(fetchGitBundleIntoLocalRef({ localDir: host, bundlePath, exportRef, + importedRef: createImportedGitRef("invalid"), baseSha: published, shallowPath })) + .rejects.toThrow("Invalid Git shallow boundaries"); + expect(await git(host, ["rev-list", "HEAD"])).toBe(before); + await writeFile(path.join(sandbox, ".git", "shallow"), `${published}\n`.repeat(2000)); + await expect(execFile("sh", ["-c", buildRemoteGitDeltaBundleScript({ + remoteDir: sandbox, baseSha: published, exportRef, bundlePath, shallowPath, forceFullBundle: true, + })])).rejects.toThrow("byte limit"); + }); + it("falls back to a full self-contained bundle when the sandbox lacks baseSha", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-git-full-")); cleanupDirs.push(rootDir); diff --git a/packages/adapter-utils/src/git-workspace-sync.ts b/packages/adapter-utils/src/git-workspace-sync.ts index 278a9642131..10f0ec97169 100644 --- a/packages/adapter-utils/src/git-workspace-sync.ts +++ b/packages/adapter-utils/src/git-workspace-sync.ts @@ -695,22 +695,120 @@ export async function deleteLocalGitRef(input: { }).catch(() => undefined); } +export const GIT_SHALLOW_METADATA_EXIT_CODE = 47; +export const GIT_SHALLOW_METADATA_ERROR_CODE = "WORKSPACE_RESTORE_INVALID_GIT_METADATA"; + +/** Invalid restore metadata is an integrity rejection, not a dropped transfer. */ +export class GitShallowMetadataError extends Error { + readonly code = GIT_SHALLOW_METADATA_ERROR_CODE; +} + +export const GIT_SHALLOW_MAX_BYTES = 64 * 1024; +export const GIT_SHALLOW_MAX_BOUNDARIES = 256; + export async function fetchGitBundleIntoLocalRef(input: { localDir: string; bundlePath: string; exportRef: string; importedRef: string; baseSha: string; + /** Shallow boundaries accompanying a full bundle, which cannot encode them. */ + shallowPath?: string; }): Promise { const bundleSize = (await fs.stat(input.bundlePath).catch(() => null))?.size ?? 0; if (bundleSize === 0) { return input.baseSha; } - await runLocalGit(input.localDir, ["fetch", "--force", input.bundlePath, `${input.exportRef}:${input.importedRef}`], { - timeout: 60_000, - maxBuffer: 1024 * 1024, - }); + const shallowSize = input.shallowPath ? (await fs.stat(input.shallowPath)).size : 0; + if (shallowSize > GIT_SHALLOW_MAX_BYTES) { + throw new GitShallowMetadataError(`Git shallow metadata exceeds its byte limit (${shallowSize} > ${GIT_SHALLOW_MAX_BYTES})`); + } + const shallow = input.shallowPath ? await fs.readFile(input.shallowPath, "utf8") : ""; + const shallowBytes = Buffer.byteLength(shallow); + if (shallowBytes > GIT_SHALLOW_MAX_BYTES) + throw new GitShallowMetadataError(`Git shallow metadata exceeds its byte limit (${shallowBytes} > ${GIT_SHALLOW_MAX_BYTES})`); + if (shallow.trim()) { + const boundaries = [...new Set(shallow.trim().split("\n"))]; + if (boundaries.length > GIT_SHALLOW_MAX_BOUNDARIES) throw new GitShallowMetadataError(`Git shallow metadata exceeds its boundary limit (${boundaries.length} > ${GIT_SHALLOW_MAX_BOUNDARIES})`); + let deadline = Date.now() + 60_000; + const importGit = (cwd: string, args: string[], options: NonNullable[2]> = {}) => { + const remaining = deadline - Date.now(); + if (remaining <= 0) throw new Error("Git shallow history import timed out"); + return runLocalGit(cwd, args, { ...options, timeout: Math.min(options.timeout ?? 15_000, remaining) }); + }; + const objectFormat = (await importGit(input.localDir, ["rev-parse", "--show-object-format"])).stdout.trim(); + const oid = objectFormat === "sha256" ? /^[a-f0-9]{64}$/ : /^[a-f0-9]{40}$/; + if (!boundaries.every((boundary) => oid.test(boundary))) throw new GitShallowMetadataError(`Invalid Git shallow boundaries for host object format ${objectFormat}`); + const staging = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-git-bundle-")); + try { + // Git bundles have no shallow protocol. Unpack into a private repository + // carrying the sender's boundaries, then let ordinary Git fetch negotiate + // which boundaries the host actually needs. Never overwrite host shallow + // metadata or disable connectivity checks. + await importGit(input.localDir, ["clone", "--bare", "--shared", input.localDir, staging], { timeout: 60_000 }); + const advertised = await importGit(staging, ["bundle", "unbundle", input.bundlePath], { + timeout: 60_000, maxBuffer: 1024 * 1024, + }); + const head = advertised.stdout.split("\n").find((line) => line.endsWith(` ${input.exportRef}`))?.split(" ")[0]; + if (!head || !oid.test(head)) throw new Error("Git bundle does not advertise the requested export ref"); + const retainedBoundaries = (await fs.readFile(path.join(staging, "shallow"), "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + return ""; + })).trim().split("\n").filter(Boolean); + // Boundary inspection has its own small budget. It must not consume the + // transfer budget needed to save a valid large bundle after the probes. + const probeStarted = Date.now(), probeDeadline = probeStarted + 15_000; + const probeGit = (args: string[]) => { + const remaining = probeDeadline - Date.now(); + if (remaining <= 0) throw new Error("Git shallow boundary inspection timed out"); + return runLocalGit(staging, args, { timeout: remaining }); + }; + const knownCommits = new Map(); + const hasCommit = async (commit: string): Promise => { + if (knownCommits.has(commit)) return knownCommits.get(commit)!; + try { + await probeGit(["rev-parse", "--verify", "--quiet", `${commit}^{commit}`]); + knownCommits.set(commit, true); + return true; + } catch (error) { + if ((error as { code?: unknown }).code === 1) { + knownCommits.set(commit, false); + return false; + } + throw error; + } + }; + for (const boundary of boundaries) { + // A shallow sender can stop before history the host already holds. + // Keep that connection when possible; adding its boundary unconditionally + // would hide a real merge-base and reject concurrent host changes. + if (!await hasCommit(boundary)) continue; // An unrelated tip is not in this bundle. + const parents = await probeGit(["rev-parse", `${boundary}^@`]); + let missingParent = false; + for (const parent of parents.stdout.trim().split("\n").filter(Boolean)) { + if (!await hasCommit(parent)) { + missingParent = true; + break; + } + } + if (missingParent) retainedBoundaries.push(boundary); + } + deadline += Date.now() - probeStarted; + if (retainedBoundaries.length) await fs.writeFile(path.join(staging, "shallow"), `${[...new Set(retainedBoundaries)].join("\n")}\n`); + await importGit(staging, ["update-ref", input.exportRef, head]); + await importGit(input.localDir, ["fetch", "--force", "--update-shallow", staging, `${input.exportRef}:${input.importedRef}`], { + timeout: 60_000, maxBuffer: 1024 * 1024, + }); + } finally { + await fs.rm(staging, { recursive: true, force: true }); + } + } else { + await runLocalGit(input.localDir, ["fetch", "--force", input.bundlePath, `${input.exportRef}:${input.importedRef}`], { + timeout: 60_000, + maxBuffer: 1024 * 1024, + }); + } const importedHead = await runLocalGit(input.localDir, ["rev-parse", input.importedRef], { timeout: 10_000, maxBuffer: 16 * 1024, @@ -750,6 +848,8 @@ export function buildRemoteGitDeltaBundleScript(input: { * failed because the host lacked the bundle's prerequisite. */ forceFullBundle?: boolean; + /** Export the sender's shallow boundaries alongside a full bundle. */ + shallowPath?: string; }): string { const remoteDir = shellQuote(input.remoteDir); const bundlePath = shellQuote(input.bundlePath); @@ -758,6 +858,7 @@ export function buildRemoteGitDeltaBundleScript(input: { const statusPath = input.statusPath ? shellQuote(input.statusPath) : null; const cleanupParts = [ `rm -f ${bundlePath}`, + ...(input.shallowPath ? [`rm -f ${shellQuote(input.shallowPath)}`] : []), ...(statusPath ? [`rm -f ${statusPath}`] : []), `git -C ${remoteDir} update-ref -d ${exportRef} >/dev/null 2>&1 || true`, ]; @@ -767,6 +868,13 @@ export function buildRemoteGitDeltaBundleScript(input: { input.cleanupBundle ? "trap cleanup EXIT" : "", `mkdir -p ${shellQuote(path.posix.dirname(input.bundlePath))}`, `rm -f ${bundlePath}`, + ...(input.shallowPath ? [ + `shallow_file=$(git -C ${remoteDir} rev-parse --path-format=absolute --git-path shallow)`, + `if [ -f "$shallow_file" ]; then`, + ` dd if="$shallow_file" of=${shellQuote(input.shallowPath)} bs=${GIT_SHALLOW_MAX_BYTES + 1} count=1 2>/dev/null`, + `else : > ${shellQuote(input.shallowPath)}; fi`, + `[ "$(wc -c < ${shellQuote(input.shallowPath)})" -le ${GIT_SHALLOW_MAX_BYTES} ] || { printf '%s\\n' 'Git shallow metadata exceeds its byte limit' >&2; exit ${GIT_SHALLOW_METADATA_EXIT_CODE}; }`, + ] : []), // Choose the bundle boundary. A thin bundle `HEAD --not ` records // baseSha as a prerequisite the importer (host) must already hold. That // assumption breaks in two real cases, and then `git fetch` on the host diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts index 854f76e9102..799ecfb82ca 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts @@ -932,6 +932,50 @@ describe("sandbox managed runtime", () => { expect(await git(host, ["diff", "--cached", "--name-only"])).toBe(""); }); + it("restores a separately fetched shallow branch whose boundary the host lacks", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-shallow-restore-")); + cleanupDirs.push(root); + const host = path.join(root, "host"); + const remote = path.join(root, "remote"); + const origin = path.join(root, "origin"); + await mkdir(host); + await git(host, ["init"]); + await git(host, ["config", "user.name", "Paperclip Test"]); + await git(host, ["config", "user.email", "test@paperclip.dev"]); + await writeFile(path.join(host, "tracked.txt"), "base\n"); + await git(host, ["add", "."]); + await git(host, ["commit", "-m", "base"]); + const baseline = await git(host, ["rev-parse", "HEAD"]); + await git(root, ["clone", host, origin]); + await git(origin, ["config", "user.name", "Paperclip Test"]); + await git(origin, ["config", "user.email", "test@paperclip.dev"]); + for (const content of ["published", "review fix"]) { + await writeFile(path.join(origin, "tracked.txt"), content); + await git(origin, ["commit", "-am", content]); + } + const publishedHead = await git(origin, ["rev-parse", "HEAD"]); + const prepared = await prepareSandboxManagedRuntime({ + ...sandboxRuntimeInput(sandboxSpec("shallow", remote), "test-adapter", makeFilesystemClient(), host), + workspaceRestoreRetryDelaysMs: [], + }); + // Like checking out a PR in a fresh task workspace: only its tip is fetched, + // and the sandbox still has its independent original shallow boundary. + await git(remote, ["fetch", "--depth=1", origin, publishedHead]); + await git(remote, ["checkout", "--detach", "FETCH_HEAD"]); + await writeFile(path.join(remote, "evidence.txt"), "saved result\n"); + await writeFile(path.join(host, "local-only.txt"), "concurrent host work\n"); + + await prepared.restoreWorkspace(); + + expect(await readFile(path.join(host, "tracked.txt"), "utf8")).toBe("review fix"); + expect(await readFile(path.join(host, "evidence.txt"), "utf8")).toBe("saved result\n"); + expect(await readFile(path.join(host, "local-only.txt"), "utf8")).toBe("concurrent host work\n"); + expect(await git(host, ["cat-file", "-t", publishedHead])).toBe("commit"); + // Imported shallow history must not truncate the host's existing history. + expect(await git(host, ["rev-parse", "HEAD^"])).toBe(baseline); + expect(await git(host, ["fsck", "--connectivity-only"])).not.toContain("missing"); + }); + it("restores a sandbox reset to an ancestor with the matching clean working tree", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-reset-")); cleanupDirs.push(rootDir); @@ -4244,7 +4288,7 @@ describe("sandbox git-bundle export transport", () => { expect(capture.readFilePaths.some((remotePath) => remotePath.endsWith("git-delta.bundle"))).toBe(true); }); - it("retries the full bundle through the native branch when the delta misses its prerequisite", async () => { + it.each([[false, false, false], [true, false, false], [false, true, false], [false, false, true]])("restores native bundles with concurrent work %s, oversized transfer %s, oversized source %s", async (concurrentHostCommit, oversizedTransfer, oversizedSource) => { const capture: TransportCapture = { syncOutOperations: [], readFilePaths: [] }; const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bundle-retry-")); cleanupDirs.push(rootDir); @@ -4268,19 +4312,61 @@ describe("sandbox git-bundle export transport", () => { const stagedBase = await git(localWorkspaceDir, ["rev-parse", "HEAD"]); const client = makeTransportClient(true, capture); + let shallowProgressCalls = 0, completedShallowRead = false; + if (oversizedTransfer) { + const read = client.readFile.bind(client); + client.readFile = async (remotePath, options) => { + if (remotePath.endsWith("git-shallow.txt")) { + shallowProgressCalls++; + await options?.onProgress?.(0, 64 * 1024 + 1); + completedShallowRead = true; + } + return read(remotePath, options); + }; + } + let sourceCapChecks = 0; + if (oversizedSource) { + const run = client.run.bind(client); + client.run = async (command, options) => { + if (command.includes("git-shallow.txt") && command.includes("wc -c")) sourceCapChecks++; + return run(command, options); + }; + } const prepared = await prepareSandboxManagedRuntime(sandboxRuntimeInput(gitSpec(remoteWorkspaceDir), "test-adapter", client, localWorkspaceDir)); await commitInSandbox(remoteWorkspaceDir); const sandboxHead = await git(remoteWorkspaceDir, ["rev-parse", "HEAD"]); + if (oversizedSource) await writeFile(path.join(remoteWorkspaceDir, ".git", "shallow"), `${sandboxHead}\n`.repeat(2000)); + // The host drops below the staged base commit and prunes it. The delta // bundle names that boundary commit as a prerequisite the host no longer // holds, so the import fails and forces the full-bundle retry. await git(localWorkspaceDir, ["reset", "--hard", firstCommit]); + if (concurrentHostCommit) { + await writeFile(path.join(localWorkspaceDir, "host-only.txt"), "concurrent host commit\n"); + await git(localWorkspaceDir, ["add", "host-only.txt"]); + await git(localWorkspaceDir, ["commit", "-m", "host work after reset"]); + } + const hostHead = await git(localWorkspaceDir, ["rev-parse", "HEAD"]); await git(localWorkspaceDir, ["reflog", "expire", "--expire=now", "--all"]); await git(localWorkspaceDir, ["gc", "--prune=now"]); await expect(git(localWorkspaceDir, ["cat-file", "-e", `${stagedBase}^{commit}`])).rejects.toThrow(); + if (oversizedSource) { + await expect(prepared.restoreWorkspace()).rejects.toMatchObject({ code: "WORKSPACE_RESTORE_INVALID_GIT_METADATA" }); + expect(sourceCapChecks).toBe(1); + expect(capture.readFilePaths.some(p => p.endsWith("git-shallow.txt"))).toBe(false); + expect(await git(localWorkspaceDir, ["rev-parse", "HEAD"])).toBe(hostHead); + return; + } + if (oversizedTransfer) { + await expect(prepared.restoreWorkspace()).rejects.toThrow("byte limit"); + expect(shallowProgressCalls).toBe(1); + expect(completedShallowRead).toBe(false); + expect(await git(localWorkspaceDir, ["rev-parse", "HEAD"])).toBe(hostHead); + return; + } await prepared.restoreWorkspace(); // Two bundle exports rode native file mappings: the delta attempt and the @@ -4293,6 +4379,13 @@ describe("sandbox git-bundle export transport", () => { // The full bundle was self-contained: the host repository now holds the // sandbox head commit. await expect(git(localWorkspaceDir, ["cat-file", "-e", `${sandboxHead}^{commit}`])).resolves.toBe(""); + if (concurrentHostCommit) { + expect(await readFile(path.join(localWorkspaceDir, "host-only.txt"), "utf8")).toBe("concurrent host commit\n"); + expect(await readFile(path.join(localWorkspaceDir, "remote-only.txt"), "utf8")).toBe("from sandbox\n"); + expect((await git(localWorkspaceDir, ["rev-list", "--parents", "-1", "HEAD"])).split(" ").slice(1)) + .toEqual([hostHead, sandboxHead]); + expect(await git(localWorkspaceDir, ["fsck", "--connectivity-only"])).not.toContain("missing"); + } }); it("reports the real transferred bytes for the native git-history export and workspace restore", async () => { diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index 5f9bc27f50b..8396a13371f 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -14,6 +14,9 @@ import path from "node:path"; import { promisify } from "node:util"; import { buildRemoteGitDeltaBundleScript, + GIT_SHALLOW_METADATA_EXIT_CODE, + GIT_SHALLOW_MAX_BYTES, + GitShallowMetadataError, isMissingGitPrerequisiteError, createImportedGitRef, createRemoteGitExportRef, @@ -2052,6 +2055,8 @@ export async function prepareSandboxManagedRuntime(input: { const remoteWorkspaceStatusPath = path.posix.join(runtimeRootDir, "workspace-status.txt"); const exportRef = createRemoteGitExportRef("sandbox"); const localBundlePath = path.join(tempDir, "git-delta.bundle"); + const remoteShallowPath = path.posix.join(runtimeRootDir, "git-shallow.txt"); + const localShallowPath = path.join(tempDir, "git-shallow.txt"); // Export the sandbox history and import it into the host workspace. // The delta bundle assumes the host holds the bundle's boundary @@ -2060,17 +2065,28 @@ export async function prepareSandboxManagedRuntime(input: { // a full, self-contained bundle from the still-live sandbox rather // than discard the completed run. const exportAndImport = async (forceFullBundle: boolean): Promise => withWorkspaceRestoreStep("git_export", async () => { - await input.client.run( - `sh -c ${shellQuote(buildRemoteGitDeltaBundleScript({ - remoteDir: workspaceRemoteDir, - baseSha: gitSnapshot.headCommit, - exportRef, - bundlePath: remoteGitBundle, - statusPath: forceFullBundle ? undefined : remoteWorkspaceStatusPath, - forceFullBundle, - }))}`, - { timeoutMs: input.spec.timeoutMs }, - ); + try { + await input.client.run( + `sh -c ${shellQuote(buildRemoteGitDeltaBundleScript({ + remoteDir: workspaceRemoteDir, + baseSha: gitSnapshot.headCommit, + exportRef, + bundlePath: remoteGitBundle, + statusPath: forceFullBundle ? undefined : remoteWorkspaceStatusPath, + forceFullBundle, + shallowPath: forceFullBundle ? remoteShallowPath : undefined, + }))}`, + { timeoutMs: input.spec.timeoutMs }, + ); + } catch (error) { + // Restore the owned producer refusal across clients that + // expose process status rather than the typed host error. + const status = error as { exitCode?: unknown; code?: unknown; timedOut?: unknown } | null; + if (status?.timedOut !== true && (status?.exitCode === GIT_SHALLOW_METADATA_EXIT_CODE + || status?.code === GIT_SHALLOW_METADATA_EXIT_CODE)) + throw new GitShallowMetadataError(`Git shallow metadata exceeds its byte limit (> ${GIT_SHALLOW_MAX_BYTES})`); + throw error; + } const gitExport = makeTransferProgress( restoreSink, "Exporting git history", @@ -2110,6 +2126,18 @@ export async function prepareSandboxManagedRuntime(input: { await fs.writeFile(localBundlePath, bundleBuffer); } await input.client.remove(remoteGitBundle).catch(() => undefined); + if (forceFullBundle) { + const shallow = toBuffer(await input.client.readFile(remoteShallowPath, { + onProgress: (transferred, total) => { + if (transferred > GIT_SHALLOW_MAX_BYTES || (total !== null && total > GIT_SHALLOW_MAX_BYTES)) { + throw new GitShallowMetadataError(`Git shallow metadata exceeds its byte limit (${Math.max(transferred, total ?? 0)} > ${GIT_SHALLOW_MAX_BYTES})`); + } + }, + })); + if (shallow.byteLength > GIT_SHALLOW_MAX_BYTES) throw new GitShallowMetadataError(`Git shallow metadata exceeds its byte limit (${shallow.byteLength} > ${GIT_SHALLOW_MAX_BYTES})`); + await fs.writeFile(localShallowPath, shallow); + await input.client.remove(remoteShallowPath).catch(() => undefined); + } if (!forceFullBundle) { remoteWorkspaceStatus = await input.client.readFile(remoteWorkspaceStatusPath) .then((bytes) => toBuffer(bytes).toString("utf8").trim()) @@ -2123,6 +2151,7 @@ export async function prepareSandboxManagedRuntime(input: { exportRef, importedRef: importedRef!, baseSha: gitSnapshot.headCommit, + shallowPath: forceFullBundle ? localShallowPath : undefined, })); }); diff --git a/packages/adapter-utils/src/workspace-restore-merge.test.ts b/packages/adapter-utils/src/workspace-restore-merge.test.ts index 57d22515d83..446a893f027 100644 --- a/packages/adapter-utils/src/workspace-restore-merge.test.ts +++ b/packages/adapter-utils/src/workspace-restore-merge.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { GitShallowMetadataError } from "./git-workspace-sync.js"; import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js"; import { captureDirectorySnapshot, @@ -197,6 +198,12 @@ describe("workspace restore merge", () => { expect(describeWorkspaceRestoreFailure(classifyWorkspaceRestoreFailure(new Error(message)))).not.toContain("private"); }); + it("keeps invalid metadata distinct from an omittable unsafe archive", () => { + const error = new GitShallowMetadataError("Invalid Git shallow boundaries /private/source"); + expect(classifyWorkspaceRestoreFailure(error)).toBe("restore_invalid_metadata"); + expect(describeWorkspaceRestoreFailure(classifyWorkspaceRestoreFailure(error))).not.toContain("private"); + }); + it("preserves the generic policy for other outbound command failures", () => { expect(classifyWorkspaceRestoreFailure(new Error("Daytona outbound symlink-escape guard command failed (exit 1): transport failed"))).toBe("restore_failed"); }); diff --git a/packages/adapter-utils/src/workspace-restore-merge.ts b/packages/adapter-utils/src/workspace-restore-merge.ts index f15960f046f..a93574d7429 100644 --- a/packages/adapter-utils/src/workspace-restore-merge.ts +++ b/packages/adapter-utils/src/workspace-restore-merge.ts @@ -308,6 +308,7 @@ export type WorkspaceRestoreFailureCode = | "restore_permission_denied" | "restore_lock_timeout" | "restore_unsafe_archive" + | "restore_invalid_metadata" | "restore_failed"; /** @@ -330,6 +331,7 @@ export type WorkspaceRestoreOutcome = */ export function classifyWorkspaceRestoreFailure(error: unknown): WorkspaceRestoreFailureCode { const code = error && typeof error === "object" ? (error as NodeJS.ErrnoException).code : undefined; + if (code === "WORKSPACE_RESTORE_INVALID_GIT_METADATA") return "restore_invalid_metadata"; if (code === "EACCES" || code === "EPERM") return "restore_permission_denied"; if (code === WORKSPACE_RESTORE_LOCK_TIMEOUT_CODE) return "restore_lock_timeout"; const message = error instanceof Error ? error.message : ""; @@ -361,6 +363,8 @@ export function describeWorkspaceRestoreFailure(code: WorkspaceRestoreFailureCod return "the restore timed out waiting for the workspace merge lock"; case "restore_unsafe_archive": return "the archive contains an unsafe link or path; workspace repair is required"; + case "restore_invalid_metadata": + return "the restore metadata is invalid; workspace repair is required"; case "restore_failed": return "the restore failed"; } diff --git a/packages/db/src/migrations/0416_salty_domino.sql b/packages/db/src/migrations/0416_salty_domino.sql new file mode 100644 index 00000000000..4a1ab2e228d --- /dev/null +++ b/packages/db/src/migrations/0416_salty_domino.sql @@ -0,0 +1,6 @@ +DROP INDEX "heartbeat_runs_cancelled_workspace_preservation_idx";--> statement-breakpoint +CREATE INDEX "heartbeat_runs_cancelled_workspace_preservation_idx" ON "heartbeat_runs" USING btree ("id","company_id") WHERE "heartbeat_runs"."runtime_mode" = 'native' AND "heartbeat_runs"."status" = 'cancelled' + AND "heartbeat_runs"."runner_profile_json"->'nativeWorkspaceSync'->>'state' = 'prepared' + AND "heartbeat_runs"."result_json"->'nativeCancellation'->>'dispatchState' = 'acknowledged' + AND ("heartbeat_runs"."result_json"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1' + AND "heartbeat_runs"."result_json"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_invalid_metadata', 'restore_failed')) IS NOT TRUE; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0411_snapshot.json b/packages/db/src/migrations/meta/0416_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0411_snapshot.json rename to packages/db/src/migrations/meta/0416_snapshot.json index 084f5255a1d..70f64c333dd 100644 --- a/packages/db/src/migrations/meta/0411_snapshot.json +++ b/packages/db/src/migrations/meta/0416_snapshot.json @@ -1,6 +1,6 @@ { - "id": "ff56ee93-8cfc-465a-b26a-1c12d10773ff", - "prevId": "c02426e4-de1a-4ce3-a541-99bb970c272a", + "id": "bbaeecd8-ff3f-4e70-a069-291bf3dc97bb", + "prevId": "dce41524-29a3-4552-ba5d-f0abefbf1ea1", "version": "7", "dialect": "postgresql", "tables": { @@ -1351,6 +1351,39 @@ "method": "btree", "with": {} }, + "activity_log_company_entity_created_idx": { + "name": "activity_log_company_entity_created_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "activity_log_company_user_issue_actor_idx": { "name": "activity_log_company_user_issue_actor_idx", "columns": [ @@ -19046,7 +19079,46 @@ "name": "file_inventory", "type": "jsonb", "primaryKey": false, - "notNull": true + "notNull": true, + "default": "'[]'::jsonb" + }, + "storage": { + "name": "storage", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "upload": { + "name": "upload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "restore": { + "name": "restore", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "maintenance_at": { + "name": "maintenance_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "restore_retry_at": { + "name": "restore_retry_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false }, "created_at": { "name": "created_at", @@ -19077,6 +19149,38 @@ "concurrently": false, "method": "btree", "with": {} + }, + "company_skill_snapshots_migration_idx": { + "name": "company_skill_snapshots_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_snapshots_maintenance_idx": { + "name": "company_skill_snapshots_maintenance_idx", + "columns": [ + { + "expression": "maintenance_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"maintenance_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -19966,6 +20070,25 @@ "primaryKey": false, "notNull": false }, + "package_snapshot_id": { + "name": "package_snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "package_file_map": { + "name": "package_file_map", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, "revision_number": { "name": "revision_number", "type": "integer", @@ -20024,6 +20147,52 @@ } }, "indexes": { + "company_skill_versions_migration_idx": { + "name": "company_skill_versions_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_versions\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_package_snapshot_idx": { + "name": "company_skill_versions_package_snapshot_idx", + "columns": [ + { + "expression": "package_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_repository_snapshot_idx": { + "name": "company_skill_versions_repository_snapshot_idx", + "columns": [ + { + "expression": "repository_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "company_skill_versions_skill_revision_idx": { "name": "company_skill_versions_skill_revision_idx", "columns": [ @@ -20135,6 +20304,19 @@ "onDelete": "restrict", "onUpdate": "no action" }, + "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk": { + "name": "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk", + "tableFrom": "company_skill_versions", + "tableTo": "company_skill_repository_snapshots", + "columnsFrom": [ + "package_snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, "company_skill_versions_author_agent_id_agents_id_fk": { "name": "company_skill_versions_author_agent_id_agents_id_fk", "tableFrom": "company_skill_versions", @@ -20580,6 +20762,78 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.skill_package_deletions": { + "name": "skill_package_deletions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_key": { + "name": "archive_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_key": { + "name": "manifest_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delete_after": { + "name": "delete_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "skill_package_deletions_due_idx": { + "name": "skill_package_deletions_due_idx", + "columns": [ + { + "expression": "delete_after", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_transfer_runs": { "name": "company_transfer_runs", "schema": "", @@ -33771,6 +34025,34 @@ } }, "indexes": { + "heartbeat_runs_native_unpresented_response_idx": { + "name": "heartbeat_runs_native_unpresented_response_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "native_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" in ('succeeded', 'failed', 'timed_out') AND \"heartbeat_runs\".\"finished_at\" is not null AND jsonb_typeof(\"heartbeat_runs\".\"result_json\" #> '{presentationDecision,commentId}') is distinct from 'string'", + "concurrently": false, + "method": "btree", + "with": {} + }, "heartbeat_runs_cancelled_workspace_preservation_idx": { "name": "heartbeat_runs_cancelled_workspace_preservation_idx", "columns": [ @@ -33788,7 +34070,7 @@ } ], "isUnique": false, - "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed')) IS NOT TRUE", + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_invalid_metadata', 'restore_failed')) IS NOT TRUE", "concurrently": false, "method": "btree", "with": {} @@ -36475,6 +36757,40 @@ "method": "btree", "with": {} }, + "issue_comments_review_response_receipt_idx": { + "name": "issue_comments_review_response_receipt_idx", + "columns": [ + { + "expression": "created_by_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "author_agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"issue_comments\".\"author_type\" = 'agent' AND \"issue_comments\".\"metadata\"->>'authorizationReason' = 'allow_chat_run_presentation'", + "concurrently": false, + "method": "btree", + "with": {} + }, "issue_comments_company_idx": { "name": "issue_comments_company_idx", "columns": [ diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index e637a1f7a26..73f2d86b5e5 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2899,6 +2899,13 @@ "when": 1793289638018, "tag": "0415_zippy_zemo", "breakpoints": true + }, + { + "idx": 416, + "version": "7", + "when": 1793320102041, + "tag": "0416_salty_domino", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/heartbeat_runs.ts b/packages/db/src/schema/heartbeat_runs.ts index 9e215f825f0..28443732d95 100644 --- a/packages/db/src/schema/heartbeat_runs.ts +++ b/packages/db/src/schema/heartbeat_runs.ts @@ -1,3 +1,4 @@ +import { WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared"; import { sql } from "drizzle-orm"; import { type AnyPgColumn, @@ -19,6 +20,14 @@ import { agents } from "./agents.js"; import { agentWakeupRequests } from "./agent_wakeup_requests.js"; import { issues } from "./issues.js"; +// Literal allowlist shared by bounded projections and the matching partial index. +// Constants are quoted here so PostgreSQL can prove the index predicate even +// when callers use prepared statements. +export const workspaceRestoreFailureCodesSql = sql.join( + WORKSPACE_RESTORE_FAILURE_CODES.map(code => sql.raw(`'${code.replaceAll("'", "''")}'`)), + sql`, `, +); + export const heartbeatRuns = pgTable( "heartbeat_runs", { @@ -119,7 +128,7 @@ export const heartbeatRuns = pgTable( AND ${table.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' AND ${table.resultJson}->'nativeCancellation'->>'dispatchState' = 'acknowledged' AND (${table.resultJson}->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1' - AND ${table.resultJson}->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed')) IS NOT TRUE`), + AND ${table.resultJson}->>'workspaceRestoreFailure' IN (${workspaceRestoreFailureCodesSql})) IS NOT TRUE`), retainedRecoveryAdoptionIdx: index("heartbeat_runs_retained_recovery_adoption_idx") .on(table.id).where(sql`${table.resultJson}->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1' and not (coalesce(${table.resultJson}, '{}'::jsonb) ? 'systemRecoveryAdoption') diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 2e7e342dad3..9632226c874 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -137,7 +137,7 @@ export { routineDocuments } from "./routine_documents.js"; export { documentAnnotationThreads } from "./document_annotation_threads.js"; export { documentAnnotationComments } from "./document_annotation_comments.js"; export { documentAnnotationAnchorSnapshots } from "./document_annotation_anchor_snapshots.js"; -export { heartbeatRuns } from "./heartbeat_runs.js"; +export { heartbeatRuns, workspaceRestoreFailureCodesSql } from "./heartbeat_runs.js"; export { heartbeatRunEvents } from "./heartbeat_run_events.js"; export { providerTraceRecords } from "./provider_trace_records.js"; export { completionContracts } from "./completion_contracts.js"; diff --git a/packages/shared/src/workspace-restore.ts b/packages/shared/src/workspace-restore.ts index 65da9708969..26e32555e20 100644 --- a/packages/shared/src/workspace-restore.ts +++ b/packages/shared/src/workspace-restore.ts @@ -3,6 +3,7 @@ export const WORKSPACE_RESTORE_FAILURE_CODES = [ "restore_permission_denied", "restore_lock_timeout", "restore_unsafe_archive", + "restore_invalid_metadata", "restore_failed", ] as const; diff --git a/server/src/services/activity.ts b/server/src/services/activity.ts index 60ef56c4e84..b256bf358ff 100644 --- a/server/src/services/activity.ts +++ b/server/src/services/activity.ts @@ -13,6 +13,7 @@ import { environments, heartbeatRunEvents, heartbeatRuns, + workspaceRestoreFailureCodesSql, issueComments, issueDocuments, issues, @@ -109,7 +110,7 @@ export function activityService(db: Db) { ) ) end, 'workspaceRestoreFailure', case when ${heartbeatRuns.resultJson} ->> 'workspaceRestoreFailure' - in ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed') + in (${workspaceRestoreFailureCodesSql}) then ${heartbeatRuns.resultJson} -> 'workspaceRestoreFailure' end, 'workspaceRestorePath', case when length(${heartbeatRuns.resultJson} ->> 'workspaceRestorePath') <= 180 then ${heartbeatRuns.resultJson} -> 'workspaceRestorePath' end, diff --git a/server/src/services/heartbeat/run-state.ts b/server/src/services/heartbeat/run-state.ts index 954b27d0f0a..af4d682291e 100644 --- a/server/src/services/heartbeat/run-state.ts +++ b/server/src/services/heartbeat/run-state.ts @@ -26,6 +26,7 @@ import { agentTaskSessions, heartbeatRunEvents, heartbeatRuns, + workspaceRestoreFailureCodesSql, issues, } from "@paperclipai/db"; import { notFound } from "../../errors.js"; @@ -238,7 +239,7 @@ const heartbeatRunSafeResultJsonColumn = sql | null>` 'workspaceRestoreRecovery', case when ${heartbeatRuns.resultJson} #>> '{workspaceRestoreRecovery,schema}' = 'paperclip.workspace-restore-recovery.v1' then jsonb_build_object('schema', 'paperclip.workspace-restore-recovery.v1') end, 'workspaceRestoreFailure', case when ${heartbeatRuns.resultJson} ->> 'workspaceRestoreFailure' - in ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed') + in (${workspaceRestoreFailureCodesSql}) then ${heartbeatRuns.resultJson} -> 'workspaceRestoreFailure' end, 'cancellation', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'cancellation') = 'object' then jsonb_strip_nulls(jsonb_build_object( diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index b93c94aa897..5c3268f6afc 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -3,7 +3,7 @@ import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; -import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { environmentLeases, heartbeatRuns, workspaceRestoreFailureCodesSql, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { recordRequiredWorkspaceRestoreFailure } from "../legacy-execution-recovery.js"; import { hasRequiredWorkspaceRecovery, nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js"; import { logActivity } from "../activity-log.js"; @@ -165,7 +165,7 @@ function pendingCancelledWorkspace() { AND ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' AND ${heartbeatRuns.resultJson}->'nativeCancellation'->>'dispatchState' = 'acknowledged' AND (${heartbeatRuns.resultJson}->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1' - AND ${heartbeatRuns.resultJson}->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed')) IS NOT TRUE`; + AND ${heartbeatRuns.resultJson}->>'workspaceRestoreFailure' IN (${workspaceRestoreFailureCodesSql})) IS NOT TRUE`; } export function registerCancelledWorkspacePreservationWork(db: Db, diff --git a/server/src/services/native-runtime/native-workspace-best-effort.test.ts b/server/src/services/native-runtime/native-workspace-best-effort.test.ts index f067117c09c..7d58a3b4a31 100644 --- a/server/src/services/native-runtime/native-workspace-best-effort.test.ts +++ b/server/src/services/native-runtime/native-workspace-best-effort.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; +import { GitShallowMetadataError } from "@paperclipai/adapter-utils/git-workspace-sync"; import type { Db } from "@paperclipai/db"; const log = vi.hoisted(() => vi.fn()); vi.mock("../heartbeat-run-events.js", () => ({ appendHeartbeatRunEvent: log })); @@ -33,6 +34,15 @@ describe("best effort native workspace export", () => { expect(JSON.stringify(log.mock.calls[0][1])).not.toContain("private"); }); + it("keeps rejected Git metadata behind the workspace persistence barrier", async () => { + const { db, select } = fixture(); + const error = new GitShallowMetadataError("Invalid Git shallow boundaries for host object format sha1"); + await expect(restoreNativeWorkspaceBestEffort({ db, runId: "run", restore: async () => { throw error; } })) + .rejects.toBe(error); + expect(select).not.toHaveBeenCalled(); + expect(log).not.toHaveBeenCalled(); + }); + it("does not hide transport failures or ownership loss", async () => { const { db, select } = fixture(); const restore = async () => { throw new Error("transport unavailable"); }; diff --git a/server/src/services/native-runtime/native-workspace-failure.ts b/server/src/services/native-runtime/native-workspace-failure.ts index 36eade3f3be..44144152289 100644 --- a/server/src/services/native-runtime/native-workspace-failure.ts +++ b/server/src/services/native-runtime/native-workspace-failure.ts @@ -14,6 +14,7 @@ export function classifyNativeWorkspaceFailure(error: unknown): { const message = error instanceof Error ? error.message.trim() : ""; const code: NativeWorkspaceFailureCode = message === "workspace_sync_out_unrecoverable" || message.includes("daytona_sandbox_not_found") + || classifyWorkspaceRestoreFailure(error) === "restore_invalid_metadata" ? "workspace_sync_out_unrecoverable" : message === "workspace_sync_out_unsafe_archive" || classifyWorkspaceRestoreFailure(error) === "restore_unsafe_archive" ? "workspace_sync_out_unsafe_archive" diff --git a/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts b/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts index a1c5ae1ec3a..46d6c7cdde4 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { GitShallowMetadataError } from "@paperclipai/adapter-utils/git-workspace-sync"; import type { Db } from "@paperclipai/db"; const sync = vi.hoisted(() => ({ resume: vi.fn() })); @@ -56,6 +57,13 @@ describe("native workspace finalization without a manual export repair", () => { expect(result).toMatchObject({ status: "failed", exitCode: 1, stderr: `${code}\n` }); }); + it("fails copyback for invalid Git metadata instead of successfully omitting saved files", async () => { + sync.resume.mockRejectedValueOnce(new GitShallowMetadataError("Git shallow metadata exceeds its byte limit")); + const result = await resumeNativeWorkspaceFinalization({ db: fixtureDb(), runId: "run", environmentRuntime: {} as never }); + expect(result).toMatchObject({ status: "failed", exitCode: 1, stderr: "workspace_sync_out_unrecoverable\n" }); + expect(sync.resume).toHaveBeenCalledOnce(); + }); + it("does not treat a missing workspace reference as an omitted unsafe archive", async () => { sync.resume.mockResolvedValueOnce(false); const result = await resumeNativeWorkspaceFinalization({ db: fixtureDb(), runId: "run", environmentRuntime: {} as never }); diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.ts index a648e433d5f..f604c01f147 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.ts @@ -1,3 +1,4 @@ +import { classifyNativeWorkspaceFailure } from "./native-workspace-failure.js"; import { recordNativeFinalizationFailure } from "./native-run-finalizer.js"; import { inspectPluginWorkspaceForFinalization } from "./native-plugin-workspace-finalization.js"; import type { WorkspaceWorkers } from "../plugin-workspaces.js"; @@ -237,12 +238,8 @@ export async function attemptNativeWorkspaceFinalization(input: { }; } catch (error) { await ownership.assertHeld(); - const code = - error instanceof Error && - (error.message === "workspace_sync_out_unrecoverable" || - error.message.includes("daytona_sandbox_not_found")) - ? "workspace_sync_out_unrecoverable" - : "workspace_sync_out_failed"; + const code = classifyNativeWorkspaceFailure(error).code === "workspace_sync_out_unrecoverable" + ? "workspace_sync_out_unrecoverable" : "workspace_sync_out_failed"; return workspaceSyncFailure(code); } } diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts index ab423804e38..d8331c96c28 100644 --- a/server/src/services/sandbox-stop-and-retain.ts +++ b/server/src/services/sandbox-stop-and-retain.ts @@ -80,20 +80,23 @@ export async function prepareSandboxStopAndRetain(db: Db, lease: Lease) { } /** Historical release rows can lack a stop receipt. Re-inspect the exact - * recorded allocation using stop-only dispatch; never infer that it stopped. */ + * recorded allocation using stop-only dispatch; never infer that it stopped. + * The preservation owner verifies source eligibility and exclusive allocation + * ownership before calling. Lease reuse policy does not authorize destruction. */ export async function prepareHistoricalSandboxStopVerification(db: Db, lease: typeof environmentLeases.$inferSelect) { if (lease.status !== "released" || !lease.heartbeatRunId || !lease.providerLeaseId || - !["retain_on_failure", "ephemeral"].includes(lease.leasePolicy) || lease.metadata?.pendingCleanupInFlight === true) + lease.metadata?.pendingCleanupInFlight === true) return null; return prepareStopAndRetain(db, lease, "released"); } /** A repair-only resume invalidates the old stop receipt atomically. This is - * separate from ordinary release; a released timestamp alone cannot grant it. */ + * separate from ordinary release; a released timestamp alone cannot grant it. + * Source eligibility belongs to the preservation owner, including accepted + * native exports whose original allocation policy is reusable. */ export async function prepareStoppedSandboxForRecovery(db: Db, lease: typeof environmentLeases.$inferSelect) { if (!hasRemoteTerminationReceipt(lease) || lease.status !== "released" - || (lease.metadata?.remoteExecutionTermination as { state?: string })?.state !== "stopped" - || !["retain_on_failure", "ephemeral"].includes(lease.leasePolicy)) return null; + || (lease.metadata?.remoteExecutionTermination as { state?: string })?.state !== "stopped") return null; return prepareStopAndRetain(db, lease, "released"); } diff --git a/server/src/services/workspace-recovery-source.test.ts b/server/src/services/workspace-recovery-source.test.ts index 92e62fa74d9..e1f2ac475d9 100644 --- a/server/src/services/workspace-recovery-source.test.ts +++ b/server/src/services/workspace-recovery-source.test.ts @@ -1,10 +1,13 @@ -import { randomUUID } from "node:crypto"; +import { WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared"; +import { createHeartbeatRunState } from "./heartbeat/run-state.js"; +import { activityService } from "./activity.js"; +import { randomBytes, randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import express from "express"; import request from "supertest"; import { eq, sql } from "drizzle-orm"; import { activityLog, agents, agentWakeupRequests, applyPendingMigrations, companies, createDb, environmentLeases, environments, - heartbeatRuns, issueRecoveryActions, issues, workspaceOperations } from "@paperclipai/db"; + heartbeatRuns, issueRecoveryActions, issues, workspaceOperations, completionContracts, nativeRunResults, nativeRunFinalizations } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; import { inspectWorkspaceRecoverySource } from "./workspace-recovery-source.js"; @@ -76,6 +79,38 @@ describe("workspace source verification diagnostics", () => { }; } + it.each(WORKSPACE_RESTORE_FAILURE_CODES)("preserves %s through both run API projections", async code => { + const f = await seed(); + await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreFailure: code, opaque: randomBytes(100_000).toString("hex"), + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] } } }) + .where(eq(heartbeatRuns.id, f.runId)); + const [runs, activity] = await Promise.all([ + createHeartbeatRunState(db).getRun(f.runId), + activityService(db).runsForIssue(f.companyId, f.issueId), + ]); + expect(runs?.resultJson?.opaque).toBeUndefined(); + const activityResult = activity[0].resultJson; + expect({ run: runs?.resultJson?.workspaceRestoreFailure, + activity: activityResult && "workspaceRestoreFailure" in activityResult ? activityResult.workspaceRestoreFailure : undefined }) + .toEqual({ run: code, activity: code }); + }); + + it("keeps a cancelled metadata-repair hold out of the preservation index while retaining ordinary work", async () => { + const f = await seed(); + const [index] = await db.execute(sql`select pg_get_expr(indpred, indrelid) as predicate + from pg_index where indexrelid = 'heartbeat_runs_cancelled_workspace_preservation_idx'::regclass`); + await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { + nativeCancellation: { dispatchState: "acknowledged" }, workspaceRestoreFailure: "restore_invalid_metadata", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] }, + } }).where(eq(heartbeatRuns.id, f.runId)); + const candidates = () => db.execute(sql`select id from heartbeat_runs where id = ${f.runId} + and ${sql.raw(String(index.predicate))}`); + expect(await candidates()).toHaveLength(0); + await db.update(heartbeatRuns).set({ resultJson: { nativeCancellation: { dispatchState: "acknowledged" } } }) + .where(eq(heartbeatRuns.id, f.runId)); + expect(await candidates()).toHaveLength(1); + }); + it.each(["lease_policy_not_retained", "source_lease_not_bound"])("identifies the rejected contract: %s", async reason => { const f = await seed(); if (reason === "lease_policy_not_retained") { @@ -91,6 +126,34 @@ describe("workspace source verification diagnostics", () => { }); }); + it.each(["rejected", "failed_result", "unselected", "shared_allocation", "active_writer"])("keeps reusable export guards: %s", async kind => { + const f = await seed(); + const contractId = randomUUID(), resultId = randomUUID(); + await db.insert(completionContracts).values({ id: contractId, companyId: f.companyId, issueId: f.issueId, + revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", + completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: {}, canonicalSha256: "contract", createdByActorType: "system", createdByActorId: "test" }); + await db.update(heartbeatRuns).set({ completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId)); + await db.insert(nativeRunResults).values({ id: resultId, companyId: f.companyId, issueId: f.issueId, runId: f.runId, + completionContractId: contractId, schemaStatus: kind === "rejected" ? "rejected" : "accepted", + serverFingerprint: "result", canonicalSha256: "result", resultJson: { + terminal: { runTerminalState: kind === "failed_result" ? "failed" : "succeeded" } } }); + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, + phase: "terminal_failure", resultId: kind === "unselected" ? null : resultId }); + await db.update(environmentLeases).set({ leasePolicy: "reuse_by_environment" }).where(eq(environmentLeases.id, f.lease.id)); + if (kind === "shared_allocation") await db.insert(environmentLeases).values({ companyId: f.companyId, + environmentId: f.environment.id, provider: "daytona", providerLeaseId: f.lease.providerLeaseId, + leasePolicy: "reuse_by_environment", status: "released" }); + if (kind === "active_writer") await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + issueId: f.issueId, status: "running" }); + const before = await snapshot(f); + await expect(inspectWorkspaceRecoverySource(db, f)).rejects.toMatchObject({ + code: kind === "shared_allocation" ? "allocation_ownership_ambiguous" + : kind === "active_writer" ? "workspace_writer_active" : "source_binding_invalid", + }); + expect(await snapshot(f)).toEqual(before); + }); + it("inspects a resolved retained source through HTTP without revalidating or changing recorded recovery", async () => { const f = await seed(); await db.update(environmentLeases).set({ leasePolicy: "reuse_by_environment" }).where(eq(environmentLeases.id, f.lease.id)); diff --git a/server/src/services/workspace-recovery-source.ts b/server/src/services/workspace-recovery-source.ts index 7b98811ab12..5afb3274608 100644 --- a/server/src/services/workspace-recovery-source.ts +++ b/server/src/services/workspace-recovery-source.ts @@ -1,5 +1,5 @@ import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; -import { environmentLeases, environments, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { environmentLeases, environments, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; import { adapterExecutionControls } from "./adapter-execution-control.js"; import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; import { SystemRecoveryFailure } from "./system-recovery-state.js"; @@ -98,6 +98,19 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec if (coordinator?.leaseOwner || adapterExecutionControls.has(run.id) || (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date())) throw new SystemRecoveryFailure("agent_stop_unconfirmed", "recheck_evidence"); + // Reusable allocations can still owe copyback for a completed native result. + // This authorizes only preservation of the exact descriptor, never reuse for + // agent execution. Stop verification and physical ownership checks remain below. + const acceptedExportResult = reference && workspaceExportCauses.includes(action.cause) + && run.runtimeMode === "native" && coordinator?.issueId === input.issueId + && coordinator.resultId && run.completionContractId + ? (await db.select({ id: nativeRunResults.id }).from(nativeRunResults).where(and( + eq(nativeRunResults.companyId, input.companyId), eq(nativeRunResults.issueId, input.issueId), + eq(nativeRunResults.runId, run.id), eq(nativeRunResults.id, coordinator.resultId), + eq(nativeRunResults.completionContractId, run.completionContractId), + eq(nativeRunResults.schemaStatus, "accepted"), + sql`${nativeRunResults.resultJson}#>>'{terminal,runTerminalState}' = 'succeeded'`, + )).limit(1))[0] : null; const sources = []; for (const leaseId of leaseIds as string[]) { const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId), @@ -109,7 +122,10 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec if (!lease.environmentId) throw invalid("lease_environment_missing"); if (!lease.providerLeaseId) throw invalid("lease_allocation_missing"); if (!lease.provider || lease.provider === "local") throw invalid("lease_provider_invalid"); - if (!["retain_on_failure", "ephemeral"].includes(lease.leasePolicy)) throw invalid("lease_policy_not_retained", lease.leasePolicy); + const reusableExport = descriptor && acceptedExportResult + && ["reuse_by_environment", "reuse_by_execution_workspace"].includes(lease.leasePolicy); + if (!["retain_on_failure", "ephemeral"].includes(lease.leasePolicy) && !reusableExport) + throw invalid("lease_policy_not_retained", lease.leasePolicy); if (typeof remoteCwd !== "string" || !remoteCwd.startsWith("/")) throw invalid("lease_cwd_invalid"); if (descriptor && lease.providerLeaseId !== descriptor.providerLeaseId) throw invalid("allocation_reference_mismatch"); if (descriptor && remoteCwd !== descriptor.remoteCwd) throw invalid("cwd_reference_mismatch"); diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 74af69eda36..5d044015334 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -9,6 +9,8 @@ import { executionWorkspaceService } from "./execution-workspaces.js"; import { workspaceOperationService } from "./workspace-operations.js"; import { instanceSettingsService } from "./instance-settings.js"; import { projectTaskInterruption } from "./task-interruption-presentation.js"; +import * as nativeWorkspaceSync from "./native-runtime/native-workspace-sync.js"; +import { GitShallowMetadataError } from "@paperclipai/adapter-utils/git-workspace-sync"; import { readSystemRecovery } from "./system-recovery-state.js"; import { adoptSystemRecovery } from "./system-recovery.js"; import { getStorageService } from "../storage/index.js"; @@ -496,8 +498,11 @@ describe("automatic retained workspace recovery", () => { expect((await Promise.all(observed)).some(rows => rows.some(row => row.status === "succeeded"))).toBe(true); }); - it("re-enters only the existing finalizer for an accepted result, once per intent, preserving failure history", async () => { - const f = await seed(); + it.each(["retain_on_failure", "reuse_by_environment", "reuse_by_execution_workspace"] as const)("re-enters only the accepted result finalizer for %s, preserving files and failure history", async leasePolicy => { + const f = await seed("native", true, "keep_running"); + await db.update(environmentLeases).set({ leasePolicy }).where(eq(environmentLeases.id, f.lease.id)); + await db.update(issueRecoveryActions).set({ cause: "native_workspace_sync_out_retry_exhausted" }).where(eq(issueRecoveryActions.id, f.actionId)); + await db.update(heartbeatRuns).set({ resultJson: {} }).where(eq(heartbeatRuns.id, f.runId)); const contractId = randomUUID(), resultId = randomUUID(); await db.insert(completionContracts).values({ id: contractId, companyId: f.companyId, issueId: f.issueId, revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", @@ -510,7 +515,24 @@ describe("automatic retained workspace recovery", () => { await db.update(nativeRunFinalizations).set({ resultId, attempt: 3, failureCode: "native_workspace_sync_out_retry_exhausted" }) .where(eq(nativeRunFinalizations.runId, f.runId)); expect(await resumePreservedResultFinalization(db, f.companyId, f.runId)).toBe(false); + const priorAction = await action(f.actionId); + await db.update(issueRecoveryActions).set({ ownerType: "system", evidence: { ...priorAction.evidence, + systemRecovery: failSystemRecovery(createSystemRecovery(f.runId), + new SystemRecoveryFailure("source_binding_invalid", "service_required")), + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: f.runId, status: "blocked", + stage: "verifying_stop", attempt: 1, attemptLimit: 6, requestIds: [] }, + } }).where(eq(issueRecoveryActions.id, f.actionId)); + const request = { ...f.input, actorId: "board", idempotencyKey: randomUUID() }; + await requestWorkspaceRecovery(request); + const admitted = await action(f.actionId); + await requestWorkspaceRecovery(request); + expect(await action(f.actionId)).toEqual(admitted); + expect(admitted.ownerType).toBe("system"); await recoverRetainedWorkspace(f.input); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n"); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); const attempts = await Promise.all([resumePreservedResultFinalization(db, f.companyId, f.runId), resumePreservedResultFinalization(db, f.companyId, f.runId)]); expect(attempts.filter(Boolean)).toHaveLength(1); @@ -583,6 +605,20 @@ describe("automatic retained workspace recovery", () => { sourceRunIds: [laterId], cwd: next })).toEqual([]); }, 30_000); + it("retains files and the recovery barrier when native export rejects Git metadata", async () => { + const f = await seed(); + const restore = vi.spyOn(nativeWorkspaceSync, "resumeNativeWorkspaceSync") + .mockRejectedValueOnce(new GitShallowMetadataError("Git shallow metadata exceeds its byte limit")); + try { await recoverRetainedWorkspace(f.input); } finally { restore.mockRestore(); } + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "exhausted", + failure: { code: "snapshot_verification_failed", retryClass: "service_required" }, preservationOperationIds: [] }); + expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false); + expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n"); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); + expect((await run(f.runId)).status).toBe("failed"); + }, 30_000); + it("reconciles an uploaded snapshot after a lost upload acknowledgement without exporting again", async () => { const f = await seed("legacy", false); const target = targets.get(f.lease.id)!; diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index 989c74335de..1eb9b1bf959 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -1,3 +1,4 @@ +import { classifyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { notifyNativeResultCommit } from "./native-result-commit-notifications.js"; import { notifyNativeWorkspaceFinalization } from "./native-workspace-finalization-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; @@ -271,6 +272,8 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ } const classification = classifyNativeWorkspaceFailure(error); const failure = error instanceof SystemRecoveryFailure ? error + : classifyWorkspaceRestoreFailure(error) === "restore_invalid_metadata" + ? new SystemRecoveryFailure("snapshot_verification_failed", "service_required") : classification.code === "workspace_sync_out_unsafe_archive" ? new SystemRecoveryFailure("archive_unsafe", "service_required") : classification.code === "workspace_sync_out_unrecoverable" diff --git a/ui/src/components/TaskChatThread.test.tsx b/ui/src/components/TaskChatThread.test.tsx index 44ae1ad8933..11c8f99d1a9 100644 --- a/ui/src/components/TaskChatThread.test.tsx +++ b/ui/src/components/TaskChatThread.test.tsx @@ -1401,7 +1401,8 @@ describe("TaskChatThread runtime transcript selection", () => { }); - it.each(["matching", "document_only", "absent", "other_revision"] as const)("reports a restore failure with %s saved-plan evidence", (evidence) => { + it.each((["matching", "document_only", "absent", "other_revision"] as const).flatMap(evidence => + ["restore_unsafe_archive", "restore_invalid_metadata"].map(code => [evidence, code] as const)))("reports %s saved-plan evidence for %s without offering another provider turn", (evidence, code) => { if (evidence !== "absent") planState.data = planDocument(); renderThread({ issueId: "issue-1", @@ -1414,7 +1415,7 @@ describe("TaskChatThread runtime transcript selection", () => { runId: "restore-run", runtimeMode: "legacy", status: "failed", errorCode: "workspace_restore_failed", agentId: "agent-1", agentName: "Runner", adapterType: "grok_local", createdAt: "2026-08-25T18:00:00.000Z", startedAt: "2026-08-25T18:00:00.000Z", finishedAt: "2026-08-25T18:00:02.000Z", - resultJson: { workspaceRestoreFailure: "restore_unsafe_archive", workspaceRestorePath: ".claude/skills/paperclip", finalResponseRecorded: false, + resultJson: { workspaceRestoreFailure: code, workspaceRestorePath: ".claude/skills/paperclip", finalResponseRecorded: false, ...(evidence === "document_only" ? { savedPlanRevisionId: "revision-3" } : {}), }, }], diff --git a/ui/src/components/TaskChatThread.tsx b/ui/src/components/TaskChatThread.tsx index 1effc474b45..973d92c1d24 100644 --- a/ui/src/components/TaskChatThread.tsx +++ b/ui/src/components/TaskChatThread.tsx @@ -1831,7 +1831,8 @@ export function TaskChatThread(props: TaskChatThreadProps) { runId: source.status === "cancelled" ? undefined : source.id, ...(branchUnavailable || aiRequest?.status === "pending" ? { retryable: false } : {}), ...(restoreFailed ? { - retryable: meta?.resultJson?.workspaceRestoreFailure !== "restore_unsafe_archive", + retryable: meta?.resultJson?.workspaceRestoreFailure !== "restore_unsafe_archive" + && meta?.resultJson?.workspaceRestoreFailure !== "restore_invalid_metadata", collapsible: true, runHref: meta?.agentId ? `/agents/${encodeURIComponent(agentMap?.get(meta.agentId)?.urlKey ?? meta.agentId)}/runs/${encodeURIComponent(source.id)}` : undefined, planHref: savedPlan ? "#document-plan" : undefined,