diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index fd0e4f61ab8..bc245fedc63 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2724,6 +2724,21 @@ again automatically; after that it names Retry. Holds exhausted by earlier releases carry no marker and are re-armed on the worker's first pass after deploy, with no migration. +An exhausted save whose last failure happened while preserving files +(`export_interrupted`) under an older workspace capture +policy is retried once under the current policy. The capture policy has a +version (`REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION`); each recovery generation +records the version it saves with, and generations written before the field +existed count as version 1. A release that makes the capture accept workspaces +an earlier release refused bumps the version. The recovery worker then re-arms +each such hold with one new bounded generation, under the same run-over checks +as the stop-proof re-arm, records the `capturePolicyRecovery` marker and +`issue.workspace_recovery_rearmed` activity with reason +`capture_policy_changed`, and settles the hold like any completed recovery. +The new generation carries the current version, so the rule never re-arms it +again; a later failure follows the normal exhausted handoff. Holds saved by +earlier releases need no migration. + An exhausted workspace export records the exact Blocked status version it creates, bound to its source run, result, contract and identity. Recovery may restore the accepted passive wait only while that receipt still owns the task diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index 911815e092f..176354127d2 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -353,6 +353,25 @@ finalization from the accepted result. Confirm the run's native phase and `resultJson.finalizationPhase` are `committed`, there is no `nextAttemptAt`, and no workspace operation is still running. The accepted provider result is reused. +## Symlinks in remote workspace saves + +The remote workspace save (`remote-workspace-snapshot-archive.ts`) records +symlinks and never follows them. A link whose target resolves inside the +workspace, such as `node_modules/.bin/tool -> ../tool/cli.js`, is saved as a +link and restored as the same link. A link that is absolute or leaves the +workspace through `..`, such as a headless Chromium profile's +`SingletonSocket -> /tmp/...` or a link into a sandbox-only cache, has no +portable meaning: the save records its path in the archive manifest's +`excluded` list, omits it, and continues. Such a link no longer fails the save +with `remote_workspace_archive_unsafe_symlink`. Restore materializes only the +manifest entries, so an omitted link is absent after restore, and restore still +refuses any archive whose manifest names an absolute or escaping link. Omitting +a tracked link shows as a deletion in `git status` after restore. + +Saves that recovery exhausted before this policy (capture policy version 1) are +retried once automatically; see the capture-policy re-arm in +`doc/execution-semantics.md`. + ## Automatic unsafe archive recovery An unsafe link in a native workspace must not fail a completed task or require diff --git a/server/src/services/remote-workspace-snapshot-archive-environments.test.ts b/server/src/services/remote-workspace-snapshot-archive-environments.test.ts index a90c301adf0..6e6eab919e2 100644 --- a/server/src/services/remote-workspace-snapshot-archive-environments.test.ts +++ b/server/src/services/remote-workspace-snapshot-archive-environments.test.ts @@ -47,7 +47,7 @@ describe("reinstallable environments in portable workspace archives", () => { expect(await fs.readlink(path.join(venv, "bin", "python"))).toBe(process.execPath); }); - it("keeps a deliberately tracked virtual environment and still rejects other absolute links", async () => { + it("keeps a deliberately tracked virtual environment and drops other absolute links", async () => { const cwd = await source(), venv = path.join(cwd, "env"); await fs.mkdir(venv); await fs.writeFile(path.join(venv, "pyvenv.cfg"), "home = /usr/bin\n"); await git(cwd, "init", "-q", "-b", "main"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "tracked environment"); @@ -55,8 +55,11 @@ describe("reinstallable environments in portable workspace archives", () => { expect(await fs.readFile(path.join(restored, "env", "pyvenv.cfg"), "utf8")).toBe("home = /usr/bin\n"); await fs.rm(path.join(root, "snapshot.pcws")); await fs.rm(restored, { recursive: true }); // A directory without the PEP 405 marker is ordinary work, not an environment. + // Its absolute interpreter link is recorded and omitted, never followed. await fs.mkdir(path.join(cwd, "tools")); await fs.symlink(process.execPath, path.join(cwd, "tools", "node")); - await expect(capture(cwd)).rejects.toThrow("unsafe_symlink"); + const snapshot = await capture(cwd); + expect(await excluded(snapshot.archivePath)).toEqual(["tools/node"]); + expect(await fs.readdir(path.join(await restore(snapshot), "tools"))).toEqual([]); }); it("does not treat a linked pyvenv.cfg as an environment marker", async () => { @@ -65,6 +68,11 @@ describe("reinstallable environments in portable workspace archives", () => { await fs.writeFile(path.join(cwd, "pyvenv.cfg"), "home = /usr/bin\n"); await fs.symlink("../pyvenv.cfg", path.join(venv, "pyvenv.cfg")); await fs.symlink(process.execPath, path.join(venv, "bin", "python")); - await expect(capture(cwd)).rejects.toThrow("unsafe_symlink"); + // The directory is saved as ordinary work; only its absolute link is omitted. + const snapshot = await capture(cwd); + expect(await excluded(snapshot.archivePath)).toEqual([".venv/bin/python"]); + const restored = await restore(snapshot); + expect(await fs.readlink(path.join(restored, ".venv", "pyvenv.cfg"))).toBe("../pyvenv.cfg"); + expect(await fs.readdir(path.join(restored, ".venv", "bin"))).toEqual([]); }); }); diff --git a/server/src/services/remote-workspace-snapshot-archive.test.ts b/server/src/services/remote-workspace-snapshot-archive.test.ts index 1028a63063b..ef290dbe637 100644 --- a/server/src/services/remote-workspace-snapshot-archive.test.ts +++ b/server/src/services/remote-workspace-snapshot-archive.test.ts @@ -895,10 +895,55 @@ describe("portable remote workspace archives", () => { expect(await git(parent, "rev-parse", "HEAD")).toBe(head); }, ); - it("rejects an external symlink before creating an archive", async () => { - const cwd = await source(); await fs.symlink("../outside", path.join(cwd, "unsafe")); - await expect(capture(cwd)).rejects.toThrow("unsafe_symlink"); - await expect(fs.access(path.join(root, "snapshot.pcws"))).rejects.toMatchObject({ code: "ENOENT" }); + /** + * The staging shape (Nov 10): a review workspace with a dependency tree + * whose `.bin` links stay inside it, a headless Chromium profile whose + * SingletonSocket links by absolute path into the sandbox's /tmp, and a link + * that escapes the workspace with `..`. The save keeps the confined link, + * records and drops the other two without following them, and restores the + * same tree. + */ + it.each(["local", "serialized"])("keeps confined links and drops outside links in %s capture", async mode => { + const cwd = await source(), outside = path.join(root, "outside"), privateBytes = "SYNTHETIC_OUTSIDE_SECRET\n"; + await fs.mkdir(outside); await fs.writeFile(path.join(outside, "secret.txt"), privateBytes); + await fs.writeFile(path.join(cwd, "saved.txt"), "review evidence\n"); + await fs.mkdir(path.join(cwd, "node_modules", "tool"), { recursive: true }); + await fs.mkdir(path.join(cwd, "node_modules", ".bin")); + await fs.writeFile(path.join(cwd, "node_modules", "tool", "cli.js"), "#!/usr/bin/env node\n", { mode: 0o755 }); + await fs.symlink("../tool/cli.js", path.join(cwd, "node_modules", ".bin", "tool")); + await fs.mkdir(path.join(cwd, ".review-chrome")); + await fs.symlink(path.join(outside, "secret.txt"), path.join(cwd, ".review-chrome", "SingletonSocket")); + await fs.symlink("../outside/secret.txt", path.join(cwd, "escape")); + let snapshot: Awaited>; + if (mode === "serialized") { + const helper = path.join(root, "helper.cjs"), archivePath = path.join(root, "snapshot.pcws"); + await fs.writeFile(helper, remoteWorkspaceArchiveScript()); + snapshot = { archivePath, ...JSON.parse((await execute(process.execPath, [helper, "capture", cwd, archivePath], { env })).stdout) }; + } else snapshot = await capture(cwd); + const archive = await fs.readFile(snapshot.archivePath), start = Buffer.from("PCWS1\n").length; + const manifest = JSON.parse(archive.subarray(start + 4, start + 4 + archive.readUInt32BE(start)).toString()); + expect(manifest.excluded).toEqual([".review-chrome/SingletonSocket", "escape"]); + expect(manifest.entries.filter((entry: { type: string }) => entry.type === "symlink")) + .toEqual([{ path: "node_modules/.bin/tool", type: "symlink", target: "../tool/cli.js" }]); + // The link is never followed: the outside bytes are not in the archive. + expect(archive.includes(Buffer.from(privateBytes))).toBe(false); + // The source workspace is unchanged. + expect(await fs.readlink(path.join(cwd, "escape"))).toBe("../outside/secret.txt"); + const restored = await restore(snapshot); + expect(await fs.readFile(path.join(restored, "saved.txt"), "utf8")).toBe("review evidence\n"); + expect(await fs.readlink(path.join(restored, "node_modules", ".bin", "tool"))).toBe("../tool/cli.js"); + expect(await fs.readFile(path.join(restored, "node_modules", ".bin", "tool"), "utf8")).toBe("#!/usr/bin/env node\n"); + expect(await fs.readdir(path.join(restored, ".review-chrome"))).toEqual([]); + await expect(fs.lstat(path.join(restored, "escape"))).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("still refuses an archive that names an outside link on restore", async () => { + const manifest = { schema: "paperclip.remote-workspace-archive.v1", entries: [{ path: "escape", type: "symlink", target: "../outside" }] }; + const bytes = Buffer.from(JSON.stringify(manifest)), length = Buffer.alloc(4); length.writeUInt32BE(bytes.length); + const archivePath = path.join(root, "unsafe.pcws"), targetDir = path.join(root, "target"); + await fs.writeFile(archivePath, Buffer.concat([Buffer.from("PCWS1\n"), length, bytes])); + await expect(restoreRemoteWorkspaceArchive({ archivePath, targetDir, + expectedManifestDigest: createHash("sha256").update(bytes).digest("hex") })).rejects.toThrow("remote_workspace_archive_unsafe_symlink"); + await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" }); }); it("redacts a real missing-source error in the serialized capture helper", async () => { const helper = path.join(root, "archive.cjs"), archivePath = path.join(root, "snapshot.pcws"); diff --git a/server/src/services/remote-workspace-snapshot-archive.ts b/server/src/services/remote-workspace-snapshot-archive.ts index 2c8be6bbcf4..990d92f1fc8 100644 --- a/server/src/services/remote-workspace-snapshot-archive.ts +++ b/server/src/services/remote-workspace-snapshot-archive.ts @@ -1,5 +1,13 @@ import { Worker } from "node:worker_threads"; +/** + * Bump when the capture policy starts accepting workspaces that an earlier + * release refused. Saves that recovery exhausted under an older version are + * retried once under the new one (workspace-recovery-capture-policy). + * 2: absolute or escaping symlinks are dropped and recorded instead of refusing the save. + */ +export const REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION = 2; + /** * A portable, streaming workspace archive. This function is intentionally * self-contained: the same implementation runs locally and in a sandbox. @@ -460,8 +468,12 @@ export async function workspaceArchive(input: { entries.push({ path: next, type: "directory", mode: stat.mode & 0o777 }); await visit(next, inRepository); } else if (stat.isSymbolicLink()) { + // Links are recorded, never followed. A link that resolves inside the + // workspace is kept. One that is absolute or leaves the workspace (a + // browser profile's socket in /tmp, a sandbox cache, a `..` escape) + // has no portable meaning: record its path as excluded and keep saving. const link = await fs.readlink(source); - if (!safeLink(next, link)) fail("unsafe_symlink"); + if (!safeLink(next, link)) { excludedPaths.add(next); continue; } entries.push({ path: next, type: "symlink", target: link }); } else if (stat.isFile()) { const entry: Entry = { path: next, type: "file" }; diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index 3ac0b8b7807..945dbcbcc37 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -857,22 +857,51 @@ fs.unlinkSync(${JSON.stringify(kill)});process.kill(process.pid,'SIGKILL');}retu expect(await fs.readdir(outside)).toEqual(["workspace.pcws"]); }); - it.each(["split", "one-shot"] as const)("retains a closed capture validation failure with %s output", async outputMode => { - const privateName = "SYNTHETIC_PRIVATE_FILENAME", outside = path.join(scratch, privateName); - await fs.writeFile(outside, "outside source bytes\n"); + it.skipIf(process.platform === "win32").each(["split", "one-shot"] as const)("retains a closed capture validation failure with %s output", async outputMode => { + const privateName = "SYNTHETIC_PRIVATE_FILENAME"; await fs.writeFile(path.join(workspace, "saved.txt"), "retained source bytes\n"); - await fs.symlink(`../${privateName}`, path.join(workspace, "link")); + await exec("mkfifo", [path.join(workspace, privateName)]); const { remote, execute } = target({}, outputMode), operationId = randomUUID(); await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId })) - .rejects.toThrow(/^remote_workspace_archive_unsafe_symlink$/); + .rejects.toThrow(/^remote_workspace_archive_unsupported_entry$/); expect(execute).toHaveBeenCalledTimes(2); - expect(await fs.readFile(outside, "utf8")).toBe("outside source bytes\n"); expect(await fs.readFile(path.join(workspace, "saved.txt"), "utf8")).toBe("retained source bytes\n"); - expect(await fs.readlink(path.join(workspace, "link"))).toBe(`../${privateName}`); + expect((await fs.lstat(path.join(workspace, privateName))).isFIFO()).toBe(true); const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId); expect(await fs.readdir(directory)).toEqual(["archive.cjs"]); }); + /** + * The staging shape through the sandbox command path: a `node_modules/.bin` link + * inside the workspace, a Chromium profile link by absolute path into /tmp, + * and a `..` link out of the workspace. The save completes; only the + * confined link is restored. + */ + it.each(["split", "one-shot"] as const)("saves a workspace with outside links through the sandbox helper with %s output", async outputMode => { + const outside = path.join(scratch, "sandbox-tmp"); await fs.mkdir(outside); + await fs.writeFile(path.join(outside, "SingletonSocket"), "SYNTHETIC_SANDBOX_ONLY\n"); + await fs.writeFile(path.join(workspace, "saved.txt"), "review evidence\n"); + await fs.mkdir(path.join(workspace, "node_modules", "tool"), { recursive: true }); + await fs.mkdir(path.join(workspace, "node_modules", ".bin")); + await fs.writeFile(path.join(workspace, "node_modules", "tool", "cli.js"), "cli\n"); + await fs.symlink("../tool/cli.js", path.join(workspace, "node_modules", ".bin", "tool")); + await fs.mkdir(path.join(workspace, ".review-chrome")); + await fs.symlink(path.join(outside, "SingletonSocket"), path.join(workspace, ".review-chrome", "SingletonSocket")); + await fs.symlink("../sandbox-tmp/SingletonSocket", path.join(workspace, "escape")); + const { remote } = target({}, outputMode); + const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + try { + const restoredRoot = path.join(scratch, "restored"); + await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: restoredRoot, archivePath: captured.archivePath, + manifestDigest: captured.manifestDigest, restorationIdentity: "d".repeat(64) }); + expect(await fs.readFile(path.join(restoredRoot, "saved.txt"), "utf8")).toBe("review evidence\n"); + expect(await fs.readlink(path.join(restoredRoot, "node_modules", ".bin", "tool"))).toBe("../tool/cli.js"); + expect(await fs.readdir(path.join(restoredRoot, ".review-chrome"))).toEqual([]); + await expect(fs.lstat(path.join(restoredRoot, "escape"))).rejects.toMatchObject({ code: "ENOENT" }); + expect((await fs.readFile(captured.archivePath)).includes(Buffer.from("SYNTHETIC_SANDBOX_ONLY"))).toBe(false); + } finally { await captured.cleanup(); } + }); + it.each(["split", "one-shot"] as const)("retains a closed filesystem failure for a missing canonical archive with %s output", async outputMode => { const operationId = randomUUID(), canonical = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId, "captured"); await fs.mkdir(canonical, { recursive: true }); diff --git a/server/src/services/system-recovery-state.ts b/server/src/services/system-recovery-state.ts index 2e7d99a1865..00f110948c9 100644 --- a/server/src/services/system-recovery-state.ts +++ b/server/src/services/system-recovery-state.ts @@ -2,6 +2,7 @@ import { createHash, randomUUID } from "node:crypto"; import { z } from "zod"; import { ENVIRONMENT_LEASE_POLICIES, WORKSPACE_SOURCE_VERIFICATION_REASONS, type WorkspaceSourceVerificationFailure } from "@paperclipai/shared"; +import { REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION } from "./remote-workspace-snapshot-archive.js"; export const SYSTEM_RECOVERY_POLICY_VERSION = 1; export const SYSTEM_RECOVERY_MAX_RETRIES = 5; @@ -69,6 +70,9 @@ export const systemRecoverySchema = z.object({ }).nullable(), preservationOperationIds: z.array(z.string().uuid()), freshAttemptRunId: z.string().uuid().nullable(), + /** The workspace capture policy this generation saves with. Absent on + * generations written before the field existed; those used version 1. */ + capturePolicyVersion: z.number().int().positive().optional(), }); export type SystemRecoveryState = z.infer; @@ -85,6 +89,7 @@ export function createSystemRecovery(sourceRunId: string, intentId: string | nul phase: "verify_execution", state: "scheduled", attemptId: null, attempts: 0, retriesUsed: 0, firstFailureAt: null, nextAttemptAt: null, deadlineAt: null, providerCooldownAt: null, failure: null, preservationOperationIds: [], freshAttemptRunId: null, + capturePolicyVersion: REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION, }; } @@ -127,6 +132,25 @@ export function failSystemRecovery( }; } +/** + * A save that failed while preserving files under an older workspace capture + * policy. The current policy may accept what the older one refused (for + * example a workspace with a symlink out of it), so recovery retries it once + * under the current policy (workspace-recovery-capture-policy). The new + * generation carries the current version, so the retry is bounded per policy. + */ +export const CAPTURE_POLICY_RECOVERY_KEY = "capturePolicyRecovery"; +export const CAPTURE_POLICY_RECOVERY_SCHEMA = "paperclip.capture-policy-recovery.v1"; +// Capture refusals surface as export_interrupted. archive_unsafe names a restore +// refusal, which the capture policy does not change, so it is not re-armed. +export const CAPTURE_POLICY_REARM_CODES = ["export_interrupted"] as const; + +export function isCapturePolicyRearmable(state: Pick | null | undefined) { + return state?.state === "exhausted" && state.phase === "preserve_files" + && (CAPTURE_POLICY_REARM_CODES as readonly string[]).includes(state.failure?.code ?? "") + && (state.capturePolicyVersion ?? 1) < REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION; +} + /** * An exhausted save whose last failure was an unproven agent stop is not a * permanent refusal: once the source run has ended, a later attempt can prove diff --git a/server/src/services/system-recovery.ts b/server/src/services/system-recovery.ts index 020f2354d9e..b9b372ee58b 100644 --- a/server/src/services/system-recovery.ts +++ b/server/src/services/system-recovery.ts @@ -5,8 +5,8 @@ import { writeExecutionControl } from "./execution-control-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { createSystemRecovery, exhaustedSystemRecoveryAction, readSystemRecovery, type SystemRecoveryState } from "./system-recovery-state.js"; import { workspaceRecoverySourceEligible, type WorkspaceRecoverySourceIdentity } from "./workspace-recovery-source.js"; -import { retainedSourceFallbackDueAt, retainedSourceFallbackPendingCondition, stopProofRearmDueAt, - stopProofRearmPendingCondition } from "./workspace-recovery-fallback-discovery.js"; +import { capturePolicyRearmDueAt, capturePolicyRearmPendingCondition, retainedSourceFallbackDueAt, + retainedSourceFallbackPendingCondition, stopProofRearmDueAt, stopProofRearmPendingCondition } from "./workspace-recovery-fallback-discovery.js"; export async function adoptSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { return db.transaction(async tx => { @@ -123,6 +123,7 @@ export async function nextWorkspaceRecoveryAt(db: Db): Promise { const rows = await db.select({ at: sql`case when ${retainedSourceFallbackPendingCondition()} then ${retainedSourceFallbackDueAt()} when ${stopProofRearmPendingCondition()} then ${stopProofRearmDueAt()} + when ${capturePolicyRearmPendingCondition()} then ${capturePolicyRearmDueAt()} when jsonb_typeof(${issueRecoveryActions.evidence}#>'{systemRecovery,nextAttemptAt}') = 'number' then (${issueRecoveryActions.evidence}#>>'{systemRecovery,nextAttemptAt}')::bigint else 0 end` }) .from(issueRecoveryActions).innerJoin(heartbeatRuns, and( @@ -135,6 +136,7 @@ export async function nextWorkspaceRecoveryAt(db: Db): Promise { sql`(${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('scheduled', 'running', 'awaiting_evidence') or ${retainedSourceFallbackPendingCondition()} or ${stopProofRearmPendingCondition()} + or ${capturePolicyRearmPendingCondition()} or (${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' = 'complete' and ${issueRecoveryActions.evidence}->>'bootstrapPreparationSettledAt' is not null and coalesce(${issueRecoveryActions.evidence}->>'bootstrapPreparationContinuation', 'pending') = 'pending' diff --git a/server/src/services/workspace-recovery-capture-policy.test.ts b/server/src/services/workspace-recovery-capture-policy.test.ts new file mode 100644 index 00000000000..37eef096ca0 --- /dev/null +++ b/server/src/services/workspace-recovery-capture-policy.test.ts @@ -0,0 +1,114 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, executionWorkspaces, issueRecoveryActions } from "@paperclipai/db"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import { REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION } from "./remote-workspace-snapshot-archive.js"; +import { readSystemRecovery } from "./system-recovery-state.js"; +import { nextWorkspaceRecoveryAt } from "./system-recovery.js"; +import { readWorkspaceRecovery } from "./workspace-recovery-receipt.js"; +import { reconcileRetainedWorkspaces, recoverRetainedWorkspace, retainedWorkspaceCandidates } from "./workspace-recovery.js"; +import { acceptedRemoteExport, action, connectionString, db, useWorkspaceRecoveryFixture } from "./__tests__/workspace-recovery-fixture.js"; + +const targets = vi.hoisted(() => new Map()); +vi.mock("./environment-execution-target.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, resolveEnvironmentExecutionTarget: async (input: Parameters[0]) => + (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; +}); + +const adminText = /sandbox|environment|administrator|board member|owner must|inspect the run/i; + +describe("saves exhausted under an older workspace capture policy", () => { + useWorkspaceRecoveryFixture(targets); + beforeEach(() => { + vi.stubEnv("DATABASE_URL", connectionString); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "63".repeat(32)); + }); + afterEach(() => { vi.unstubAllEnvs(); }); + + /** + * The staging shape (Nov 10): the run's save was refused with + * `remote_workspace_archive_unsafe_symlink` because a headless Chromium + * profile in the workspace linked its SingletonSocket by absolute path into + * /tmp. Automatic recovery then spent its whole budget on the same refusal, + * recorded as `export_interrupted` with cause `provider_error`, and handed + * the hold to the board. This is the exact evidence that release saved. + */ + async function exhaustedCaptureHold(policyVersion?: number) { + const f = await acceptedRemoteExport("retained"); + const outside = path.join(path.dirname(f.remote), "sandbox-tmp"); + await mkdir(outside, { recursive: true }); + await writeFile(path.join(outside, "SingletonSocket"), "SYNTHETIC_SANDBOX_ONLY\n"); + await mkdir(path.join(f.remote, "node_modules", "tool"), { recursive: true }); + await mkdir(path.join(f.remote, "node_modules", ".bin")); + await writeFile(path.join(f.remote, "node_modules", "tool", "cli.js"), "cli\n"); + await symlink("../tool/cli.js", path.join(f.remote, "node_modules", ".bin", "tool")); + await mkdir(path.join(f.remote, ".review-chrome")); + await symlink(path.join(outside, "SingletonSocket"), path.join(f.remote, ".review-chrome", "SingletonSocket")); + await symlink("../sandbox-tmp/SingletonSocket", path.join(f.remote, "escape")); + const prior = await action(f.actionId); + const current = readSystemRecovery(prior.evidence)!; + const firstFailureAt = Date.now() - 60 * 60_000; + const nextAction = "Automatic recovery of saved work stopped after 6 attempts (last failure: export_interrupted, cause: provider_error). " + + "Retry to try again; the agent will not repeat its work."; + const { capturePolicyVersion: _omitted, ...saved } = current as typeof current & { capturePolicyVersion?: number }; + await db.update(issueRecoveryActions).set({ ownerType: "board", nextAction, evidence: { ...prior.evidence, + systemRecovery: { ...saved, ...(policyVersion === undefined ? {} : { capturePolicyVersion: policyVersion }), + revision: 18, phase: "preserve_files", state: "exhausted", attempts: 6, retriesUsed: 5, + attemptId: randomUUID(), firstFailureAt, deadlineAt: firstFailureAt + 30 * 60_000, nextAttemptAt: null, + failure: { code: "export_interrupted", retryClass: "retry_transient", evidenceFingerprint: "9".repeat(64), + cause: { code: null, label: "provider_error", status: null } } }, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: f.runId, status: "blocked", stage: "restoring_files", + attempt: 6, attemptLimit: 6, retryable: true, message: nextAction, requestIds: [] } } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + return f; + } + + it("retries once under the new policy and saves the files without a person", async () => { + const f = await exhaustedCaptureHold(); + + // The recovery worker discovers the hold and wakes for it. + expect(await nextWorkspaceRecoveryAt(db)).toBeLessThanOrEqual(Date.now()); + expect((await retainedWorkspaceCandidates(db, undefined, false, true)).map(row => row.action.id)).toContain(f.actionId); + const onRecovered = vi.fn(async () => undefined); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, onRecovered, true) ?? []); + + const settled = await action(f.actionId); + const state = readSystemRecovery(settled.evidence)!; + expect(state).toMatchObject({ state: "complete", capturePolicyVersion: REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION }); + expect(state.preservationOperationIds.length).toBeGreaterThan(0); + expect(readWorkspaceRecovery(settled.evidence)?.status).toBe("recovered"); + expect(settled.evidence.capturePolicyRecovery).toMatchObject({ schema: "paperclip.capture-policy-recovery.v1", + policyVersion: REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION }); + expect(settled.ownerType).toBe("system"); + expect(String(settled.nextAction)).not.toMatch(adminText); + expect(onRecovered).toHaveBeenCalledTimes(1); + + const authority = readRemoteWorkspaceAuthority((await db.select().from(executionWorkspaces) + .where(eq(executionWorkspaces.id, f.workspaceId)))[0]!.metadata)!; + expect(authority).toMatchObject({ dirty: false }); + expect(authority.snapshotId).not.toBeNull(); + // The source workspace is untouched; nothing followed the outside links. + expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n"); + + const rearmed = await db.select().from(activityLog).where(eq(activityLog.action, "issue.workspace_recovery_rearmed")); + expect(rearmed).toEqual([expect.objectContaining({ actorType: "system", entityId: f.issueId, runId: f.runId, + details: expect.objectContaining({ reason: "capture_policy_changed" }) })]); + }, 60_000); + + it("does not re-arm a save that the current policy already exhausted", async () => { + const f = await exhaustedCaptureHold(REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION); + expect((await retainedWorkspaceCandidates(db, undefined, false, true)).map(row => row.action.id)).not.toContain(f.actionId); + const next = await nextWorkspaceRecoveryAt(db); + expect(next === null || next > Date.now()).toBe(true); + await recoverRetainedWorkspace(f.input); + const held = await action(f.actionId); + expect(readSystemRecovery(held.evidence)).toMatchObject({ state: "exhausted", failure: { code: "export_interrupted" } }); + expect(held.evidence.capturePolicyRecovery).toBeUndefined(); + expect(f.runtime.resumeRunLease).not.toHaveBeenCalled(); + }, 60_000); +}); diff --git a/server/src/services/workspace-recovery-capture-policy.ts b/server/src/services/workspace-recovery-capture-policy.ts new file mode 100644 index 00000000000..afc7e69652b --- /dev/null +++ b/server/src/services/workspace-recovery-capture-policy.ts @@ -0,0 +1,76 @@ +import { and, eq } from "drizzle-orm"; +import { heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { logger } from "../middleware/logger.js"; +import { persistActivity, publishActivity } from "./activity-log.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; +import { REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION } from "./remote-workspace-snapshot-archive.js"; +import { CAPTURE_POLICY_RECOVERY_KEY, CAPTURE_POLICY_RECOVERY_SCHEMA, createSystemRecovery, isCapturePolicyRearmable, + readSystemRecovery, type SystemRecoveryState } from "./system-recovery-state.js"; +import type { WorkspaceRecoverySourceIdentity } from "./workspace-recovery-source.js"; +import { sourceRunProvablyOver } from "./workspace-recovery-stop-proof.js"; + +/** User-facing text while the retried save runs. It names no person who must act. */ +export const CAPTURE_POLICY_REARM_MESSAGE = "Paperclip is saving the previous run's files again with its updated save rules. " + + "No provider turn is submitted."; + +/** + * Retries, once per capture policy version, a save that recovery exhausted + * while preserving files under an older policy. The same saved workspace can + * succeed now (for example one with a symlink out of the workspace, which an + * older policy refused and the current one records and drops). The new + * generation repeats the strict stop proof and the save under the current + * policy and records that version, so it is never re-armed again by this + * rule. Holds exhausted by older releases carry no version and are re-armed on + * the first pass, with no migration. Returns whether a new generation was written. + */ +export async function rearmExhaustedCapturePolicy(db: Db, input: WorkspaceRecoverySourceIdentity, observed: SystemRecoveryState, + now = Date.now()): Promise { + if (!isCapturePolicyRearmable(observed)) return false; + const outcome = await db.transaction(async tx => { + // Lock order shared with the fallback and the stop-proof re-arm: task, coordinator, run, recovery action. + const [issue] = await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, input.companyId), + eq(issues.id, input.issueId))).for("update"); + const [coordinator] = await tx.select({ leaseOwner: nativeRunFinalizations.leaseOwner }) + .from(nativeRunFinalizations).where(and(eq(nativeRunFinalizations.companyId, input.companyId), + eq(nativeRunFinalizations.runId, input.runId))).for("update"); + const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.id, input.runId))).for("update"); + const [action] = await tx.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, input.companyId), + eq(issueRecoveryActions.sourceIssueId, input.issueId), eq(issueRecoveryActions.id, input.actionId))).for("update"); + if (!issue || !run || !action) return null; + const system = readSystemRecovery(action.evidence); + if (!system || system.generationId !== observed.generationId || system.revision !== observed.revision + || !isCapturePolicyRearmable(system) || system.sourceRunId !== run.id) return null; + if (!["active", "escalated"].includes(action.status) + || action.evidence.runId !== run.id || action.evidence.executionReconciliation || action.evidence.retainedSourceFallback) return null; + // Nothing still executes or finalizes the source run. The new attempt + // still proves the sandbox stop itself before it reads any file. + const at = new Date(now); + if (!sourceRunProvablyOver(run, coordinator, at)) return null; + const next: SystemRecoveryState = { ...createSystemRecovery(run.id), revision: system.revision + 1, + preservationOperationIds: system.preservationOperationIds, providerCooldownAt: system.providerCooldownAt, + nextAttemptAt: system.providerCooldownAt }; + const marker = { schema: CAPTURE_POLICY_RECOVERY_SCHEMA, policyVersion: REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION, + previousPolicyVersion: system.capturePolicyVersion ?? 1, generationId: next.generationId, + previousGenerationId: system.generationId, rearmedAt: now }; + const prior = action.evidence.workspaceRecovery as { requestIds?: unknown } | undefined; + await notifyIssueRecovery(tx, { companyId: input.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.workspaceRecovery); + await tx.update(issueRecoveryActions).set({ ownerType: "system", ownerAgentId: null, ownerUserId: null, + nextAction: CAPTURE_POLICY_REARM_MESSAGE, updatedAt: at, + evidence: { ...action.evidence, systemRecovery: next, [CAPTURE_POLICY_RECOVERY_KEY]: marker, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: run.id, status: "queued", stage: "verifying_stop", + attempt: 0, attemptLimit: 6, retryable: true, message: CAPTURE_POLICY_REARM_MESSAGE, + requestIds: Array.isArray(prior?.requestIds) ? prior.requestIds : [] } }, + }).where(and(eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, action.id))); + return persistActivity(tx as unknown as Db, { companyId: input.companyId, actorType: "system", actorId: "workspace-recovery", + action: "issue.workspace_recovery_rearmed", entityType: "issue", entityId: input.issueId, runId: run.id, + details: { recoveryActionId: action.id, reason: "capture_policy_changed", failureCode: system.failure?.code ?? null, + policyVersion: marker.policyVersion, previousPolicyVersion: marker.previousPolicyVersion, + previousGenerationId: system.generationId, generationId: next.generationId } }); + }); + if (!outcome) return false; + try { publishActivity(outcome.publication); } catch { logger.warn({ runId: input.runId }, "Recovery activity delivery deferred"); } + return true; +} diff --git a/server/src/services/workspace-recovery-fallback-discovery.ts b/server/src/services/workspace-recovery-fallback-discovery.ts index 6cba725d77b..adfe86eaf31 100644 --- a/server/src/services/workspace-recovery-fallback-discovery.ts +++ b/server/src/services/workspace-recovery-fallback-discovery.ts @@ -1,6 +1,7 @@ import { sql } from "drizzle-orm"; import { issueRecoveryActions } from "@paperclipai/db"; -import { isExhaustedStopProofWait, STOP_PROOF_MAX_REARMS, STOP_PROOF_RECOVERY_KEY, STOP_PROOF_RECOVERY_SCHEMA, STOP_PROOF_WAIT_CODE, +import { REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION } from "./remote-workspace-snapshot-archive.js"; +import { CAPTURE_POLICY_REARM_CODES, isExhaustedStopProofWait, STOP_PROOF_MAX_REARMS, STOP_PROOF_RECOVERY_KEY, STOP_PROOF_RECOVERY_SCHEMA, STOP_PROOF_WAIT_CODE, stopProofRearmsSpent, type SystemRecoveryState } from "./system-recovery-state.js"; import { canonicalExecutionUuid } from "./execution-identity.js"; @@ -84,7 +85,12 @@ export function stopProofRearmPendingCondition() { export function stopProofRearmDueAt() { return sql`greatest(case when ${stopProofMarker}->>'schema' = ${STOP_PROOF_RECOVERY_SCHEMA} and jsonb_typeof(${stopProofMarker}->'nextRearmAt') = 'number' then (${stopProofMarker}->>'nextRearmAt')::bigint else 0 end, - coalesce((select (extract(epoch from source.controller_lease_expires_at) * 1000)::bigint from heartbeat_runs source + ${sourceRunLeasesExpireAt()})`; +} + +/** The latest expiry of a controller or finalizer lease the source run holds (0 = none). */ +function sourceRunLeasesExpireAt() { + return sql`greatest(coalesce((select (extract(epoch from source.controller_lease_expires_at) * 1000)::bigint from heartbeat_runs source where source.company_id = ${issueRecoveryActions.companyId} and source.id = ${sourceRunId}), 0), coalesce((select max((extract(epoch from finalizer.lease_expires_at) * 1000)::bigint) from native_run_finalizations finalizer where finalizer.company_id = ${issueRecoveryActions.companyId} and finalizer.run_id = ${sourceRunId} @@ -96,6 +102,33 @@ export function stopProofRearmDueCondition(now: number) { } const list = (values: readonly string[]) => sql.join(values.map(value => sql`${value}`), sql`, `); +const capturePolicyVersion = sql`case when jsonb_typeof(${issueRecoveryActions.evidence}#>'{systemRecovery,capturePolicyVersion}') = 'number' + then (${issueRecoveryActions.evidence}#>>'{systemRecovery,capturePolicyVersion}')::int else 1 end`; + +/** An exhausted save whose files a previous capture policy refused, not yet + * retried under the current policy (isCapturePolicyRearmable). Saves exhausted + * by older releases carry no version and are due at once, with no migration. */ +export function capturePolicyRearmPendingCondition() { + return sql`(${issueRecoveryActions.status} in ('active', 'escalated') + and ${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' = 'exhausted' + and ${issueRecoveryActions.evidence}#>>'{systemRecovery,phase}' = 'preserve_files' + and ${issueRecoveryActions.evidence}#>>'{systemRecovery,failure,code}' in (${list(CAPTURE_POLICY_REARM_CODES)}) + and ${capturePolicyVersion} < ${REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION} + and not exists (select 1 from native_run_finalizations finalizer + where finalizer.company_id = ${issueRecoveryActions.companyId} and finalizer.run_id = ${sourceRunId} + and finalizer.lease_owner is not null and coalesce(finalizer.lease_expires_at <= now(), true)) + and ${issueRecoveryActions.evidence}->'retainedSourceFallback' is null + and ${issueRecoveryActions.evidence}->'executionReconciliation' is null)`; +} + +export function capturePolicyRearmDueAt() { + return sourceRunLeasesExpireAt(); +} + +export function capturePolicyRearmDueCondition(now: number) { + return sql`(${capturePolicyRearmPendingCondition()} and ${capturePolicyRearmDueAt()} <= ${now}::bigint)`; +} + const check = sql`${issueRecoveryActions.evidence}->${RETAINED_SOURCE_FALLBACK_CHECK_KEY}`; // New stop evidence (any change to an allocation of the source run, such as a // late provider stop receipt) after the last deferred check reconsiders the diff --git a/server/src/services/workspace-recovery-fallback.test.ts b/server/src/services/workspace-recovery-fallback.test.ts index 2d440101d72..13a95cff050 100644 --- a/server/src/services/workspace-recovery-fallback.test.ts +++ b/server/src/services/workspace-recovery-fallback.test.ts @@ -1,3 +1,4 @@ +import { REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION } from "./remote-workspace-snapshot-archive.js"; import { randomUUID } from "node:crypto"; import { execFile } from "node:child_process"; import { cp, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; @@ -507,7 +508,8 @@ describe("retained source fallback", () => { generationId: randomUUID(), intentId: null, phase: "preserve_files", state: "exhausted", attemptId: randomUUID(), attempts: 6, retriesUsed: 5, firstFailureAt: now - 60_000, nextAttemptAt: null, deadlineAt: now - 1, providerCooldownAt: null, failure: { code: "export_interrupted", retryClass: "retry_transient", evidenceFingerprint: "c".repeat(64) }, - preservationOperationIds: [], freshAttemptRunId: null }, + // Exhausted under the current capture policy, so only the sibling rule applies. + preservationOperationIds: [], freshAttemptRunId: null, capturePolicyVersion: REMOTE_WORKSPACE_CAPTURE_POLICY_VERSION }, workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: f.runId, status: "blocked", stage: "restoring_files", attempt: 6, attemptLimit: 6, retryable: true, requestIds: [] } } }); const ids = async () => (await retainedWorkspaceCandidates(db, undefined, false, true)).map(row => row.action.id); diff --git a/server/src/services/workspace-recovery-stop-proof.ts b/server/src/services/workspace-recovery-stop-proof.ts index 704bb144eb0..4c862086419 100644 --- a/server/src/services/workspace-recovery-stop-proof.ts +++ b/server/src/services/workspace-recovery-stop-proof.ts @@ -9,6 +9,18 @@ import { createSystemRecovery, isExhaustedStopProofWait, readStopProofRecovery, STOP_PROOF_RECOVERY_KEY, STOP_PROOF_RECOVERY_SCHEMA, type SystemRecoveryState } from "./system-recovery-state.js"; import { terminalWorkspaceRunStates, type WorkspaceRecoverySourceIdentity } from "./workspace-recovery-source.js"; +/** + * The source run ended and nothing still executes or finalizes it, so a new + * recovery generation may start. Shared by every exhausted-hold re-arm. The + * attempt refuses any recorded finalizer owner (inspectWorkspaceRecoverySource); + * a stale one is released by the abandoned-owner settlement first. + */ +export function sourceRunProvablyOver(run: typeof heartbeatRuns.$inferSelect, + coordinator: { leaseOwner: string | null } | undefined, at: Date) { + return terminalWorkspaceRunStates.includes(run.status) && Boolean(run.finishedAt) && !adapterExecutionControls.has(run.id) + && !(run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > at) && !coordinator?.leaseOwner; +} + /** Delay before re-arm n+1, measured from re-arm n. The first re-arm is due at once. */ const STOP_PROOF_REARM_DELAYS_MS = [30 * 60_000, 2 * 60 * 60_000, 8 * 60 * 60_000]; @@ -52,11 +64,7 @@ export async function rearmExhaustedStopProof(db: Db, input: WorkspaceRecoverySo // The agent is provably gone: its run ended and nothing still executes or // finalizes it. The new attempt still proves the sandbox stop itself. const at = new Date(now); - if (!terminalWorkspaceRunStates.includes(run.status) || !run.finishedAt || adapterExecutionControls.has(run.id) - || (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > at) - // The attempt refuses any recorded finalizer owner (inspectWorkspaceRecoverySource); - // a stale one is released by the abandoned-owner settlement first. - || coordinator?.leaseOwner) return null; + if (!sourceRunProvablyOver(run, coordinator, at)) return null; const marker = readStopProofRecovery(action.evidence); const rearms = marker?.rearms ?? 0; if (rearms >= STOP_PROOF_MAX_REARMS || (marker?.nextRearmAt ?? 0) > now) return null; diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index 5be4f8bf03c..fc0078036d0 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -36,7 +36,8 @@ import { adoptHistoricalWorkspaceRecoveries } from "./workspace-recovery-backfil import { canAdoptNativeSourceRetention, retainNativeRecoverySource, settleRecoveredNativeSourceRetention } from "./native-runtime/native-source-retention.js"; import { saveRemoteWorkspace } from "./remote-workspaces.js"; import { continueFromLastSavedWorkspace, isUnverifiableSourceFailure, recordRetainedSourceFallbackDeferral, settleExhaustedRetainedHold } from "./workspace-recovery-fallback.js"; -import { retainedSourceFallbackDueCondition, stopProofRearmDueCondition } from "./workspace-recovery-fallback-discovery.js"; +import { capturePolicyRearmDueCondition, retainedSourceFallbackDueCondition, stopProofRearmDueCondition } from "./workspace-recovery-fallback-discovery.js"; +import { rearmExhaustedCapturePolicy } from "./workspace-recovery-capture-policy.js"; import { rearmExhaustedStopProof } from "./workspace-recovery-stop-proof.js"; import { settleRecoveredWorkspaceRestoreHolds } from "./workspace-restore-hold-settlement.js"; @@ -209,6 +210,9 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ // An exhausted wait for stop proof is re-armed, bounded, once the run is over. if (initial?.state === "exhausted" && await rearmExhaustedStopProof(db, input, initial)) initial = await loadSystemRecovery(db, input); + // A save exhausted under an older capture policy is retried once under the current one. + if (initial?.state === "exhausted" && await rearmExhaustedCapturePolicy(db, input, initial)) + initial = await loadSystemRecovery(db, input); if (initial?.state === "exhausted" && !canAdoptNativeSourceRetention(initial)) { // A budget spent before this hold existed still names its receipt, so the // board confirmation in its handoff can release it (RC-50 wave databases). @@ -628,7 +632,8 @@ export function retainedWorkspaceCandidates(db: Db, cursor?: string, unfinishedO and coalesce(${issueRecoveryActions.evidence}->>'bootstrapPreparationContinuation', 'pending') = 'pending'))` : undefined, obligationsOnly ? sql`(${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('scheduled', 'running', 'awaiting_evidence') or (${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' = 'complete' and (${cleanup} or ${interruptedPreparation})) or ${waitingInput} - or ${retainedSourceFallbackDueCondition(Date.now())} or ${stopProofRearmDueCondition(Date.now())})` : undefined, + or ${retainedSourceFallbackDueCondition(Date.now())} or ${stopProofRearmDueCondition(Date.now())} + or ${capturePolicyRearmDueCondition(Date.now())})` : undefined, sql`(${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('scheduled', 'running', 'awaiting_evidence', 'complete', 'exhausted') and (${issueRecoveryActions.evidence}#>'{systemRecovery,nextAttemptAt}' = 'null'::jsonb or ${issueRecoveryActions.evidence}#>'{systemRecovery,nextAttemptAt}' <= to_jsonb(${Date.now()}::bigint)))`,