Merge remote-tracking branch 'origin/master' into codex/provider-routing-connections

* origin/master:
  fix(tasks): surface Codex ChatGPT model rejection (#15299)
  fix(runner): continue restart-interrupted Codex turns (#15297)
  fix(agents): grant configuration access by default (#15283)
  Guard routine UUID lookups without narrowing valid inputs (#15313)
  fix(ssh): transport project repositories as their own git checkouts (#14782)
  feat(exe-dev): copy a source VM with exe.dev cp (#14975)
  Diagnose native model rejections and preserve repairable reviews (#15304)
  Verify native semantic input against its raw wire digest (#15301)
  Retry pooled execution projection reads after disconnects (#15300)
  Fix browser polling for unsaved agent chats (#15298)
  test(ui): keep initial reasoning with the saved reply (#15107)
  docs: add a product feature map (#15288)

packages/adapter-utils/src/remote-managed-runtime.ts

@@ -1,5 +1,5 @@
 import path from "node:path";
-import { GIT_ARCHIVE_EXCLUDES } from "./git-workspace-sync.js";
+import { GIT_ARCHIVE_EXCLUDES, PROJECT_REPOSITORIES_DIR } from "./git-workspace-sync.js";
 import {
   type SshRemoteExecutionSpec,
   prepareWorkspaceForSshExecution,
@@ -13,7 +13,7 @@ import {
   type SandboxAdditionalSource,
   type SandboxManagedRuntimeAssetRestoreContext,
 } from "./sandbox-managed-runtime.js";
-import { captureDirectorySnapshot } from "./workspace-restore-merge.js";
+import { captureDirectorySnapshot, type DirectorySnapshot } from "./workspace-restore-merge.js";
 import type { RuntimeProgressSink } from "./runtime-progress.js";
 
 // The fixed heavy-directory excludes every referenced project drops,
@@ -145,13 +145,27 @@ export async function prepareRemoteManagedRuntime(input: {
         workspaceExclude: input.workspaceExclude,
       })
     : null;
+  const projectRepositories = preparedWorkspace?.repositories ?? [];
   const baselineSnapshot = preparedWorkspace
     ? await captureDirectorySnapshot(input.workspaceLocalDir, {
         exclude: preparedWorkspace.gitBacked
-          ? [...GIT_ARCHIVE_EXCLUDES, ".paperclip-runtime"]
+          ? [
+              ...GIT_ARCHIVE_EXCLUDES,
+              ".paperclip-runtime",
+              ...(projectRepositories.length > 0 ? [PROJECT_REPOSITORIES_DIR] : []),
+            ]
           : [".paperclip-runtime", ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])],
       })
     : null;
+  const repositoryBaselines: Array<{ path: string; baselineSnapshot: DirectorySnapshot }> = [];
+  for (const repository of projectRepositories) {
+    repositoryBaselines.push({
+      path: repository,
+      baselineSnapshot: await captureDirectorySnapshot(path.join(input.workspaceLocalDir, repository), {
+        exclude: [...GIT_ARCHIVE_EXCLUDES, ".paperclip-runtime"],
+      }),
+    });
+  }
 
   const assetDirs: Record<string, string> = {};
   try {
@@ -177,6 +191,7 @@ export async function prepareRemoteManagedRuntime(input: {
         baselineSnapshot,
         restoreGitHistory: preparedWorkspace.gitBacked,
         onProgress: input.onProgress,
+        repositories: repositoryBaselines,
       });
     }
     throw error;
@@ -244,6 +259,7 @@ export async function prepareRemoteManagedRuntime(input: {
           baselineSnapshot,
           restoreGitHistory: preparedWorkspace.gitBacked,
           onProgress,
+          repositories: repositoryBaselines,
         });
       }
       for (const asset of input.assets ?? []) {

packages/adapter-utils/src/ssh-fixture.test.ts

@@ -21,6 +21,16 @@ import {
 import { prepareRemoteManagedRuntime } from "./remote-managed-runtime.js";
 
 const SSH_FIXTURE_TEST_TIMEOUT_MS = 30_000;
+const UNREACHABLE_SSH_SPEC = {
+  host: "127.0.0.1",
+  port: 1,
+  username: "nobody",
+  remoteWorkspacePath: "/nonexistent",
+  remoteCwd: "/nonexistent",
+  privateKey: null,
+  knownHosts: null,
+  strictHostKeyChecking: false,
+} as const;
 let sshEnvLabUnsupportedReason: string | null = null;
 
 // One entry per fixture root directory, registered at creation time so
@@ -850,6 +860,205 @@ describe("ssh env-lab fixture", () => {
     await expect(readFile(path.join(localRepo, "run-b.txt"), "utf8")).resolves.toBe("from run b\n");
   }, SSH_FIXTURE_TEST_TIMEOUT_MS);
 
+  it("round-trips project repositories nested in .paperclip-repositories across consecutive SSH runs", async () => {
+    const rootDir = await createFixtureRootDir();
+    const statePath = path.join(rootDir, "state.json");
+    const localRepo = path.join(rootDir, "local-workspace");
+    const nestedRelative = ".paperclip-repositories/frontend-0123456789ab";
+    const nestedRepo = path.join(localRepo, nestedRelative);
+
+    await mkdir(localRepo, { recursive: true });
+    await git(localRepo, ["init"]);
+    await git(localRepo, ["checkout", "-b", "main"]);
+    await git(localRepo, ["config", "user.name", "Paperclip Test"]);
+    await git(localRepo, ["config", "user.email", "test@paperclip.dev"]);
+    await writeFile(path.join(localRepo, "backend.txt"), "backend base\n", "utf8");
+    await git(localRepo, ["add", "backend.txt"]);
+    await git(localRepo, ["commit", "-m", "backend initial"]);
+    await writeFile(path.join(localRepo, ".git", "info", "exclude"), "\n/.paperclip-repositories/\n", { flag: "a" });
+
+    await mkdir(nestedRepo, { recursive: true });
+    await git(nestedRepo, ["init"]);
+    await git(nestedRepo, ["checkout", "-b", "main"]);
+    await git(nestedRepo, ["config", "user.name", "Paperclip Test"]);
+    await git(nestedRepo, ["config", "user.email", "test@paperclip.dev"]);
+    await writeFile(path.join(nestedRepo, "frontend.txt"), "frontend base\n", "utf8");
+    await writeFile(path.join(nestedRepo, "obsolete.txt"), "tracked\n", "utf8");
+    await git(nestedRepo, ["add", "frontend.txt", "obsolete.txt"]);
+    await git(nestedRepo, ["commit", "-m", "frontend initial"]);
+    await writeFile(path.join(nestedRepo, "frontend.txt"), "frontend dirty local\n", "utf8");
+    await rm(path.join(nestedRepo, "obsolete.txt"));
+
+    const started = await startSshEnvLabFixtureOrSkip(statePath, "SSH nested project repositories test");
+    if (!started) return;
+    const config = await buildSshEnvLabFixtureConfig(started);
+    const spec = {
+      ...config,
+      remoteCwd: started.workspaceDir,
+    } as const;
+
+    const first = await prepareRemoteManagedRuntime({
+      spec,
+      runId: "run-1",
+      adapterKey: "test-adapter",
+      workspaceLocalDir: localRepo,
+    });
+    const remoteNested = path.posix.join(first.workspaceRemoteDir, nestedRelative);
+
+    const remoteNestedStatus = await runSshCommand(
+      config,
+      `cd ${JSON.stringify(remoteNested)} && git log -1 --pretty=%s && git status --short`,
+    );
+    expect(remoteNestedStatus.stdout).toContain("frontend initial");
+    expect(remoteNestedStatus.stdout).toContain("M frontend.txt");
+    expect(remoteNestedStatus.stdout).toContain("D obsolete.txt");
+    const remoteAnchorStatus = await runSshCommand(
+      config,
+      `cd ${JSON.stringify(first.workspaceRemoteDir)} && git status --short --untracked-files=all`,
+    );
+    expect(remoteAnchorStatus.stdout).not.toContain(".paperclip-repositories");
+
+    await runSshCommand(
+      config,
+      [
+        `cd ${JSON.stringify(remoteNested)}`,
+        `git config user.name "Paperclip SSH"`,
+        `git config user.email "ssh@paperclip.dev"`,
+        `git add frontend.txt`,
+        `git commit -m "remote frontend update" >/dev/null`,
+        `printf "frontend remote dirty\\n" > frontend.txt`,
+        `cd ${JSON.stringify(first.workspaceRemoteDir)}`,
+        `git config user.name "Paperclip SSH"`,
+        `git config user.email "ssh@paperclip.dev"`,
+        `printf "backend remote\\n" > backend.txt`,
+        `git add backend.txt`,
+        `git commit -m "remote backend update" >/dev/null`,
+      ].join(" && "),
+      { timeoutMs: 30_000, maxBuffer: 256 * 1024 },
+    );
+
+    await first.restoreWorkspace();
+
+    await expect(stat(path.join(nestedRepo, ".git"))).resolves.toBeDefined();
+    expect(await git(nestedRepo, ["log", "-1", "--pretty=%s"])).toBe("remote frontend update");
+    await expect(readFile(path.join(nestedRepo, "frontend.txt"), "utf8")).resolves.toBe("frontend remote dirty\n");
+    expect(await git(localRepo, ["log", "-1", "--pretty=%s"])).toBe("remote backend update");
+    expect(await git(localRepo, ["status", "--short", "--untracked-files=all"])).not.toContain(".paperclip-repositories");
+
+    const second = await prepareRemoteManagedRuntime({
+      spec,
+      runId: "run-2",
+      adapterKey: "test-adapter",
+      workspaceLocalDir: localRepo,
+    });
+    const secondNestedLog = await runSshCommand(
+      config,
+      `cd ${JSON.stringify(path.posix.join(second.workspaceRemoteDir, nestedRelative))} && git log -1 --pretty=%s`,
+    );
+    expect(secondNestedLog.stdout.trim()).toBe("remote frontend update");
+    await second.restoreWorkspace();
+    await expect(stat(path.join(nestedRepo, ".git"))).resolves.toBeDefined();
+    await expect(readFile(path.join(nestedRepo, "frontend.txt"), "utf8")).resolves.toBe("frontend remote dirty\n");
+  }, SSH_FIXTURE_TEST_TIMEOUT_MS * 2);
+
+  it("restores project repositories through the direct SSH restore path", async () => {
+    const rootDir = await createFixtureRootDir();
+    const statePath = path.join(rootDir, "state.json");
+    const localRepo = path.join(rootDir, "local-workspace");
+    const nestedRelative = ".paperclip-repositories/frontend-0123456789ab";
+    const nestedRepo = path.join(localRepo, nestedRelative);
+
+    for (const repo of [localRepo, nestedRepo]) {
+      await mkdir(repo, { recursive: true });
+      await git(repo, ["init"]);
+      await git(repo, ["checkout", "-b", "main"]);
+      await git(repo, ["config", "user.name", "Paperclip Test"]);
+      await git(repo, ["config", "user.email", "test@paperclip.dev"]);
+      await writeFile(path.join(repo, "tracked.txt"), "base\n", "utf8");
+      await git(repo, ["add", "tracked.txt"]);
+      await git(repo, ["commit", "-m", "initial"]);
+    }
+    await writeFile(path.join(localRepo, ".git", "info", "exclude"), "\n/.paperclip-repositories/\n", { flag: "a" });
+
+    const started = await startSshEnvLabFixtureOrSkip(statePath, "SSH direct restore with project repositories test");
+    if (!started) return;
+    const config = await buildSshEnvLabFixtureConfig(started);
+    const spec = {
+      ...config,
+      remoteCwd: started.workspaceDir,
+    } as const;
+
+    const prepared = await prepareWorkspaceForSshExecution({
+      spec,
+      localDir: localRepo,
+      remoteDir: started.workspaceDir,
+    });
+    expect(prepared).toEqual({ gitBacked: true, repositories: [nestedRelative] });
+
+    await runSshCommand(
+      config,
+      [
+        `cd ${JSON.stringify(path.posix.join(started.workspaceDir, nestedRelative))}`,
+        `git config user.name "Paperclip SSH"`,
+        `git config user.email "ssh@paperclip.dev"`,
+        `printf "remote\\n" > tracked.txt`,
+        `git commit -am "remote nested update" >/dev/null`,
+      ].join(" && "),
+      { timeoutMs: 30_000, maxBuffer: 256 * 1024 },
+    );
+
+    await restoreWorkspaceFromSshExecution({
+      spec,
+      localDir: localRepo,
+      remoteDir: started.workspaceDir,
+      repositories: (prepared.repositories ?? []).map((repository) => ({ path: repository })),
+    });
+
+    await expect(stat(path.join(nestedRepo, ".git"))).resolves.toBeDefined();
+    expect(await git(nestedRepo, ["log", "-1", "--pretty=%s"])).toBe("remote nested update");
+    await expect(readFile(path.join(nestedRepo, "tracked.txt"), "utf8")).resolves.toBe("remote\n");
+    expect(await git(localRepo, ["status", "--short", "--untracked-files=all"])).toBe("");
+  }, SSH_FIXTURE_TEST_TIMEOUT_MS * 2);
+
+  it("fails closed when .paperclip-repositories holds something other than a git checkout", async () => {
+    const rootDir = await createFixtureRootDir();
+    const localRepo = path.join(rootDir, "local-workspace");
+    await mkdir(path.join(localRepo, ".paperclip-repositories", "not-a-repo"), { recursive: true });
+    await git(localRepo, ["init"]);
+    await git(localRepo, ["checkout", "-b", "main"]);
+    await git(localRepo, ["config", "user.name", "Paperclip Test"]);
+    await git(localRepo, ["config", "user.email", "test@paperclip.dev"]);
+    await writeFile(path.join(localRepo, "tracked.txt"), "base\n", "utf8");
+    await git(localRepo, ["add", "tracked.txt"]);
+    await git(localRepo, ["commit", "-m", "initial"]);
+
+    await expect(prepareWorkspaceForSshExecution({ spec: UNREACHABLE_SSH_SPEC, localDir: localRepo, remoteDir: "/nonexistent" }))
+      .rejects.toThrow("Project repository is not a Git checkout: .paperclip-repositories/not-a-repo");
+  }, SSH_FIXTURE_TEST_TIMEOUT_MS);
+
+  it("rejects inconsistent project repository restores before any SSH connection", async () => {
+    const localDir = await createFixtureRootDir();
+    const repositoryBaseline = { exclude: [".git", ".git/*", ".paperclip-runtime"], entries: new Map() };
+    const restore = (input: Partial<Parameters<typeof restoreWorkspaceFromSshExecution>[0]>) =>
+      restoreWorkspaceFromSshExecution({
+        spec: UNREACHABLE_SSH_SPEC,
+        localDir,
+        remoteDir: "/nonexistent",
+        baselineSnapshot: { exclude: [".git", ".git/*", ".paperclip-runtime", ".paperclip-repositories"], entries: new Map() },
+        restoreGitHistory: true,
+        ...input,
+      });
+
+    await expect(restore({ repositories: [{ path: "../outside", baselineSnapshot: repositoryBaseline }] }))
+      .rejects.toThrow("Invalid project repository path: ../outside");
+    await expect(restore({ repositories: [{ path: ".paperclip-repositories/frontend-0123456789ab" }] }))
+      .rejects.toThrow("Project repository has no workspace baseline: .paperclip-repositories/frontend-0123456789ab");
+    await expect(restore({
+      baselineSnapshot: { exclude: [".git", ".git/*", ".paperclip-runtime"], entries: new Map() },
+      repositories: [{ path: ".paperclip-repositories/frontend-0123456789ab", baselineSnapshot: repositoryBaseline }],
+    })).rejects.toThrow("Workspace baseline must exclude .paperclip-repositories");
+  }, SSH_FIXTURE_TEST_TIMEOUT_MS);
+
   it("preserves nested per-run files across sequential SSH restores with stale baselines", async () => {
     const rootDir = await createFixtureRootDir();
     const statePath = path.join(rootDir, "state.json");

packages/adapter-utils/src/ssh.ts

@@ -9,6 +9,7 @@ import type { CommandManagedRuntimeRunner } from "./command-managed-runtime.js";
 import {
   createUnrelatedHistoryGraftCommit,
   GIT_SYNC_COMMIT_IDENTITY_ARGS,
+  PROJECT_REPOSITORIES_DIR,
   readSanitizedOriginRemoteUrl,
 } from "./git-workspace-sync.js";
 import type { RunProcessResult } from "./server-utils.js";
@@ -1071,6 +1072,91 @@ async function removeDeletedPathsOnSsh(input: {
   });
 }
 
+const PROJECT_REPOSITORY_NAME_PATTERN = /^[a-zA-Z0-9_-]+$/;
+
+async function isLocalGitRepositoryRoot(localDir: string): Promise<boolean> {
+  try {
+    const toplevel = await runLocalGit(localDir, ["rev-parse", "--show-toplevel"], {
+      timeout: 10_000,
+      maxBuffer: 16 * 1024,
+    });
+    const [directory, repository] = await Promise.all([
+      fs.realpath(localDir),
+      fs.realpath(toplevel.stdout.trim()),
+    ]);
+    return directory === repository;
+  } catch {
+    return false;
+  }
+}
+
+async function listLocalProjectRepositories(localDir: string): Promise<string[]> {
+  const root = path.join(localDir, PROJECT_REPOSITORIES_DIR);
+  const rootStat = await fs.lstat(root).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  });
+  if (!rootStat) return [];
+  if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) {
+    throw new Error("Invalid project repositories directory");
+  }
+  const repositories: string[] = [];
+  for (const entry of await fs.readdir(root, { withFileTypes: true })) {
+    if (!entry.isDirectory() || !PROJECT_REPOSITORY_NAME_PATTERN.test(entry.name)) {
+      throw new Error("Invalid project repository directory");
+    }
+    const relative = `${PROJECT_REPOSITORIES_DIR}/${entry.name}`;
+    if (!(await isLocalGitRepositoryRoot(path.join(localDir, relative)))) {
+      throw new Error(`Project repository is not a Git checkout: ${relative}`);
+    }
+    repositories.push(relative);
+  }
+  return repositories.sort();
+}
+
+async function transportGitWorkspaceToSsh(input: {
+  spec: SshRemoteExecutionSpec;
+  localDir: string;
+  remoteDir: string;
+  snapshot: LocalGitWorkspaceSnapshot;
+  exclude?: string[];
+  onProgress?: RuntimeProgressSink;
+  progressLabel: string;
+}): Promise<void> {
+  await importGitWorkspaceToSsh({
+    spec: input.spec,
+    localDir: input.localDir,
+    remoteDir: input.remoteDir,
+    snapshot: input.snapshot,
+    onProgress: input.onProgress,
+  });
+  await syncDirectoryToSsh({
+    spec: input.spec,
+    localDir: input.localDir,
+    remoteDir: input.remoteDir,
+    exclude: [".git", ".paperclip-runtime", ...(input.exclude ?? [])],
+    onProgress: input.onProgress,
+    progressLabel: input.progressLabel,
+  });
+  await removeDeletedPathsOnSsh({
+    spec: input.spec,
+    remoteDir: input.remoteDir,
+    deletedPaths: input.snapshot.deletedPaths,
+  });
+}
+
+async function excludeProjectRepositoriesOnSsh(spec: SshConnectionConfig, remoteDir: string): Promise<void> {
+  const pattern = `/${PROJECT_REPOSITORIES_DIR}/`;
+  const excludeFile = path.posix.join(remoteDir, ".git", "info", "exclude");
+  await runSshScript(
+    spec,
+    `mkdir -p ${shellQuote(path.posix.dirname(excludeFile))} && ` +
+      `{ grep -qxF ${shellQuote(pattern)} ${shellQuote(excludeFile)} 2>/dev/null || ` +
+      `printf '\\n%s\\n' ${shellQuote(pattern)} >> ${shellQuote(excludeFile)}; }`,
+    { timeoutMs: 30_000 },
+  );
+}
+
 async function allocateLoopbackPort(host: string): Promise<number> {
   return await new Promise<number>((resolve, reject) => {
     const server = net.createServer();
@@ -1560,32 +1646,38 @@ export async function prepareWorkspaceForSshExecution(input: {
   onProgress?: RuntimeProgressSink;
   workspaceFileMode?: "all";
   workspaceExclude?: string[];
-}): Promise<{ gitBacked: boolean }> {
+}): Promise<{ gitBacked: boolean; repositories?: string[] }> {
   const remoteDir = input.remoteDir ?? input.spec.remoteCwd;
   const gitSnapshot = input.workspaceFileMode === "all" ? null : await readLocalGitWorkspaceSnapshot(input.localDir);
 
   if (gitSnapshot) {
-    await importGitWorkspaceToSsh({
+    const repositories = await listLocalProjectRepositories(input.localDir);
+    await transportGitWorkspaceToSsh({
       spec: input.spec,
       localDir: input.localDir,
       remoteDir,
       snapshot: gitSnapshot,
-      onProgress: input.onProgress,
-    });
-    await syncDirectoryToSsh({
-      spec: input.spec,
-      localDir: input.localDir,
-      remoteDir,
-      exclude: [".git", ".paperclip-runtime"],
+      exclude: repositories.length > 0 ? [PROJECT_REPOSITORIES_DIR] : [],
       onProgress: input.onProgress,
       progressLabel: "workspace",
     });
-    await removeDeletedPathsOnSsh({
-      spec: input.spec,
-      remoteDir,
-      deletedPaths: gitSnapshot.deletedPaths,
-    });
-    return { gitBacked: true };
+    if (repositories.length > 0) {
+      await excludeProjectRepositoriesOnSsh(input.spec, remoteDir);
+    }
+    for (const relative of repositories) {
+      const localDir = path.join(input.localDir, relative);
+      const snapshot = await readLocalGitWorkspaceSnapshot(localDir);
+      if (!snapshot) throw new Error(`Cannot read the Git state of project repository: ${relative}`);
+      await transportGitWorkspaceToSsh({
+        spec: input.spec,
+        localDir,
+        remoteDir: path.posix.join(remoteDir, relative),
+        snapshot,
+        onProgress: input.onProgress,
+        progressLabel: relative,
+      });
+    }
+    return { gitBacked: true, ...(repositories.length > 0 ? { repositories } : {}) };
   }
 
   await clearRemoteDirectory({
@@ -1611,8 +1703,62 @@ export async function restoreWorkspaceFromSshExecution(input: {
   baselineSnapshot?: DirectorySnapshot;
   restoreGitHistory?: boolean;
   onProgress?: RuntimeProgressSink;
+  repositories?: Array<{ path: string; baselineSnapshot?: DirectorySnapshot }>;
 }): Promise<void> {
   const remoteDir = input.remoteDir ?? input.spec.remoteCwd;
+  const repositories = input.repositories ?? [];
+  for (const repository of repositories) {
+    if (
+      path.posix.dirname(repository.path) !== PROJECT_REPOSITORIES_DIR ||
+      !PROJECT_REPOSITORY_NAME_PATTERN.test(path.posix.basename(repository.path))
+    ) {
+      throw new Error(`Invalid project repository path: ${repository.path}`);
+    }
+    if (input.baselineSnapshot && !repository.baselineSnapshot) {
+      throw new Error(`Project repository has no workspace baseline: ${repository.path}`);
+    }
+  }
+  if (
+    input.baselineSnapshot &&
+    repositories.length > 0 &&
+    !input.baselineSnapshot.exclude.includes(PROJECT_REPOSITORIES_DIR)
+  ) {
+    throw new Error(`Workspace baseline must exclude ${PROJECT_REPOSITORIES_DIR} when project repositories are restored separately`);
+  }
+  for (const repository of repositories) {
+    await restoreWorkspaceRootFromSsh({
+      spec: input.spec,
+      localDir: path.join(input.localDir, repository.path),
+      remoteDir: path.posix.join(remoteDir, repository.path),
+      baselineSnapshot: repository.baselineSnapshot,
+      restoreGitHistory: input.restoreGitHistory,
+      onProgress: input.onProgress,
+      progressLabel: repository.path,
+    });
+  }
+  await restoreWorkspaceRootFromSsh({
+    spec: input.spec,
+    localDir: input.localDir,
+    remoteDir,
+    baselineSnapshot: input.baselineSnapshot,
+    restoreGitHistory: input.restoreGitHistory,
+    onProgress: input.onProgress,
+    progressLabel: "workspace",
+    hasProjectRepositories: repositories.length > 0,
+  });
+}
+
+async function restoreWorkspaceRootFromSsh(input: {
+  spec: SshRemoteExecutionSpec;
+  localDir: string;
+  remoteDir: string;
+  baselineSnapshot?: DirectorySnapshot;
+  restoreGitHistory?: boolean;
+  onProgress?: RuntimeProgressSink;
+  progressLabel: string;
+  hasProjectRepositories?: boolean;
+}): Promise<void> {
+  const remoteDir = input.remoteDir;
   if (input.baselineSnapshot) {
     const stagingDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-sync-back-"));
     const importedRef = input.restoreGitHistory
@@ -1635,7 +1781,7 @@ export async function restoreWorkspaceFromSshExecution(input: {
         localDir: stagingDir,
         exclude: input.baselineSnapshot.exclude,
         onProgress: input.onProgress,
-        progressLabel: "workspace",
+        progressLabel: input.progressLabel,
       });
       await mergeDirectoryWithBaseline({
         baseline: input.baselineSnapshot,
@@ -1666,6 +1812,7 @@ export async function restoreWorkspaceFromSshExecution(input: {
   const gitSnapshot = await readLocalGitWorkspaceSnapshot(input.localDir);
 
   if (gitSnapshot) {
+    const projectRepositoryEntries = input.hasProjectRepositories ? [PROJECT_REPOSITORIES_DIR] : [];
     await exportGitWorkspaceFromSsh({
       spec: input.spec,
       remoteDir,
@@ -1676,10 +1823,10 @@ export async function restoreWorkspaceFromSshExecution(input: {
       spec: input.spec,
       remoteDir,
       localDir: input.localDir,
-      exclude: [".git", ".paperclip-runtime"],
-      preserveLocalEntries: [".git"],
+      exclude: [".git", ".paperclip-runtime", ...projectRepositoryEntries],
+      preserveLocalEntries: [".git", ...projectRepositoryEntries],
       onProgress: input.onProgress,
-      progressLabel: "workspace",
+      progressLabel: input.progressLabel,
     });
     return;
   }
@@ -1690,7 +1837,7 @@ export async function restoreWorkspaceFromSshExecution(input: {
     localDir: input.localDir,
     exclude: [".paperclip-runtime"],
     onProgress: input.onProgress,
-    progressLabel: "workspace",
+    progressLabel: input.progressLabel,
   });
 }
 

packages/paperclip-runner/docs/durable-recovery.md

@@ -95,6 +95,15 @@ welcome and every control envelope against the authenticated connection,
 runner, environment lease, run, normalized session, turn, item, protocol,
 lease ID, expiry, and revocation metadata before applying an ACK or command.
 
+Semantic input integrity uses SHA-256 of the complete transmitted input's
+canonical JSON, with UTF-16 object-key order and JavaScript number formatting.
+The controller checks this proof after authentication and exact run/session/
+turn/item correlation, before committing, dispatching, or acknowledging the
+input. Receipt redaction has a separate digest; a redacted digest cannot stand
+in for wire integrity, including when only a protected field changes. A failed
+integrity proof keeps the existing operator-required recovery fence. Updating
+the verifier does not clear a previously failed run or replay its work.
+
 The daemon captures and removes the bootstrap environment variable before it
 parses arguments or starts child work. Secret buffers are overwritten when they
 are dropped. It resolves the destination once before sending a bearer value and

packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs

@@ -652,6 +652,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
         .iter()
         .any(|value| value == "--require-codex-home-auth");
     let durable_turn_ids = args.iter().any(|value| value == "--durable-turn-ids");
+    let hold_first_durable_turn = args
+        .iter()
+        .any(|value| value == "--hold-first-durable-turn");
     let durable_tool_ids = args.iter().any(|value| value == "--durable-tool-ids");
     let expected_canonical_task_context_file =
         argument(&args, "--expected-canonical-task-context-file");
@@ -1348,6 +1351,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
                         .checked_add(1)
                         .ok_or("fake provider turn sequence exhausted")?;
                 }
+                let hold_turn = hold_turn || (hold_first_durable_turn && state.next_turn == 1);
                 if reject_second_turn_start && turn_start_count == 2 {
                     send(json!({
                         "method": "warning",

packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs

@@ -2103,17 +2103,29 @@ impl CodexCommandExecutor {
                                 "providerTurnId": previous_active_turn_id,
                                 "status": "failed",
                                 "providerTerminalObserved": false,
+                                "error": if provider_label == "codex" { json!({
+                                    "code": "provider_turn_lost_on_restore",
+                                    "recoverable": true,
+                                    "message": "The runner restored the conversation after process loss, but the previous turn is no longer active.",
+                                }) } else { Value::Null },
                             }),
                         };
-                        let outcome = terminal_events(
-                            state,
-                            "turn.failed",
-                            state.goal.as_ref().map(|goal| goal.status.as_str()),
-                        );
-                        state.extend_terminal_events(with_terminal_outcome(
-                            vec![provider_terminal],
-                            outcome,
-                        ))?;
+                        if provider_label == "codex" {
+                            // No provider result was observed. Preserve the lost
+                            // turn as a fact, but leave the run open for the
+                            // controller's bounded same-conversation recovery.
+                            state.push_terminal_event(provider_terminal)?;
+                        } else {
+                            let outcome = terminal_events(
+                                state,
+                                "turn.failed",
+                                state.goal.as_ref().map(|goal| goal.status.as_str()),
+                            );
+                            state.extend_terminal_events(with_terminal_outcome(
+                                vec![provider_terminal],
+                                outcome,
+                            ))?;
+                        }
                     }
                 } else {
                     state.push_event(reconciled)?;

packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs

@@ -3484,6 +3484,65 @@ fn durable_backend_replays_pending_tool_calls_without_mutating_the_event_queue()
     fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
 }
 
+#[test]
+fn restart_lost_turn_retains_a_recoverable_cause_without_inventing_a_result() {
+    let directory = temporary_directory("restart-lost-turn");
+    let config = provider_config(
+        &directory,
+        &["--durable-turn-ids", "--linger-after-turn-start"],
+    );
+    let mut first = CodexCommandExecutor::new(&directory);
+    first
+        .execute(&command(
+            "prepare",
+            1,
+            "run.prepare",
+            json!({"provider": config}),
+        ))
+        .unwrap();
+    first
+        .execute(&command("open", 2, "session.open", json!({})))
+        .unwrap();
+    first
+        .execute(&command(
+            "start",
+            3,
+            "turn.start",
+            json!({"text": "Watch the existing check run."}),
+        ))
+        .unwrap();
+    // The process disappeared while its command was in flight. Codex restores
+    // the conversation, but cannot restore the running turn.
+    drop(first);
+    fs::write(
+        directory.join("fake-state.json"),
+        serde_json::to_vec(&json!({
+            "threadId": "codex-thread-1", "activeTurnId": null, "nextTurn": 1,
+        }))
+        .unwrap(),
+    )
+    .unwrap();
+    let mut restored = CodexCommandExecutor::new(&directory);
+    restored
+        .execute(&command("snapshot", 4, "session.snapshot", json!({})))
+        .unwrap();
+    let events = poll_and_ack(&mut restored).unwrap();
+    let interrupted = events
+        .iter()
+        .find(|event| event.event_type == "turn.failed")
+        .unwrap();
+    assert_eq!(
+        interrupted.payload["error"]["code"],
+        "provider_turn_lost_on_restore"
+    );
+    assert_eq!(interrupted.payload["error"]["recoverable"], true);
+    assert!(!events
+        .iter()
+        .any(|event| event.event_type == "run.result.proposed"));
+    restored.shutdown().unwrap();
+    fs::remove_dir_all(directory).unwrap();
+}
+
 #[test]
 fn durable_backend_settles_pending_tools_when_recovery_finds_the_turn_ended() {
     let directory = temporary_directory("durable-tool-ended-offline");

packages/paperclip-runner/runner/crates/runner-core/tests/semantic_input_digest.rs

@@ -0,0 +1,18 @@
+use paperclip_runner_core::provider_bridge::semantic_value_digest;
+use serde_json::Value;
+
+#[test]
+fn semantic_input_digests_match_the_authenticated_controller_fixtures() {
+    let fixtures: Value = serde_json::from_str(include_str!(
+        "../../../../test/fixtures/semantic-input-digests.json"
+    ))
+    .expect("parse shared semantic input digest fixtures");
+    for fixture in fixtures.as_array().expect("digest fixture array") {
+        assert_eq!(
+            semantic_value_digest(&fixture["input"]),
+            fixture["digest"].as_str().expect("fixture digest"),
+            "fixture {}",
+            fixture["name"]
+        );
+    }
+}

packages/paperclip-runner/src/backends/harness-driver-backend.ts

@@ -673,6 +673,7 @@ class HarnessNativeSession implements NativeSession {
     // The provider has no active turn to interrupt yet. Do not launch work
     // whose events cancellation would suppress and leave the owner waiting.
     if (this.#explicitlyCancelled) throw new Error("native_session_cancelled");
+    this.#terminal = null;
     try {
       const started = await this.#session.startTurn(input);
       this.#assertProtocolIntegrity();

packages/paperclip-runner/src/contracts/native-session-backend.ts

@@ -63,7 +63,7 @@ export interface PersistedNativeSession {
   terminal?: PrpTerminalState | null;
   activeTurnId?: string | null;
   terminalTurns?: PersistedHarnessTurnTerminal[];
-  /** Durable at-most-once marker for a resultless terminal recovery turn. */
+  /** At-most-once marker for resultless disposition repair or restart continuation. */
   dispositionOnlyRecoveryConsumed?: boolean;
   dispositionOnlyRecoveryTurnId?: string | null;
   pendingRuntimeRequests?: HarnessRuntimeRequest[];
@@ -77,6 +77,31 @@ export interface NativeSessionRecoveryResult {
   reason?: string;
 }
 
+/** Runner-owned cause: the process was restored, but its active turn was lost. */
+export const NATIVE_RESTART_INTERRUPTION_CODE = "provider_turn_lost_on_restore";
+
+export function isNativeRestartInterruption(error: unknown): boolean {
+  if (!error || typeof error !== "object" || Array.isArray(error)) return false;
+  const failure = error as Record<string, unknown>;
+  return failure.code === NATIVE_RESTART_INTERRUPTION_CODE && failure.recoverable === true;
+}
+
+/** The terminal fingerprint survives a crash between interruption and continuation. */
+export function nativeRestartInterruptedTurnId(snapshot: Pick<PersistedNativeSession,
+  "driverKind" | "activeTurnId" | "semanticResult" | "terminalTurns" | "dispositionOnlyRecoveryConsumed" | "pendingRuntimeRequests" | "goal"
+>): string | null {
+  if (snapshot.driverKind !== "codex_app_server" || snapshot.activeTurnId != null ||
+      snapshot.semanticResult != null || snapshot.dispositionOnlyRecoveryConsumed ||
+      snapshot.goal != null || snapshot.pendingRuntimeRequests?.length) return null;
+  const terminal = Array.isArray(snapshot.terminalTurns) ? snapshot.terminalTurns.at(-1) : null;
+  if (!terminal || typeof terminal.fingerprint !== "string" || !terminal.turnId) return null;
+  try {
+    const fingerprint = JSON.parse(terminal.fingerprint);
+    return fingerprint?.terminalState === "failed" && fingerprint.result === null &&
+      isNativeRestartInterruption(fingerprint.error) ? terminal.turnId : null;
+  } catch { return null; }
+}
+
 /**
  * Cancellation is a synchronous authority transition followed by passive
  * provider cleanup. Once `cancel` returns, the provider session must no longer

packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts

@@ -26,6 +26,7 @@ import { validatePrpEvent } from "../protocol/replay-contract.js";
 import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js";
 import {
   DurablePrpControlPlane,
+  durableRecoveryInternals,
   SemanticToolNotDispatchedError,
   inspectWarmRunTransition,
   spawnRunner,
@@ -1170,7 +1171,17 @@ async function receiveSecure(
   return value;
 }
 
-function semanticInputEvent(sourceSeq = 1): Record<string, unknown> {
+const semanticInputDigestFixtures = JSON.parse(readFileSync(
+  new URL("../../test/fixtures/semantic-input-digests.json", import.meta.url),
+  "utf8",
+)) as Array<{ name: string; input: Record<string, unknown>; digest: string }>;
+
+function rawSemanticInputDigest(input: unknown): string {
+  return `sha256:${createHash("sha256")
+    .update(durableRecoveryInternals.canonicalJson(input)).digest("hex")}`;
+}
+
+function semanticInputEvent(sourceSeq = 1, input: Record<string, unknown> = {}): Record<string, unknown> {
   return {
     protocol: "paperclip.runner",
     version: 1,
@@ -1210,11 +1221,11 @@ function semanticInputEvent(sourceSeq = 1): Record<string, unknown> {
           },
           idempotencyKey: null,
           content: {
-            digest: digestPaperclipSemanticContent({}),
+            digest: rawSemanticInputDigest(input),
             redactionDisposition: "digest_only",
             references: [],
           },
-          input: {},
+          input,
         },
       },
     },
@@ -1693,6 +1704,144 @@ describe.sequential("DurablePrpControlPlane", () => {
     },
   );
 
+  it.each(semanticInputDigestFixtures.flatMap((fixture) =>
+    ["semantic_tool.input", "mcp_app.tool_input"].map((eventType) => ({ ...fixture, eventType })),
+  ))(
+    "accepts the Rust wire digest and preserves exact input: $eventType / $name",
+    async ({ input, digest, eventType }) => {
+      // The JSON transport normalizes negative zero before the receiver sees it.
+      const wireInput = JSON.parse(JSON.stringify(input)) as Record<string, unknown>;
+      const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-raw-input-"));
+      const onProtocolIntegrityError = vi.fn();
+      const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } }));
+      const core = new DurablePrpControlPlane({
+        stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest,
+        onProtocolIntegrityError, onSemanticToolInput,
+      });
+      try {
+        await core.start();
+        const client = (await authenticate(core, core.issueBootstrapTicket()))!;
+        const event = semanticInputEvent(1, wireInput);
+        (event.payload as any).eventType = eventType;
+        const semantic = (event.payload as any).payload.semantic_tool;
+        // This shared golden also runs against the Rust producer's hash.
+        semantic.content.digest = digest;
+        expect(rawSemanticInputDigest(input)).toBe(digest);
+        sendSecure(client, event);
+        const frames = [await receiveSecure(client), await receiveSecure(client)];
+        expect(frames).toContainEqual(expect.objectContaining({
+          kind: "ack", payload: expect.objectContaining({ ackedSourceSeq: 1 }),
+        }));
+        const command = frames.find((frame) => frame?.kind === "command")!.payload as any;
+        expect(command.payload.inputDigest).toBe(digest.slice("sha256:".length));
+        expect(onSemanticToolInput).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ input: wireInput }));
+        expect(core.store.state.committedEvents).toHaveLength(1);
+        expect(core.store.state.committedEvents[0]?.envelope).toEqual(event);
+        expect(onProtocolIntegrityError).not.toHaveBeenCalled();
+        client.socket.destroy();
+      } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); }
+    },
+  );
+
+  it.each([
+    "redacted_digest", "protected_field_tampering", "forged_digest",
+    "over_node_limit", "over_depth_limit",
+  ] as const)(
+    "rejects invalid raw semantic integrity without commit, dispatch or ACK: %s",
+    async (fault) => {
+      const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-raw-integrity-"));
+      const onProtocolIntegrityError = vi.fn();
+      const onCommittedEvent = vi.fn(async () => undefined);
+      const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } }));
+      const core = new DurablePrpControlPlane({
+        stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest,
+        onProtocolIntegrityError, onCommittedEvent, onSemanticToolInput,
+      });
+      try {
+        await core.start();
+        const client = (await authenticate(core, core.issueBootstrapTicket()))!;
+        const input = { password: "fixture-before", safe: true };
+        const event = semanticInputEvent(1, input);
+        const semantic = (event.payload as any).payload.semantic_tool;
+        const redactedDigest = digestPaperclipSemanticContent(input);
+        expect(redactedDigest).not.toBe(semantic.content.digest);
+        if (fault === "redacted_digest") semantic.content.digest = redactedDigest;
+        if (fault === "forged_digest") semantic.content.digest = `sha256:${"0".repeat(64)}`;
+        if (fault === "protected_field_tampering") {
+          semantic.input = { ...input, password: "fixture-after" };
+          // Redacted receipt hashes intentionally cannot prove byte integrity.
+          expect(digestPaperclipSemanticContent(semantic.input)).toBe(redactedDigest);
+        }
+        if (fault === "over_node_limit") semantic.input = { values: Array(10_001).fill(0) };
+        if (fault === "over_depth_limit") {
+          let nested: unknown = {};
+          for (let depth = 0; depth < 65; depth += 1) nested = { child: nested };
+          semantic.input = nested;
+        }
+        if (fault === "over_node_limit" || fault === "over_depth_limit") {
+          // Each fixture object has one key, so JSON.stringify is canonical.
+          // Hash independently of the bounded helper: the limits themselves
+          // must reject even a matching raw digest and keep the permanent fence.
+          semantic.content.digest = `sha256:${createHash("sha256")
+            .update(JSON.stringify(semantic.input)).digest("hex")}`;
+        }
+        sendSecure(client, event);
+        await expect(receiveSecure(client)).resolves.toBeNull();
+        expect(onProtocolIntegrityError).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
+          code: "native_event_replay_conflict", reason: "semantic_input_digest_mismatch",
+          recovery: "operator_required",
+        }));
+        expect(core.store.state.ackedSourceSeq).toBe(0);
+        expect(core.store.state.committedEvents).toEqual([]);
+        expect(onCommittedEvent).not.toHaveBeenCalled();
+        expect(onSemanticToolInput).not.toHaveBeenCalled();
+        expect(JSON.stringify(onProtocolIntegrityError.mock.calls)).not.toContain("fixture-before");
+        expect(JSON.stringify(onProtocolIntegrityError.mock.calls)).not.toContain("fixture-after");
+        const replay = (await authenticate(core, client.leaseToken!))!;
+        sendSecure(replay, semanticInputEvent());
+        await expect(receiveSecure(replay)).resolves.toBeNull();
+        expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1);
+        expect(onSemanticToolInput).not.toHaveBeenCalled();
+      } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); }
+    },
+  );
+
+  it.each(["authentication", "envelope", "event", "correlation"] as const)(
+    "rejects raw semantic input outside its exact authenticated scope: %s",
+    async (boundary) => {
+      const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-raw-scope-"));
+      const onProtocolIntegrityError = vi.fn();
+      const onCommittedEvent = vi.fn(async () => undefined);
+      const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } }));
+      const core = new DurablePrpControlPlane({
+        stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest,
+        onProtocolIntegrityError, onCommittedEvent, onSemanticToolInput,
+      });
+      try {
+        await core.start();
+        const wrongRunId = "00000000-0000-4000-8000-000000000999";
+        const client = await authenticate(core, core.issueBootstrapTicket(),
+          boundary === "authentication" ? { ...identity, runId: wrongRunId } : identity);
+        if (boundary === "authentication") {
+          expect(client).toBeNull();
+        } else {
+          const event = semanticInputEvent(1, { password: "fixture-only" });
+          const payload = event.payload as any;
+          if (boundary === "envelope") event.runId = wrongRunId;
+          if (boundary === "event") payload.runId = wrongRunId;
+          if (boundary === "correlation") payload.payload.semantic_tool.correlation.runId = wrongRunId;
+          sendSecure(client!, event);
+          await expect(receiveSecure(client!)).resolves.toBeNull();
+        }
+        expect(core.store.state.ackedSourceSeq).toBe(0);
+        expect(core.store.state.committedEvents).toEqual([]);
+        expect(onCommittedEvent).not.toHaveBeenCalled();
+        expect(onSemanticToolInput).not.toHaveBeenCalled();
+        expect(onProtocolIntegrityError).not.toHaveBeenCalled();
+      } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); }
+    },
+  );
+
   it("latches an authenticated semantic integrity fault without ACK or dispatch and still permits suspension", async () => {
     const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-integrity-"));
     const onProtocolIntegrityError = vi.fn();
@@ -1854,7 +2003,7 @@ describe.sequential("DurablePrpControlPlane", () => {
       ).semantic_tool as Record<string, unknown>;
       semantic.input = { changed: true };
       (semantic.content as Record<string, unknown>).digest =
-        digestPaperclipSemanticContent(semantic.input);
+        rawSemanticInputDigest(semantic.input);
       sendSecure(client, changed);
       await expect(receiveSecure(client)).resolves.toBeNull();
       expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1);
@@ -2238,13 +2387,13 @@ describe.sequential("DurablePrpControlPlane", () => {
       const event = semanticInputEvent();
       const semantic = ((event.payload as any).payload as any).semantic_tool;
       semantic.input = args;
-      semantic.content.digest = digestPaperclipSemanticContent(args);
+      semantic.content.digest = rawSemanticInputDigest(args);
       sendSecure(client, event);
       const frames = [await receiveSecure(client), await receiveSecure(client)];
       const wire = frames.find((frame) => frame?.kind === "command")!.payload as any;
       expect(wire.payload.result.content).toBe(content);
       expect(wire.payload.input).toBeUndefined();
-      expect(wire.payload.inputDigest).toBe(digestPaperclipSemanticContent(args).slice("sha256:".length));
+      expect(wire.payload.inputDigest).toBe(rawSemanticInputDigest(args).slice("sha256:".length));
       expect(Buffer.byteLength(JSON.stringify(wire))).toBeLessThan(1024 * 1024);
       sendSecure(client, { protocol: "paperclip.runner", version: 1, kind: "command_result", payload: {
         commandId: wire.commandId, controllerSeq: wire.controllerSeq, commandType: wire.type,

packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts

@@ -35,7 +35,6 @@ import {
   validatePrpEvent,
   type PrpEvent,
 } from "../protocol/replay-contract.js";
-import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js";
 import {
   type DurableRecoveryCommittedEvent,
   type DurableRecoveryCoreCommand,
@@ -416,6 +415,18 @@ function canonicalDigest(value: unknown): string {
   return createHash("sha256").update(canonicalJson(value)).digest("hex");
 }
 
+function matchesSemanticInputDigest(input: unknown, digest: unknown): boolean {
+  try {
+    // Wire integrity covers the complete input, including protected fields.
+    // Receipt redaction can erase those differences and has a separate hash.
+    return digest === `sha256:${canonicalDigest(input)}`;
+  } catch {
+    // Canonicalization bounds must keep the same permanent integrity fence.
+    // Never attach an input-derived error or payload to the diagnostic.
+    return false;
+  }
+}
+
 function exactIdentity(value: unknown): value is DurableRecoveryIdentity {
   return (
     isRecord(value) &&
@@ -3141,8 +3152,10 @@ export class DurablePrpControlPlane {
     if (
       isSemanticInput &&
       semantic !== undefined &&
-      (semantic.content as Record<string, unknown>).digest !==
-        digestPaperclipSemanticContent(semantic.input)
+      !matchesSemanticInputDigest(
+        semantic.input,
+        (semantic.content as Record<string, unknown>).digest,
+      )
     ) {
       // Only the authenticated, schema-valid, exactly correlated input may
       // permanently fail its owner. Never commit, dispatch, or ACK these bytes.

packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts

@@ -17,7 +17,7 @@ import type {
   PersistedHarnessSession,
   PersistedHarnessTurnTerminal,
 } from "../../contracts/harness-driver.js";
-import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js";
+import { NativeSessionProtocolIntegrityError, nativeRestartInterruptedTurnId } from "../../contracts/native-session-backend.js";
 import { HarnessReconciliationError } from "../../contracts/harness-driver.js";
 import {
   CODEX_CODEX_PROTOCOL_VERSION,
@@ -528,6 +528,12 @@ export class CodexAppServerDriver implements HarnessDriver {
         turns.forEach((turn, index) => {
           if (terminalIds.has(text(turn.id))) lastKnownTerminalIndex = index;
         });
+        if (nativeRestartInterruptedTurnId({
+          ...snapshot, semanticResult: null,
+        }) && (!providerHistoryIsArray || lastKnownTerminalIndex < 0)) {
+          await cancellation.wait(cancellation.close());
+          return { recovered: false, reason: "restart interruption history is incomplete" };
+        }
         // Releasing a consumed marker requires both an actual history array
         // and a checkpointed terminal that anchors its ordering. An array that
         // omits every durable terminal may be truncated or inconsistent, so

packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts

@@ -45,6 +45,29 @@ import {
 import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js";
 
 describe("Codex app-server Codex driver", () => {
+  it("does not resend a restart continuation when its history anchor is missing", async () => {
+    const first = new FakeCodexTransport();
+    const second = new FakeCodexTransport();
+    const driver = makeDriver([first, second]);
+    const original = await driver.openSession({
+      runId: "run-restart-anchor", normalizedSessionId: "session-restart-anchor", workingDirectory: WORKSPACE,
+    });
+    await original.startTurn({ message: { role: "user", text: "Finish the request." } });
+    first.push("turn/completed", { threadId: "thread-1", turn: {
+      id: "turn-1", status: "failed", error: { code: "provider_turn_lost_on_restore", recoverable: true }, items: [],
+    } });
+    await collectUntilTerminal(original.events());
+    const snapshot = await original.snapshot();
+    await original.close({ reason: "controller lost before continuation checkpoint" });
+    second.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE,
+      turns: [{ id: "uncheckpointed-turn", status: "completed", items: [] }],
+    } };
+    await expect(driver.recoverSession?.(snapshot)).resolves.toMatchObject({
+      recovered: false, reason: "restart interruption history is incomplete",
+    });
+    expect(second.calls.some(call => call.method === "turn/start")).toBe(false);
+  });
+
   it.each([null, "checkpointed-prior-turn"])("recovers an autonomous goal turn beyond checkpoint %s", async (checkpointTurnId) => {
     const first = new FakeCodexTransport();
     const second = new FakeCodexTransport();

packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts

@@ -1,4 +1,5 @@
 import { HarnessReconciliationError } from "../../contracts/harness-driver.js";
+import { isNativeRestartInterruption } from "../../contracts/native-session-backend.js";
 import type { PrpStructuredRunResult } from "../../protocol/replay-contract.js";
 import { boundedCodexPayload as boundedPayload, isRetainableCodexPayload } from "./codex-boundaries.js";
 import type { CodexSessionState } from "./codex-session-state.js";
@@ -168,6 +169,13 @@ export function mapTerminalTurn(
       { turnId },
     );
     finalize(state, status);
+    if (status === "failed" && isNativeRestartInterruption(turn.error) && state.result === null &&
+        !state.dispositionOnlyRecoveryConsumed) {
+      // Orchestration may use the paid restart attempt for one same-thread
+      // continuation. Its accepted turn id uses the existing durable marker.
+      state.terminal = false;
+      state.dispositionOnlyRecoveryAvailable = true;
+    }
   }
 
 function terminalFingerprint(

packages/paperclip-runner/src/live/runnerd-codex-transport.ts

@@ -3583,20 +3583,31 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
       if (method === "thread/turns/list") {
         data = turns.map(turn => ({ ...turn, items: [], itemsView: "notLoaded" }));
       } else {
-        if (params.turnId !== this.#turnId) throw new Error("codex_history_unavailable: requested turn is outside the retained runner event window");
+        if (!turns.some(turn => turn.id === params.turnId)) throw new Error("codex_history_unavailable: requested turn is outside the retained runner event window");
         const items = new Map<string, Record<string, unknown>>();
+        let semanticResultItem: Record<string, unknown> | null = null;
         let observedTurn = "";
         let observedStart = false;
         for (const event of this.#core?.store.state.committedEvents ?? []) {
+          if (event.envelope.runId !== this.#core?.store.state.identity.runId) continue;
           const payload = record(record(event.envelope.payload).payload);
           if (event.eventType === "turn.started") observedTurn = String(payload.providerTurnId ?? payload.turnId ?? record(payload.turn).id ?? "");
           if (event.eventType === "turn.started" && observedTurn === params.turnId) observedStart = true;
+          if (event.eventType === "run.result.proposed" && observedTurn === params.turnId) {
+            // Reconciliation can precede notification delivery. Recover the
+            // runner's authoritative result with the exact retained turn,
+            // rather than launching work again just to obtain a disposition.
+            const id = `runner-result-${event.sourceSeq}`;
+            semanticResultItem = { turnId: observedTurn, item: { id, type: "agentMessage", text: JSON.stringify(payload) } };
+          }
           if (event.eventType !== "item.completed" || observedTurn !== params.turnId) continue;
           const item = record(rehydrateRunnerdItemNotification(payload, this.#threadId, observedTurn).item);
           if (typeof item.id === "string") items.set(item.id, { turnId: observedTurn, item });
         }
         if (!observedStart) throw new Error("codex_history_incomplete: requested turn start is outside the retained runner event window");
         data = [...items.values()];
+        // Runner authority wins over schema-shaped prose in an assistant item.
+        if (semanticResultItem) data.push(semanticResultItem);
       }
       if (params.sortDirection === "desc") data.reverse();
       const offset = params.cursor == null ? 0 : Number(params.cursor);
@@ -3657,8 +3668,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
           await new Promise((resolveWait) => setTimeout(resolveWait, 10));
         }
         if (terminal !== undefined) {
+          const payload = record(record(terminal.envelope.payload).payload);
           recoveredTurns.push({
             id: this.#turnId,
+            // Reconciliation must retain the cause, including the runner's
+            // explicit process-loss marker, rather than inventing error:null.
+            error: payload.error ?? record(payload.turn).error ?? null,
             status:
               terminal.eventType === "turn.completed"
                 ? "completed"
@@ -3670,6 +3685,24 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
           });
         }
       }
+      // A controller can lose the checkpoint after a continuation is accepted.
+      // Keep its prior terminal as the history anchor, so driver recovery can
+      // adopt the later accepted turn instead of submitting it again. Items
+      // remain lazy and fail closed if their start left the retained window.
+      const priorTerminals = new Map<string, Record<string, unknown>>();
+      for (const event of this.#core?.store.state.committedEvents ?? []) {
+        if (event.envelope.runId !== this.#core?.store.state.identity.runId ||
+            !["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) continue;
+        const payload = record(record(event.envelope.payload).payload);
+        const turnId = payload.providerTurnId ?? payload.turnId ?? record(payload.turn).id;
+        if (typeof turnId !== "string" || !turnId || turnId === this.#turnId) continue;
+        priorTerminals.set(turnId, {
+          id: turnId,
+          status: event.eventType.slice("turn.".length),
+          error: payload.error ?? record(payload.turn).error ?? null,
+        });
+      }
+      recoveredTurns.unshift(...priorTerminals.values());
       this.#recoveryTurnBindingPending = false;
       this.#pumpEvents();
       return {

packages/paperclip-runner/src/live/runnerd-restart-continuation.test.ts

@@ -0,0 +1,125 @@
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { expect, it, vi } from "vitest";
+import { createRunnerdNativeSessionBackend } from "../backends/codex-native-backend.js";
+import type { NativeExecutionInputV1 } from "../contracts/native-execution.js";
+import type { PersistedNativeSession } from "../contracts/native-session-backend.js";
+import type { PrpEvent } from "../protocol/replay-contract.js";
+import { executeNativeSession } from "../native-session-runtime.js";
+import { createRunnerdCodexTransport, defaultCapabilityRunnerdBinary } from "./runnerd-codex-transport.js";
+
+it.each(["none", "before-start", "after-start"] as const)("completes after the real runner and provider are killed (second crash: %s)", async (crash) => {
+  const root = await mkdtemp(join(tmpdir(), "runner-restart-continuation-"));
+  const identity = { runId: "run-restart", sessionId: "session-restart", companyId: "company-restart", issueId: "issue-restart", agentId: "agent-restart" };
+  const execution: NativeExecutionInputV1 = {
+    schema: "paperclip.native-execution-input.v1",
+    binding: { runId: identity.runId, companyId: identity.companyId, issueId: identity.issueId, agentId: identity.agentId, executionWorkspaceId: "workspace-restart" },
+    session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1 },
+    task: { identifier: "TEST-1", title: "Finish checks", description: null, prompt: "Watch the already running checks and finish the request.", workMode: "standard" },
+    workspace: { cwd: root, repoUrl: null, repoRef: null, branchName: null },
+    provider: { kind: "codex", model: null },
+    completionContract: { id: "contract-restart", sha256: "digest", schemaVersion: "paperclip.completion-contract.v1", contract: {
+      revision: "1", objective: "Finish checks", criteria: [{ id: "checks", requirement: "Check the outcome" }],
+    } },
+    interactionResponses: [], credentialBindings: [],
+  };
+  const options = {
+    runnerBinary: defaultCapabilityRunnerdBinary(),
+    codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"),
+    codexArgs: ["--state-file", join(root, "fake-state.json"), "--call-log", join(root, "calls.log"), "--durable-turn-ids", "--hold-first-durable-turn"],
+    stateDirectory: root,
+    prpIdentity: { runnerInstanceId: "runner-restart", environmentLeaseId: "lease-restart", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-restart", itemId: "item-restart" },
+  };
+  const first = createRunnerdCodexTransport(options);
+  let restored: ReturnType<typeof createRunnerdCodexTransport> | undefined;
+  const alive = (pid: number | null) => {
+    if (!pid) return false;
+    try { process.kill(pid, 0); return true; } catch { return false; }
+  };
+  const kill = (pid: number | null) => {
+    if (!pid) return;
+    try { process.kill(pid, "SIGKILL"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; }
+  };
+  try {
+    const backend = createRunnerdNativeSessionBackend(execution, { runnerInstanceId: "runner-restart", transportFactory: () => first.transport });
+    const session = await backend.openSession({ identity, workingDirectory: root });
+    await session.startTurn({ message: { role: "user", text: execution.task.prompt } });
+    const persisted = await session.snapshot();
+    expect(persisted.activeTurnId).toBe("provider-turn-1");
+    const durableEvents: PrpEvent[] = [];
+    const initialEvents = session.events()[Symbol.asyncIterator]();
+    while ((durableEvents.at(-1)?.sourceSeq ?? 0) < Number(persisted.cursor)) {
+      const event = await initialEvents.next();
+      if (event.done) throw new Error("initial event history ended early");
+      durableEvents.push(event.value);
+    }
+    const owner = first.evidence();
+    await first.detachControllerForRestart();
+    kill(owner.runnerPid);
+    kill(owner.providerPid);
+    await vi.waitFor(() => { expect(alive(owner.runnerPid)).toBe(false); expect(alive(owner.providerPid)).toBe(false); });
+    // Simulate Codex's process-loss history: the thread remains, its in-memory
+    // turn does not. Keep the turn counter to prevent provider identity reuse.
+    const providerState = JSON.parse(await readFile(join(root, "fake-state.json"), "utf8"));
+    providerState.activeTurnId = null;
+    await writeFile(join(root, "fake-state.json"), JSON.stringify(providerState));
+    restored = createRunnerdCodexTransport({ ...options, resumeDynamicTools: [], resumeActiveTurnId: persisted.activeTurnId });
+    let recovering = createRunnerdNativeSessionBackend(execution, { runnerInstanceId: "runner-restart", transportFactory: () => restored!.transport });
+    const events: PrpEvent[] = [];
+    const checkpoints: PersistedNativeSession[] = [];
+    let recoveryCheckpoint = persisted;
+    let failCheckpoint = crash !== "none";
+    const interruptedController = new Error("controller lost before continuation submission");
+    const execute = () => executeNativeSession({
+      input: execution, backend: recovering, persistedSession: recoveryCheckpoint, resumeInterruptedTurn: true,
+      runnerInstanceId: "runner-restart", controlPlaneInstanceId: "controller-restart", timeoutMs: 15_000,
+      controlPlane: {
+        async openRun() {},
+        async checkpointSession(snapshot) {
+          if (failCheckpoint && crash === "after-start" && snapshot.dispositionOnlyRecoveryTurnId === "provider-turn-2") {
+            failCheckpoint = false;
+            throw interruptedController;
+          }
+          checkpoints.push(structuredClone(snapshot));
+          recoveryCheckpoint = structuredClone(snapshot);
+          if (failCheckpoint && crash === "before-start" && snapshot.terminal?.runTerminalState === "failed") {
+            failCheckpoint = false;
+            throw interruptedController;
+          }
+        },
+        async appendEvent(event) { events.push(event); durableEvents.push(event); return { cursor: durableEvents.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; },
+        async replayEvents(request) {
+          const source = durableEvents.filter(event => event.sourceInstanceId === request.sourceInstanceId);
+          return { events: source.filter(event => event.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: source.at(-1)?.sourceSeq ?? 0 };
+        },
+        async completeRun() {},
+      },
+    });
+    if (crash !== "none") {
+      await expect(execute()).rejects.toBe(interruptedController);
+      await restored.transport.close().catch(() => undefined);
+      restored = createRunnerdCodexTransport({ ...options, resumeDynamicTools: [], resumeActiveTurnId: recoveryCheckpoint.activeTurnId });
+      recovering = createRunnerdNativeSessionBackend(execution, { runnerInstanceId: "runner-restart", transportFactory: () => restored!.transport });
+    }
+    const completion = await execute();
+    expect(completion).toMatchObject({ providerSessionId: persisted.providerSessionId, turnId: "provider-turn-2", terminal: { runTerminalState: "succeeded" } });
+    expect(events.filter(event => event.eventType === "turn.failed")).toHaveLength(1);
+    expect(events.filter(event => event.eventType === "run.terminal")).toHaveLength(1);
+    if (crash !== "after-start") expect(checkpoints.some(snapshot => snapshot.activeTurnId === "provider-turn-2" && snapshot.terminal === null)).toBe(true);
+    expect((await readFile(join(root, "calls.log"), "utf8")).match(/^turn\/start$/gm)).toHaveLength(2);
+    const journal = JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8"));
+    const submissions = journal.commands.filter((command: { type: string }) => command.type === "turn.start");
+    expect(submissions).toHaveLength(2);
+    expect(submissions[1].payload.text).toContain("Reconcile any unfinished tool or command");
+  } finally {
+    await restored?.transport.close().catch(() => undefined);
+    for (const bundle of [first, restored]) {
+      if (!bundle) continue;
+      const evidence = bundle.evidence();
+      kill(evidence.runnerPid);
+      kill(evidence.providerPid);
+    }
+    await rm(root, { recursive: true, force: true });
+  }
+}, 30_000);

packages/paperclip-runner/src/native-session-runtime.test.ts

@@ -225,6 +225,71 @@ function highestContiguous(events: PrpEvent[]): number {
 }
 
 describe("executeNativeSession recovery", () => {
+  it.each([
+    { label: "continues a restart-lost Codex turn", allowed: true, error: { code: "provider_turn_lost_on_restore", recoverable: true } },
+    { label: "requires the control plane restart claim", allowed: false, error: { code: "provider_turn_lost_on_restore", recoverable: true } },
+    { label: "preserves real provider errors", allowed: true, error: { code: "model_not_found", recoverable: false } },
+    { label: "does not reopen historical error-null failures", allowed: true, error: null },
+  ])("$label without replaying the task", async ({ label, allowed, error }) => {
+    const checkpoint: PersistedNativeSession = {
+      backendKind: "runner", driverKind: "codex_app_server", sessionId: "thread-recovery", identity,
+      providerSessionId: "provider-recovery", cursor: "0", activeTurnId: "turn-recovery",
+      terminalTurns: [], semanticResult: null, pendingRuntimeRequests: [], lineage: [],
+    };
+    const failure = error;
+    const interrupted: PersistedNativeSession = {
+      ...checkpoint, activeTurnId: null,
+      terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }],
+    };
+    const startTurn = vi.fn<NativeSession["startTurn"]>(async () => ({ turnId: "turn-continuation" }));
+    const session: NativeSession = {
+      identity: () => identity,
+      async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; },
+      async *events() {
+        yield runnerEvent(1, "turn.failed", { status: "failed", error: failure });
+        if (!startTurn.mock.calls.length) return;
+        yield { ...runnerEvent(2, "turn.completed", { status: "completed" }), turnId: "turn-continuation" };
+      },
+      startTurn,
+      async result() { return startTurn.mock.calls.length ? { result, terminal, turnId: "turn-continuation" } : null; },
+      async snapshot() { return structuredClone(interrupted); },
+      async close() {},
+    };
+    const events: PrpEvent[] = [];
+    const checkpoints: PersistedNativeSession[] = [];
+    const backend: NativeSessionBackend = {
+      async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; },
+      async openSession() { throw new Error("must preserve the provider session"); },
+      async recoverSession() { return { recovered: true, session }; },
+    };
+    const port: ControlPlanePort = {
+      async openRun() {},
+      async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); },
+      async appendEvent(event) { events.push(event); return { cursor: events.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; },
+      async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+      async completeRun() {},
+    };
+    const execution = executeNativeSession({
+      input, backend, controlPlane: port, persistedSession: checkpoint,
+      runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery",
+      resumeInterruptedTurn: allowed,
+    });
+    if (label !== "continues a restart-lost Codex turn") {
+      await expect(execution).rejects.toMatchObject({ code: "native_provider_terminal_failed" });
+      expect(startTurn).not.toHaveBeenCalled();
+      return;
+    }
+    await expect(execution).resolves.toMatchObject({ result, turnId: "turn-continuation", terminal: { runTerminalState: "succeeded" } });
+    expect(startTurn).toHaveBeenCalledOnce();
+    const submitted = startTurn.mock.calls[0]![0] as Parameters<NativeSession["startTurn"]>[0];
+    expect(submitted.continuation).toBe(true);
+    expect(submitted.message.text).toContain("Reconcile any unfinished tool or command");
+    expect(submitted.message.text).not.toContain(input.task.prompt);
+    expect(events.filter(event => event.eventType === "turn.failed")).toHaveLength(1);
+    expect(events.filter(event => event.eventType === "run.terminal")).toHaveLength(1);
+    expect(checkpoints.some(snapshot => snapshot.terminalTurns?.some(turn => turn.turnId === "turn-recovery"))).toBe(true);
+  });
+
   it.each([undefined, 0, 7 * 24 * 60 * 60 * 1000, 30 * 24 * 60 * 60 * 1000])(
     "honors long-lived turn duration independently of operation bounds (%s)",
     async (turnTimeoutMs) => {
@@ -6234,7 +6299,12 @@ describe("executeNativeSession recovery", () => {
     expect(openRun).not.toHaveBeenCalled();
   });
 
-  it.each([false, true])("replaces a provider session that already ended with a failed terminal (prepared: %s)", async (preparedMode) => {
+  it.each([
+    { preparedMode: false, interrupted: false },
+    { preparedMode: true, interrupted: false },
+    { preparedMode: false, interrupted: true },
+    { preparedMode: true, interrupted: true },
+  ])("replaces a failed provider with full task context (prepared: $preparedMode, interrupted: $interrupted)", async ({ preparedMode, interrupted }) => {
     const digest = "0".repeat(64);
     const skill = {
       key: "company/recovery-skill",
@@ -6286,6 +6356,7 @@ describe("executeNativeSession recovery", () => {
         } as const;
     const checkpoint: PersistedNativeSession = {
       backendKind: "mock",
+      driverKind: "codex_app_server",
       sessionId: "driver-failed",
       identity,
       providerSessionId: "provider-failed",
@@ -6299,7 +6370,10 @@ describe("executeNativeSession recovery", () => {
         runTerminalState: "failed",
         reportedWorkDisposition: "yielded",
       },
-      terminalTurns: [{ turnId: "turn-failed", fingerprint: "failed" }],
+      terminalTurns: [{ turnId: "turn-failed", fingerprint: interrupted ? JSON.stringify({
+        terminalState: "failed", result: null,
+        error: { code: "provider_turn_lost_on_restore", recoverable: true },
+      }) : "failed" }],
       pendingRuntimeRequests: [],
       lineage: [],
     };
@@ -6335,8 +6409,8 @@ describe("executeNativeSession recovery", () => {
       async close() {},
     };
     const recoverSession = vi.fn(async () => ({
-      recovered: true as const,
-      session: replacementSession,
+      recovered: false as const,
+      reason: "provider thread is unavailable",
     }));
     const openReplacementSession = vi.fn(async () => replacementSession);
     const onContinuityBreak = vi.fn(async () => undefined);
@@ -6397,7 +6471,7 @@ describe("executeNativeSession recovery", () => {
     ).resolves.toMatchObject({ providerSessionId: "provider-replacement" });
 
     expect(getFreshSessionHandoff).toHaveBeenCalledOnce();
-    expect(recoverSession).not.toHaveBeenCalled();
+    expect(recoverSession).toHaveBeenCalledTimes(interrupted ? 1 : 0);
     expect(openReplacementSession).toHaveBeenCalledOnce();
     const replacementEnvelope = JSON.parse(
       startTurn.mock.calls[0]![0].message.text,
@@ -6419,7 +6493,7 @@ describe("executeNativeSession recovery", () => {
     expect(JSON.stringify(replacementEnvelope)).not.toContain("ONLY_NEW_COMMENT");
     expect(startTurn.mock.calls[0]![0]).not.toHaveProperty("continuation");
     expect(onContinuityBreak).toHaveBeenCalledWith({
-      reason: "provider session ended with a failed terminal",
+      reason: interrupted ? "provider thread is unavailable" : "provider session ended with a failed terminal",
       previousDriverSessionId: "driver-failed",
       previousProviderSessionId: "provider-failed",
       replacementDriverSessionId: "driver-replacement",

packages/paperclip-runner/src/native-session-runtime.ts

@@ -23,6 +23,8 @@ import {
   NativeSessionCloseUnrecoverableError,
   NativeSessionCleanupQuarantinedError,
   NativeSessionProtocolIntegrityError,
+  isNativeRestartInterruption,
+  nativeRestartInterruptedTurnId,
 } from "./contracts/native-session-backend.js";
 import {
   validatePrpStructuredRunResult,
@@ -184,6 +186,8 @@ export interface NativeSessionGoalControl {
 }
 
 export interface ExecuteNativeSessionOptions {
+  /** The control plane proved the old local runner stopped and charged a recovery attempt. */
+  resumeInterruptedTurn?: boolean;
   /** Read bounded task history only after recovery requires a fresh conversation. */
   getFreshSessionHandoff?: () => Promise<string | null>;
   /** Durable launch intent, after cleanup admission and before provider calls. */
@@ -865,6 +869,7 @@ async function consumeTurn(
     requestId: string;
   },
   initialGoal?: HarnessThreadGoal | null,
+  resumeInterruptedTurn?: (event: PrpEvent, signal: AbortSignal) => Promise<boolean>,
 ) {
   let timer: ReturnType<typeof setTimeout> | undefined;
   const appendAbort = new AbortController();
@@ -1174,6 +1179,8 @@ async function consumeTurn(
       }
       if (isTurnTerminal(event)) {
         if (providerFailure) throw providerFailure;
+        if (semanticResultProposal === null && governedResult === null &&
+            inputTimers.size === 0 && await resumeInterruptedTurn?.(event, appendAbort.signal)) continue;
         return {
           event,
           eventCount,
@@ -1871,6 +1878,12 @@ export async function executeNativeSession(
     previousProviderSessionId: string | null;
   } | null = null;
   let reconciledRecoveryCheckpoint: PersistedNativeSession | null = null;
+  const checkpointedInterruption = persistedSession && nativeRestartInterruptedTurnId(persistedSession);
+  if (checkpointedInterruption && persistedSession) {
+    // The turn failed because its process vanished; the conversation is still
+    // usable. Keep its terminal fingerprint so replay cannot execute it twice.
+    persistedSession = { ...persistedSession, terminal: null };
+  }
   await options.onSessionAdmission?.();
   if (options.existingSession) {
     if (options.existingSession.attachRun === undefined) {
@@ -2157,6 +2170,42 @@ export async function executeNativeSession(
       await persistCheckpoint(snapshot, signal);
     };
     const recoveredSnapshot = await session.snapshot();
+    const restartContinuation = () => ({
+      message: { role: "user" as const, text: JSON.stringify({
+        schema: "paperclip.native-continuation.v1",
+        events: [
+          "Paperclip restarted and restored this same conversation. The previous turn was interrupted.",
+          "Continue the current user request from the recorded progress and preserved workspace.",
+          "Reconcile any unfinished tool or command before proceeding: inspect its current state and recorded outcomes.",
+          "Do not repeat completed work or blindly rerun an action whose outcome is unknown. If an external action cannot be reconciled, report the specific blocker.",
+        ].join("\n"),
+        completion: {
+          revision: input.completionContract.contract.revision,
+          criterionIds: input.completionContract.contract.criteria.map(criterion => criterion.id),
+        },
+      }) },
+      continuation: true as const,
+      requestedCollaborationMode: "executionMode" in input ? input.executionMode : "default" as const,
+    });
+    let restartContinuationStarted = false;
+    const resumeInterruptedTurn = async (event: PrpEvent, signal: AbortSignal) => {
+      if (restartContinuationStarted || !recovered || !options.resumeInterruptedTurn ||
+          input.provider.kind !== "codex" || options.sessionGoalControl || options.resumeSessionGoalHeartbeat ||
+          recoveredSnapshot.goal || persistedSession?.semanticResult || persistedSession?.pendingRuntimeRequests?.length ||
+          event.eventType !== "turn.failed" || !isNativeRestartInterruption(event.payload.error) ||
+          event.turnId !== persistedSession?.activeTurnId) return false;
+      const snapshot = await session.snapshot();
+      signal.throwIfAborted();
+      if (nativeRestartInterruptedTurnId(snapshot) !== event.turnId || snapshot.pendingRuntimeRequests?.length) return false;
+      restartContinuationStarted = true;
+      // Persist the old terminal before launching. Recovery inspects provider
+      // history to adopt an accepted continuation if its checkpoint was lost.
+      await persistCheckpoint(snapshot, signal);
+      signal.throwIfAborted();
+      await session.startTurn(restartContinuation());
+      await checkpoint(signal);
+      return true;
+    };
     const recoveredActiveTurnId = recovered
       ? (recoveredSnapshot.activeTurnId ?? null)
       : (persistedSession?.activeTurnId ?? null);
@@ -2294,6 +2343,7 @@ export async function executeNativeSession(
                   }
                 : undefined,
               recoveredSnapshot.goal,
+              resumeInterruptedTurn,
             )
           : Promise.resolve({
               event: recoveryTerminal,
@@ -2333,6 +2383,10 @@ export async function executeNativeSession(
             await session.goal({ action: "get", requestId });
           }
           await checkpoint();
+        } else if (shouldStartFreshTurn && recovered && checkpointedInterruption) {
+          restartContinuationStarted = true;
+          await session.startTurn(restartContinuation());
+          await checkpoint();
         } else if (shouldStartFreshTurn) {
           let modelEnvelope = recovered
             ? buildNativeModelEnvelope(input, { resumedSession: true })

packages/paperclip-runner/test/fixtures/semantic-input-digests.json

@@ -0,0 +1,42 @@
+[
+  {
+    "name": "plain document",
+    "input": {
+      "body": "Plain fixture document"
+    },
+    "digest": "sha256:2ba38770004c4e7672a5097c4d8e3241c62750b21e034903796fb035fcf9a2a4"
+  },
+  {
+    "name": "protected fields",
+    "input": {
+      "password": "fixture-only-not-a-real-credential",
+      "safe": true
+    },
+    "digest": "sha256:98ecc80f0e6d2d4b99ce3620d18a21392356f99def55ed599c1b1c002ccd2d47"
+  },
+  {
+    "name": "credential-like document text",
+    "input": {
+      "body": "Example Bearer fixture_token_123456\nhttps://example.test/callback?token=fixture-only"
+    },
+    "digest": "sha256:aa7f429cc5dd63e85fd11aa93e0659fa1c7d5041f5f35843fef62092ff86fdc0"
+  },
+  {
+    "name": "canonical Unicode and numbers",
+    "input": {
+      "": "private-use key",
+      "😀": "astral key",
+      "values": [
+        -0,
+        0.000001,
+        9.99999e-7,
+        100000000000000000000,
+        1e+21,
+        true,
+        null,
+        "日本語\n\"text\""
+      ]
+    },
+    "digest": "sha256:3104a5e5db1fe2c6dcdaa0bfcf362d7bd36441e7e08905a48ae03be06eea9dc4"
+  }
+]

packages/plugins/sandbox-providers/exe-dev/README.md

@@ -23,7 +23,7 @@ Configure exe.dev from `Instance Settings -> Environments`, not from the plugin'
 
 To use the provider successfully, the environment/host needs all of the following:
 
-- An exe.dev API token that allows the lifecycle commands the provider uses: `new`, `ls`, and `rm`. `whoami` and `help` are recommended for manual debugging. `restart` is only needed if you extend the provider to restart retained VMs.
+- An exe.dev API token that allows the lifecycle commands the provider uses: `new`, `ls`, and `rm`, plus `cp` when `sourceVm` is set. exe.dev answers 403 for a command the token does not list. `whoami` and `help` are recommended for manual debugging. `restart` is only needed if you extend the provider to restart retained VMs.
 - SSH access from the Paperclip host to the resulting `*.exe.xyz` VMs.
 - An SSH private key that exe.dev already recognizes. You can either:
   - paste the private key into the environment config via `sshPrivateKey`
@@ -37,6 +37,7 @@ Operational notes:
 - Reusable leases keep the VM alive between runs. exe.dev does not expose a documented "stop and later resume" command in the public CLI docs, so `reuseLease: true` means "retain the VM" rather than "suspend it."
 - The provisioning path uses `https://exe.dev/exec`, which exe.dev documents as a command-style HTTPS API with a 30-second request timeout. Typical `new` calls are expected to fit inside that limit; command execution itself does not use `/exec`.
 - Probes still create and delete a real exe.dev VM through `/exec`, and so do the `new`/`rm` calls inside the normal acquire/release lifecycle. Treat all of those as real provisioning cost, not just probes.
+- Set `sourceVm` ("Source VM" in the form) to the name of an existing exe.dev VM to copy it for each run with `exe.dev cp`, disk and config included, instead of creating a fresh VM with `exe.dev new`. This lets you prepare one VM with your toolchain and caches and start every run from it. `cp` accepts only the VM name, `cpu`, `memory`, and `disk`, so config validation rejects `sourceVm` together with `image`, `command`, `comment`, `env`, `integrations`, `tags`, `setupScript`, or `prompt`. The default setup script does not run either, so the source VM must already have Node 24.11+ and accept the configured SSH key. Do not keep secrets on the source VM: every copy inherits its disk. `cp` also goes through `/exec`, so a large disk copy must finish inside the same 30-second limit.
 - exe.dev runs `--setup-script` as the unprivileged `exedev` user, not as root. That user has passwordless `sudo`, so any system-level steps in a custom `setupScript` must invoke `sudo` explicitly (for example `sudo apt-get install -y …`). When you omit `setupScript`, the plugin supplies a default that installs Node 24 via the official nodesource script — Paperclip's sandbox callback bridge is a Node program, so the VM needs `node` on `PATH` before the bridge can launch.
 
 ## Local development

packages/plugins/sandbox-providers/exe-dev/src/manifest.ts

@@ -31,7 +31,7 @@ const manifest: PaperclipPluginManifestV1 = {
             type: "string",
             format: "secret-ref",
             description:
-              "Paste your exe.dev API token, or pick a saved Paperclip secret. Create one at exe.dev → Settings → API tokens with `/exec` scope (`new`, `ls`, `rm`).",
+              "Paste your exe.dev API token, or pick a saved Paperclip secret. Create one at exe.dev → Settings → API tokens with `/exec` scope (`new`, `ls`, `rm`, plus `cp` if you use Source VM).",
           },
           sshPrivateKey: {
             type: "string",
@@ -99,6 +99,14 @@ const manifest: PaperclipPluginManifestV1 = {
             "x-paperclip-group": "VM resources",
           },
           // ---- Advanced: VM creation ----
+          sourceVm: {
+            type: "string",
+            title: "Source VM",
+            description:
+              "Name of an existing exe.dev VM to copy for each run with `exe.dev cp`, disk and config included. Leave blank to create a fresh VM with `exe.dev new`. Your API token must allow `cp`: tokens list their permitted commands, and one without `cp` fails with a 403. When set, leave image, command, env, integrations, tags, setup script, prompt, and comment empty: `cp` cannot apply them, and saving fails until they are cleared.",
+            "x-paperclip-advanced": true,
+            "x-paperclip-group": "VM creation",
+          },
           command: {
             type: "string",
             description: "Optional container command passed to `exe.dev new --command`.",

packages/plugins/sandbox-providers/exe-dev/src/plugin.test.ts

@@ -106,6 +106,7 @@ describe("exe.dev sandbox provider plugin", () => {
         integrations: ["github"],
         tags: ["prod", "sandbox"],
         setupScript: null,
+        sourceVm: null,
         prompt: null,
         timeoutMs: 450000,
         reuseLease: true,
@@ -319,6 +320,57 @@ describe("exe.dev sandbox provider plugin", () => {
     });
   });
 
+  it("copies the source VM with exe.dev cp instead of creating a new VM", async () => {
+    fetchMock.mockResolvedValueOnce(
+      new Response(JSON.stringify({
+        vm_name: "paperclip-env1-run1",
+        ssh_dest: "paperclip-env1-run1.exe.xyz",
+        status: "running",
+      }), { status: 200 }),
+    );
+    queueSpawnResult({ stdout: "/home/exedev\nbash\n" });
+    queueSpawnResult({});
+
+    const lease = await plugin.definition.onEnvironmentAcquireLease?.({
+      driverKey: "exe-dev",
+      companyId: "company-1",
+      environmentId: "env-1",
+      runId: "run-1",
+      config: {
+        apiKey: "api-key",
+        sourceVm: " golden-vm ",
+        image: "ubuntu:22.04",
+        env: { FOO: "bar" },
+        cpu: 4,
+        memory: "8GB",
+        disk: "40GB",
+      },
+    });
+
+    expect(String(fetchMock.mock.calls[0]?.[1]?.body ?? "")).toBe(
+      "cp 'golden-vm' 'paperclip-env1-run1' --json --cpu='4' --memory='8GB' --disk='40GB'",
+    );
+    expect(lease).toMatchObject({ providerLeaseId: "paperclip-env1-run1" });
+  });
+
+  it("rejects VM creation settings that exe.dev cp cannot apply when sourceVm is set", async () => {
+    const result = await plugin.definition.onEnvironmentValidateConfig?.({
+      driverKey: "exe-dev",
+      config: {
+        apiKey: "api-key",
+        sourceVm: "golden-vm",
+        image: "ubuntu:22.04",
+        env: { FOO: "bar" },
+        setupScript: "echo hi",
+      },
+    });
+
+    expect(result?.ok).toBe(false);
+    expect(result?.errors).toContain(
+      "sourceVm copies an existing VM with `exe.dev cp`, which cannot apply image, env, setupScript. Clear these settings or clear sourceVm.",
+    );
+  });
+
   it("uses a pasted sshPrivateKey when connecting to the VM", async () => {
     fetchMock.mockResolvedValueOnce(
       new Response(JSON.stringify({
@@ -860,6 +912,12 @@ describe("exe-dev manifest form defaults", () => {
     expect(properties.disk?.default).toBe("20GB");
   });
 
+  it("explains the cp token permission on the Source VM field", () => {
+    const sourceVm = properties.sourceVm as { title?: string; description?: string } | undefined;
+    expect(sourceVm?.title).toBe("Source VM");
+    expect(sourceVm?.description).toMatch(/token must allow `cp`.*403/);
+  });
+
   it("declares no default on secret-ref fields, which would be persisted as a company secret", () => {
     for (const prop of Object.values(properties)) {
       if (prop.format === "secret-ref") {

packages/plugins/sandbox-providers/exe-dev/src/plugin.ts

@@ -24,6 +24,7 @@ interface ExeDevDriverConfig {
   apiKey: string | null;
   apiUrl: string;
   namePrefix: string;
+  sourceVm: string | null;
   image: string | null;
   command: string | null;
   cpu: number | null;
@@ -246,6 +247,7 @@ function parseDriverConfig(raw: Record<string, unknown>): ExeDevDriverConfig {
     apiKey: parseOptionalString(raw.apiKey),
     apiUrl: normalizeApiUrl(parseOptionalString(raw.apiUrl)),
     namePrefix: normalizeNamePrefix(parseOptionalString(raw.namePrefix)),
+    sourceVm: parseOptionalString(raw.sourceVm),
     image: parseOptionalString(raw.image),
     command: parseOptionalString(raw.command),
     cpu: parseOptionalInteger(raw.cpu),
@@ -313,10 +315,23 @@ function resolveSetupScript(config: ExeDevDriverConfig): string | null {
   return trimmed.length > 0 ? config.setupScript : null;
 }
 
+function buildCopyCommand(sourceVm: string, config: ExeDevDriverConfig, vmName: string): string {
+  return [
+    "cp",
+    shellQuote(sourceVm),
+    shellQuote(vmName),
+    "--json",
+    ...buildFlag("cpu", config.cpu),
+    ...buildFlag("memory", config.memory),
+    ...buildFlag("disk", config.disk),
+  ].join(" ");
+}
+
 function buildCreateCommand(
   config: ExeDevDriverConfig,
   vmName: string,
 ): string {
+  if (config.sourceVm) return buildCopyCommand(config.sourceVm, config, vmName);
   return [
     "new",
     "--json",
@@ -774,6 +789,23 @@ const plugin = definePlugin({
     warnings.push(
       "The Paperclip host must have SSH access to the created exe.dev VM, and its SSH key must be registered with exe.dev. The API token only covers provisioning.",
     );
+    if (config.sourceVm) {
+      const ignored = Object.entries({
+        image: config.image,
+        command: config.command,
+        comment: config.comment,
+        env: Object.keys(config.env).length > 0,
+        integrations: config.integrations.length > 0,
+        tags: config.tags.length > 0,
+        setupScript: config.setupScript,
+        prompt: config.prompt,
+      }).filter(([, value]) => value).map(([key]) => key);
+      if (ignored.length > 0) {
+        errors.push(
+          `sourceVm copies an existing VM with \`exe.dev cp\`, which cannot apply ${ignored.join(", ")}. Clear these settings or clear sourceVm.`,
+        );
+      }
+    }
     if (config.reuseLease) {
       warnings.push("reuseLease keeps the VM alive between runs; this provider does not suspend retained VMs.");
     }