Files
PaperClipAI/server/src/services/native-runtime/native-harness-backup-stamp.ts
T
Dotta bcc6fe7a44 fix(runner): restore multi-turn remote sessions (#12840)
## Thinking Path

> - Paperclip manages AI agents and their work.
> - The runner executes agent turns on local and remote providers.
> - A remote per-turn session must save its state before Paperclip
releases its sandbox.
> - The session runtime returned after 100 milliseconds while the remote
checkpoint still ran.
> - The next turn also checked the local state path instead of the
verified remote backup.
> - This pull request waits for the bounded remote close and accepts
only a verified suspended backup.
> - The benefit is reliable multi-turn execution without weaker identity
checks.

## Linked Issues or Issue Description

**What happened?**

A successful remote agent turn released its sandbox before the runner
saved the verified continuation backup. The next turn failed with
`runner_state_identity_mismatch`.

**Expected behavior**

Paperclip must finish the bounded remote checkpoint before it releases
the sandbox. A later turn must validate and restore the digest-matched
suspended backup.

**Steps to reproduce**

1. Run a native ACPX Claude Plan test in a non-reusable Daytona sandbox.
2. Reject the first plan to start a second turn.
3. Observe that the second turn fails before provider execution.

**Paperclip version or commit**

The failure reproduced at `13775a90b078ff64872f50961ea1b83d575e7bc6`.

**Deployment mode**

GitHub Actions with a Daytona sandbox.

## What Changed

- Wait for the internally bounded remote runner close and checkpoint
before the host returns.
- Preserve the existing short cleanup bound for other providers.
- Validate remote continuation lifecycle from a complete digest-verified
backup when local runner state is absent.
- Keep corrupt, non-suspended, mismatched, and unverified state
fail-closed.
- Make native Plan completion and accepted-Plan wake prompts
deterministic.

## Verification

- A prior 45-cell local campaign passed 44 cells. The only failure was
the OpenCode Plan prompt variance fixed here.
- A focused OpenCode local Plan rerun passed.
- ACPX Claude Daytona message and question cells passed.
- Focused regressions cover delayed checkpoint close and verified remote
backup lifecycle.
- GitHub Build and the focused ACPX Claude Daytona Plan cell will
validate this exact head.

## Risks

Remote runnerd sessions now wait for their internally bounded
close/checkpoint path before returning; generic provider cleanup retains
the existing 100 millisecond bound. Durable run success still cannot be
reversed. The environment release guard still blocks sandbox destruction
when no verified backup stamp exists.

## Model Used

OpenAI Codex, GPT-5.6, extended reasoning, with code execution and
GitHub Actions inspection.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` / `Closes
#` / `Refs #` OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change and contains no internal task
id
- [ ] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [ ] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [ ] All Paperclip CI gates are green
- [ ] Greptile is 5/5 with no open findings
- [ ] I will address all Greptile and reviewer comments before
requesting merge
2026-09-05 06:25:06 -05:00

229 lines
7.7 KiB
TypeScript

import { createHash } from "node:crypto";
import { lstatSync, readFileSync, readdirSync, readlinkSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
export interface NativeHarnessBackupStamp {
schema: "paperclip.native-harness-backup-stamp.v2";
normalizedSessionId: string;
runnerInstanceId: string;
sessionScopeSha256: string;
sourceProviderLeaseId: string;
authorizedProviderLeaseId: string;
manifestSha256: string;
completedAt: string;
}
function stateBase(): string {
return resolve(
process.env.PAPERCLIP_RUNNER_STATE_DIR ??
resolve(
resolvePaperclipInstanceRoot(),
"runtime",
"paperclip-runner",
"durable-sessions",
),
);
}
function stateRootFromDigest(digest: string): string | null {
if (!/^[0-9a-f]{64}$/.test(digest)) return null;
const base = stateBase();
const root = resolve(base, digest);
return dirname(root) === base ? root : null;
}
function isRealDirectory(path: string): boolean {
try {
const metadata = lstatSync(path);
return metadata.isDirectory() && !metadata.isSymbolicLink();
} catch {
return false;
}
}
function isRealFile(path: string): boolean {
try {
const metadata = lstatSync(path);
return metadata.isFile() && !metadata.isSymbolicLink();
} catch {
return false;
}
}
function digestDirectory(directory: string): { sha256: string; bytes: number } {
const hash = createHash("sha256");
let bytes = 0;
const visit = (current: string, relative: string) => {
const entries = readdirSync(current, { withFileTypes: true }).sort(
(left, right) => left.name.localeCompare(right.name),
);
if (entries.length === 0) hash.update(`directory:${relative}\0`);
for (const entry of entries) {
const path = resolve(current, entry.name);
const relativePath = relative ? `${relative}/${entry.name}` : entry.name;
const stats = lstatSync(path);
if (entry.isDirectory()) {
hash.update(`directory:${relativePath}:${stats.mode & 0o777}\0`);
visit(path, relativePath);
} else if (entry.isSymbolicLink()) {
hash.update(`symlink:${relativePath}:${readlinkSync(path)}\0`);
} else if (entry.isFile()) {
const contents = readFileSync(path);
bytes += contents.byteLength;
hash.update(
`file:${relativePath}:${stats.mode & 0o777}:${contents.byteLength}\0`,
);
hash.update(contents);
} else {
throw new Error(
`runner_harness_backup_unsupported_entry:${relativePath}`,
);
}
}
};
visit(directory, "");
return { sha256: `sha256:${hash.digest("hex")}`, bytes };
}
export function createNativeHarnessBackupStamp(input: {
manifestPath: string;
sessionScopeId: string;
authorizedProviderLeaseId: string;
normalizedSessionId: string;
runnerInstanceId: string;
completedAt: string;
}): NativeHarnessBackupStamp {
if (!input.authorizedProviderLeaseId) {
throw new Error("runner_harness_backup_lease_missing");
}
const sessionScopeSha256 = createHash("sha256")
.update(input.sessionScopeId)
.digest("hex");
const root = stateRootFromDigest(sessionScopeSha256);
if (!root || !isRealDirectory(root)) {
throw new Error("runner_harness_backup_scope_invalid");
}
const resolvedManifestPath = resolve(input.manifestPath);
const allowedManifestPaths = ["current", "previous"].map((candidate) =>
resolve(root, "failover-backups", candidate, "manifest.json"),
);
if (
!allowedManifestPaths.includes(resolvedManifestPath) ||
!isRealDirectory(dirname(resolvedManifestPath)) ||
!isRealFile(resolvedManifestPath)
) {
throw new Error("runner_harness_backup_scope_invalid");
}
const manifestBytes = readFileSync(input.manifestPath);
const manifest = JSON.parse(manifestBytes.toString("utf8")) as Record<
string,
unknown
>;
if (
manifest.schema !== "paperclip.native-harness-backup.v1" ||
manifest.normalizedSessionId !== input.normalizedSessionId ||
manifest.runnerInstanceId !== input.runnerInstanceId ||
manifest.completedAt !== input.completedAt ||
typeof manifest.sourceProviderLeaseId !== "string" ||
!manifest.sourceProviderLeaseId
) {
throw new Error("runner_harness_backup_scope_invalid");
}
return {
schema: "paperclip.native-harness-backup-stamp.v2",
normalizedSessionId: input.normalizedSessionId,
runnerInstanceId: input.runnerInstanceId,
sessionScopeSha256,
sourceProviderLeaseId: manifest.sourceProviderLeaseId,
authorizedProviderLeaseId: input.authorizedProviderLeaseId,
manifestSha256: `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`,
completedAt: input.completedAt,
};
}
export function verifyNativeHarnessBackupStamp(
value: unknown,
expectedProviderLeaseId: string,
): boolean {
if (!expectedProviderLeaseId) return false;
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
const stamp = value as Record<string, unknown>;
if (
stamp.schema !== "paperclip.native-harness-backup-stamp.v2" ||
typeof stamp.normalizedSessionId !== "string" ||
!stamp.normalizedSessionId ||
typeof stamp.runnerInstanceId !== "string" ||
!stamp.runnerInstanceId ||
typeof stamp.sessionScopeSha256 !== "string" ||
typeof stamp.sourceProviderLeaseId !== "string" ||
!stamp.sourceProviderLeaseId ||
stamp.authorizedProviderLeaseId !== expectedProviderLeaseId ||
typeof stamp.manifestSha256 !== "string" ||
!stamp.manifestSha256.startsWith("sha256:")
)
return false;
const root = stateRootFromDigest(stamp.sessionScopeSha256);
if (!root || !isRealDirectory(root)) return false;
const backupRoot = resolve(root, "failover-backups");
for (const candidate of [
resolve(backupRoot, "current"),
resolve(backupRoot, "previous"),
]) {
const manifestPath = resolve(candidate, "manifest.json");
if (!isRealDirectory(candidate) || !isRealFile(manifestPath)) continue;
try {
const bytes = readFileSync(manifestPath);
const manifestSha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
if (manifestSha256 !== stamp.manifestSha256) continue;
const manifest = JSON.parse(bytes.toString("utf8")) as Record<
string,
unknown
>;
if (
manifest.schema !== "paperclip.native-harness-backup.v1" ||
manifest.normalizedSessionId !== stamp.normalizedSessionId ||
manifest.runnerInstanceId !== stamp.runnerInstanceId ||
manifest.sourceProviderLeaseId !== stamp.sourceProviderLeaseId ||
!Array.isArray(manifest.directories) ||
manifest.directories.length === 0
)
continue;
let valid = true;
for (const entry of manifest.directories) {
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
valid = false;
break;
}
const declared = entry as Record<string, unknown>;
if (
typeof declared.name !== "string" ||
!/^[A-Za-z0-9._-]+$/.test(declared.name) ||
typeof declared.sha256 !== "string" ||
typeof declared.bytes !== "number"
) {
valid = false;
break;
}
const directory = resolve(candidate, declared.name);
if (!isRealDirectory(directory)) {
valid = false;
break;
}
const actual = digestDirectory(directory);
if (
actual.sha256 !== declared.sha256 ||
actual.bytes !== declared.bytes
) {
valid = false;
break;
}
}
if (valid) return true;
} catch {
// Try the previous atomically-published backup.
}
}
return false;
}