mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-06 23:20:53 +01:00
## Thinking Path > - Paperclip manages AI agents and their work. > - The runner executes agent turns on local and remote providers. > - A remote per-turn session must save its state before Paperclip releases its sandbox. > - The session runtime returned after 100 milliseconds while the remote checkpoint still ran. > - The next turn also checked the local state path instead of the verified remote backup. > - This pull request waits for the bounded remote close and accepts only a verified suspended backup. > - The benefit is reliable multi-turn execution without weaker identity checks. ## Linked Issues or Issue Description **What happened?** A successful remote agent turn released its sandbox before the runner saved the verified continuation backup. The next turn failed with `runner_state_identity_mismatch`. **Expected behavior** Paperclip must finish the bounded remote checkpoint before it releases the sandbox. A later turn must validate and restore the digest-matched suspended backup. **Steps to reproduce** 1. Run a native ACPX Claude Plan test in a non-reusable Daytona sandbox. 2. Reject the first plan to start a second turn. 3. Observe that the second turn fails before provider execution. **Paperclip version or commit** The failure reproduced at `13775a90b078ff64872f50961ea1b83d575e7bc6`. **Deployment mode** GitHub Actions with a Daytona sandbox. ## What Changed - Wait for the internally bounded remote runner close and checkpoint before the host returns. - Preserve the existing short cleanup bound for other providers. - Validate remote continuation lifecycle from a complete digest-verified backup when local runner state is absent. - Keep corrupt, non-suspended, mismatched, and unverified state fail-closed. - Make native Plan completion and accepted-Plan wake prompts deterministic. ## Verification - A prior 45-cell local campaign passed 44 cells. The only failure was the OpenCode Plan prompt variance fixed here. - A focused OpenCode local Plan rerun passed. - ACPX Claude Daytona message and question cells passed. - Focused regressions cover delayed checkpoint close and verified remote backup lifecycle. - GitHub Build and the focused ACPX Claude Daytona Plan cell will validate this exact head. ## Risks Remote runnerd sessions now wait for their internally bounded close/checkpoint path before returning; generic provider cleanup retains the existing 100 millisecond bound. Durable run success still cannot be reversed. The environment release guard still blocks sandbox destruction when no verified backup stamp exists. ## Model Used OpenAI Codex, GPT-5.6, extended reasoning, with code execution and GitHub Actions inspection. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change and contains no internal task id - [ ] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open findings - [ ] I will address all Greptile and reviewer comments before requesting merge
229 lines
7.7 KiB
TypeScript
229 lines
7.7 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { lstatSync, readFileSync, readdirSync, readlinkSync } from "node:fs";
|
|
import { dirname, resolve } from "node:path";
|
|
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
|
|
|
|
export interface NativeHarnessBackupStamp {
|
|
schema: "paperclip.native-harness-backup-stamp.v2";
|
|
normalizedSessionId: string;
|
|
runnerInstanceId: string;
|
|
sessionScopeSha256: string;
|
|
sourceProviderLeaseId: string;
|
|
authorizedProviderLeaseId: string;
|
|
manifestSha256: string;
|
|
completedAt: string;
|
|
}
|
|
|
|
function stateBase(): string {
|
|
return resolve(
|
|
process.env.PAPERCLIP_RUNNER_STATE_DIR ??
|
|
resolve(
|
|
resolvePaperclipInstanceRoot(),
|
|
"runtime",
|
|
"paperclip-runner",
|
|
"durable-sessions",
|
|
),
|
|
);
|
|
}
|
|
|
|
function stateRootFromDigest(digest: string): string | null {
|
|
if (!/^[0-9a-f]{64}$/.test(digest)) return null;
|
|
const base = stateBase();
|
|
const root = resolve(base, digest);
|
|
return dirname(root) === base ? root : null;
|
|
}
|
|
|
|
function isRealDirectory(path: string): boolean {
|
|
try {
|
|
const metadata = lstatSync(path);
|
|
return metadata.isDirectory() && !metadata.isSymbolicLink();
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function isRealFile(path: string): boolean {
|
|
try {
|
|
const metadata = lstatSync(path);
|
|
return metadata.isFile() && !metadata.isSymbolicLink();
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function digestDirectory(directory: string): { sha256: string; bytes: number } {
|
|
const hash = createHash("sha256");
|
|
let bytes = 0;
|
|
const visit = (current: string, relative: string) => {
|
|
const entries = readdirSync(current, { withFileTypes: true }).sort(
|
|
(left, right) => left.name.localeCompare(right.name),
|
|
);
|
|
if (entries.length === 0) hash.update(`directory:${relative}\0`);
|
|
for (const entry of entries) {
|
|
const path = resolve(current, entry.name);
|
|
const relativePath = relative ? `${relative}/${entry.name}` : entry.name;
|
|
const stats = lstatSync(path);
|
|
if (entry.isDirectory()) {
|
|
hash.update(`directory:${relativePath}:${stats.mode & 0o777}\0`);
|
|
visit(path, relativePath);
|
|
} else if (entry.isSymbolicLink()) {
|
|
hash.update(`symlink:${relativePath}:${readlinkSync(path)}\0`);
|
|
} else if (entry.isFile()) {
|
|
const contents = readFileSync(path);
|
|
bytes += contents.byteLength;
|
|
hash.update(
|
|
`file:${relativePath}:${stats.mode & 0o777}:${contents.byteLength}\0`,
|
|
);
|
|
hash.update(contents);
|
|
} else {
|
|
throw new Error(
|
|
`runner_harness_backup_unsupported_entry:${relativePath}`,
|
|
);
|
|
}
|
|
}
|
|
};
|
|
visit(directory, "");
|
|
return { sha256: `sha256:${hash.digest("hex")}`, bytes };
|
|
}
|
|
|
|
export function createNativeHarnessBackupStamp(input: {
|
|
manifestPath: string;
|
|
sessionScopeId: string;
|
|
authorizedProviderLeaseId: string;
|
|
normalizedSessionId: string;
|
|
runnerInstanceId: string;
|
|
completedAt: string;
|
|
}): NativeHarnessBackupStamp {
|
|
if (!input.authorizedProviderLeaseId) {
|
|
throw new Error("runner_harness_backup_lease_missing");
|
|
}
|
|
const sessionScopeSha256 = createHash("sha256")
|
|
.update(input.sessionScopeId)
|
|
.digest("hex");
|
|
const root = stateRootFromDigest(sessionScopeSha256);
|
|
if (!root || !isRealDirectory(root)) {
|
|
throw new Error("runner_harness_backup_scope_invalid");
|
|
}
|
|
const resolvedManifestPath = resolve(input.manifestPath);
|
|
const allowedManifestPaths = ["current", "previous"].map((candidate) =>
|
|
resolve(root, "failover-backups", candidate, "manifest.json"),
|
|
);
|
|
if (
|
|
!allowedManifestPaths.includes(resolvedManifestPath) ||
|
|
!isRealDirectory(dirname(resolvedManifestPath)) ||
|
|
!isRealFile(resolvedManifestPath)
|
|
) {
|
|
throw new Error("runner_harness_backup_scope_invalid");
|
|
}
|
|
const manifestBytes = readFileSync(input.manifestPath);
|
|
const manifest = JSON.parse(manifestBytes.toString("utf8")) as Record<
|
|
string,
|
|
unknown
|
|
>;
|
|
if (
|
|
manifest.schema !== "paperclip.native-harness-backup.v1" ||
|
|
manifest.normalizedSessionId !== input.normalizedSessionId ||
|
|
manifest.runnerInstanceId !== input.runnerInstanceId ||
|
|
manifest.completedAt !== input.completedAt ||
|
|
typeof manifest.sourceProviderLeaseId !== "string" ||
|
|
!manifest.sourceProviderLeaseId
|
|
) {
|
|
throw new Error("runner_harness_backup_scope_invalid");
|
|
}
|
|
return {
|
|
schema: "paperclip.native-harness-backup-stamp.v2",
|
|
normalizedSessionId: input.normalizedSessionId,
|
|
runnerInstanceId: input.runnerInstanceId,
|
|
sessionScopeSha256,
|
|
sourceProviderLeaseId: manifest.sourceProviderLeaseId,
|
|
authorizedProviderLeaseId: input.authorizedProviderLeaseId,
|
|
manifestSha256: `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`,
|
|
completedAt: input.completedAt,
|
|
};
|
|
}
|
|
|
|
export function verifyNativeHarnessBackupStamp(
|
|
value: unknown,
|
|
expectedProviderLeaseId: string,
|
|
): boolean {
|
|
if (!expectedProviderLeaseId) return false;
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
const stamp = value as Record<string, unknown>;
|
|
if (
|
|
stamp.schema !== "paperclip.native-harness-backup-stamp.v2" ||
|
|
typeof stamp.normalizedSessionId !== "string" ||
|
|
!stamp.normalizedSessionId ||
|
|
typeof stamp.runnerInstanceId !== "string" ||
|
|
!stamp.runnerInstanceId ||
|
|
typeof stamp.sessionScopeSha256 !== "string" ||
|
|
typeof stamp.sourceProviderLeaseId !== "string" ||
|
|
!stamp.sourceProviderLeaseId ||
|
|
stamp.authorizedProviderLeaseId !== expectedProviderLeaseId ||
|
|
typeof stamp.manifestSha256 !== "string" ||
|
|
!stamp.manifestSha256.startsWith("sha256:")
|
|
)
|
|
return false;
|
|
const root = stateRootFromDigest(stamp.sessionScopeSha256);
|
|
if (!root || !isRealDirectory(root)) return false;
|
|
const backupRoot = resolve(root, "failover-backups");
|
|
for (const candidate of [
|
|
resolve(backupRoot, "current"),
|
|
resolve(backupRoot, "previous"),
|
|
]) {
|
|
const manifestPath = resolve(candidate, "manifest.json");
|
|
if (!isRealDirectory(candidate) || !isRealFile(manifestPath)) continue;
|
|
try {
|
|
const bytes = readFileSync(manifestPath);
|
|
const manifestSha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
if (manifestSha256 !== stamp.manifestSha256) continue;
|
|
const manifest = JSON.parse(bytes.toString("utf8")) as Record<
|
|
string,
|
|
unknown
|
|
>;
|
|
if (
|
|
manifest.schema !== "paperclip.native-harness-backup.v1" ||
|
|
manifest.normalizedSessionId !== stamp.normalizedSessionId ||
|
|
manifest.runnerInstanceId !== stamp.runnerInstanceId ||
|
|
manifest.sourceProviderLeaseId !== stamp.sourceProviderLeaseId ||
|
|
!Array.isArray(manifest.directories) ||
|
|
manifest.directories.length === 0
|
|
)
|
|
continue;
|
|
let valid = true;
|
|
for (const entry of manifest.directories) {
|
|
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
|
valid = false;
|
|
break;
|
|
}
|
|
const declared = entry as Record<string, unknown>;
|
|
if (
|
|
typeof declared.name !== "string" ||
|
|
!/^[A-Za-z0-9._-]+$/.test(declared.name) ||
|
|
typeof declared.sha256 !== "string" ||
|
|
typeof declared.bytes !== "number"
|
|
) {
|
|
valid = false;
|
|
break;
|
|
}
|
|
const directory = resolve(candidate, declared.name);
|
|
if (!isRealDirectory(directory)) {
|
|
valid = false;
|
|
break;
|
|
}
|
|
const actual = digestDirectory(directory);
|
|
if (
|
|
actual.sha256 !== declared.sha256 ||
|
|
actual.bytes !== declared.bytes
|
|
) {
|
|
valid = false;
|
|
break;
|
|
}
|
|
}
|
|
if (valid) return true;
|
|
} catch {
|
|
// Try the previous atomically-published backup.
|
|
}
|
|
}
|
|
return false;
|
|
}
|