mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 02:44:50 +01:00
fix(runner): restore non-reusable sandbox continuations
This commit is contained in:
1 parent
50d5ecea39
commit
7cf2560bc5
2 files changed
+111
-33
No files matched your search
@@ -65,6 +65,7 @@ import {
|
||||
steerNativeSession,
|
||||
syncRemoteRunnerDirectoryOut,
|
||||
verifyNativeHarnessBackup,
|
||||
shouldRestoreNativeHarnessBackupIntoSandbox,
|
||||
} from "./native-session-executor.js";
|
||||
|
||||
describe("remote provider pack manifest", () => {
|
||||
@@ -236,6 +237,30 @@ describe("verified native harness backups", () => {
|
||||
},
|
||||
} as unknown as NativeExecutionInputV1;
|
||||
|
||||
it("restores a verified continuation into an intentionally fresh non-reusable sandbox", () => {
|
||||
expect(
|
||||
shouldRestoreNativeHarnessBackupIntoSandbox({
|
||||
acquisitionOutcome: "created",
|
||||
reusableLeaseConfigured: false,
|
||||
backupAvailable: true,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldRestoreNativeHarnessBackupIntoSandbox({
|
||||
acquisitionOutcome: "created",
|
||||
reusableLeaseConfigured: true,
|
||||
backupAvailable: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldRestoreNativeHarnessBackupIntoSandbox({
|
||||
acquisitionOutcome: "created",
|
||||
reusableLeaseConfigured: false,
|
||||
backupAvailable: false,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts a complete digest-matched backup and rejects corruption", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "paperclip-harness-backup-"));
|
||||
try {
|
||||
|
||||
@@ -1095,6 +1095,18 @@ type VerifiedHarnessBackup = {
|
||||
bytes: number;
|
||||
};
|
||||
|
||||
export function shouldRestoreNativeHarnessBackupIntoSandbox(input: {
|
||||
acquisitionOutcome: "created" | "resumed" | "replacement" | null;
|
||||
reusableLeaseConfigured: boolean | null | undefined;
|
||||
backupAvailable: boolean;
|
||||
}): boolean {
|
||||
return (
|
||||
input.acquisitionOutcome === "created" &&
|
||||
input.reusableLeaseConfigured === false &&
|
||||
input.backupAvailable
|
||||
);
|
||||
}
|
||||
|
||||
function compatibleNativeHarnessBackupManifests(input: {
|
||||
root: string;
|
||||
execution: NativeExecutionInput;
|
||||
@@ -4273,6 +4285,40 @@ export function createRunnerdBackend(input: {
|
||||
});
|
||||
};
|
||||
|
||||
const recordHarnessBackupStampForCurrentLease = async (
|
||||
backup: VerifiedHarnessBackup,
|
||||
) => {
|
||||
if (remoteTarget?.transport !== "sandbox" || !remoteTarget.leaseId) {
|
||||
return;
|
||||
}
|
||||
const leaseRow = await input.db
|
||||
.select({ metadata: environmentLeases.metadata })
|
||||
.from(environmentLeases)
|
||||
.where(eq(environmentLeases.id, remoteTarget.leaseId))
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!leaseRow) throw new Error("runner_harness_backup_lease_missing");
|
||||
const stamp = createNativeHarnessBackupStamp({
|
||||
manifestPath: resolve(backup.root, "manifest.json"),
|
||||
normalizedSessionId: backup.manifest.normalizedSessionId,
|
||||
runnerInstanceId: backup.manifest.runnerInstanceId,
|
||||
completedAt: backup.manifest.completedAt,
|
||||
});
|
||||
const updated = await input.db
|
||||
.update(environmentLeases)
|
||||
.set({
|
||||
metadata: {
|
||||
...(leaseRow.metadata ?? {}),
|
||||
nativeHarnessBackup: stamp,
|
||||
},
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(environmentLeases.id, remoteTarget.leaseId))
|
||||
.returning({ id: environmentLeases.id })
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!updated) throw new Error("runner_harness_backup_lease_missing");
|
||||
};
|
||||
|
||||
const restoreVerifiedHarnessBackup = async () => {
|
||||
if (!remoteTarget || !remoteCommandRunner) {
|
||||
throw new Error("runner_harness_backup_unavailable");
|
||||
@@ -4319,6 +4365,12 @@ export function createRunnerdBackend(input: {
|
||||
) {
|
||||
throw new Error("runner_harness_state_mismatch");
|
||||
}
|
||||
// A deliberately non-reusable environment receives a fresh provider lease
|
||||
// for every turn. Stamp that new lease as soon as the verified host backup
|
||||
// has been restored so a later provider/bootstrap failure can still clean
|
||||
// the ephemeral sandbox up without discarding the only durable copy.
|
||||
await recordHarnessBackupStampForCurrentLease(backup);
|
||||
return backup;
|
||||
};
|
||||
|
||||
const materializeRemoteHarnessLaunchState = async () => {
|
||||
@@ -4390,6 +4442,17 @@ export function createRunnerdBackend(input: {
|
||||
remoteCommandRunner
|
||||
) {
|
||||
const acquisitionRecordedAtMs = Date.now();
|
||||
const backupAvailable = harnessBackupCandidates(root).some(
|
||||
(candidate) => existsSync(resolve(candidate, "manifest.json")),
|
||||
);
|
||||
const restoreIntoCreatedSandbox =
|
||||
shouldRestoreNativeHarnessBackupIntoSandbox({
|
||||
acquisitionOutcome:
|
||||
sandboxLeaseAcquisition?.outcome ?? null,
|
||||
reusableLeaseConfigured:
|
||||
remoteTarget.reusableLeaseConfigured,
|
||||
backupAvailable,
|
||||
});
|
||||
await input.trace?.record({
|
||||
name: "sandbox.lease.acquisition",
|
||||
startedAtMs: acquisitionRecordedAtMs,
|
||||
@@ -4400,7 +4463,8 @@ export function createRunnerdBackend(input: {
|
||||
lifecycleMode: input.execution.session.lifecyclePolicy.mode,
|
||||
outcome: sandboxLeaseAcquisition?.outcome ?? "unknown",
|
||||
stateSource:
|
||||
sandboxLeaseAcquisition?.outcome === "replacement"
|
||||
sandboxLeaseAcquisition?.outcome === "replacement" ||
|
||||
restoreIntoCreatedSandbox
|
||||
? "verified_failover_backup"
|
||||
: sandboxLeaseAcquisition?.outcome === "resumed"
|
||||
? "sandbox_filesystem"
|
||||
@@ -4447,6 +4511,22 @@ export function createRunnerdBackend(input: {
|
||||
},
|
||||
},
|
||||
);
|
||||
} else if (restoreIntoCreatedSandbox) {
|
||||
await measureNativeRunnerSpan(
|
||||
input.trace,
|
||||
"sandbox.lease.replacement",
|
||||
restoreVerifiedHarnessBackup,
|
||||
{
|
||||
attributes: {
|
||||
provider: remoteTarget.providerKey ?? "sandbox",
|
||||
harness: input.execution.session.driverKind,
|
||||
lifecycleMode:
|
||||
input.execution.session.lifecyclePolicy.mode,
|
||||
outcome: "created",
|
||||
reason: "reuse_disabled",
|
||||
},
|
||||
},
|
||||
);
|
||||
} else {
|
||||
const reuseStartedAtMs = Date.now();
|
||||
const state = await inspectRemoteHarnessState();
|
||||
@@ -4457,11 +4537,7 @@ export function createRunnerdBackend(input: {
|
||||
state.runnerState,
|
||||
reuseStartedAtMs,
|
||||
);
|
||||
} else if (
|
||||
harnessBackupCandidates(root).some((candidate) =>
|
||||
existsSync(resolve(candidate, "manifest.json")),
|
||||
)
|
||||
) {
|
||||
} else if (backupAvailable) {
|
||||
// A continuation that has a durable backup but no recorded reusable
|
||||
// lease was not provider-confirmed lost. Never silently create a new
|
||||
// provider session from that ambiguous state.
|
||||
@@ -4689,34 +4765,11 @@ export function createRunnerdBackend(input: {
|
||||
remoteTarget?.transport === "sandbox" &&
|
||||
remoteTarget.leaseId
|
||||
) {
|
||||
const leaseRow = await input.db
|
||||
.select({ metadata: environmentLeases.metadata })
|
||||
.from(environmentLeases)
|
||||
.where(eq(environmentLeases.id, remoteTarget.leaseId))
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!leaseRow)
|
||||
throw new Error("runner_harness_backup_lease_missing");
|
||||
const stamp = createNativeHarnessBackupStamp({
|
||||
manifestPath: resolve(currentRoot, "manifest.json"),
|
||||
normalizedSessionId: manifest.normalizedSessionId,
|
||||
runnerInstanceId: manifest.runnerInstanceId,
|
||||
completedAt: manifest.completedAt,
|
||||
await recordHarnessBackupStampForCurrentLease({
|
||||
root: currentRoot,
|
||||
manifest,
|
||||
bytes: backupSpanAttributes.bytesTransferred,
|
||||
});
|
||||
const updated = await input.db
|
||||
.update(environmentLeases)
|
||||
.set({
|
||||
metadata: {
|
||||
...(leaseRow.metadata ?? {}),
|
||||
nativeHarnessBackup: stamp,
|
||||
},
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(environmentLeases.id, remoteTarget.leaseId))
|
||||
.returning({ id: environmentLeases.id })
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!updated)
|
||||
throw new Error("runner_harness_backup_lease_missing");
|
||||
}
|
||||
} catch (error) {
|
||||
if (existsSync(currentRoot)) {
|
||||
|
||||
Reference in new issue
Block a user