fix(runner): persist warm Daytona workspaces

This commit is contained in:
Dotta committed 2026-09-05 10:52:22 -05:00
1 parent 3da58b185e
commit ee11cd65d9
44 files changed
+4019 -359

No files matched your search

@@ -561,6 +561,18 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
mode: "shared_workspace",
},
});
const workspaceSyncStamp = {
schema: "paperclip.native-workspace-stamp/v1",
workspaceId: seeded.executionWorkspaceId,
providerLeaseId: "sandbox-exact-resume",
remoteCwd: "/workspace",
hostSha256: "a".repeat(64),
finalizedRunId: seeded.runId,
};
await environmentService(db).updateLeaseMetadata(first.lease.id, {
...(first.lease.metadata ?? {}),
nativeWorkspaceSync: workspaceSyncStamp,
});
await runtimeWithPlugin.releaseRunLeases(
seeded.runId,
"released",
@@ -592,9 +604,17 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" });
expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume");
expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed" });
expect(workerManager.call.mock.calls.filter((call) => call[1] === "environmentAcquireLease"))
.toHaveLength(1);
expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({
outcome: "resumed",
});
expect(acquired.lease.metadata?.nativeWorkspaceSync).toEqual(
workspaceSyncStamp,
);
expect(
workerManager.call.mock.calls.filter(
(call) => call[1] === "environmentAcquireLease",
),
).toHaveLength(1);
});
it("destroys a disposable paperclip_runner sandbox after the turn", async () => {
@@ -1,5 +1,9 @@
import { describe, expect, it } from "vitest";
import { resolveNativeSandboxLifecycle } from "../services/heartbeat.js";
import {
providerResourceDispositionForTerminalRun,
resolveNativeSandboxLifecycle,
resolveReusableSandboxLifecycle,
} from "../services/heartbeat.js";
const reusableSandbox = {
kind: "remote" as const,
@@ -21,6 +25,19 @@ describe("paperclip_runner sandbox lifecycle", () => {
});
});
it("keeps the same warm reusable sandbox for a legacy adapter", () => {
expect(
resolveReusableSandboxLifecycle({
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 },
target: reusableSandbox,
}),
).toEqual({
runnerProcess: "warm",
sandboxResource: "keep_running",
failoverBackup: "verified",
});
});
it("stops and reuses a per-turn reusable sandbox", () => {
expect(resolveNativeSandboxLifecycle({
adapterType: "paperclip_runner",
@@ -71,4 +88,22 @@ describe("paperclip_runner sandbox lifecycle", () => {
target: { kind: "local" },
})).toBeNull();
});
it("keeps a warm sandbox only after a successful turn", () => {
expect(
providerResourceDispositionForTerminalRun("keep_running", "succeeded"),
).toBe("keep_running");
expect(
providerResourceDispositionForTerminalRun("keep_running", "failed"),
).toBe("stop_and_retain");
expect(
providerResourceDispositionForTerminalRun("keep_running", "cancelled"),
).toBe("stop_and_retain");
expect(
providerResourceDispositionForTerminalRun("keep_running", "timed_out"),
).toBe("stop_and_retain");
expect(providerResourceDispositionForTerminalRun("destroy", "failed")).toBe(
"destroy",
);
});
});
@@ -0,0 +1,189 @@
import { mkdtemp, readdir, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
directorySnapshotSha256,
serializeDirectorySnapshot,
} from "@paperclipai/adapter-utils/workspace-restore-merge";
import {
classifyNativeWorkspaceInbound,
nativeWorkspaceSyncInternals,
readNativeWorkspaceSyncReference,
} from "../services/native-runtime/native-workspace-sync.js";
const digest = "a".repeat(64);
describe("native workspace sync durable metadata", () => {
const originalPaperclipHome = process.env.PAPERCLIP_HOME;
const originalPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
const cleanupDirs: string[] = [];
afterEach(async () => {
if (originalPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = originalPaperclipHome;
if (originalPaperclipInstanceId === undefined)
delete process.env.PAPERCLIP_INSTANCE_ID;
else process.env.PAPERCLIP_INSTANCE_ID = originalPaperclipInstanceId;
await Promise.all(
cleanupDirs.splice(0).map((directory) =>
rm(directory, { recursive: true, force: true }),
),
);
});
it("classifies fresh, warm, replacement, and same-run recovery inputs", () => {
expect(
classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition: "created",
hasPriorStamp: false,
}),
).toBe("host_current");
expect(
classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition: "resumed",
hasPriorStamp: true,
}),
).toBe("adopt_remote");
expect(
classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition: "resumed",
hasPriorStamp: false,
}),
).toBe("host_current");
expect(
classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition: "replacement",
hasPriorStamp: true,
}),
).toBe("host_current");
expect(
classifyNativeWorkspaceInbound({
kind: "existing_run",
restartRecovery: true,
sameProviderLease: true,
}),
).toBe("adopt_remote");
expect(
classifyNativeWorkspaceInbound({
kind: "existing_run",
restartRecovery: true,
sameProviderLease: false,
}),
).toBe("durable_seed");
expect(() =>
classifyNativeWorkspaceInbound({
kind: "existing_run",
restartRecovery: false,
sameProviderLease: true,
}),
).toThrow("native_workspace_sync_unexpected_existing_descriptor");
});
it("reads backward-compatible references and the resource disposition", () => {
const base = {
schema: "paperclip.native-workspace-sync/v1",
state: "prepared",
descriptorSha256: digest,
baselineSha256: digest,
finalHostSha256: null,
workspaceId: "workspace-1",
leaseId: "lease-1",
providerLeaseId: "sandbox-1",
remoteCwd: "/workspace",
};
expect(readNativeWorkspaceSyncReference(base)).toEqual({
...base,
resourceDisposition: null,
});
expect(
readNativeWorkspaceSyncReference({
...base,
resourceDisposition: "keep_running",
}),
).toEqual({ ...base, resourceDisposition: "keep_running" });
expect(
readNativeWorkspaceSyncReference({
...base,
resourceDisposition: "delete_everything",
}),
).toBeNull();
});
it("rejects traversal before constructing a durable state path", () => {
expect(() =>
nativeWorkspaceSyncInternals.descriptorPath("../run", digest),
).toThrow("native_workspace_sync_invalid_run_id");
expect(() =>
nativeWorkspaceSyncInternals.descriptorPath("run-1", "../descriptor"),
).toThrow("native_workspace_sync_descriptor_digest_invalid");
});
it("writes one immutable descriptor when the same state is replayed", async () => {
const paperclipHome = await mkdtemp(
path.join(os.tmpdir(), "paperclip-native-workspace-sync-"),
);
cleanupDirs.push(paperclipHome);
process.env.PAPERCLIP_HOME = paperclipHome;
process.env.PAPERCLIP_INSTANCE_ID = "descriptor-test";
const baseline = {
exclude: [".paperclip-runtime"],
entries: new Map([
["continuity.txt", { kind: "file" as const, mode: 0o644, hash: digest }],
]),
};
const descriptor = {
schema: "paperclip.native-workspace-sync/v1" as const,
binding: {
runId: "run-idempotent",
companyId: "company-1",
workspaceId: "workspace-1",
leaseId: "lease-1",
providerLeaseId: "sandbox-1",
localCwd: path.join(paperclipHome, "workspace"),
remoteCwd: "/workspace",
},
state: "prepared" as const,
baselineSha256: directorySnapshotSha256(baseline),
baseline: serializeDirectorySnapshot(baseline),
gitSnapshot: null,
seed: null,
createdAt: "2026-01-01T00:00:00.000Z",
finalizedAt: null,
finalHostSha256: null,
resourceDisposition: "keep_running" as const,
};
const first = await nativeWorkspaceSyncInternals.writeDescriptor(
descriptor,
);
const second = await nativeWorkspaceSyncInternals.writeDescriptor(
descriptor,
);
expect(second).toEqual(first);
const files = await readdir(
path.dirname(
nativeWorkspaceSyncInternals.descriptorPath(
descriptor.binding.runId,
first.descriptorSha256,
),
),
);
expect(files.filter((file) => file.endsWith(".json"))).toEqual([
`descriptor.${first.descriptorSha256}.json`,
]);
await expect(
nativeWorkspaceSyncInternals.readDescriptor({
runId: descriptor.binding.runId,
reference: first,
}),
).resolves.toMatchObject({ descriptor });
});
});
@@ -2060,6 +2060,11 @@ function createSandboxEnvironmentDriver(
...(reusableLease?.metadata?.nativeHarnessBackup
? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }
: {}),
...(providerLease && reusableLease?.metadata?.nativeWorkspaceSync
? {
nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync,
}
: {}),
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
};
try {
@@ -2282,6 +2287,13 @@ function createSandboxEnvironmentDriver(
...(reusableLease?.metadata?.nativeHarnessBackup
? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }
: {}),
...(reusableLease &&
providerLease.providerLeaseId === reusableLease.providerLeaseId &&
reusableLease.metadata?.nativeWorkspaceSync
? {
nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync,
}
: {}),
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
};
try {
@@ -3154,6 +3166,7 @@ const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set([
"sandboxProviderPlugin",
"sandboxLeaseAcquisition",
"nativeHarnessBackup",
"nativeWorkspaceSync",
]);
// Drop the host-internal and per-lease runtime keys from a sandbox config
+248 -22
View File
@@ -142,6 +142,9 @@ import {
materializeLegacyQuestionResponseWakeProjection,
materializeNativeInteractionResponses,
NativeCancellationPendingRecoveryError,
prepareNativeWorkspaceSync,
readNativeWorkspaceSyncReference,
recordNativeFinalizationFailure,
type NativeRestartRecoveryClaim,
rebindNativeSessionCheckpoint,
reconcileNativeFinalizations,
@@ -1746,12 +1749,53 @@ export function leaseReleaseStatusForRunStatus(
return status === "failed" || status === "timed_out" ? "failed" : "released";
}
export function providerResourceDispositionForTerminalRun(
desired: ProviderResourceDisposition | undefined,
status: string | null | undefined,
): ProviderResourceDisposition | undefined {
return desired === "keep_running" && status !== "succeeded"
? "stop_and_retain"
: desired;
}
export interface NativeSandboxLifecycle {
runnerProcess: "per_turn" | "warm";
sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
failoverBackup: "verified";
}
export function resolveReusableSandboxLifecycle(input: {
lifecyclePolicy:
| { mode: "per_turn"; idleTimeoutMs: null }
| { mode: "warm"; idleTimeoutMs: number };
target: {
kind: "local" | "remote";
transport?: string;
reusableLeaseConfigured?: boolean;
effectiveCapabilities?: { reusableLeases: boolean } | null;
} | null;
}): NativeSandboxLifecycle | null {
if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") {
return null;
}
const reusableLease =
input.target.reusableLeaseConfigured === true &&
input.target.effectiveCapabilities?.reusableLeases === true;
if (input.lifecyclePolicy.mode === "warm" && !reusableLease) {
throw new Error("runner_warm_lifecycle_requires_reusable_provider_lease");
}
return {
runnerProcess: input.lifecyclePolicy.mode,
sandboxResource:
input.lifecyclePolicy.mode === "warm"
? "keep_running"
: reusableLease
? "stop_and_reuse"
: "destroy_after_turn",
failoverBackup: "verified",
};
}
export function resolveNativeSandboxLifecycle(input: {
adapterType: string;
lifecyclePolicy:
@@ -1770,22 +1814,7 @@ export function resolveNativeSandboxLifecycle(input: {
input.target.transport !== "sandbox"
)
return null;
const reusableLease =
input.target.reusableLeaseConfigured === true &&
input.target.effectiveCapabilities?.reusableLeases === true;
if (input.lifecyclePolicy.mode === "warm" && !reusableLease) {
throw new Error("runner_warm_lifecycle_requires_reusable_provider_lease");
}
return {
runnerProcess: input.lifecyclePolicy.mode,
sandboxResource:
input.lifecyclePolicy.mode === "warm"
? "keep_running"
: reusableLease
? "stop_and_reuse"
: "destroy_after_turn",
failoverBackup: "verified",
};
return resolveReusableSandboxLifecycle(input);
}
export function applyPersistedExecutionWorkspaceConfig(input: {
@@ -8170,6 +8199,18 @@ class NativeSessionResumeScheduledError extends Error {
}
}
class NativeWorkspaceFinalizeScheduledError extends Error {
constructor(
readonly original: unknown,
readonly terminalFailure: boolean,
readonly reasonCode:
"workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",
) {
super("Native workspace finalization recovery has been scheduled.");
this.name = "NativeWorkspaceFinalizeScheduledError";
}
}
type WorkspaceReadyCommentWriter = {
addComment: (
issueId: string,
@@ -12945,7 +12986,10 @@ export function heartbeatService(
// A result committed before the old controller stopped outranks process
// recovery. Finish its durable workspace/status suffix before deciding
// whether any provider authority needs to be reopened.
await reconcileNativeFinalizations(db);
await reconcileNativeFinalizations(db, undefined, {
environmentRuntime,
onWorkspaceSettled: settleRecoveredNativeWorkspace,
});
const intent = await readHotRestartIntent().catch((error) => {
logger.warn(
{ err: error },
@@ -16854,6 +16898,41 @@ export function heartbeatService(
return blocked;
}
async function settleRecoveredNativeWorkspace(input: {
runId: string;
companyId: string;
agentId: string;
succeeded: boolean;
}) {
const settledRun = await getRun(input.runId);
const workspaceSyncReference = readNativeWorkspaceSyncReference(
parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync,
);
await releaseEnvironmentLeasesForRun({
runId: input.runId,
companyId: input.companyId,
agentId: input.agentId,
status: settledRun?.status,
failureReason: settledRun?.error ?? undefined,
providerResourceDisposition: input.succeeded
? (workspaceSyncReference?.resourceDisposition ?? "stop_and_retain")
: "stop_and_retain",
});
await releaseRuntimeServicesForRun(input.runId).catch(() => undefined);
await finalizeAgentStatus(
input.agentId,
input.succeeded ? "succeeded" : "failed",
input.succeeded
? null
: (settledRun?.error ?? "native_workspace_sync_out_failed"),
{
wasFirstHeartbeat: settledRun
? timerClaimWasFirstHeartbeat(settledRun)
: undefined,
},
).catch(() => undefined);
}
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) {
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
const now = new Date();
@@ -16861,7 +16940,10 @@ export function heartbeatService(
// Complete persisted native results before generic orphan recovery. The
// reconciler reads the durable workspace barrier and persisted runtime
// mode, never the current feature flag.
await reconcileNativeFinalizations(db).catch((error) => {
await reconcileNativeFinalizations(db, undefined, {
environmentRuntime,
onWorkspaceSettled: settleRecoveredNativeWorkspace,
}).catch((error) => {
logger.warn(
{ err: error },
"failed to reconcile persisted native finalizations before orphan reaping",
@@ -17876,6 +17958,10 @@ export function heartbeatService(
activeRunExecutions.add(run.id);
let runScratch: HeartbeatRunScratch | null = null;
let nativeSessionResumeScheduled = false;
let nativeWorkspaceFinalizeScheduled = false;
let nativeWorkspaceSync: Awaited<
ReturnType<typeof prepareNativeWorkspaceSync>
> = null;
let providerResourceDispositionForRun:
ProviderResourceDisposition | undefined;
let nativeLifecycleTelemetryForRun:
@@ -19659,6 +19745,9 @@ export function heartbeatService(
driver: selectedEnvironment.driver,
leaseId: activeEnvironmentLease.lease.id,
workspaceRealization,
sandboxLeaseAcquisition:
activeEnvironmentLease.lease.metadata?.sandboxLeaseAcquisition ??
null,
...(typeof activeEnvironmentLease.lease.metadata?.remoteCwd === "string"
? {
remoteCwd: activeEnvironmentLease.lease.metadata.remoteCwd,
@@ -20363,6 +20452,9 @@ export function heartbeatService(
agentId: heartbeatRuns.agentId,
runnerInstanceId: heartbeatRuns.runnerInstanceId,
nativeSessionId: heartbeatRuns.nativeSessionId,
processPid: heartbeatRuns.processPid,
processGroupId: heartbeatRuns.processGroupId,
processStartedAt: heartbeatRuns.processStartedAt,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
})
.from(heartbeatRuns)
@@ -20718,6 +20810,21 @@ export function heartbeatService(
? previousNativeRun.runnerInstanceId
: (lockedRun.runnerInstanceId ?? nativeRunnerInstanceId),
nativeSessionId: lockedRun.nativeSessionId ?? nativeSessionId,
processPid:
lockedRun.processPid ??
(previousNativeRun?.nativeSessionId === nativeSessionId
? previousNativeRun.processPid
: null),
processGroupId:
lockedRun.processGroupId ??
(previousNativeRun?.nativeSessionId === nativeSessionId
? previousNativeRun.processGroupId
: null),
processStartedAt:
lockedRun.processStartedAt ??
(previousNativeRun?.nativeSessionId === nativeSessionId
? previousNativeRun.processStartedAt
: null),
nativeIssueId: lockedRun.nativeIssueId ?? issueRef.id,
driverKind:
lockedRun.driverKind ??
@@ -20745,7 +20852,30 @@ export function heartbeatService(
})
.onConflictDoNothing();
});
nativeWorkspaceSync = await prepareNativeWorkspaceSync({
db,
runId: run.id,
companyId: agent.companyId,
workspaceId: nativeExecutionWorkspaceId,
workspaceLocalDir: executionWorkspace.cwd,
target: executionTarget,
lease: activeEnvironmentLease.lease,
restartRecovery: runOptions.nativeRestartRecovery,
resourceDisposition: providerResourceDispositionForRun,
});
} else {
const legacyWarmLifecycle =
executionTarget?.kind === "remote" &&
executionTarget.transport === "sandbox" &&
executionTarget.runnerLifecyclePolicy?.mode === "warm"
? resolveReusableSandboxLifecycle({
lifecyclePolicy: executionTarget.runnerLifecyclePolicy,
target: executionTarget,
})
: null;
if (legacyWarmLifecycle?.sandboxResource === "keep_running") {
providerResourceDispositionForRun = "keep_running";
}
await db
.update(heartbeatRuns)
.set({
@@ -21315,19 +21445,26 @@ export function heartbeatService(
// If recording the barrier itself fails, propagate as a run failure
// rather than silently leaving dependents stranded behind a missing
// finalize row.
if (nativeWorkspaceSync) {
await nativeWorkspaceSync.restoreWorkspace();
}
await recordWorkspaceFinalize("succeeded");
if (adapterResult.nativeFinalization) {
adapterResult.nativeFinalization.workspaceFinalizeStatus =
"succeeded";
try {
await finalizeNativeRun({
const finalized = await finalizeNativeRun({
db,
runId: run.id,
workspaceFinalizeStatus: "succeeded",
preserveProviderAttempt: Boolean(nativeWorkspaceSync),
});
await dispatchPendingNativeStatusWakeups({
companyId: run.companyId,
});
if (finalized.phase === "committed") {
await nativeWorkspaceSync?.cleanup();
}
} catch (finalizeErr) {
logger.warn(
{ err: finalizeErr, runId: run.id },
@@ -21379,6 +21516,40 @@ export function heartbeatService(
);
}
if (nativeRuntimeResolution.kind === "native") {
const proposedResult = await db
.select({ resultId: nativeRunFinalizations.resultId })
.from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, run.id))
.limit(1)
.then((rows) => rows[0]?.resultId ?? null);
if (proposedResult && nativeWorkspaceSync) {
const workspaceFailureMessage =
adapterErr instanceof Error ? adapterErr.message : "";
const unrecoverable =
workspaceFailureMessage ===
"workspace_sync_out_unrecoverable" ||
workspaceFailureMessage.includes("daytona_sandbox_not_found");
const failure = await recordNativeFinalizationFailure({
db,
runId: run.id,
error: new Error(
unrecoverable
? "native_workspace_sync_out_unrecoverable"
: "native_workspace_sync_out_failed",
),
projectRunStatus: true,
failureScope: "workspace",
permanent: unrecoverable,
});
nativeWorkspaceFinalizeScheduled = true;
throw new NativeWorkspaceFinalizeScheduledError(
adapterErr,
failure.phase === "terminal_failure",
unrecoverable
? "workspace_sync_out_unrecoverable"
: "workspace_sync_out_failed",
);
}
try {
await finalizeNativeRun({
db,
@@ -22064,6 +22235,45 @@ export function heartbeatService(
}
return;
}
if (err instanceof NativeWorkspaceFinalizeScheduledError) {
const coordinator = await db
.select({
nextAttemptAt: nativeRunFinalizations.nextAttemptAt,
attempt: nativeRunFinalizations.attempt,
})
.from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, run.id))
.limit(1)
.then((rows) => rows[0] ?? null);
await appendRunEvent(run, {
eventType: "lifecycle",
stream: "system",
level: err.terminalFailure ? "error" : "warn",
message: err.terminalFailure
? "native result is durable, but the sandbox containing unexported workspace changes is unrecoverable"
: "native result is durable; workspace copy-back will retry without another provider turn",
payload: {
attempt: coordinator?.attempt ?? null,
nextAttemptAt: coordinator?.nextAttemptAt?.toISOString() ?? null,
fallbackSuppressed: true,
retryReasonCode: err.reasonCode,
},
}).catch(() => undefined);
if (err.terminalFailure) {
// The durable coordinator already failed the run, blocked the
// issue, and cleared its execution lock. Let ordinary teardown
// release the now-useless lease and return the agent to service.
nativeWorkspaceFinalizeScheduled = false;
providerResourceDispositionForRun = "stop_and_retain";
await finalizeAgentStatus(
run.agentId,
"failed",
"native_workspace_sync_out_unrecoverable",
{ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) },
).catch(() => undefined);
}
return;
}
const message = redactCurrentUserText(
err instanceof Error ? err.message : "Unknown adapter failure",
await getCurrentUserRedactionOptions(),
@@ -22456,7 +22666,11 @@ export function heartbeatService(
// terminal". When the teardown reaches this point with the run still
// running or queued, force a terminal status before the lease is
// released, so the UI never shows a finished task as "Live".
if (latestRun && !nativeSessionResumeScheduled) {
if (
latestRun &&
!nativeSessionResumeScheduled &&
!nativeWorkspaceFinalizeScheduled
) {
latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch(
(terminalizeErr) => {
logger.error(
@@ -22467,7 +22681,15 @@ export function heartbeatService(
},
);
}
if (!nativeSessionResumeScheduled) {
// Warm retention is earned only by a fully successful turn. A failed,
// cancelled, or timed-out run stops the reusable sandbox so the next
// acquisition must revalidate and explicitly resume it.
providerResourceDispositionForRun =
providerResourceDispositionForTerminalRun(
providerResourceDispositionForRun,
latestRun?.status,
);
if (!nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled) {
await releaseEnvironmentLeasesForRun({
runId: run.id,
companyId: run.companyId,
@@ -22539,7 +22761,11 @@ export function heartbeatService(
}
}
activeRunExecutions.delete(run.id);
if (!nativeSessionResumeScheduled && !shutdownInProgress) {
if (
!nativeSessionResumeScheduled &&
!nativeWorkspaceFinalizeScheduled &&
!shutdownInProgress
) {
await startNextQueuedRunForAgent(run.agentId);
}
}
@@ -9,4 +9,5 @@ export * from "./paperclip-control-plane-port.js";
export * from "./native-run-finalizer.js";
export * from "./native-finalization-reconciler.js";
export * from "./native-restart-recovery.js";
export * from "./native-workspace-sync.js";
export * from "./status-arbiter.js";
@@ -24,6 +24,11 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { issueService } from "../issues.js";
import { emitAgentTaskRun } from "../agent-task-run-telemetry.js";
import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js";
import {
cleanupNativeWorkspaceSync,
readNativeWorkspaceSyncReference,
} from "./native-workspace-sync.js";
import type { EnvironmentRuntimeService } from "../environment-runtime.js";
import { classifyNativeEvidence } from "./evidence-classifier.js";
import { recordNativeWorkAssessment } from "./work-assessments.js";
import {
@@ -373,19 +378,33 @@ export async function claimNativeSessionResumptions(input: {
}
/** Recovery is keyed only by persisted mode/coordinator state, never the live flag. */
export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
const rows = await db.select({
runId: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
issueId: nativeRunFinalizations.issueId,
issueStatus: issues.status,
issueStatusVersion: issues.statusVersion,
issueDecisionId: issues.lastStatusDecisionId,
coordinatorPhase: nativeRunFinalizations.phase,
assessmentId: nativeRunFinalizations.assessmentId,
decisionId: nativeRunFinalizations.decisionId,
})
export async function reconcileNativeFinalizations(
db: Db,
runIds?: string[],
options: {
environmentRuntime?: EnvironmentRuntimeService;
onWorkspaceSettled?: (input: {
runId: string;
companyId: string;
agentId: string;
succeeded: boolean;
}) => Promise<void>;
} = {},
) {
const rows = await db
.select({
runId: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
issueId: nativeRunFinalizations.issueId,
issueStatus: issues.status,
issueStatusVersion: issues.statusVersion,
issueDecisionId: issues.lastStatusDecisionId,
coordinatorPhase: nativeRunFinalizations.phase,
assessmentId: nativeRunFinalizations.assessmentId,
decisionId: nativeRunFinalizations.decisionId,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
})
.from(heartbeatRuns)
.innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id))
.innerJoin(issues, and(
@@ -617,14 +636,59 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
if (!recoveryDecision.effects.some((effect) => effect.kind === "resume_workspace_operation")) {
throw new Error("native_reconciliation_workspace_resume_policy_missing");
}
const operation = await resumeNativeWorkspaceFinalization({ db, runId: row.runId });
const workspaceFinalizeStatus = operation.status === "succeeded" ? "succeeded" : "failed";
const operation = await resumeNativeWorkspaceFinalization({
db,
runId: row.runId,
environmentRuntime: options.environmentRuntime,
});
const workspaceFinalizeStatus =
operation.status === "succeeded" ? "succeeded" : "failed";
if (workspaceFinalizeStatus === "failed") {
const unrecoverable = operation.stderrExcerpt?.includes(
"workspace_sync_out_unrecoverable",
);
const failure = await recordNativeFinalizationFailure({
db,
runId: row.runId,
error: new Error(
unrecoverable
? "native_workspace_sync_out_unrecoverable"
: "native_workspace_sync_out_failed",
),
projectRunStatus: true,
failureScope: "workspace",
permanent: unrecoverable,
});
results.push({
...failure,
reconciliationAction: "resume_workspace_operation" as const,
workspaceOperationId: operation.id,
workspaceFinalizeStatus,
reconciliationDecision: recoveryDecision,
});
if (failure.phase === "terminal_failure") {
await options
.onWorkspaceSettled?.({
runId: row.runId,
companyId: row.companyId,
agentId: row.agentId,
succeeded: false,
})
.catch(() => undefined);
}
continue;
}
try {
const finalized = await finalizeNativeRun({
db,
runId: row.runId,
workspaceFinalizeStatus,
projectRunStatus: true,
preserveProviderAttempt: Boolean(
readNativeWorkspaceSyncReference(
record(row.runnerProfileJson).nativeWorkspaceSync,
),
),
});
results.push({
...finalized,
@@ -633,6 +697,20 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
workspaceFinalizeStatus,
reconciliationDecision: recoveryDecision,
});
if (
workspaceFinalizeStatus === "succeeded" &&
finalized.phase === "committed"
) {
await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined);
await options
.onWorkspaceSettled?.({
runId: row.runId,
companyId: row.companyId,
agentId: row.agentId,
succeeded: true,
})
.catch(() => undefined);
}
} catch (error) {
const failure = await recordNativeFinalizationFailure({
db,
@@ -659,12 +737,29 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
throw new Error("native_reconciliation_replay_policy_invalid");
}
try {
results.push(await finalizeNativeRun({
const finalized = await finalizeNativeRun({
db,
runId: row.runId,
workspaceFinalizeStatus: barrier.status as "succeeded" | "failed",
projectRunStatus: true,
}));
preserveProviderAttempt: Boolean(
readNativeWorkspaceSyncReference(
record(row.runnerProfileJson).nativeWorkspaceSync,
),
),
});
results.push(finalized);
if (finalized.phase === "committed") {
await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined);
await options
.onWorkspaceSettled?.({
runId: row.runId,
companyId: row.companyId,
agentId: row.agentId,
succeeded: true,
})
.catch(() => undefined);
}
} catch (error) {
results.push(await recordNativeFinalizationFailure({
db,
@@ -279,6 +279,93 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a
expect(run?.status).toBe("running");
});
it("bounds workspace-only retries without consuming the provider attempt", async () => {
const fixture = await seedNativeRun();
await db.insert(nativeRunFinalizations).values({
runId: fixture.runId,
companyId,
issueId: fixture.issueId,
phase: "observed",
attempt: 1,
});
for (
let expectedAttempt = 1;
expectedAttempt <= 3;
expectedAttempt += 1
) {
await recordNativeFinalizationFailure({
db,
runId: fixture.runId,
error: new Error("native_workspace_sync_out_failed"),
projectRunStatus: true,
failureScope: "workspace",
});
const coordinator = await db
.select()
.from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, fixture.runId))
.then((rows) => rows[0]!);
expect(coordinator.attempt).toBe(1);
expect(coordinator.failureDetail).toMatchObject({
workspaceFinalizeAttempt: expectedAttempt,
});
expect(coordinator.phase).toBe(
expectedAttempt === 3 ? "terminal_failure" : "retryable_failure",
);
}
await expect(
db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, fixture.runId))
.then((rows) => rows[0]?.status),
).resolves.toBe("failed");
});
it("blocks immediately when the sandbox with unexported changes is gone", async () => {
const fixture = await seedNativeRun();
await db.insert(nativeRunFinalizations).values({
runId: fixture.runId,
companyId,
issueId: fixture.issueId,
phase: "workspace_finalizing",
attempt: 1,
resultId: null,
});
const failure = await recordNativeFinalizationFailure({
db,
runId: fixture.runId,
error: new Error("native_workspace_sync_out_unrecoverable"),
projectRunStatus: true,
failureScope: "workspace",
permanent: true,
});
expect(failure).toMatchObject({
phase: "terminal_failure",
failureCode: "native_workspace_sync_out_unrecoverable",
nextAttemptAt: null,
attempt: 1,
});
await expect(
db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, fixture.runId))
.then((rows) => rows[0]?.status),
).resolves.toBe("failed");
await expect(
db
.select({ status: issues.status })
.from(issues)
.where(eq(issues.id, fixture.issueId))
.then((rows) => rows[0]?.status),
).resolves.toBe("blocked");
});
it("emits exactly one event for a cancel_continuations write (trap 2: :685/:518 overlap)", async () => {
const fixture = await seedNativeRun();
// Build the minimal real rows commitNativeStatusDecision's foreign keys
@@ -329,7 +416,10 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a
toStatus: "cancelled",
reasonCode: "cancellation_issue_authorized",
unblockDescriptor: null,
effects: [{ kind: "release_checkout" }, { kind: "cancel_continuations" }],
effects: [
{ kind: "release_checkout" },
{ kind: "cancel_continuations" },
],
};
const callsBefore = mockTelemetryClient.track.mock.calls.length;
@@ -126,7 +126,11 @@ async function acceptedInteractionFromRun(input: {
.then((rows) => rows[0] ?? null);
}
async function claimCoordinator(input: { db: Db; runId: string }) {
async function claimCoordinator(input: {
db: Db;
runId: string;
preserveProviderAttempt?: boolean;
}) {
const leaseOwner = `native-finalizer:${randomUUID()}`;
const now = new Date();
const claimed = await input.db.transaction(async (tx) => {
@@ -152,23 +156,33 @@ async function claimCoordinator(input: { db: Db; runId: string }) {
}
if (coordinator.phase === "terminal_failure") throw new Error("native_finalization_terminal_failure");
if (
coordinator.leaseOwner
&& coordinator.leaseExpiresAt
&& coordinator.leaseExpiresAt > now
&& coordinator.leaseOwner !== leaseOwner
) throw new Error("native_finalization_lease_busy");
const [updated] = await tx.update(nativeRunFinalizations).set({
leaseOwner,
leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),
attempt: coordinator.attempt + 1,
phase: coordinator.phase === "retryable_failure"
? coordinator.assessmentId ? "arbitrating" : "workspace_finalizing"
: coordinator.phase,
failureCode: null,
failureDetail: null,
nextAttemptAt: null,
updatedAt: now,
}).where(eq(nativeRunFinalizations.runId, input.runId)).returning();
coordinator.leaseOwner &&
coordinator.leaseExpiresAt &&
coordinator.leaseExpiresAt > now &&
coordinator.leaseOwner !== leaseOwner
)
throw new Error("native_finalization_lease_busy");
const [updated] = await tx
.update(nativeRunFinalizations)
.set({
leaseOwner,
leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),
attempt: input.preserveProviderAttempt
? coordinator.attempt
: coordinator.attempt + 1,
phase:
coordinator.phase === "retryable_failure"
? coordinator.assessmentId
? "arbitrating"
: "workspace_finalizing"
: coordinator.phase,
failureCode: null,
failureDetail: null,
nextAttemptAt: null,
updatedAt: now,
})
.where(eq(nativeRunFinalizations.runId, input.runId))
.returning();
if (!updated) throw new Error("native_finalization_claim_failed");
return { coordinator: updated, leaseOwner };
});
@@ -183,6 +197,8 @@ async function recordRetryableFailure(input: {
message: string;
nextAction: string;
projectRunStatus?: boolean;
failureScope?: "provider" | "workspace";
permanent?: boolean;
}) {
const now = new Date();
const nextAttemptAt = new Date(now.getTime() + 30_000);
@@ -228,50 +244,87 @@ async function recordRetryableFailure(input: {
const supersededByNewerRun = Boolean(
latestDecisionRun && latestDecisionRun.createdAt > input.run.createdAt,
);
const exhausted = input.coordinator.attempt >= 3;
const phase = supersededByNewerRun || exhausted
? "terminal_failure" as const
: "retryable_failure" as const;
const priorFailureDetail = record(input.coordinator.failureDetail);
const workspaceFinalizeAttempt =
input.failureScope === "workspace"
? (typeof priorFailureDetail.workspaceFinalizeAttempt === "number" &&
Number.isInteger(priorFailureDetail.workspaceFinalizeAttempt) &&
priorFailureDetail.workspaceFinalizeAttempt >= 0
? priorFailureDetail.workspaceFinalizeAttempt
: 0) + 1
: null;
const exhausted =
input.permanent === true ||
(workspaceFinalizeAttempt !== null
? workspaceFinalizeAttempt >= 3
: input.coordinator.attempt >= 3);
const phase =
supersededByNewerRun || exhausted
? ("terminal_failure" as const)
: ("retryable_failure" as const);
const failureCode = supersededByNewerRun
? "native_finalization_superseded"
: exhausted
? "native_finalization_retry_exhausted"
: input.failureCode;
await tx.update(nativeRunFinalizations).set({
phase,
leaseOwner: null,
leaseExpiresAt: null,
failureCode,
failureDetail: {
message: input.message.slice(0, 2_000),
originalFailureCode: input.failureCode,
recoveryOwner: supersededByNewerRun
? { kind: "none", reason: "newer_native_decision" }
: exhausted
? { kind: "board" }
: { kind: "agent", agentId: input.run.agentId },
nextAction: input.nextAction,
},
nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt,
updatedAt: now,
}).where(eq(nativeRunFinalizations.runId, input.run.id));
const projectsTerminalStatus = exhausted && !supersededByNewerRun && input.projectRunStatus;
const [updatedRun] = await tx.update(heartbeatRuns).set({
...(projectsTerminalStatus ? {
status: exhaustedRunStatus,
finishedAt: input.run.finishedAt ?? now,
} : {}),
nativePhase: phase,
nativePhaseUpdatedAt: now,
resultJson: {
...record(input.run.resultJson),
finalizationPhase: phase,
: input.permanent
? input.failureCode
: exhausted
? input.failureScope === "workspace"
? "native_workspace_sync_out_retry_exhausted"
: "native_finalization_retry_exhausted"
: input.failureCode;
await tx
.update(nativeRunFinalizations)
.set({
phase,
leaseOwner: null,
leaseExpiresAt: null,
failureCode,
originalFailureCode: input.failureCode,
nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt.toISOString(),
},
updatedAt: now,
}).where(eq(heartbeatRuns.id, input.run.id)).returning();
failureDetail: {
message: input.message.slice(0, 2_000),
originalFailureCode: input.failureCode,
...(workspaceFinalizeAttempt === null
? {}
: { workspaceFinalizeAttempt }),
recoveryOwner: supersededByNewerRun
? { kind: "none", reason: "newer_native_decision" }
: exhausted
? { kind: "board" }
: { kind: "agent", agentId: input.run.agentId },
nextAction: input.nextAction,
},
nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt,
updatedAt: now,
})
.where(eq(nativeRunFinalizations.runId, input.run.id));
const projectsTerminalStatus =
exhausted && !supersededByNewerRun && input.projectRunStatus;
const [updatedRun] = await tx
.update(heartbeatRuns)
.set({
...(projectsTerminalStatus
? {
status:
input.failureScope === "workspace"
? "failed"
: exhaustedRunStatus,
finishedAt: input.run.finishedAt ?? now,
}
: {}),
nativePhase: phase,
nativePhaseUpdatedAt: now,
resultJson: {
...record(input.run.resultJson),
finalizationPhase: phase,
failureCode,
originalFailureCode: input.failureCode,
nextAttemptAt:
supersededByNewerRun || exhausted
? null
: nextAttemptAt.toISOString(),
},
updatedAt: now,
})
.where(eq(heartbeatRuns.id, input.run.id))
.returning();
if (projectsTerminalStatus) terminalRunToEmit = updatedRun ?? null;
if (supersededByNewerRun) {
await issueRecoveryActionService(tx as unknown as Db).resolveActiveForIssue({
@@ -284,9 +337,16 @@ async function recordRetryableFailure(input: {
resolutionNote: "A newer native run already committed the authoritative issue decision; the stale finalizer was retired without changing issue state.",
}, tx);
} else if (exhausted) {
await issueService(tx as unknown as Db).update(input.coordinator.issueId, {
status: "in_review",
}, tx);
await issueService(tx as unknown as Db).update(
input.coordinator.issueId,
{
status:
input.permanent && input.failureScope === "workspace"
? "blocked"
: "in_review",
},
tx,
);
}
if (!supersededByNewerRun) {
await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({
@@ -301,10 +361,15 @@ async function recordRetryableFailure(input: {
evidence: {
runId: input.run.id,
coordinatorAttempt: input.coordinator.attempt,
...(workspaceFinalizeAttempt === null
? {}
: { workspaceFinalizeAttempt }),
originalFailureCode: input.failureCode,
},
nextAction: exhausted
? `Finalization retry budget exhausted. ${input.nextAction}`
? input.permanent
? input.nextAction
: `Finalization retry budget exhausted. ${input.nextAction}`
: input.nextAction,
wakePolicy: exhausted
? null
@@ -328,6 +393,8 @@ export async function recordNativeFinalizationFailure(input: {
runId: string;
error: unknown;
projectRunStatus?: boolean;
failureScope?: "provider" | "workspace";
permanent?: boolean;
}) {
const [run, coordinator] = await Promise.all([
input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
@@ -344,8 +411,15 @@ export async function recordNativeFinalizationFailure(input: {
coordinator,
failureCode,
message,
nextAction: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",
nextAction:
input.failureScope === "workspace"
? input.permanent
? "Restore the exact sandbox containing the unexported workspace changes, or resolve the task manually from durable evidence."
: "Retry workspace export and merge from the retained sandbox; do not submit another provider turn."
: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",
projectRunStatus: input.projectRunStatus,
failureScope: input.failureScope,
permanent: input.permanent,
});
}
@@ -395,12 +469,23 @@ export async function finalizeNativeRun(input: {
workspaceFinalizeStatus: "succeeded" | "failed";
/** Reconciliation owns terminal run projection; the live heartbeat does it afterward. */
projectRunStatus?: boolean;
/** Workspace-only replay must not consume the provider recovery budget. */
preserveProviderAttempt?: boolean;
failpoint?: NativeStatusCommitFailpoint;
}) {
const run = await input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
.limit(1).then((rows) => rows[0] ?? null);
if (!run || run.runtimeMode !== "native") throw new Error("native_finalization_run_missing");
const claim = await claimCoordinator({ db: input.db, runId: input.runId });
const run = await input.db
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.runId))
.limit(1)
.then((rows) => rows[0] ?? null);
if (!run || run.runtimeMode !== "native")
throw new Error("native_finalization_run_missing");
const claim = await claimCoordinator({
db: input.db,
runId: input.runId,
preserveProviderAttempt: input.preserveProviderAttempt,
});
const coordinator = claim.coordinator;
if (!claim.leaseOwner && coordinator.phase === "committed") {
if (input.projectRunStatus) await projectCommittedRun({ db: input.db, run, coordinator });
@@ -2880,7 +2880,7 @@ describe("native warm session supervision", () => {
);
});
it("rehydrates a runnerd warm session from its checkpoint under a fresh run authority", async () => {
it("reattaches a live runnerd warm session under a fresh run authority", async () => {
const stateBase = await mkdtemp(
join(tmpdir(), "paperclip-runnerd-warm-authority-"),
);
@@ -2889,9 +2889,7 @@ describe("native warm session supervision", () => {
process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase;
process.env.PAPERCLIP_HOME = stateBase;
const firstClose = vi.fn(async () => undefined);
const secondClose = vi.fn(async () => undefined);
const firstSession = { close: firstClose };
const secondSession = { close: secondClose };
const first = {
...execution,
binding: {
@@ -2947,18 +2945,8 @@ describe("native warm session supervision", () => {
return result;
})
.mockImplementationOnce(async (options) => {
expect(options.existingSession).toBeUndefined();
expect(options.persistedSession).toEqual(
expect.objectContaining({
identity: expect.objectContaining({
runId: second.binding.runId,
sessionId: second.session.normalizedSessionId,
}),
providerSessionId: "provider-runnerd-warm",
activeTurnId: null,
}),
);
options.onSession?.(secondSession);
expect(options.existingSession).toBe(firstSession);
expect(options.persistedSession).toBeUndefined();
return result;
});
@@ -3014,10 +3002,10 @@ describe("native warm session supervision", () => {
runnerInstanceId: "runner-runnerd-warm",
useRunnerd: true,
});
expect(firstClose).toHaveBeenCalledWith({
reason: "warm native session authority epoch rotated",
});
await vi.waitFor(() => expect(secondClose).toHaveBeenCalled(), {
expect(firstClose).not.toHaveBeenCalled();
await vi.waitFor(() => expect(firstClose).toHaveBeenCalledWith({
reason: "warm native session idle timeout",
}), {
timeout: 500,
});
} finally {
@@ -5392,9 +5380,13 @@ describe("runnerd provider runtime wiring", () => {
const firstOptions = state.createBackend.mock.calls[0]![1];
const continuationOptions = state.createBackend.mock.calls[1]![1];
await expect(firstOptions.dynamicToolHandler!({})).rejects.toThrow(
"native_tool_authority_epoch_revoked",
);
// The retained runner backend owns one stable callback. After run.attach,
// that callback routes through the session-scope authority registry to
// the new run; stale provider calls are rejected earlier by runnerd's
// turn identity boundary.
await expect(firstOptions.dynamicToolHandler!({})).resolves.toEqual({
runId: continuation.binding.runId,
});
await expect(
continuationOptions.dynamicToolHandler!({}),
).resolves.toEqual({ runId: continuation.binding.runId });
@@ -2623,7 +2623,13 @@ export async function nativeProviderRecoveryEvidence(input: {
const durableEvents = await input.db
.select({ eventType: heartbeatRunEvents.eventType })
.from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, input.runId));
.where(
and(
eq(heartbeatRunEvents.runId, input.runId),
inArray(heartbeatRunEvents.eventType, [...PROVIDER_DURABLE_EVENT_TYPES]),
),
)
.limit(1);
const providerEventsExist = durableEvents.some((event) =>
PROVIDER_DURABLE_EVENT_TYPES.has(event.eventType),
);
@@ -4171,23 +4177,7 @@ async function executePaperclipNativeSessionWithinScope(
entry.busy = true;
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
entry.idleTimer = null;
if (input.useRunnerd) {
// A retained driver closes over the tool authority from the run that
// created it. Preserve provider continuity through its durable
// checkpoint, but rebuild the runnerd backend so the next run gets a
// fresh, immutable authority epoch. Reusing the live driver would
// either retain stale authority or require rebinding old callbacks.
warmNativeSessions.delete(warmSessionId);
await entry.session.close({
reason: "warm native session authority epoch rotated",
});
persistedWarmSession = loadWarmNativeCheckpoint(
input.execution,
warmConfigDigest,
);
} else {
existingWarmSession = entry.session;
}
existingWarmSession = entry.session;
}
} else {
persistedWarmSession = loadWarmNativeCheckpoint(
@@ -7235,6 +7225,13 @@ async function createRunnerdBackendWithinSessionClaim(
input.restartRecovery?.kind === "reattach_existing_runner"
? input.restartRecovery.process
: null;
const executeCurrentToolAuthority = (
call: Parameters<SessionToolAuthorityEpoch["execute"]>[0],
) => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current) throw new Error("native_session_tool_authority_unavailable");
return current.execute(call);
};
const backend = createNativeSessionBackend(runnerExecution, {
runnerInstanceId: input.runnerInstanceId,
environment: effectiveRunnerEnvironment,
@@ -7243,8 +7240,8 @@ async function createRunnerdBackendWithinSessionClaim(
: "local_filesystem",
onSpawn: input.onSpawn,
dynamicTools,
dynamicToolHandler: (call) => authorityEpoch.execute(call),
acpxDynamicToolHandler: (call) => authorityEpoch.execute(call),
dynamicToolHandler: executeCurrentToolAuthority,
acpxDynamicToolHandler: executeCurrentToolAuthority,
opencodeRuntimeDirectory: resolve(
resolvePaperclipInstanceRoot(),
"runtime",
@@ -7419,7 +7416,7 @@ async function createRunnerdBackendWithinSessionClaim(
turnId: `turn-${input.execution.binding.runId}`,
itemId: `item-${input.execution.binding.runId}`,
},
controlPlaneRegistration: (authority) =>
controlPlaneRegistration: (authority, attachmentIdentity) =>
measureNativeRunnerSpan(
input.trace,
"runner.transport.connect",
@@ -7446,7 +7443,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
registerRunnerPrpAuthority({
companyId: input.execution.binding.companyId,
runId: input.execution.binding.runId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
authority,
}),
);
@@ -7469,7 +7468,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
resolvePaperclipRunnerTransport({
target,
runId: input.execution.binding.runId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused",
runnerPublicUrl: input.runnerPublicUrl,
runnerCaBundlePath: input.runnerCaBundlePath,
@@ -7505,7 +7506,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
resolvePaperclipRunnerTransport({
target,
runId: input.execution.binding.runId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused",
runnerPublicUrl: input.runnerPublicUrl,
runnerCaBundlePath: input.runnerCaBundlePath,
@@ -7541,7 +7544,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
registerRunnerPrpAuthority({
companyId: input.execution.binding.companyId,
runId: input.execution.binding.runId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
authority,
}),
{ parentName: "runner.transport.connect" },
@@ -10,6 +10,13 @@ import {
} from "@paperclipai/db";
import { workspaceOperationService } from "../workspace-operations.js";
import { inspectManagedGitWorktreeBranch } from "../workspace-runtime.js";
import { environmentService } from "../environments.js";
import type { EnvironmentRuntimeService } from "../environment-runtime.js";
import { resolveEnvironmentExecutionTarget } from "../environment-execution-target.js";
import {
readNativeWorkspaceSyncReference,
resumeNativeWorkspaceSync,
} from "./native-workspace-sync.js";
function record(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
@@ -21,14 +28,27 @@ function readString(value: unknown) {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function workspaceSyncFailure(
code: "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",
) {
return {
status: "failed" as const,
exitCode: 1,
stderr: `${code}\n`,
metadata: { workspaceSync: { code } },
};
}
/**
* Resume the real workspace-finalization action for a result-bearing native
* run. This service observes the workspace; it never fabricates a successful
* marker. The caller decides whether a failed observation is retryable.
* run. Durable sandbox-backed runs export and merge the remote workspace;
* older runs retain the local workspace validation path. The caller owns the
* durable retry and failure policy.
*/
export async function resumeNativeWorkspaceFinalization(input: {
db: Db;
runId: string;
environmentRuntime?: EnvironmentRuntimeService;
}) {
const bound = await input.db.select({
companyId: heartbeatRuns.companyId,
@@ -57,7 +77,12 @@ export async function resumeNativeWorkspaceFinalization(input: {
eq(workspaceOperations.phase, "workspace_finalize"),
)).orderBy(desc(workspaceOperations.createdAt)).limit(1).then((rows) => rows[0] ?? null);
const persistedInput = record(record(bound.runnerProfileJson).nativeExecutionInput);
const persistedInput = record(
record(bound.runnerProfileJson).nativeExecutionInput,
);
const nativeWorkspaceSync = readNativeWorkspaceSyncReference(
record(bound.runnerProfileJson).nativeWorkspaceSync,
);
const binding = record(persistedInput.binding);
const workspaceId = previous?.executionWorkspaceId
?? bound.issueWorkspaceId
@@ -87,6 +112,88 @@ export async function resumeNativeWorkspaceFinalization(input: {
: "workspace_directory",
},
run: async () => {
if (nativeWorkspaceSync) {
if (!input.environmentRuntime) {
return {
status: "failed",
exitCode: 1,
stderr:
"Native workspace finalization cannot access the environment runtime.\n",
};
}
const environmentsSvc = environmentService(input.db);
const lease = await environmentsSvc.getLeaseById(
nativeWorkspaceSync.leaseId,
);
const environment = lease?.environmentId
? await environmentsSvc.getById(lease.environmentId)
: null;
if (
!lease ||
!environment ||
lease.companyId !== bound.companyId ||
environment.id !== lease.environmentId
) {
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
}
if (
lease.status === "expired" ||
lease.status === "failed" ||
lease.status === "pending_cleanup" ||
!lease.providerLeaseId ||
lease.providerLeaseId !== nativeWorkspaceSync.providerLeaseId
) {
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
}
const target = await resolveEnvironmentExecutionTarget({
db: input.db,
companyId: bound.companyId,
adapterType: "paperclip_runner",
environment,
leaseId: lease.id,
leaseMetadata: lease.metadata,
lease,
environmentRuntime: input.environmentRuntime,
});
if (!target) {
return {
status: "failed",
exitCode: 1,
stderr: "Native workspace finalization target is unavailable.\n",
};
}
try {
const restored = await resumeNativeWorkspaceSync({
db: input.db,
runId: input.runId,
target,
});
if (!restored) {
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
}
return {
status: "succeeded",
exitCode: 0,
system:
"Native workspace finalization restored the remote workspace.\n",
metadata: {
workspaceSync: {
schema: nativeWorkspaceSync.schema,
workspaceId: nativeWorkspaceSync.workspaceId,
leaseId: nativeWorkspaceSync.leaseId,
},
},
};
} catch (error) {
const code =
error instanceof Error &&
(error.message === "workspace_sync_out_unrecoverable" ||
error.message.includes("daytona_sandbox_not_found"))
? "workspace_sync_out_unrecoverable"
: "workspace_sync_out_failed";
return workspaceSyncFailure(code);
}
}
if (!cwd) {
return {
status: "failed",
@@ -0,0 +1,980 @@
import fs from "node:fs/promises";
import { createReadStream } from "node:fs";
import path from "node:path";
import { createHash, randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { environmentLeases, heartbeatRuns } from "@paperclipai/db";
import type { EnvironmentLease } from "@paperclipai/shared";
import {
prepareAdapterExecutionTargetRuntime,
type AdapterExecutionTarget,
type PreparedAdapterExecutionTargetRuntime,
} from "@paperclipai/adapter-utils/execution-target";
import type { GitWorkspaceSnapshot } from "@paperclipai/adapter-utils/git-workspace-sync";
import {
directorySnapshotSha256,
parseDirectorySnapshot,
serializeDirectorySnapshot,
type DirectorySnapshot,
type SerializedDirectorySnapshot,
} from "@paperclipai/adapter-utils/workspace-restore-merge";
import type {
WorkspaceDurableSeedPaths,
WorkspaceInboundMode,
} from "@paperclipai/adapter-utils/sandbox-managed-runtime";
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
import { parseObject } from "../../adapters/utils.js";
import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js";
const DESCRIPTOR_SCHEMA = "paperclip.native-workspace-sync/v1";
const STAMP_SCHEMA = "paperclip.native-workspace-stamp/v1";
const STATE_ROOT_NAME = "native-workspace-sync";
const DESCRIPTOR_NAME = "descriptor";
const WORKSPACE_SEED_NAME = "workspace-seed.tar";
const GIT_SEED_NAME = "git-seed.tar";
const REMOTE_STAMP_NAME = "workspace-sync-v1.json";
const MAX_DESCRIPTOR_BYTES = 64 * 1024 * 1024;
const SAFE_SEGMENT_RE = /^[A-Za-z0-9_-]+$/;
type NativeWorkspaceSyncState = "prepared" | "finalized";
export type NativeWorkspaceResourceDisposition =
"keep_running" | "stop_and_retain" | "destroy";
interface NativeWorkspaceSyncDescriptor {
schema: typeof DESCRIPTOR_SCHEMA;
binding: {
runId: string;
companyId: string;
workspaceId: string;
leaseId: string;
providerLeaseId: string;
localCwd: string;
remoteCwd: string;
};
state: NativeWorkspaceSyncState;
baselineSha256: string;
baseline: SerializedDirectorySnapshot;
gitSnapshot: GitWorkspaceSnapshot | null;
seed: {
workspaceArchiveSha256: string;
gitArchiveSha256: string | null;
} | null;
createdAt: string;
finalizedAt: string | null;
finalHostSha256: string | null;
resourceDisposition: NativeWorkspaceResourceDisposition | null;
}
export interface NativeWorkspaceSyncReference {
schema: typeof DESCRIPTOR_SCHEMA;
state: NativeWorkspaceSyncState;
descriptorSha256: string;
baselineSha256: string;
finalHostSha256: string | null;
workspaceId: string;
leaseId: string;
providerLeaseId: string;
remoteCwd: string;
resourceDisposition: NativeWorkspaceResourceDisposition | null;
}
export interface PreparedNativeWorkspaceSync {
mode: WorkspaceInboundMode;
reference: NativeWorkspaceSyncReference;
restoreWorkspace(): Promise<void>;
cleanup(): Promise<void>;
}
export type NativeWorkspaceInboundEvidence =
| {
kind: "existing_run";
restartRecovery: boolean;
sameProviderLease: boolean;
}
| {
kind: "new_run";
acquisition: "created" | "resumed" | "replacement" | null;
hasPriorStamp: boolean;
};
export function classifyNativeWorkspaceInbound(
evidence: NativeWorkspaceInboundEvidence,
): WorkspaceInboundMode {
if (evidence.kind === "existing_run") {
if (!evidence.restartRecovery) {
throw new Error("native_workspace_sync_unexpected_existing_descriptor");
}
return evidence.sameProviderLease ? "adopt_remote" : "durable_seed";
}
return evidence.acquisition === "resumed" && evidence.hasPriorStamp
? "adopt_remote"
: "host_current";
}
function canonicalJson(value: unknown): string {
if (Array.isArray(value)) {
return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`;
}
if (value && typeof value === "object") {
const record = value as Record<string, unknown>;
return `{${Object.keys(record)
.sort()
.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
.join(",")}}`;
}
return JSON.stringify(value);
}
function sha256(value: unknown): string {
return createHash("sha256").update(canonicalJson(value)).digest("hex");
}
function requireSafeSegment(value: string, label: string): string {
if (!SAFE_SEGMENT_RE.test(value)) {
throw new Error(`native_workspace_sync_invalid_${label}`);
}
return value;
}
function descriptorDirectory(runId: string): string {
return path.join(
resolvePaperclipInstanceRoot(),
STATE_ROOT_NAME,
requireSafeSegment(runId, "run_id"),
);
}
function descriptorPath(runId: string, descriptorSha256: string): string {
if (!/^[0-9a-f]{64}$/.test(descriptorSha256)) {
throw new Error("native_workspace_sync_descriptor_digest_invalid");
}
return path.join(
descriptorDirectory(runId),
`${DESCRIPTOR_NAME}.${descriptorSha256}.json`,
);
}
function legacyDescriptorPath(runId: string): string {
return path.join(descriptorDirectory(runId), `${DESCRIPTOR_NAME}.json`);
}
function durableSeedPaths(runId: string): {
workspaceArchivePath: string;
gitArchivePath: string;
} {
const directory = descriptorDirectory(runId);
return {
workspaceArchivePath: path.join(directory, WORKSPACE_SEED_NAME),
gitArchivePath: path.join(directory, GIT_SEED_NAME),
};
}
async function sha256File(filePath: string): Promise<string> {
return await new Promise((resolveDigest, rejectDigest) => {
const digest = createHash("sha256");
const stream = createReadStream(filePath);
stream.on("data", (chunk) => digest.update(chunk));
stream.on("error", rejectDigest);
stream.on("end", () => resolveDigest(digest.digest("hex")));
});
}
async function verifiedDurableSeed(input: {
runId: string;
seed: NonNullable<NativeWorkspaceSyncDescriptor["seed"]>;
gitSnapshot: GitWorkspaceSnapshot | null;
}): Promise<WorkspaceDurableSeedPaths> {
try {
const paths = durableSeedPaths(input.runId);
const workspaceStat = await fs.lstat(paths.workspaceArchivePath);
if (workspaceStat.isSymbolicLink() || !workspaceStat.isFile()) {
throw new Error("workspace_sync_out_unrecoverable");
}
if (
(await sha256File(paths.workspaceArchivePath)) !==
input.seed.workspaceArchiveSha256
) {
throw new Error("workspace_sync_out_unrecoverable");
}
if (input.gitSnapshot) {
const gitStat = await fs.lstat(paths.gitArchivePath);
if (
gitStat.isSymbolicLink() ||
!gitStat.isFile() ||
!input.seed.gitArchiveSha256 ||
(await sha256File(paths.gitArchivePath)) !== input.seed.gitArchiveSha256
) {
throw new Error("workspace_sync_out_unrecoverable");
}
} else if (input.seed.gitArchiveSha256 !== null) {
throw new Error("workspace_sync_out_unrecoverable");
}
return {
workspaceArchivePath: paths.workspaceArchivePath,
workspaceArchiveSha256: input.seed.workspaceArchiveSha256,
gitArchivePath: input.gitSnapshot ? paths.gitArchivePath : null,
gitArchiveSha256: input.seed.gitArchiveSha256,
};
} catch (error) {
if (
error instanceof Error &&
error.message === "workspace_sync_out_unrecoverable"
) {
throw error;
}
throw new Error("workspace_sync_out_unrecoverable", { cause: error });
}
}
async function ensurePrivateDirectory(dir: string): Promise<void> {
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
const stat = await fs.lstat(dir);
if (stat.isSymbolicLink() || !stat.isDirectory()) {
throw new Error("native_workspace_sync_state_root_invalid");
}
}
async function writeDescriptor(
descriptor: NativeWorkspaceSyncDescriptor,
): Promise<NativeWorkspaceSyncReference> {
const dir = descriptorDirectory(descriptor.binding.runId);
await ensurePrivateDirectory(dir);
const body = `${canonicalJson(descriptor)}\n`;
const descriptorSha256 = createHash("sha256").update(body).digest("hex");
const finalPath = descriptorPath(descriptor.binding.runId, descriptorSha256);
const existingBody = await fs.readFile(finalPath, "utf8").catch(() => null);
if (existingBody !== null && existingBody !== body) {
throw new Error("native_workspace_sync_descriptor_digest_collision");
}
const tempPath = path.join(dir, `${DESCRIPTOR_NAME}.${randomUUID()}.tmp`);
try {
if (existingBody === null) {
await fs.writeFile(tempPath, body, { encoding: "utf8", mode: 0o600 });
await fs.rename(tempPath, finalPath);
}
} finally {
await fs.rm(tempPath, { force: true }).catch(() => undefined);
}
return {
schema: DESCRIPTOR_SCHEMA,
state: descriptor.state,
descriptorSha256,
baselineSha256: descriptor.baselineSha256,
finalHostSha256: descriptor.finalHostSha256,
workspaceId: descriptor.binding.workspaceId,
leaseId: descriptor.binding.leaseId,
providerLeaseId: descriptor.binding.providerLeaseId,
remoteCwd: descriptor.binding.remoteCwd,
resourceDisposition: descriptor.resourceDisposition,
};
}
export function readNativeWorkspaceSyncReference(
value: unknown,
): NativeWorkspaceSyncReference | null {
const candidate = parseObject(value);
if (
candidate.schema !== DESCRIPTOR_SCHEMA ||
(candidate.state !== "prepared" && candidate.state !== "finalized") ||
typeof candidate.descriptorSha256 !== "string" ||
!/^[0-9a-f]{64}$/.test(candidate.descriptorSha256) ||
typeof candidate.baselineSha256 !== "string" ||
!/^[0-9a-f]{64}$/.test(candidate.baselineSha256) ||
(candidate.finalHostSha256 !== null &&
candidate.finalHostSha256 !== undefined &&
(typeof candidate.finalHostSha256 !== "string" ||
!/^[0-9a-f]{64}$/.test(candidate.finalHostSha256))) ||
typeof candidate.workspaceId !== "string" ||
typeof candidate.leaseId !== "string" ||
typeof candidate.providerLeaseId !== "string" ||
typeof candidate.remoteCwd !== "string" ||
(candidate.resourceDisposition !== null &&
candidate.resourceDisposition !== undefined &&
candidate.resourceDisposition !== "keep_running" &&
candidate.resourceDisposition !== "stop_and_retain" &&
candidate.resourceDisposition !== "destroy")
) {
return null;
}
return {
schema: DESCRIPTOR_SCHEMA,
state: candidate.state,
descriptorSha256: candidate.descriptorSha256,
baselineSha256: candidate.baselineSha256,
finalHostSha256:
typeof candidate.finalHostSha256 === "string"
? candidate.finalHostSha256
: null,
workspaceId: candidate.workspaceId,
leaseId: candidate.leaseId,
providerLeaseId: candidate.providerLeaseId,
remoteCwd: candidate.remoteCwd,
resourceDisposition:
candidate.resourceDisposition === "keep_running" ||
candidate.resourceDisposition === "stop_and_retain" ||
candidate.resourceDisposition === "destroy"
? candidate.resourceDisposition
: null,
};
}
function parseGitSnapshot(
value: unknown,
): GitWorkspaceSnapshot | null | undefined {
if (value === null) return null;
const candidate = parseObject(value);
const paths = [
candidate.overlayPaths,
candidate.deletedPaths,
candidate.ignoredPaths,
];
if (
typeof candidate.headCommit !== "string" ||
(candidate.branchName !== null &&
typeof candidate.branchName !== "string") ||
!paths.every(
(entries) =>
Array.isArray(entries) &&
entries.every(
(entry) =>
typeof entry === "string" &&
!path.posix.isAbsolute(entry) &&
!path.win32.isAbsolute(entry) &&
!entry.split(/[\\/]/).some((segment) => segment === ".."),
),
)
) {
return undefined;
}
return {
headCommit: candidate.headCommit,
branchName: candidate.branchName as string | null,
overlayPaths: [...(candidate.overlayPaths as string[])],
deletedPaths: [...(candidate.deletedPaths as string[])],
ignoredPaths: [...(candidate.ignoredPaths as string[])],
};
}
async function readDescriptor(input: {
runId: string;
reference: NativeWorkspaceSyncReference;
}): Promise<{
descriptor: NativeWorkspaceSyncDescriptor;
baseline: DirectorySnapshot;
}> {
const contentAddressedPath = descriptorPath(
input.runId,
input.reference.descriptorSha256,
);
// Early v1 writers used descriptor.json. Prefer the immutable,
// content-addressed name, but keep the digest-verified legacy path readable
// so an interrupted upgrade can still finish its already-proposed result.
const filePath = await fs
.lstat(contentAddressedPath)
.then(() => contentAddressedPath)
.catch(async (error: NodeJS.ErrnoException) => {
if (error.code !== "ENOENT") throw error;
const legacyPath = legacyDescriptorPath(input.runId);
await fs.lstat(legacyPath);
return legacyPath;
});
const stat = await fs.lstat(filePath);
if (
stat.isSymbolicLink() ||
!stat.isFile() ||
stat.size > MAX_DESCRIPTOR_BYTES
) {
throw new Error("native_workspace_sync_descriptor_invalid");
}
const body = await fs.readFile(filePath, "utf8");
if (
createHash("sha256").update(body).digest("hex") !==
input.reference.descriptorSha256
) {
throw new Error("native_workspace_sync_descriptor_digest_mismatch");
}
const parsed = JSON.parse(body) as unknown;
const candidate = parseObject(parsed);
const binding = parseObject(candidate.binding);
const baseline = parseDirectorySnapshot(candidate.baseline);
const gitSnapshot = parseGitSnapshot(candidate.gitSnapshot);
const rawSeed =
candidate.seed === null || candidate.seed === undefined
? null
: parseObject(candidate.seed);
const seed =
rawSeed === null
? null
: typeof rawSeed.workspaceArchiveSha256 === "string" &&
/^[0-9a-f]{64}$/.test(rawSeed.workspaceArchiveSha256) &&
(rawSeed.gitArchiveSha256 === null ||
(typeof rawSeed.gitArchiveSha256 === "string" &&
/^[0-9a-f]{64}$/.test(rawSeed.gitArchiveSha256)))
? {
workspaceArchiveSha256: rawSeed.workspaceArchiveSha256,
gitArchiveSha256: rawSeed.gitArchiveSha256 as string | null,
}
: undefined;
const finalHostSha256 =
typeof candidate.finalHostSha256 === "string" &&
/^[0-9a-f]{64}$/.test(candidate.finalHostSha256)
? candidate.finalHostSha256
: candidate.finalHostSha256 === null
? null
: undefined;
const resourceDisposition =
candidate.resourceDisposition === "keep_running" ||
candidate.resourceDisposition === "stop_and_retain" ||
candidate.resourceDisposition === "destroy"
? candidate.resourceDisposition
: candidate.resourceDisposition === null
? null
: undefined;
if (
candidate.schema !== DESCRIPTOR_SCHEMA ||
(candidate.state !== "prepared" && candidate.state !== "finalized") ||
!baseline ||
gitSnapshot === undefined ||
seed === undefined ||
binding.runId !== input.runId ||
binding.workspaceId !== input.reference.workspaceId ||
binding.leaseId !== input.reference.leaseId ||
binding.providerLeaseId !== input.reference.providerLeaseId ||
binding.remoteCwd !== input.reference.remoteCwd ||
typeof binding.companyId !== "string" ||
typeof binding.localCwd !== "string" ||
!path.isAbsolute(binding.localCwd) ||
typeof binding.remoteCwd !== "string" ||
!path.posix.isAbsolute(binding.remoteCwd) ||
typeof candidate.baselineSha256 !== "string" ||
candidate.baselineSha256 !== input.reference.baselineSha256 ||
directorySnapshotSha256(baseline) !== candidate.baselineSha256 ||
typeof candidate.createdAt !== "string" ||
!Number.isFinite(Date.parse(candidate.createdAt)) ||
candidate.state !== input.reference.state ||
finalHostSha256 === undefined ||
finalHostSha256 !== input.reference.finalHostSha256 ||
resourceDisposition === undefined ||
resourceDisposition !== input.reference.resourceDisposition ||
(candidate.state === "prepared" &&
(candidate.finalizedAt !== null || finalHostSha256 !== null)) ||
(candidate.state === "finalized" &&
(typeof candidate.finalizedAt !== "string" ||
!Number.isFinite(Date.parse(candidate.finalizedAt)) ||
finalHostSha256 === null))
) {
throw new Error("native_workspace_sync_descriptor_binding_mismatch");
}
return {
descriptor: {
schema: DESCRIPTOR_SCHEMA,
binding: {
runId: binding.runId as string,
companyId: binding.companyId,
workspaceId: binding.workspaceId as string,
leaseId: binding.leaseId as string,
providerLeaseId: binding.providerLeaseId as string,
localCwd: binding.localCwd,
remoteCwd: binding.remoteCwd as string,
},
state: candidate.state,
baselineSha256: candidate.baselineSha256,
baseline: candidate.baseline as SerializedDirectorySnapshot,
gitSnapshot,
seed,
createdAt: candidate.createdAt,
finalizedAt:
typeof candidate.finalizedAt === "string"
? candidate.finalizedAt
: null,
finalHostSha256,
resourceDisposition,
},
baseline,
};
}
async function persistRunReference(
db: Db,
runId: string,
reference: NativeWorkspaceSyncReference,
): Promise<void> {
await db.transaction(async (tx) => {
const run = await tx
.select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.for("update")
.limit(1)
.then((rows) => rows[0] ?? null);
if (!run) throw new Error("native_workspace_sync_run_missing");
await tx
.update(heartbeatRuns)
.set({
runnerProfileJson: {
...parseObject(run.runnerProfileJson),
nativeWorkspaceSync: reference,
},
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, runId));
});
}
async function persistLeaseStamp(input: {
db: Db;
leaseId: string;
stamp: Record<string, unknown>;
}): Promise<void> {
await input.db.transaction(async (tx) => {
const lease = await tx
.select({ metadata: environmentLeases.metadata })
.from(environmentLeases)
.where(eq(environmentLeases.id, input.leaseId))
.for("update")
.limit(1)
.then((rows) => rows[0] ?? null);
if (!lease) throw new Error("native_workspace_sync_lease_missing");
await tx
.update(environmentLeases)
.set({
metadata: {
...parseObject(lease.metadata),
nativeWorkspaceSync: input.stamp,
},
updatedAt: new Date(),
})
.where(eq(environmentLeases.id, input.leaseId));
});
}
function providerLeaseIdFor(input: {
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
lease: EnvironmentLease;
}): string {
const providerLeaseId =
input.target.sandboxLeaseAcquisition?.providerLeaseId ??
input.lease.providerLeaseId;
if (!providerLeaseId) {
throw new Error("native_workspace_sync_provider_lease_missing");
}
return providerLeaseId;
}
function shellQuote(value: string): string {
return `'${value.replaceAll("'", "'\\''")}'`;
}
async function writeRemoteStamp(input: {
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
stamp: Record<string, unknown>;
}): Promise<void> {
if (!input.target.runner) {
throw new Error("native_workspace_sync_runner_missing");
}
const runtimeDir = path.posix.join(
input.target.remoteCwd,
".paperclip-runtime",
"paperclip-runner",
);
const stampPath = path.posix.join(runtimeDir, REMOTE_STAMP_NAME);
const tempPath = `${stampPath}.tmp`;
const encoded = Buffer.from(
`${canonicalJson(input.stamp)}\n`,
"utf8",
).toString("base64");
const result = await input.target.runner.execute({
command: input.target.shellCommand ?? "sh",
args: [
"-c",
`umask 077; mkdir -p ${shellQuote(runtimeDir)} && printf %s ${shellQuote(encoded)} | base64 -d > ${shellQuote(tempPath)} && mv ${shellQuote(tempPath)} ${shellQuote(stampPath)}`,
],
cwd: "/",
timeoutMs: 15_000,
bypassSession: true,
});
if (result.timedOut || result.exitCode !== 0) {
throw new Error("native_workspace_sync_remote_stamp_failed");
}
}
async function remoteStampMatches(input: {
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
expected: Record<string, unknown>;
}): Promise<boolean> {
if (!input.target.runner) return false;
const stampPath = path.posix.join(
input.target.remoteCwd,
".paperclip-runtime",
"paperclip-runner",
REMOTE_STAMP_NAME,
);
const result = await input.target.runner.execute({
command: input.target.shellCommand ?? "sh",
args: [
"-c",
`test -f ${shellQuote(stampPath)} && cat ${shellQuote(stampPath)}`,
],
cwd: "/",
timeoutMs: 15_000,
bypassSession: true,
});
if (result.timedOut || result.exitCode !== 0) return false;
try {
return (
canonicalJson(JSON.parse(result.stdout)) === canonicalJson(input.expected)
);
} catch {
return false;
}
}
function leaseStamp(input: {
lease: EnvironmentLease;
workspaceId: string;
providerLeaseId: string;
remoteCwd: string;
}): Record<string, unknown> | null {
const stamp = parseObject(
parseObject(input.lease.metadata).nativeWorkspaceSync,
);
if (
stamp.schema !== STAMP_SCHEMA ||
stamp.workspaceId !== input.workspaceId ||
stamp.providerLeaseId !== input.providerLeaseId ||
stamp.remoteCwd !== input.remoteCwd ||
typeof stamp.hostSha256 !== "string" ||
!/^[0-9a-f]{64}$/.test(stamp.hostSha256)
) {
return null;
}
return stamp;
}
async function prepareRuntime(input: {
runId: string;
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
workspaceLocalDir: string;
mode: WorkspaceInboundMode;
baseline?: DirectorySnapshot;
gitSnapshot?: GitWorkspaceSnapshot | null;
durableSeed?: WorkspaceDurableSeedPaths;
}): Promise<PreparedAdapterExecutionTargetRuntime> {
return prepareAdapterExecutionTargetRuntime({
runId: input.runId,
target: input.target,
adapterKey: "paperclip-runner",
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir: input.target.remoteCwd,
workspaceInboundMode: input.mode,
workspaceDurableSeed: input.durableSeed,
workspaceBaseline: input.baseline,
workspaceGitSnapshot: input.gitSnapshot,
});
}
async function finalizePreparedRuntime(input: {
db: Db;
runId: string;
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
runtime: PreparedAdapterExecutionTargetRuntime;
descriptor: NativeWorkspaceSyncDescriptor;
}): Promise<NativeWorkspaceSyncReference> {
await input.runtime.restoreWorkspace();
const finalSnapshot =
await import("@paperclipai/adapter-utils/workspace-restore-merge").then(
({ captureDirectorySnapshot }) =>
captureDirectorySnapshot(input.descriptor.binding.localCwd, {
exclude: input.runtime.workspaceSyncSnapshot?.baseline.exclude ?? [],
}),
);
const finalHostSha256 = directorySnapshotSha256(finalSnapshot);
const stamp = {
schema: STAMP_SCHEMA,
workspaceId: input.descriptor.binding.workspaceId,
providerLeaseId: input.descriptor.binding.providerLeaseId,
remoteCwd: input.descriptor.binding.remoteCwd,
hostSha256: finalHostSha256,
finalizedRunId: input.runId,
};
await writeRemoteStamp({ target: input.target, stamp });
const finalizedDescriptor: NativeWorkspaceSyncDescriptor = {
...input.descriptor,
state: "finalized",
finalizedAt: new Date().toISOString(),
finalHostSha256,
};
const reference = await writeDescriptor(finalizedDescriptor);
await persistRunReference(input.db, input.runId, reference);
await persistLeaseStamp({
db: input.db,
leaseId: input.descriptor.binding.leaseId,
stamp,
});
return reference;
}
export async function prepareNativeWorkspaceSync(input: {
db: Db;
runId: string;
companyId: string;
workspaceId: string;
workspaceLocalDir: string;
target: AdapterExecutionTarget | null;
lease: EnvironmentLease;
restartRecovery?: NativeRestartRecoveryClaim;
resourceDisposition?: NativeWorkspaceResourceDisposition;
}): Promise<PreparedNativeWorkspaceSync | null> {
if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") {
return null;
}
const target = input.target;
const providerLeaseId = providerLeaseIdFor({ target, lease: input.lease });
const run = await input.db
.select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.runId))
.limit(1)
.then((rows) => rows[0] ?? null);
if (!run) throw new Error("native_workspace_sync_run_missing");
const existingReference = readNativeWorkspaceSyncReference(
parseObject(run.runnerProfileJson).nativeWorkspaceSync,
);
let runtime: PreparedAdapterExecutionTargetRuntime;
let descriptor: NativeWorkspaceSyncDescriptor;
let mode: WorkspaceInboundMode = "host_current";
const seedPaths = durableSeedPaths(input.runId);
await ensurePrivateDirectory(descriptorDirectory(input.runId));
if (existingReference) {
const existing = await readDescriptor({
runId: input.runId,
reference: existingReference,
});
if (
existing.descriptor.binding.companyId !== input.companyId ||
existing.descriptor.binding.workspaceId !== input.workspaceId ||
path.resolve(existing.descriptor.binding.localCwd) !==
path.resolve(input.workspaceLocalDir) ||
existing.descriptor.binding.remoteCwd !== target.remoteCwd
) {
throw new Error("native_workspace_sync_descriptor_binding_mismatch");
}
const sameProviderLease =
existing.descriptor.binding.providerLeaseId === providerLeaseId;
mode = classifyNativeWorkspaceInbound({
kind: "existing_run",
restartRecovery: Boolean(input.restartRecovery),
sameProviderLease,
});
const durableSeed =
mode === "durable_seed"
? existing.descriptor.seed
? await verifiedDurableSeed({
runId: input.runId,
seed: existing.descriptor.seed,
gitSnapshot: existing.descriptor.gitSnapshot,
})
: (() => {
throw new Error("workspace_sync_out_unrecoverable");
})()
: undefined;
runtime = await prepareRuntime({
runId: input.runId,
target,
workspaceLocalDir: input.workspaceLocalDir,
mode,
baseline: existing.baseline,
gitSnapshot: existing.descriptor.gitSnapshot,
durableSeed,
});
descriptor = {
...existing.descriptor,
binding: {
...existing.descriptor.binding,
leaseId: target.leaseId ?? input.lease.id,
providerLeaseId,
},
};
} else {
const acquisition = target.sandboxLeaseAcquisition?.outcome ?? null;
const priorStamp = leaseStamp({
lease: input.lease,
workspaceId: input.workspaceId,
providerLeaseId,
remoteCwd: target.remoteCwd,
});
mode = classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition,
hasPriorStamp: priorStamp !== null,
});
runtime = await prepareRuntime({
runId: input.runId,
target,
workspaceLocalDir: input.workspaceLocalDir,
mode,
durableSeed: {
workspaceArchivePath: seedPaths.workspaceArchivePath,
gitArchivePath: seedPaths.gitArchivePath,
},
});
const currentSnapshot = runtime.workspaceSyncSnapshot;
if (!currentSnapshot) {
throw new Error("native_workspace_sync_snapshot_missing");
}
if (acquisition === "resumed" && priorStamp) {
const currentHostSha256 = directorySnapshotSha256(
currentSnapshot.baseline,
);
const verifiedWarmAdoption =
priorStamp.hostSha256 === currentHostSha256 &&
(await remoteStampMatches({ target, expected: priorStamp }));
if (verifiedWarmAdoption) {
mode = "adopt_remote";
} else {
mode = "host_current";
runtime = await prepareRuntime({
runId: input.runId,
target,
workspaceLocalDir: input.workspaceLocalDir,
mode,
baseline: currentSnapshot.baseline,
gitSnapshot: currentSnapshot.gitSnapshot,
durableSeed: {
workspaceArchivePath: seedPaths.workspaceArchivePath,
gitArchivePath: seedPaths.gitArchivePath,
},
});
}
}
const snapshot = runtime.workspaceSyncSnapshot;
if (!snapshot) throw new Error("native_workspace_sync_snapshot_missing");
const now = new Date().toISOString();
const workspaceArchiveSha256 = await sha256File(
seedPaths.workspaceArchivePath,
);
const gitArchiveSha256 = snapshot.gitSnapshot
? await sha256File(seedPaths.gitArchivePath)
: null;
descriptor = {
schema: DESCRIPTOR_SCHEMA,
binding: {
runId: input.runId,
companyId: input.companyId,
workspaceId: input.workspaceId,
leaseId: target.leaseId ?? input.lease.id,
providerLeaseId,
localCwd: path.resolve(input.workspaceLocalDir),
remoteCwd: target.remoteCwd,
},
state: "prepared",
baselineSha256: directorySnapshotSha256(snapshot.baseline),
baseline: serializeDirectorySnapshot(snapshot.baseline),
gitSnapshot: snapshot.gitSnapshot,
seed: { workspaceArchiveSha256, gitArchiveSha256 },
createdAt: now,
finalizedAt: null,
finalHostSha256: null,
resourceDisposition: input.resourceDisposition ?? null,
};
}
let reference = await writeDescriptor(descriptor);
await persistRunReference(input.db, input.runId, reference);
let restorePromise: Promise<void> | null = null;
return {
mode,
get reference() {
return reference;
},
restoreWorkspace: async () => {
if (!restorePromise) {
restorePromise = finalizePreparedRuntime({
db: input.db,
runId: input.runId,
target,
runtime,
descriptor,
})
.then((finalizedReference) => {
reference = finalizedReference;
})
.catch((error) => {
restorePromise = null;
throw error;
});
}
await restorePromise;
},
cleanup: () => cleanupNativeWorkspaceSync(input.runId),
};
}
export async function resumeNativeWorkspaceSync(input: {
db: Db;
runId: string;
target: AdapterExecutionTarget;
}): Promise<boolean> {
if (input.target.kind !== "remote" || input.target.transport !== "sandbox") {
throw new Error("workspace_sync_out_unrecoverable");
}
const run = await input.db
.select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.runId))
.limit(1)
.then((rows) => rows[0] ?? null);
const reference = readNativeWorkspaceSyncReference(
parseObject(run?.runnerProfileJson).nativeWorkspaceSync,
);
if (!reference) return false;
const existing = await readDescriptor({ runId: input.runId, reference });
if (
existing.descriptor.state === "finalized" &&
existing.descriptor.finalHostSha256
) {
return true;
}
const providerLeaseId =
input.target.sandboxLeaseAcquisition?.providerLeaseId ??
reference.providerLeaseId;
if (
providerLeaseId !== reference.providerLeaseId ||
input.target.remoteCwd !== reference.remoteCwd
) {
throw new Error("workspace_sync_out_unrecoverable");
}
const runtime = await prepareRuntime({
runId: input.runId,
target: input.target,
workspaceLocalDir: existing.descriptor.binding.localCwd,
mode: "adopt_remote",
baseline: existing.baseline,
gitSnapshot: existing.descriptor.gitSnapshot,
});
await finalizePreparedRuntime({
db: input.db,
runId: input.runId,
target: input.target,
runtime,
descriptor: existing.descriptor,
});
return true;
}
export async function cleanupNativeWorkspaceSync(runId: string): Promise<void> {
await fs.rm(descriptorDirectory(runId), { recursive: true, force: true });
}
export const nativeWorkspaceSyncInternals = {
descriptorPath,
legacyDescriptorPath,
durableSeedPaths,
writeDescriptor,
readDescriptor,
readReference: readNativeWorkspaceSyncReference,
sha256,
};