import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js"; import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js"; import { flushBudgetEvaluationWork } from "../budgets.js"; import { NativeSessionCloseUnrecoverableError, type PrpEvent } from "../../vendor/paperclip-runner/index.js"; import { randomUUID } from "node:crypto"; import { mkdtemp, rm, readFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { agents, budgetPolicies, companies, completionContracts, nativeRunResults, costEvents, agentWakeupRequests, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { trace } from "@opentelemetry/api"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { createRunPreparationTiming, measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js"; import { recoveryPostgresSupport as support, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; import * as pluginWorkspaces from "../plugin-workspaces.js"; import * as legacyRecovery from "../legacy-execution-recovery.js"; import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; import { createAdapterExecutionControl } from "../adapter-execution-control.js"; import { executionWorkspaceService } from "../execution-workspaces.js"; import { workspaceOperationService } from "../workspace-operations.js"; import { instanceSettingsService } from "../instance-settings.js"; import { executePaperclipNativeSession, NativeCancellationPendingRecoveryError, NativeControllerDetachedForRestartError, nativeSessionFailureSourceCode, nativeSessionFailureDisposition } from "../native-runtime/native-session-executor.js"; import { NativeRunnerOwnershipUnverifiedError } from "../native-runtime/native-runner-ownership.js"; import { NativeWorkspaceFinalizationBusyError, NativeWorkspaceFinalizationOwnershipLostError } from "../native-runtime/native-workspace-finalization-ownership.js"; import { resolveHeartbeatNativeRuntimeMode } from "../native-runtime/runtime-mode.js"; import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js"; import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js"; import { createHeartbeatRunState } from "./run-state.js"; import { buildEffectiveRunSessionConfigMetadata } from "./workspaces.js"; import { revokeHeartbeatRunGatewayTokens } from "./run-preparation.js"; import { executeHeartbeatRuntime, type HeartbeatRuntimeExecutionInput } from "./runtime-execution.js"; import { prepareNativeHeartbeatRun } from "../native-runtime/prepare-native-run.js"; import { buildNativeCompletionContract } from "../native-runtime/completion-contracts.js"; import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js"; import { CONTROL_PLANE_CONFORMANCE_RESULT } from "../../vendor/paperclip-runner/testing.js"; import type { ControlPlanePort, NativeExecutionInputV1 } from "@paperclipai/paperclip-runner"; const postResultProvider = vi.hoisted(() => ({ execute: vi.fn() })); vi.mock("../../vendor/paperclip-runner/index.js", async original => ({ ...await original(), executeNativeSession: postResultProvider.execute, })); // Keep real accounting, workspace barriers, and task-session persistence. Providers // and gateway delivery are external boundaries; browser acceptance covers their wiring. vi.mock("../native-runtime/native-session-executor.js", async original => ({ ...await original(), executePaperclipNativeSession: vi.fn(), })); vi.mock("./run-preparation.js", async original => ({ ...await original(), buildPaperclipRuntimeMcpServers: vi.fn(async () => []), createAdapterRuntimeToolAccess: vi.fn(() => null), createManagedMcpRunConfig: vi.fn(async () => null), revokeHeartbeatRunGatewayTokens: vi.fn(async () => {}), })); describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () => { let database: Awaited>; let db: Db; let home: string; beforeAll(async () => { home = await mkdtemp(join(tmpdir(), "paperclip-runtime-execution-")); vi.stubEnv("PAPERCLIP_HOME", home); database = await startRecoveryPostgres("paperclip-runtime-execution"); db = createDb(database.connectionString); await db.execute(sql`set client_min_messages = warning`); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); }); beforeEach(async () => { await db.execute(sql`truncate table companies restart identity cascade`); vi.clearAllMocks(); vi.mocked(executePaperclipNativeSession).mockResolvedValue({ exitCode: 0, signal: null, timedOut: false }); }); async function fixture(native = false, provider: "codex" | "grok_bot" | "openai_dot" = "codex") { const [company] = await db.insert(companies).values({ name: "Runtime execution", issuePrefix: "EXE" }).returning(); const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Executing agent", status: "running", adapterType: native ? "paperclip_runner" : "process", adapterConfig: provider === "codex" ? { provider } : { provider, lifecycleMode: "per_turn", allowUnmeteredProvider: true, [provider === "grok_bot" ? "externalBindingId" : "dotBindingId"]: randomUUID() } }).returning(); const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Complete this task", assigneeAgentId: agent.id }).returning(); const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: native ? "native" : "legacy" }).returning(); const state = createHeartbeatRunState(db); const executionControl = createAdapterExecutionControl(); const adapter = { type: "process", label: "Test process", testEnvironment: vi.fn(), execute: vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false })) }; let instructionSave: Record | null = null; const nativeExecution = native ? buildNativeExecutionInput({ companyId: company.id, agentId: agent.id, runId: run.id, issue, taskPrompt: issue.title, workspace: { id: run.id, cwd: home, repoUrl: null, repoRef: null, branchName: null }, normalizedSessionId: null, provider, runtimeContext: nativeRuntimeContextFixture(), dotBinding: { bindingId: randomUUID(), bindingGeneration: 1, companyId: company.id, agentId: agent.id, acceptByUnixMs: Date.now() + 60_000, expiresAtUnixMs: Date.now() + 120_000 }, completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: issue.title, criteria: [{ id: "output", requirement: issue.title }] } }, }) : null; const input: HeartbeatRuntimeExecutionInput = { run, agent, options: {}, task: { issueRef: issue, issueContext: null, context: {}, issueId: issue.id, taskKey: issue.id, executionContinuation: null }, runtime: { nativeExecution, nativeRunnerInstanceId: native ? randomUUID() : null, getNativeFreshSessionHandoff: undefined, nativeRuntimeResolution: resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: native, grokEnabled: true, dotEnabled: true, runtimeConfig: {}, adapterConfig: agent.adapterConfig, agent, issue, target: null, workspaceId: null }), adapter, runtimeForAdapter: { taskKey: issue.id, sessionId: null, sessionParams: null, sessionDisplayId: null }, getFreshSessionHandoff: undefined, runOptions: {}, }, workspace: { persistedExecutionWorkspace: null, executionWorkspace: { baseCwd: home, cwd: home, source: "agent_home", projectId: null, workspaceId: null, repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false }, executionTarget: null, nativeWorkspaceSync: null, workspaceRestoreSource: undefined, remoteExecution: null, workspaceOperationRecorder: workspaceOperationService(db).createRecorder({ companyId: company.id, heartbeatRunId: run.id, issueId: issue.id }), }, config: { runtimeConfig: {}, resolvedConfig: {}, managedAiRuntime: undefined, resolvedInstanceSettings: { experimental: { enableWorkspaceBranchReconcileForward: false, enableWorkspaceDirtyQuarantineRepair: false } }, configuredTaskEnvironment: {}, adapterEnv: {}, agentIdentity: undefined, runtimeEnv: {}, useHostGitHub: false, githubSelection: { configured: false }, }, session: { runtimeSessionParamsForAdapter: null, configuredModel: null, sessionConfigMetadata: await buildEffectiveRunSessionConfigMetadata({ adapterType: agent.adapterType, effectiveAdapterConfig: {}, agentRuntimeConfig: {}, issueOverrides: null, workspaceConfig: null, environment: null, environmentEnv: null, projectEnv: null, routineEnv: null, runtimeSkills: [] }), goalCheckpointSession: { current: null }, }, instructions: { instructionCopy: null, getInstructionSave: () => instructionSave, nativeInstructionWorkingCopy: () => undefined, collectStoppedInstructions: async () => { instructionSave = { state: "saved", revisionId: "during-execution" }; }, nativeInstructionReservation: null, releaseInstructionCopy: vi.fn(async () => {}), }, trace: { providerTraceCapture: null, nativeRunnerPreparationSpans: [], attestedQuestionResponseAtMs: null, attemptStartedAtMs: Date.now(), environmentAcquireStartedAtMs: Date.now(), environmentRealizeEndedAtMs: Date.now() }, control: { executionControl, executionPhaseContext: { onExecutionPhase: executionControl.phases.enter }, dispatchResolvedInteractionContinuationWithAtomicGate: async dispatch => ({ dispatched: true, resultPromise: dispatch(vi.fn()) }), }, output: { identityRedactor: createAgentIdentityRedactor(), appendIdentityRedactedLog: vi.fn(async () => {}), onLog: vi.fn(async () => {}), onAdapterEvent: vi.fn(async () => {}), onAdapterMeta: vi.fn(async () => {}) }, services: { ...state, executionWorkspacesSvc: executionWorkspaceService(db), instanceSettings: instanceSettingsService(db), envOrchestrator: { releaseForRun: vi.fn() }, cancelRunInternal: vi.fn(), enqueueWakeup: vi.fn(), persistRunProcessMetadata: vi.fn(), recordCurrentHeartbeatRunRuntimeProgress: vi.fn(), dispatchPendingNativeStatusWakeups: vi.fn(), }, effects: { onUsageCaptureReady: vi.fn(), onNativeDispatchStarted: vi.fn(), onLegacyAdapterEntered: vi.fn(), onWorkspaceRestoreFailure: vi.fn(), onProviderResourceDisposition: vi.fn(), onNativeWorkspaceFinalizeScheduled: vi.fn(), onNativeSessionResumeScheduled: vi.fn(), onNativeOwnershipHeld: vi.fn() }, }; return { input, adapter, run }; } it.each(["current", "expired", "replaced-owner", "generation", "attempt", "boot", "pid", "process-start", "stopped", "unknown-code", "permission", "io", "diagnostic-sink-failed", "diagnostic-lock-busy"] as const)( "preserves post-result diagnostics without changing recovery authority (%s)", async scenario => { const { input } = await fixture(true); const issue = input.task.issueRef!; const [run] = await db.update(heartbeatRuns).set({ nativeIssueId: issue.id, contextSnapshot: { issueId: issue.id }, costAccountingPending: true }).where(eq(heartbeatRuns.id, input.run.id)).returning(); input.run = run; const prepared = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issue.id)); const completion = buildNativeCompletionContract(issue, { revision: contract.revision }); await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: issue.id, phase: "observed" }); const execution: NativeExecutionInputV1 = { schema: "paperclip.native-execution-input.v1", interactionResponses: [], credentialBindings: [], binding: { companyId: run.companyId, agentId: run.agentId, issueId: issue.id, runId: run.id, executionWorkspaceId: run.id }, completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: completion }, provider: { kind: "codex", model: "test-model" }, task: { identifier: issue.id, title: issue.title, description: null, prompt: "Local post-result fault", workMode: "standard" }, workspace: { cwd: home, repoUrl: null, repoRef: null, branchName: null }, session: { normalizedSessionId: prepared.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, }; input.runtime.nativeExecution = execution; input.runtime.nativeRunnerInstanceId = prepared.runnerInstanceId; input.workspace.nativeWorkspaceSync = { mode: "host_current", reference: { schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: run.id, leaseId: randomUUID(), providerLeaseId: "controlled-local", remoteCwd: home, resourceDisposition: null }, restoreWorkspace: vi.fn(async () => {}), cleanup: vi.fn(async () => {}) }; const diagnosticExpected = ["current", "stopped", "unknown-code", "permission", "io", "diagnostic-sink-failed"].includes(scenario); let faultInjected = false, lockHeld = false, settledWhileLocked = false; let releaseLock: () => void = () => {}; let lockTask: Promise | undefined; let diagnosticLockError: { code?: string; message?: string } | undefined; if (scenario === "diagnostic-lock-busy") { const transaction = db.transaction.bind(db); let armed = false; const transactionSpy = vi.spyOn(db, "transaction").mockImplementation(async (body, config) => { const result = await transaction(async tx => { // Observe real query errors; the executor still executes its actual // diagnostic SELECT against PostgreSQL and the competing row lock. if (faultInjected) { const select = tx.select.bind(tx); vi.spyOn(tx, "select").mockImplementation(fields => { const selection = fields === undefined ? select() : select(fields); const from = selection.from.bind(selection); selection.from = ((table: Parameters[0]) => { const query = from(table); if (table === nativeRunFinalizations && fields?.runId === nativeRunFinalizations.runId) { const lock = query.for.bind(query); query.for = ((...args: Parameters) => lock(...args).catch(error => { diagnosticLockError = error.cause ?? error; throw error; })) as typeof query.for; } return query; }) as typeof selection.from; return selection as ReturnType; }); } return body(tx); }, config); // The first transaction after the injected I/O failure publishes the // control deadline. Hold the row only after that transaction commits. if (faultInjected && !armed) { armed = true; let ready!: () => void; const acquired = new Promise(resolve => { ready = resolve; }); const release = new Promise(resolve => { releaseLock = resolve; }); lockTask = transaction(async locker => { await locker.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, run.id)).for("update"); lockHeld = true; ready(); await release; lockHeld = false; }); await acquired; } return result; }); onTestFinished(async () => { releaseLock(); await lockTask; transactionSpy.mockRestore(); }); } postResultProvider.execute.mockReset().mockImplementationOnce(async (options: { controlPlane: ControlPlanePort }) => { await options.controlPlane.openRun({ identity: { companyId: run.companyId, issueId: issue.id, runId: run.id, agentId: run.agentId, sessionId: prepared.normalizedSessionId }, backendKind: "mock", sourceInstanceId: prepared.runnerInstanceId }); await options.controlPlane.completeRun({ result: { ...CONTROL_PLANE_CONFORMANCE_RESULT, completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim, contractRevision: completion.revision } }, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: prepared.turnId }); const [owner] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); const controller = await currentNativeControllerIdentity(); expect(owner).toMatchObject({ phase: "workspace_finalizing", attempt: 1, controllerGeneration: 1, controllerBootId: controller.bootId, controllerPid: controller.pid, controllerProcessStartedAt: controller.processStartedAt }); expect(owner.leaseOwner).toBeTruthy(); expect(owner.leaseExpiresAt!.getTime()).toBeGreaterThan(Date.now()); const changed = scenario === "expired" ? { leaseExpiresAt: new Date(0) } : scenario === "replaced-owner" ? { leaseOwner: "replacement-owner" } : scenario === "generation" ? { controllerGeneration: owner.controllerGeneration + 1 } : scenario === "attempt" ? { attempt: owner.attempt + 1 } : scenario === "boot" ? { controllerBootId: randomUUID() } : scenario === "pid" ? { controllerPid: controller.pid + 1 } : scenario === "process-start" ? { controllerProcessStartedAt: new Date(controller.processStartedAt.getTime() - 1000) } : null; if (changed) await db.update(nativeRunFinalizations).set(changed).where(eq(nativeRunFinalizations.runId, run.id)); if (scenario === "stopped") await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); try { await readFile(join(home, `missing-${run.id}`, "harmless-checkpoint")); } catch (error) { faultInjected = true; if (scenario === "unknown-code") (error as NodeJS.ErrnoException).code = "private-unrecognized-code"; if (scenario === "permission") (error as NodeJS.ErrnoException).code = "EACCES"; if (scenario === "io") (error as NodeJS.ErrnoException).code = "EIO"; throw error; } throw new Error("fixture must fail with ENOENT"); }); const events: unknown[] = []; let controlError: unknown; const actual = await vi.importActual("../native-runtime/native-session-executor.js"); vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async options => { try { return await actual.executePaperclipNativeSession({ ...options, useRunnerd: false, backend: { descriptor: async () => { throw new Error("Unexpected provider call"); }, openSession: async () => { throw new Error("Unexpected provider call"); } }, onEvent: async event => { events.push(event); if (event.eventType === "native.post_result_failure" && scenario === "diagnostic-sink-failed") throw new Error("private sink failure"); await options.onEvent?.(event); } }); } catch (error) { controlError = error; settledWhileLocked = lockHeld; releaseLock(); await lockTask; throw error; } }); await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ name: "NativeWorkspaceFinalizeScheduledError", reasonCode: "workspace_sync_out_failed", terminalFailure: false }); expect((controlError as Error).message).toBe("native_session_lease_lost"); if (scenario === "diagnostic-lock-busy") { expect(settledWhileLocked).toBe(true); expect(diagnosticLockError?.code).toBe("55P03"); expect(diagnosticLockError?.message).toContain("could not obtain lock"); } const source = nativeSessionFailureSourceCode(controlError); expect(source).toBe(nativeSessionFailureSourceCode(new Error("native_session_lease_lost"))); expect(nativeSessionFailureDisposition(1, new Date(0), source)).toEqual(nativeSessionFailureDisposition(1, new Date(0), nativeSessionFailureSourceCode(new Error("native_session_lease_lost")))); expect((controlError as { diagnostic?: unknown }).diagnostic).toEqual(diagnosticExpected ? { stage: "post_result_execution", category: scenario === "unknown-code" ? "execution_failure" : scenario === "permission" ? "io_permission_denied" : scenario === "io" ? "io_failure" : "io_not_found", resultAlreadyAccepted: true } : undefined); const diagnostics = events.filter(event => (event as { eventType: string }).eventType === "native.post_result_failure") as Array<{ payload: unknown }>; expect(diagnostics).toHaveLength(diagnosticExpected ? 1 : 0); if (diagnosticExpected) expect(diagnostics[0].payload).toEqual({ stage: "post_result_execution", category: scenario === "unknown-code" ? "execution_failure" : scenario === "permission" ? "io_permission_denied" : scenario === "io" ? "io_failure" : "io_not_found", resultAlreadyAccepted: true }); expect(JSON.stringify(diagnostics)).not.toContain(home); expect(JSON.stringify(diagnostics)).not.toContain("private-"); const [after] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); const results = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, run.id)); expect(results).toHaveLength(1); expect(after.resultId).toBe(results[0].id); expect(after.phase).toBe("retryable_failure"); expect(after.failureCode).toBe("native_workspace_sync_out_failed"); expect(after.nextAttemptAt).toBeInstanceOf(Date); const saved = await read(run.id); expect(saved.costAccountingPending).toBe(true); expect(saved.costAccountedAt).toBeNull(); expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id))).toHaveLength(0); expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, run.companyId))).toHaveLength(0); if (scenario === "stopped") expect(saved.status).toBe("cancelled"); expect(postResultProvider.execute).toHaveBeenCalledOnce(); expect(input.effects.onNativeWorkspaceFinalizeScheduled).toHaveBeenCalledOnce(); expect(input.services.enqueueWakeup).not.toHaveBeenCalled(); expect(input.workspace.nativeWorkspaceSync.restoreWorkspace).not.toHaveBeenCalled(); }, ); it.each(["settled", "workspace-check-failed", "terminal-unreadable"] as const)("settles an accepted local result when runner close cannot prove suspension (%s)", async scenario => { const { input } = await fixture(true); const issue = input.task.issueRef!; const [run] = await db.update(heartbeatRuns).set({ nativeIssueId: issue.id, contextSnapshot: { issueId: issue.id }, costAccountingPending: true }).where(eq(heartbeatRuns.id, input.run.id)).returning(); input.run = run; const prepared = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issue.id)); const completion = buildNativeCompletionContract(issue, { revision: contract.revision }); await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: issue.id, phase: "observed" }); input.runtime.nativeExecution = { schema: "paperclip.native-execution-input.v1", interactionResponses: [], credentialBindings: [], binding: { companyId: run.companyId, agentId: run.agentId, issueId: issue.id, runId: run.id, executionWorkspaceId: run.id }, completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: completion }, provider: { kind: "codex", model: "test-model" }, task: { identifier: issue.id, title: issue.title, description: null, prompt: "Local runner settlement fault", workMode: "standard" }, workspace: { cwd: home, repoUrl: null, repoRef: null, branchName: null }, session: { normalizedSessionId: prepared.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, } satisfies NativeExecutionInputV1; input.runtime.nativeRunnerInstanceId = prepared.runnerInstanceId; postResultProvider.execute.mockReset().mockImplementationOnce(async (options: { controlPlane: ControlPlanePort }) => { await options.controlPlane.openRun({ identity: { companyId: run.companyId, issueId: issue.id, runId: run.id, agentId: run.agentId, sessionId: prepared.normalizedSessionId }, backendKind: "mock", sourceInstanceId: prepared.runnerInstanceId }); await options.controlPlane.completeRun({ result: { ...CONTROL_PLANE_CONFORMANCE_RESULT, completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim, contractRevision: completion.revision } }, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: prepared.turnId }); if (scenario === "terminal-unreadable") { await db.update(nativeRunResults).set({ resultJson: sql`${nativeRunResults.resultJson} - 'terminal'` }).where(eq(nativeRunResults.runId, run.id)); } // The runner retired after the accepted result, but its suspend command // did not complete before the close deadline (for example, under load). throw new NativeSessionCloseUnrecoverableError({ runnerSuspended: false, providerDrained: true }, true); }); const actual = await vi.importActual("../native-runtime/native-session-executor.js"); vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async options => actual.executePaperclipNativeSession({ ...options, useRunnerd: false, backend: { descriptor: async () => { throw new Error("Unexpected provider call"); }, openSession: async () => { throw new Error("Unexpected provider call"); } } })); if (scenario === "terminal-unreadable") { // Without a rebuildable accepted result, no succeeded finalize row is written. await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ message: "native_session_lease_lost" }); const operations = await finalizations(); expect(operations.map(operation => operation.status)).toEqual(["failed"]); return; } if (scenario === "workspace-check-failed") { // A failed workspace finalization check keeps the existing failure path. const recorder = input.workspace.workspaceOperationRecorder; const record = recorder.recordOperation.bind(recorder); vi.spyOn(recorder, "recordOperation").mockImplementation(async operation => { if (operation.phase === "workspace_finalize") throw new Error("workspace check failed"); return record(operation); }); await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ message: "native_session_lease_lost", diagnostic: { category: "runner_settlement_incomplete" } }); const [after] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); expect(after).toMatchObject({ phase: "retryable_failure", leaseOwner: null, failureCode: "workspace_finalization_failed" }); return; } // Runner bookkeeping after an accepted result is not an adapter failure: // the result returns to ordinary completion like any successful turn. const execution = await executeHeartbeatRuntime(db, input); if (!execution.dispatched) throw new Error("expected a dispatched native execution"); expect(execution.adapterResult).toMatchObject({ exitCode: 0, errorMessage: null, timedOut: false, resultJson: { nativePostResultFailure: { stage: "post_result_execution", category: "runner_settlement_incomplete", resultAlreadyAccepted: true } }, nativeFinalization: { runId: run.id, terminal: { runTerminalState: "succeeded" } } }); // The executor handed over its lease, so the accepted result was finalized // now rather than after the lease TTL. const [after] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); expect(after).toMatchObject({ phase: "committed", leaseOwner: null, leaseExpiresAt: null }); const saved = await read(run.id); expect(saved.status).toBe("running"); expect(saved.errorCode).toBeNull(); const [operation] = await finalizations(); expect(operation.status).toBe("succeeded"); expect(postResultProvider.execute).toHaveBeenCalledOnce(); expect(input.services.enqueueWakeup).not.toHaveBeenCalled(); }); it("keeps workspace failure handling for other accepted-result failures on a local workspace", async () => { const { input } = await fixture(true); const issue = input.task.issueRef!; const [run] = await db.update(heartbeatRuns).set({ nativeIssueId: issue.id, contextSnapshot: { issueId: issue.id }, costAccountingPending: true }).where(eq(heartbeatRuns.id, input.run.id)).returning(); input.run = run; const prepared = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issue.id)); const completion = buildNativeCompletionContract(issue, { revision: contract.revision }); await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: issue.id, phase: "observed" }); input.runtime.nativeExecution = { schema: "paperclip.native-execution-input.v1", interactionResponses: [], credentialBindings: [], binding: { companyId: run.companyId, agentId: run.agentId, issueId: issue.id, runId: run.id, executionWorkspaceId: run.id }, completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: completion }, provider: { kind: "codex", model: "test-model" }, task: { identifier: issue.id, title: issue.title, description: null, prompt: "Local post-result I/O fault", workMode: "standard" }, workspace: { cwd: home, repoUrl: null, repoRef: null, branchName: null }, session: { normalizedSessionId: prepared.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, } satisfies NativeExecutionInputV1; input.runtime.nativeRunnerInstanceId = prepared.runnerInstanceId; postResultProvider.execute.mockReset().mockImplementationOnce(async (options: { controlPlane: ControlPlanePort }) => { await options.controlPlane.openRun({ identity: { companyId: run.companyId, issueId: issue.id, runId: run.id, agentId: run.agentId, sessionId: prepared.normalizedSessionId }, backendKind: "mock", sourceInstanceId: prepared.runnerInstanceId }); await options.controlPlane.completeRun({ result: { ...CONTROL_PLANE_CONFORMANCE_RESULT, completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim, contractRevision: completion.revision } }, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: prepared.turnId }); await readFile(join(home, `missing-${run.id}`, "harmless-checkpoint")); throw new Error("fixture must fail with ENOENT"); }); const actual = await vi.importActual("../native-runtime/native-session-executor.js"); vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async options => actual.executePaperclipNativeSession({ ...options, useRunnerd: false, backend: { descriptor: async () => { throw new Error("Unexpected provider call"); }, openSession: async () => { throw new Error("Unexpected provider call"); } } })); await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ message: "native_session_lease_lost", diagnostic: { category: "io_not_found" } }); // The executor hands its exact lease to finalization, which records the // workspace failure for repair instead of waiting out the lease TTL. const [after] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); expect(after).toMatchObject({ phase: "retryable_failure", leaseOwner: null, failureCode: "workspace_finalization_failed" }); expect((await read(run.id)).status).toBe("running"); }); const read = (id: string) => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]!); const finalizations = () => db.select().from(workspaceOperations).where(eq(workspaceOperations.phase, "workspace_finalize")); function expectCleanup(input: HeartbeatRuntimeExecutionInput) { expect(revokeHeartbeatRunGatewayTokens).toHaveBeenCalledWith({ db, companyId: input.agent.companyId, runId: input.run.id }); expect(input.instructions.releaseInstructionCopy).toHaveBeenCalledOnce(); } it("passes preparation renewal ownership to native dispatch", async () => { const { input } = await fixture(true); const acquired = vi.fn(async () => undefined); input.runtime.runOptions.onNativeExecutionLeaseAcquired = acquired; await executeHeartbeatRuntime(db, input); expect(executePaperclipNativeSession).toHaveBeenCalledWith(expect.objectContaining({ onExecutionLeaseAcquired: acquired })); // Preparation is still protected until the native executor claims its lease. expect(acquired).not.toHaveBeenCalled(); }); it.each([false, true])("preserves the dispatch gate's early return and cleanup (native: %s)", async native => { const { input, adapter } = await fixture(native); input.control.dispatchResolvedInteractionContinuationWithAtomicGate = async () => ({ dispatched: false }); expect(await executeHeartbeatRuntime(db, input)).toEqual({ dispatched: false }); expect(adapter.execute).not.toHaveBeenCalled(); expect(executePaperclipNativeSession).not.toHaveBeenCalled(); expect(input.effects.onNativeDispatchStarted).not.toHaveBeenCalled(); expect(input.effects.onLegacyAdapterEntered).not.toHaveBeenCalled(); expect(await finalizations()).toHaveLength(0); expectCleanup(input); }); it("captures usage and live instruction saves before finalizing a legacy workspace", async () => { const { input, adapter, run } = await fixture(); adapter.execute.mockImplementationOnce(async ctx => { expect(input.effects.onLegacyAdapterEntered).toHaveBeenCalledOnce(); expect(input.effects.onUsageCaptureReady).toHaveBeenCalledOnce(); await ctx.onProviderStopped?.(); return { exitCode: 0, signal: null, timedOut: false, usage: { inputTokens: 10, outputTokens: 4 }, costUsd: 0.01, provider: "test" }; }); expect(await executeHeartbeatRuntime(db, input)).toMatchObject({ dispatched: true, adapterResult: { usageComplete: true, resultJson: { instructionSave: { state: "saved", revisionId: "during-execution" } } } }); expect(await read(run.id)).toMatchObject({ status: "running", costAccountingPending: true, usageJson: { inputTokens: 10, outputTokens: 4, accountingReceiptReady: true } }); expect(await db.select().from(runUsageReceipts)).toHaveLength(1); expect(await finalizations()).toMatchObject([{ status: "succeeded" }]); expectCleanup(input); }); it.each([["local", null], ["remote", { kind: "remote", transport: "sandbox", remoteCwd: "/workspace" }]] as const)( "passes the spawn origin through the legacy adapter callback (%s)", async (expected, target) => { const { input, adapter } = await fixture(); if (target) input.workspace.executionTarget = target as typeof input.workspace.executionTarget; adapter.execute.mockImplementationOnce(async ctx => { await ctx.onSpawn?.({ pid: 4242, processGroupId: 4242, startedAt: new Date().toISOString() }); return { exitCode: 0, signal: null, timedOut: false, provider: "test" }; }); await executeHeartbeatRuntime(db, input).catch(() => undefined); expect(input.services.persistRunProcessMetadata).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ pid: 4242, targetKind: expected })); }); it("preserves Stop metadata and marks no-work accounting without manufacturing a receipt", async () => { const { input, adapter, run } = await fixture(); await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { stoppedByUser: true } }).where(eq(heartbeatRuns.id, run.id)); adapter.execute.mockResolvedValueOnce({ exitCode: 0, signal: null, timedOut: false, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }); await executeHeartbeatRuntime(db, input); expect(await read(run.id)).toMatchObject({ status: "cancelled", resultJson: { stoppedByUser: true }, usageJson: { accountingProviderWorkStarted: false } }); expect(await db.select().from(runUsageReceipts)).toHaveLength(0); expectCleanup(input); }); it("records the failed workspace and keeps the original provider error", async () => { const { input, adapter } = await fixture(); const error = new Error("provider failed"); adapter.execute.mockRejectedValueOnce(error); await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); expect(await finalizations()).toMatchObject([{ status: "failed", metadata: { errorMessage: "provider failed" } }]); expect(input.effects.onLegacyAdapterEntered).toHaveBeenCalledOnce(); expectCleanup(input); }); it.each([ [new NativeCancellationPendingRecoveryError(), null], [new NativeControllerDetachedForRestartError(), "onNativeSessionResumeScheduled"], [new NativeRunnerOwnershipUnverifiedError(), "onNativeOwnershipHeld"], [new NativeWorkspaceFinalizationBusyError(), "onNativeWorkspaceFinalizeScheduled"], [new NativeWorkspaceFinalizationOwnershipLostError(), "onNativeWorkspaceFinalizeScheduled"], ] as const)("transfers recovery ownership for %s before cleanup", async (error, effect) => { const { input } = await fixture(true); vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); input.instructions.releaseInstructionCopy = vi.fn(async () => { if (effect) expect(input.effects[effect]).toHaveBeenCalledOnce(); }); await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); expect(input.effects.onNativeDispatchStarted).toHaveBeenCalledOnce(); expect(input.effects.onLegacyAdapterEntered).not.toHaveBeenCalled(); expect(await finalizations()).toHaveLength(0); if (error instanceof NativeControllerDetachedForRestartError) { expect(revokeHeartbeatRunGatewayTokens).not.toHaveBeenCalled(); expect(input.instructions.releaseInstructionCopy).toHaveBeenCalledOnce(); } else expectCleanup(input); }); it("dispatches native execution with the selected session and persists its usage", async () => { const { input, adapter, run } = await fixture(true); vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async native => { expect(native.execution).toBe(input.runtime.nativeExecution); expect(native.runnerInstanceId).toBe(input.runtime.nativeRunnerInstanceId); await native.onUsage?.({ usage: { inputTokens: 7, outputTokens: 3 }, complete: true }); return { exitCode: 0, signal: null, timedOut: false, usageComplete: false }; }); expect(await executeHeartbeatRuntime(db, input)).toMatchObject({ dispatched: true, adapterResult: { usage: { inputTokens: 7, outputTokens: 3 }, usageComplete: true } }); expect(adapter.execute).not.toHaveBeenCalled(); expect(input.effects.onNativeDispatchStarted).toHaveBeenCalledOnce(); expect(await read(run.id)).toMatchObject({ status: "running", usageJson: { inputTokens: 7, outputTokens: 3, accountingReceiptReady: true } }); expect(await finalizations()).toMatchObject([{ status: "succeeded" }]); expectCleanup(input); }); it("returns the accepted exact native rate-card price and preserves its live budget hold", async () => { const { input, run } = await fixture(true); input.workspace.executionTarget = { kind: "remote", transport: "sandbox", remoteCwd: home }; input.runtime.nativeExecution!.provider = { kind: "codex", model: "gpt-6.1-sol" }; await db.insert(budgetPolicies).values({ companyId: run.companyId, scopeType: "agent", scopeId: input.agent.id, metric: "billed_cents", windowKind: "lifetime", amount: 100, notifyEnabled: false }); let exact: string | undefined; vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async native => { expect(native.onUsage).toBeUndefined(); const delivery = await createNativeUsageDelivery(db, { companyId: run.companyId, runId: run.id, runnerInstanceId: native.runnerInstanceId, provider: native.execution.provider, billing: { provider: "openai", biller: "openai", billingType: "metered_api" } }); const event = (sourceSeq: number, eventType: string, payload: Record): PrpEvent => ({ sourceKind: "runner", sourceInstanceId: native.runnerInstanceId, runId: run.id, sourceSeq, sourceEventId: `priced-${sourceSeq}`, turnId: "priced-turn", eventType, emittedAt: new Date().toISOString(), payload, } as PrpEvent); await delivery.capture(event(1, "usage.reported", { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 0 } })); await delivery.capture(event(2, "turn.completed", {})); const saved = (await read(run.id)).usageJson!; exact = saved.costUsdExact as string; expect(Number(exact)).toBe(1.002); expect(saved.budgetExposure).toMatchObject({ costCents: "100.2000000" }); await flushBudgetEvaluationWork(db, { companyId: run.companyId }); expect((await db.select().from(agents).where(eq(agents.id, input.agent.id)))[0].lifecycleHolds).toContain("budget"); return { exitCode: 0, signal: null, timedOut: false, usageComplete: true, costUsd: null, model: "gpt-6.1-sol" }; }); const result = await executeHeartbeatRuntime(db, input); if (!result.dispatched) throw new Error("expected native dispatch"); expect(result.adapterResult).toMatchObject({ costUsdExact: exact, costStatus: "estimated", usageComplete: true, pricingProvenance: { source: "rate_card" }, usage: { inputTokens: 1000, outputTokens: 100000 } }); }); it.each(["expected", "wrong"])("finalizes a plugin worktree through its owner (branch: %s)", async branch => { const { input } = await fixture(true); const [workspace] = await db.insert(executionWorkspaces).values({ companyId: input.agent.companyId, name: "Plugin worktree", providerType: "plugin", providerRef: "provider:opaque-worktree", cwd: "/workspaces/plugin-only", mode: "isolated_workspace", strategyType: "git_worktree", branchName: "task-branch" }).returning(); input.workspace.persistedExecutionWorkspace = workspace; const inspect = vi.spyOn(pluginWorkspaces, "finalizePluginWorkspace").mockResolvedValue({ kind: "inspection", branchName: branch === "expected" ? "task-branch" : "other-branch", headSha: "a".repeat(40), hasDirtyTrackedFiles: true, hasUntrackedFiles: false, hasLocalOnlyData: true, isMergedIntoBase: false, remotelyRecoverable: false }); try { const result = executeHeartbeatRuntime(db, input); if (branch === "expected") { await expect(result).resolves.toMatchObject({ dispatched: true }); expect(await finalizations()).toMatchObject([{ status: "succeeded" }]); } else { await expect(result).rejects.toThrow("plugin_workspace_branch_mismatch"); expect((await finalizations()).some(row => row.status === "succeeded")).toBe(false); } expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id, providerRef: workspace.providerRef }), { agentId: input.agent.id, runId: input.run.id }); } finally { inspect.mockRestore(); } }); it("retains the same-run retry signal without recording a failed workspace", async () => { const { input, run } = await fixture(true); await db.update(heartbeatRuns).set({ nativeIssueId: input.task.issueId }).where(eq(heartbeatRuns.id, run.id)); await db.insert(nativeRunFinalizations).values({ companyId: run.companyId, issueId: input.task.issueId!, runId: run.id, phase: "retryable_failure" }); const error = new Error("session connection lost"); vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ name: "NativeSessionResumeScheduledError", original: error, }); expect(input.effects.onNativeSessionResumeScheduled).toHaveBeenCalledOnce(); expect(await finalizations()).toHaveLength(0); expectCleanup(input); }); it("logs a pointer to the docs when a same-run retry follows a failed attachment", async () => { const { input, run } = await fixture(true); await db.update(heartbeatRuns).set({ nativeIssueId: input.task.issueId }).where(eq(heartbeatRuns.id, run.id)); await db.insert(nativeRunFinalizations).values({ companyId: run.companyId, issueId: input.task.issueId!, runId: run.id, phase: "retryable_failure" }); const error = new Error('native_session_recovery_failed: Error: PRP command run.attach failed: {"result":{"message":"run.attach requires a settled Codex provider session: active_turn"}}'); vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ name: "NativeSessionResumeScheduledError", original: error }); expect(input.output.onLog).toHaveBeenCalledWith("stderr", expect.stringContaining("doc/execution-semantics.md#session-settlement-after-a-run")); expect(input.output.onLog).toHaveBeenCalledWith("stderr", expect.stringContaining("run.attach requires a settled Codex provider session: active_turn")); expectCleanup(input); }); it("preserves the retained sandbox while native admission waits for its current owner", async () => { const { input } = await fixture(true); const error = new NativeControllerRecoveryWaitingError(new Date(Date.now() + 60_000)); vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ name: "NativeSessionResumeScheduledError", original: error, }); expect(input.effects.onNativeSessionResumeScheduled).toHaveBeenCalledOnce(); expect(input.effects.onProviderResourceDisposition).not.toHaveBeenCalledWith("stop_and_retain"); expect(await finalizations()).toHaveLength(0); }); it("records native restoration inside the workspace finalization operation", async () => { const { input, run } = await fixture(true); const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test"); const exporter = new InMemorySpanExporter(); const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] }); expect(trace.setGlobalTracerProvider(provider)).toBe(true); const restore = vi.fn(async () => { expect(await finalizations()).toMatchObject([{ status: "running", metadata: { owningService: "native_workspace_finalizer" }, }]); await measureWorkspaceSyncPhase("workspace.restore", async () => {}); await measureRunPhase("execution.workspace_commit", async () => {}); }); input.workspace.nativeWorkspaceSync = { mode: "host_current", reference: { schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: run.id, leaseId: randomUUID(), providerLeaseId: "sandbox", remoteCwd: "/workspace", resourceDisposition: null }, restoreWorkspace: restore, cleanup: vi.fn(async () => {}), }; try { expect(await withRunExecutionTiming(run.id, () => executeHeartbeatRuntime(db, input))).toMatchObject({ dispatched: true }); expect(restore).toHaveBeenCalledOnce(); expect(await finalizations()).toMatchObject([{ status: "succeeded", metadata: { owningService: "native_workspace_finalizer" }, }]); await provider.forceFlush(); const spans = exporter.getFinishedSpans(); const attempt = spans.find(span => span.name === "task.execution_attempt")!; expect(attempt).toBeDefined(); for (const name of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission"]) { const startup = spans.filter(span => span.name === name); expect(startup).toHaveLength(1); expect(startup[0].parentSpanContext?.spanId).toBe(attempt.spanContext().spanId); } const copyback = spans.filter(span => span.name === "execution.workspace_restore"); const commit = spans.filter(span => span.name === "execution.workspace_commit"); expect(copyback).toHaveLength(1); expect(commit).toHaveLength(1); const finalization = spans.find(span => span.name === "execution.workspace_finalize")!; expect(finalization).toBeDefined(); for (const child of [...copyback, ...commit]) expect(child.parentSpanContext?.spanId).toBe(finalization.spanContext().spanId); } finally { vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint); trace.disable(); await provider.shutdown(); } }); it.each([false, true])("ends admission timing without losing later provider spans (native: %s)", async native => { const { input, adapter, run } = await fixture(native); const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test"); const exporter = new InMemorySpanExporter(); const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] }); expect(trace.setGlobalTracerProvider(provider)).toBe(true); let release!: (result: Awaited>) => void; const providerResult = new Promise>>(resolve => { release = resolve; }); const providerWork = async () => { const result = await providerResult; await createRunPreparationTiming(run.id, "native.controller")("coordinator_release", async () => {}); return result; }; if (native) vi.mocked(executePaperclipNativeSession).mockImplementationOnce(providerWork); else adapter.execute.mockImplementationOnce(providerWork); const running = withRunExecutionTiming(run.id, () => executeHeartbeatRuntime(db, input)); try { await vi.waitFor(() => { expect(exporter.getFinishedSpans().filter(span => span.name === "execution.dispatch_admission")).toHaveLength(1); }, { timeout: 5_000 }); expect(native ? executePaperclipNativeSession : adapter.execute).toHaveBeenCalledOnce(); expect(exporter.getFinishedSpans().find(span => span.name === "task.execution_attempt")).toBeUndefined(); expect(await finalizations()).toHaveLength(0); release({ exitCode: 0, signal: null, timedOut: false }); await running; const spans = exporter.getFinishedSpans(); const attempt = spans.find(span => span.name === "task.execution_attempt")!; const later = spans.filter(span => span.name === "native.controller.coordinator_release"); expect(later).toHaveLength(1); expect(later[0].spanContext().traceId).toBe(attempt.spanContext().traceId); expect(later[0].parentSpanContext?.spanId).toBe(attempt.spanContext().spanId); } finally { release({ exitCode: 0, signal: null, timedOut: false }); await running; vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint); trace.disable(); await provider.shutdown(); } }); it.each(["codex", "grok_bot", "openai_dot"] as const)("retains user files only when %s can access the workspace after a dispatch failure", async provider => { const { input, run } = await fixture(true, provider); input.workspace.nativeWorkspaceSync = { mode: "host_current", reference: { schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: run.id, leaseId: randomUUID(), providerLeaseId: "sandbox", remoteCwd: "/workspace", resourceDisposition: "destroy" }, restoreWorkspace: vi.fn(), cleanup: vi.fn(), }; const error = new Error("runner_remote_process_identity_unavailable"); vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); input.instructions.releaseInstructionCopy = vi.fn(async () => { if (provider === "codex") expect(input.effects.onProviderResourceDisposition).toHaveBeenCalledWith("stop_and_retain"); else expect(input.effects.onProviderResourceDisposition).not.toHaveBeenCalledWith("stop_and_retain"); }); await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); expect(input.workspace.nativeWorkspaceSync.restoreWorkspace).not.toHaveBeenCalled(); expectCleanup(input); }); it("records a failed barrier if the successful finalization write fails", async () => { const { input } = await fixture(); const error = new Error("finalization write failed"); vi.spyOn(input.workspace.workspaceOperationRecorder, "recordOperation").mockRejectedValueOnce(error); await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); expect(await finalizations()).toMatchObject([{ status: "failed" }]); expectCleanup(input); }); it("retains a failed sandbox restore even when persisting its receipt throws", async () => { const { input, adapter, run } = await fixture(); input.workspace.executionTarget = { kind: "remote", transport: "sandbox", remoteCwd: home }; const evidence = { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: { message: "copy-back failed" } }; adapter.execute.mockResolvedValueOnce({ exitCode: 0, signal: null, timedOut: false, resultJson: evidence }); const order: string[] = []; input.effects.onWorkspaceRestoreFailure = vi.fn(() => { order.push("restore evidence"); }); input.effects.onProviderResourceDisposition = vi.fn(() => { order.push("retain sandbox"); }); input.instructions.releaseInstructionCopy = vi.fn(async () => { order.push("cleanup"); }); const error = new Error("restore receipt write failed"); const record = vi.spyOn(legacyRecovery, "recordLegacyWorkspaceRestoreFailure").mockImplementationOnce(async () => { order.push("receipt write"); throw error; }); try { await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); expect(record).toHaveBeenCalledWith(db, run, evidence, input.workspace.workspaceRestoreSource); expect(input.effects.onWorkspaceRestoreFailure).toHaveBeenCalledWith(evidence); expect(input.effects.onProviderResourceDisposition).toHaveBeenCalledWith("stop_and_retain"); expect(order).toEqual(["restore evidence", "retain sandbox", "receipt write", "cleanup"]); expect(await finalizations()).toMatchObject([{ status: "failed" }]); expectCleanup(input); } finally { record.mockRestore(); } }); it("runs instruction cleanup even when gateway revocation fails", async () => { const { input } = await fixture(); vi.mocked(revokeHeartbeatRunGatewayTokens).mockRejectedValueOnce(new Error("gateway unavailable")); expect(await executeHeartbeatRuntime(db, input)).toMatchObject({ dispatched: true }); expectCleanup(input); }); });