import { createHash, randomUUID } from "node:crypto"; import type { AdapterExecutionResult } from "../../adapters/index.js"; import type { NativeFinalizationResult } from "@paperclipai/shared"; import type { NativeExecutionInputV1, NativeSession, NativeSessionBackend, } from "@paperclipai/paperclip-runner"; import { createCodexNativeSessionBackend, executeNativeSession, } from "@paperclipai/paperclip-runner"; import type { Db } from "@paperclipai/db"; import { and, eq, sql } from "drizzle-orm"; import { heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { NATIVE_STATUS_ARBITER_POLICY_VERSION, type NativeAuthoritativeIssueStatus, type NativeStatusDecision, } from "./status-arbiter.js"; type ActiveNativeSession = { session: NativeSession; cancelRequested: boolean; }; const activeNativeSessions = new Map(); export function nativeSessionFailureDisposition(attempt: number, now = new Date()) { const exhausted = attempt >= 3; return { phase: exhausted ? "terminal_failure" as const : "retryable_failure" as const, failureCode: exhausted ? "native_session_retry_exhausted" as const : "native_session_interrupted" as const, nextAttemptAt: exhausted ? null : new Date(now.getTime() + 30_000), }; } function record(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; } export function cancelNativeSession(runId: string, reason: string): Promise; export function cancelNativeSession( runId: string, reason: string, options: { db: Db; scope?: "turn" | "run" | "issue"; replacementAccepted?: boolean }, ): Promise<{ dispatched: boolean; decision: NativeStatusDecision | null }>; export async function cancelNativeSession( runId: string, reason: string, options?: { db: Db; scope?: "turn" | "run" | "issue"; replacementAccepted?: boolean }, ): Promise { let decision: NativeStatusDecision | null = null; if (options) { const run = await options.db.select({ agentId: heartbeatRuns.agentId, companyId: heartbeatRuns.companyId, contextSnapshot: heartbeatRuns.contextSnapshot, }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).limit(1).then((rows) => rows[0] ?? null); const context = record(run?.contextSnapshot); const issueId = typeof context.issueId === "string" ? context.issueId : typeof context.taskId === "string" ? context.taskId : null; const issue = run && issueId ? await options.db.select({ status: issues.status }).from(issues).where(and( eq(issues.id, issueId), eq(issues.companyId, run.companyId), )).limit(1).then((rows) => rows[0] ?? null) : null; if (run && issue) { decision = resolveNativeCancellationStatus({ scope: options.scope ?? "run", priorIssueStatus: issue.status as NativeAuthoritativeIssueStatus, agentId: run.agentId, replacementAccepted: options.replacementAccepted, }); if (options.replacementAccepted && decision.effects.some((effect) => effect.kind === "accept_replacement_turn")) { await options.db.update(heartbeatRuns).set({ status: "running", continuationAttempt: sql`${heartbeatRuns.continuationAttempt} + 1`, nextAction: "Accept a replacement native turn on the existing run.", updatedAt: new Date(), }).where(eq(heartbeatRuns.id, runId)); } } } const active = activeNativeSessions.get(runId); if (!active) return options ? { dispatched: false, decision } : false; if (active.cancelRequested) return options ? { dispatched: true, decision } : true; active.cancelRequested = true; try { if (active.session.cancel) await active.session.cancel({ reason }); else if (active.session.interrupt) await active.session.interrupt({ reason }); } catch (error) { active.cancelRequested = false; throw error; } return options ? { dispatched: true, decision } : true; } /** Authenticated cancellation scope projected through the shared arbiter. */ export function resolveNativeCancellationStatus(input: { scope: "turn" | "run" | "issue"; priorIssueStatus: NativeAuthoritativeIssueStatus; agentId: string; replacementAccepted?: boolean; }): NativeStatusDecision { if (input.scope === "turn") { return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "preserve", toStatus: input.priorIssueStatus, reasonCode: input.replacementAccepted ? null : "cancellation_turn_only", unblockDescriptor: null, effects: [{ kind: "accept_replacement_turn" }], }; } if (input.scope === "run") { return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "preserve", toStatus: input.priorIssueStatus, reasonCode: "cancellation_run_only", unblockDescriptor: null, effects: [{ kind: "release_run_resources" }], }; } return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "cancelled", toStatus: "cancelled", reasonCode: "cancellation_issue_authorized", unblockDescriptor: null, effects: [{ kind: "release_checkout" }, { kind: "cancel_continuations" }], }; } export async function executePaperclipNativeSession(input: { db: Db; execution: NativeExecutionInputV1; runnerInstanceId: string; leaseOwner?: string; /** Test seam at the provider boundary; production always uses the package Codex backend. */ backend?: NativeSessionBackend; }): Promise { const leaseOwner = input.leaseOwner ?? `${input.runnerInstanceId}:${randomUUID()}`; const leaseNow = new Date(); const leaseExpiresAt = new Date(leaseNow.getTime() + 20 * 60_000); const attempt = await input.db.transaction(async (tx) => { const coordinator = await tx.select().from(nativeRunFinalizations) .where(and( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq(nativeRunFinalizations.companyId, input.execution.binding.companyId), eq(nativeRunFinalizations.issueId, input.execution.binding.issueId), )).for("update").limit(1).then((rows) => rows[0] ?? null); if (!coordinator) throw new Error("native_finalization_coordinator_missing"); if (["committed", "applied"].includes(coordinator.phase)) throw new Error("native_run_already_committed"); if ( coordinator.leaseOwner && coordinator.leaseOwner !== leaseOwner && coordinator.leaseExpiresAt && coordinator.leaseExpiresAt > leaseNow ) throw new Error("native_finalization_lease_busy"); await tx.update(nativeRunFinalizations).set({ phase: coordinator.resultId ? coordinator.phase : "observed", attempt: coordinator.attempt + 1, leaseOwner, leaseExpiresAt, failureCode: null, failureDetail: null, nextAttemptAt: null, updatedAt: leaseNow, }).where(eq(nativeRunFinalizations.runId, coordinator.runId)); await tx.update(heartbeatRuns).set({ nativePhase: coordinator.resultId ? coordinator.phase : "observed", nativePhaseUpdatedAt: leaseNow, updatedAt: leaseNow, }).where(eq(heartbeatRuns.id, coordinator.runId)); return coordinator.attempt + 1; }); const controlPlaneInstanceId = `${input.runnerInstanceId}:control`; const controlPlane = new PaperclipControlPlanePort(input.db, { companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, runId: input.execution.binding.runId, agentId: input.execution.binding.agentId, completionContractId: input.execution.completionContract.id, completionContractSha256: input.execution.completionContract.sha256, sourceInstanceId: input.runnerInstanceId, controlPlaneSourceInstanceId: controlPlaneInstanceId, }); let native: Awaited>; try { native = await executeNativeSession({ input: input.execution, backend: input.backend ?? createCodexNativeSessionBackend(input.execution, { runnerInstanceId: input.runnerInstanceId }), controlPlane, runnerInstanceId: input.runnerInstanceId, controlPlaneInstanceId, onSession: (session) => { if (session) activeNativeSessions.set(input.execution.binding.runId, { session, cancelRequested: false, }); else activeNativeSessions.delete(input.execution.binding.runId); }, }); } catch (error) { const now = new Date(); const { phase, failureCode, nextAttemptAt } = nativeSessionFailureDisposition(attempt, now); const exhausted = phase === "terminal_failure"; const message = error instanceof Error ? error.message.slice(0, 2_000) : String(error).slice(0, 2_000); await input.db.transaction(async (tx) => { const updated = await tx.update(nativeRunFinalizations).set({ phase, leaseOwner: null, leaseExpiresAt: null, failureCode, failureDetail: { message, originalFailureCode: "native_session_interrupted", recoveryOwner: { kind: "agent", agentId: input.execution.binding.agentId }, nextAction: exhausted ? "Inspect the persisted native session after its bounded resume budget was exhausted." : "Resume this same run from its persisted native provider checkpoint after the retry delay.", }, nextAttemptAt, updatedAt: now, }).where(and( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq(nativeRunFinalizations.leaseOwner, leaseOwner), )).returning({ runId: nativeRunFinalizations.runId }).then((rows) => rows[0] ?? null); if (!updated) throw new Error("native_session_lease_lost"); await tx.update(heartbeatRuns).set({ nativePhase: phase, nativePhaseUpdatedAt: now, error: message, errorCode: failureCode, updatedAt: now, }).where(eq(heartbeatRuns.id, input.execution.binding.runId)); await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({ companyId: input.execution.binding.companyId, sourceIssueId: input.execution.binding.issueId, kind: "active_run_watchdog", ownerType: "agent", ownerAgentId: input.execution.binding.agentId, cause: failureCode, fingerprint: createHash("sha256") .update(`${input.execution.binding.runId}:${failureCode}`) .digest("hex"), evidence: { runId: input.execution.binding.runId, coordinatorAttempt: attempt }, nextAction: exhausted ? "Inspect or explicitly restart the exhausted persisted native session." : "Resume the persisted native session on the same heartbeat run.", wakePolicy: nextAttemptAt ? { kind: "resume_native_run", runId: input.execution.binding.runId, notBefore: nextAttemptAt.toISOString() } : null, maxAttempts: 3, }); }); throw error; } await input.db.update(nativeRunFinalizations).set({ leaseOwner: null, leaseExpiresAt: null, updatedAt: new Date(), }).where(and( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq(nativeRunFinalizations.leaseOwner, leaseOwner), )); const finalization: NativeFinalizationResult = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: input.execution.binding.runId, issueId: input.execution.binding.issueId, companyId: input.execution.binding.companyId, result: native.result as unknown as Record, terminal: native.terminal, turnId: native.turnId, sourceInstanceId: input.runnerInstanceId, normalizedSessionId: native.normalizedSessionId, providerSessionId: native.providerSessionId, driverKind: native.driverKind, driverVersion: native.driverVersion, nativeEventCount: native.nativeEventCount, highestContiguousSourceSeq: native.highestContiguousSourceSeq, workspaceFinalizeStatus: "pending", }; return { exitCode: native.terminal.runTerminalState === "succeeded" ? 0 : 1, signal: null, timedOut: false, errorMessage: native.terminal.runTerminalState === "succeeded" ? null : `Native session ${native.terminal.runTerminalState}`, resultJson: { nativeResult: native.result as unknown as Record, nativeTerminal: native.terminal as unknown as Record, }, summary: native.result.summary, sessionId: native.normalizedSessionId, sessionDisplayId: native.providerSessionId ?? native.normalizedSessionId, provider: "openai", usageBasis: "per_run", nativeFinalization: finalization, }; }