import { createHash } from "node:crypto"; import { canonicalJson } from "../../packages/shared/src/portability-hash.js"; import { isValidNativePrpEnvelope } from "./native-event-envelope.js"; import type { StoryCheck, StoryRun } from "./everyday-observations.js"; import { pollUntil } from "./api.js"; type Row = Record; const record = (value: unknown): Row => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {}; const id = (value: unknown): value is string => typeof value === "string" && value.length > 0 && value.length <= 256 && !/[\u0000-\u001f\u007f]/u.test(value); const digest = (value: unknown) => createHash("sha256").update(canonicalJson(value)).digest("hex"); export interface RemoteRestartEvent { seq: number; sourceSeq: number; sourceEventId: string; eventType: string; sourceKind: "runner" | "control_plane"; turnId: string | null; toolExecutionId: string | null; toolCompleted: boolean; sha256: string; } export interface RemoteRestartBoundary { identity: { runId: string; companyId: string; agentId: string; issueId: string; runnerInstanceId: string; nativeSessionId: string; processPid: number; processStartedAt: string }; lease: { id: string; environmentId: string; executionWorkspaceId: string | null; providerLeaseSha256: string }; events: RemoteRestartEvent[]; } export interface RemoteRestartEvidence { before?: RemoteRestartBoundary; after?: RemoteRestartBoundary; final?: { run: StoryRun; events: RemoteRestartEvent[]; operations: Row[]; scheduledRetry: unknown; recoveryActions: Row[] }; } export async function waitForRemoteRestartBoundary(input: { deadlineAt: number; load: () => Promise<{ run: StoryRun | undefined; boundary?: RemoteRestartBoundary }>; }): Promise { const observed = await pollUntil({ label: "active remote native turn with completed tool receipt", deadlineAt: input.deadlineAt, load: input.load, accept: state => state.run?.status === "running" && state.boundary !== undefined, reject: state => !state.run || state.run.status !== "running" ? "Remote restart boundary unexercised: original run ended before fault injection" : undefined, }); return observed.boundary!; } export function projectRemoteRestartRun(run: StoryRun): StoryRun { return { id: run.id, companyId: run.companyId, agentId: run.agentId, nativeIssueId: run.nativeIssueId, status: run.status, runtimeMode: run.runtimeMode, runnerInstanceId: run.runnerInstanceId, nativeSessionId: run.nativeSessionId, nativePhase: run.nativePhase === "committed" ? "committed" : null, resultJson: { finalizationPhase: run.resultJson?.finalizationPhase === "committed" ? "committed" : null, workspaceFinalizeStatus: run.resultJson?.workspaceFinalizeStatus === "succeeded" ? "succeeded" : null, nextAttemptAt: run.resultJson?.nextAttemptAt == null ? null : "scheduled", failureCode: run.resultJson?.failureCode == null ? null : "present" } }; } /** Keep only receipt identities and hashes; never retain provider text or tool arguments. */ export function projectRemoteRestartEvents(run: StoryRun, rows: readonly Row[]): RemoteRestartEvent[] { const seen = new Set(); const cursors = new Map(); let lastSeq = 0; return rows.filter(row => record(row.payload).prpEvent !== undefined).map(row => { const event = record(record(row.payload).prpEvent); const source = event.sourceInstanceId; const runnerSource = event.sourceKind === "runner" && source === run.runnerInstanceId; const controllerSource = event.sourceKind === "control_plane" && source === `${run.runnerInstanceId}:control` && ["run.result.accepted", "run.terminal"].includes(String(event.eventType)); if (row.companyId !== run.companyId || row.runId !== run.id || !isValidNativePrpEnvelope(event, row.protocolSchemaVersion) || event.runId !== run.id || event.normalizedSessionId !== run.nativeSessionId || event.eventType !== row.eventType || (!runnerSource && !controllerSource) || !id(source) || !id(event.sourceEventId) || !Number.isSafeInteger(row.seq) || Number(row.seq) <= lastSeq || !Number.isSafeInteger(event.sourceSeq) || Number(event.sourceSeq) <= (cursors.get(source) ?? 0) || seen.has(event.sourceEventId) || event.sourceEventId !== `${source}:${run.id}:${event.sourceSeq}`) { throw new Error("Remote restart has foreign, duplicate or invalid native receipts"); } seen.add(event.sourceEventId); cursors.set(source, Number(event.sourceSeq)); lastSeq = Number(row.seq); const payload = record(event.payload); return { seq: lastSeq, sourceSeq: Number(event.sourceSeq), sourceEventId: event.sourceEventId, eventType: String(event.eventType), sourceKind: event.sourceKind as RemoteRestartEvent["sourceKind"], turnId: id(event.turnId) ? event.turnId : null, toolExecutionId: id(payload.executionId) ? payload.executionId : null, toolCompleted: event.eventType === "tool.execution.completed" && payload.status === "completed", sha256: digest(event) }; }); } export function observeRemoteRestartBoundary(run: StoryRun, rows: readonly Row[], leases: readonly Row[], scope: { companyId: string; agentId: string; issueId: string; environmentId: string; }): RemoteRestartBoundary { if (run.companyId !== scope.companyId || run.agentId !== scope.agentId || run.nativeIssueId !== scope.issueId || run.status !== "running" || run.runtimeMode !== "native" || !id(run.id) || !id(run.runnerInstanceId) || !id(run.nativeSessionId) || !Number.isSafeInteger(run.processPid) || Number(run.processPid) <= 0 || !id(run.processStartedAt) || !Number.isFinite(Date.parse(run.processStartedAt))) { throw new Error("Remote restart boundary requires the original live native process"); } const owned = leases.filter(lease => lease.companyId === scope.companyId && lease.environmentId === scope.environmentId && lease.issueId === scope.issueId && lease.heartbeatRunId === run.id && lease.provider === "daytona" && lease.status === "active" && lease.releasedAt == null && id(lease.id) && id(lease.providerLeaseId)); if (owned.length !== 1) throw new Error("Remote restart boundary requires one exact active sandbox lease"); const lease = owned[0]!; const events = projectRemoteRestartEvents(run, rows); const starts = events.filter(event => event.eventType === "turn.started" && event.sourceKind === "runner"); const completed = events.filter(event => event.toolCompleted && event.sourceKind === "runner" && event.toolExecutionId); if (starts.length !== 1 || !id(starts[0]!.turnId) || completed.length === 0 || completed.some(event => event.turnId !== starts[0]!.turnId) || events.some(event => ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted", "run.terminal"].includes(event.eventType))) { throw new Error("Remote restart boundary has no completed tool in an active original provider turn"); } return { identity: { runId: run.id, companyId: scope.companyId, agentId: scope.agentId, issueId: scope.issueId, runnerInstanceId: run.runnerInstanceId, nativeSessionId: run.nativeSessionId, processPid: Number(run.processPid), processStartedAt: run.processStartedAt }, lease: { id: String(lease.id), environmentId: scope.environmentId, executionWorkspaceId: typeof lease.executionWorkspaceId === "string" ? lease.executionWorkspaceId : null, providerLeaseSha256: digest(lease.providerLeaseId) }, events, }; } function retainsPrefix(before: readonly RemoteRestartEvent[], after: readonly RemoteRestartEvent[]): boolean { return before.length > 0 && after.length >= before.length && before.every((event, index) => digest(event) === digest(after[index])); } export function gradeRemoteRestart(evidence: RemoteRestartEvidence | undefined): StoryCheck[] { const { before, after, final } = evidence ?? {}; const check = (id: string, passed: boolean, detail: string) => ({ id, passed, detail }); const sameIdentity = Boolean(before && after && digest(before.identity) === digest(after.identity) && digest(before.lease) === digest(after.lease) && retainsPrefix(before.events, after.events)); const run = final?.run; const identity = before?.identity; const finalIdentity = Boolean(identity && run && run.id === identity.runId && run.companyId === identity.companyId && run.agentId === identity.agentId && run.nativeIssueId === identity.issueId && run.runtimeMode === "native" && run.runnerInstanceId === identity.runnerInstanceId && run.nativeSessionId === identity.nativeSessionId); const events = final?.events ?? []; const turnId = before?.events.find(event => event.eventType === "turn.started")?.turnId; const once = (kind: string, source: RemoteRestartEvent["sourceKind"]) => events.filter(event => event.eventType === kind && event.sourceKind === source).length === 1; const completedTools = events.filter(event => event.toolCompleted).map(event => event.toolExecutionId); const nativeCompletion = finalIdentity && Boolean(before && after && retainsPrefix(after.events, events)) && once("turn.submitted", "runner") && once("turn.started", "runner") && once("turn.completed", "runner") && once("run.result.accepted", "control_plane") && once("run.terminal", "control_plane") && events.filter(event => event.eventType.startsWith("turn.")).every(event => event.turnId === turnId || (event.eventType === "turn.submitted" && event.turnId === null)) && new Set(completedTools).size === completedTools.length; const operations = final?.operations ?? []; const saved = finalIdentity && run?.status === "succeeded" && run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed" && run.resultJson.workspaceFinalizeStatus === "succeeded" && run.resultJson.nextAttemptAt == null && run.resultJson.failureCode == null && operations.some(operation => operation.companyId === identity?.companyId && operation.heartbeatRunId === identity?.runId && operation.issueId === identity?.issueId && operation.phase === "workspace_finalize" && operation.status === "succeeded" && operation.executionWorkspaceId === before?.lease.executionWorkspaceId) && !operations.some(operation => ["pending", "queued", "running"].includes(String(operation.status))) && final?.scheduledRetry == null && !final?.recoveryActions.some(action => ["active", "pending"].includes(String(action.status))); return [ check("remote-restart-same-process-and-lease", sameIdentity, "The same run, native session, Runner process and sandbox lease remain live after restart."), check("remote-restart-native-completion-once", nativeCompletion, "The original provider turn completes once and retains its native receipt prefix."), check("remote-restart-files-finalized", Boolean(saved), "The original run commits its workspace receipt and leaves no active operation or recovery."), ]; }