diff --git a/doc/evals.md b/doc/evals.md index 6dd6e66ce00..52588da2c49 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -157,6 +157,11 @@ permission, and requester scope decisions through saved browser interactions. ## Validation ladder +The native Codex Daytona `recover-controller` cell in Everyday Workflows +requires active process and lease continuity, native receipt continuity and +workspace finalization. See the [restart evidence contract](../tests/runner-e2e/EVERYDAY-WORKFLOWS.md). +Catalog discovery and deterministic calibration do not establish live recovery. + The explicit-only [public MCP suite](../tests/runner-e2e/PUBLIC-MCP.md) evaluates paid assistant delegation, later retrieval, feedback, review, uncertain retries and permission boundaries. It uses the Product E2E fixtures, launcher, evidence diff --git a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md index d2c9a2e899e..567f365adeb 100644 --- a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md +++ b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md @@ -47,6 +47,31 @@ restart, a verified first download supplies the persistence checkpoint; the controller is interrupted during a subsequent revision with another queued requirement. +The native Codex Daytona restart cell now requires a completed native tool +receipt while that revision is still running. Before and after restart it pins +the same run, normalized session, Runner identity, PID/start time, and active +sandbox lease. The original receipt prefix must remain unchanged. A run that +ends before either active observation leaves the fault boundary unexercised; +it cannot pass from the final ZIP alone. The cell no longer exempts an original +Codex remote run that fails with `process_lost` and is replaced. + +Completion requires one original provider turn and control-plane terminal, +the same native receipt prefix, no duplicate completed tool execution IDs, +a committed run and successful `workspace_finalize` receipt, and no pending +workspace operation or recovery. Existing independent ZIP and preserved-download +checks still apply. These receipt checks do not prove that arbitrary external +effects with different tool IDs cannot repeat. + +`remote-restart-before.json`, `remote-restart-after.json` and +`remote-restart-final.json` retain receipt identities and hashes. They exclude +provider text, tool arguments, command output, and raw provider sandbox IDs. +The suite definition version and harness digest change with this oracle. +Its deterministic calibration is not a claim that a live Daytona run passed. + +```sh +pnpm test:e2e:runner -- --id everyday-workflows.runner-codex.daytona.recover-controller --max-automatic-retries 0 +``` + ```sh pnpm test:e2e:runner -- --list --suite everyday-workflows pnpm test:e2e:runner -- --suite everyday-workflows --environment local --max-parallel 2 diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 70281a25c54..67ba48e1a7f 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -32,6 +32,14 @@ changes. ## GitHub authorization +The Everyday remote Codex restart oracle reads only the existing public run, +event, environment-lease, workspace-operation and recovery APIs. It does not +execute commands in a provider sandbox or signal a provider process. Its new +evidence projects closed receipt metadata and hashes the provider sandbox ID +and full native event content. Provider text, tool arguments and command output +do not enter these snapshots. Existing image admission, credentials, billing, +retention and cleanup rules continue to apply. + Set `RUNNER_E2E_ALLOWED_ACTOR_IDS` to a non-empty JSON array of numeric GitHub user IDs. Keep the list equal to the owners of `.github/**` in `.github/CODEOWNERS`. For example, use `[123456,789012]`. Resolve each ID from diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index bccc1d17fb7..7dbfd7b7a86 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1367,7 +1367,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ ...(["hire-reuse", "delegate-feedback"].includes(task.id) ? [] : [`everyday-workflows.runner-opencode.local.${task.id}`]), ]), ], - definitionMetadata: { version: 6, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", + definitionMetadata: { version: 7, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", + remoteCodexRestart: "same-active-process-lease-native-prefix-and-workspace-receipt", procedureCases: { ids: ["hire-reuse", "delegate-feedback"], maximumAttemptsPerCell: 1, companyAndLeadBudgetCents: 1_000 } }, }, { diff --git a/tests/runner-e2e/everyday-flow.ts b/tests/runner-e2e/everyday-flow.ts index f3d20ad2407..3dc84a29769 100644 --- a/tests/runner-e2e/everyday-flow.ts +++ b/tests/runner-e2e/everyday-flow.ts @@ -19,6 +19,8 @@ import { submitTaskReply } from "./user-actions.js"; import { createTaskFromPromptThroughUi } from "./plan-task-ui.js"; import { waitForTaskChatRendered, waitForTaskIdentity } from "./continuation-screenshot.js"; import { hasPersistedSource, isSavedSourceCheckpoint } from "./everyday-interruption.js"; +import { collectRunEvents } from "./run-observations.js"; +import { gradeRemoteRestart, observeRemoteRestartBoundary, projectRemoteRestartEvents, projectRemoteRestartRun, waitForRemoteRestartBoundary, type RemoteRestartEvidence } from "./everyday-remote-restart.js"; import { setupAggregatorFixture } from "./aggregator-fixture.js"; import { gradeProviderChoice, gradeProviderOutcome, requireProviderAccessCard } from "./connection-routing-evidence.js"; import { setupConnectionReview } from "./connection-reviews.js"; @@ -76,6 +78,7 @@ export interface EverydayEvidence { agents: Row[]; downloads: Row[]; allowedInterruptedRuns: string[]; + remoteRestart?: RemoteRestartEvidence; } interface Input { page: Page; @@ -165,6 +168,8 @@ export async function runEverydayFlow(input: Input) { let review: Awaited> | undefined; let project = fixtures.project; const caseId = execution.task.id; + const requireRemoteRestart = caseId === "recover-controller" && execution.environment.id === "daytona" + && execution.profile.generation === "native" && execution.profile.provider === "codex"; const providerChoice = caseId === "provider-decline" || caseId === "provider-second"; const nativeProviderCase = caseId === "provider-native"; let aggregatorFixture: Awaited> | undefined; @@ -544,6 +549,19 @@ export async function runEverydayFlow(input: Input) { async function sourceReady() { await recordSource("source-saved-before-interruption", "source-before-interruption.json", true); } + async function remoteRestartBoundary(runId?: string) { + await refresh(); + const active = ev.runs.filter(run => run.status === "running" && run.nativeIssueId === parent!.id + && (!runId || run.id === runId)); + if (active.length !== 1) throw new Error("Remote restart boundary unexercised: the original run is not active"); + const run = active[0]!; + const [rows, leases] = await Promise.all([ + collectRunEvents((afterSeq, limit) => api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`)), + api.get(`/api/environments/${fixtures.environment.id}/leases`), + ]); + return observeRemoteRestartBoundary(run, rows, leases, { companyId: fixtures.company.id, + agentId: fixtures.agent.id, issueId: parent!.id, environmentId: fixtures.environment.id }); + } async function prepareStopBoundary() { // Providers can finish a short first turn before the browser can click // Stop. If that happens, submit one ordinary user follow-up through the @@ -576,6 +594,9 @@ export async function runEverydayFlow(input: Input) { "agentmail-setup-evidence.ts", "everyday-delivery.ts", "everyday-observations.ts", + "everyday-remote-restart.ts", + "native-event-envelope.ts", + "run-observations.ts", "everyday-artifact.py", "user-actions.ts", "runner.spec.ts", @@ -915,10 +936,28 @@ export async function runEverydayFlow(input: Input) { } else { await reply(LATE_REQUIREMENT); note("followup-submitted-before-interruption"); - ev.allowedInterruptedRuns.push(active.id); + if (requireRemoteRestart) { + const before = await waitForRemoteRestartBoundary({ + deadlineAt: input.deadlineAt, load: async () => { + await refresh(); + const original = ev.runs.find(run => run.id === active.id); + if (!original || original.status !== "running") return { run: original }; + const rows = await collectRunEvents((afterSeq, limit) => api.get(`/api/heartbeat-runs/${active.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); + if (!rows.some(row => row.payload?.prpEvent?.eventType === "tool.execution.completed" && row.payload.prpEvent.payload?.status === "completed")) return { run: original }; + return { run: original, boundary: await remoteRestartBoundary(active.id) }; + } }); + ev.remoteRestart = { before }; + await input.evidence("remote-restart-before.json", before); + } else ev.allowedInterruptedRuns.push(active.id); await input.restart(); note("controller-restarted"); await openParent(); + if (requireRemoteRestart) { + const after = await remoteRestartBoundary(active.id); + ev.remoteRestart!.after = after; + await input.evidence("remote-restart-after.json", after); + if (!gradeRemoteRestart(ev.remoteRestart)[0]!.passed) throw new Error("Remote restart changed the original native process, lease or receipt prefix"); + } } } if (providerChoice) { @@ -1583,6 +1622,30 @@ export async function runEverydayFlow(input: Input) { ), "Successful runs retain the native terminal contract.", ); + if (requireRemoteRestart) { + const runId = ev.remoteRestart?.before?.identity.runId; + if (!runId) throw new Error("Remote restart has no saved original run boundary"); + const final = await pollUntil({ label: "original remote run committed with workspace receipt", + deadlineAt: input.deadlineAt, load: async () => { + const [run, rows, operationRows, recovery, issue] = await Promise.all([ + api.get(`/api/heartbeat-runs/${runId}`), + collectRunEvents((afterSeq, limit) => api.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)), + api.get(`/api/heartbeat-runs/${runId}/workspace-operations`), + api.get<{ active: Row | null; actions: Row[] }>(`/api/issues/${parent!.id}/recovery-actions`), + api.get(`/api/issues/${parent!.id}`), + ]); + if (!Array.isArray(operationRows) || !Array.isArray(recovery.actions) || !("active" in recovery)) throw new Error("Remote restart finalization evidence is incomplete"); + const final = { run: projectRemoteRestartRun(run), events: projectRemoteRestartEvents(run, rows), + operations: operationRows.map(operation => ({ companyId: operation.companyId, heartbeatRunId: operation.heartbeatRunId, + issueId: operation.issueId, executionWorkspaceId: operation.executionWorkspaceId, phase: operation.phase, status: operation.status })), + scheduledRetry: issue.scheduledRetry, + recoveryActions: [recovery.active, ...recovery.actions].filter((action): action is Row => action !== null).map(action => ({ status: action.status })) }; + ev.remoteRestart!.final = final; + return final; + }, accept: final => gradeRemoteRestart({ ...ev.remoteRestart, final }).every(check => check.passed), + reject: final => ["failed", "cancelled", "timed_out"].includes(final.run.status) ? "Original remote run failed after restart" : undefined }); + await input.evidence("remote-restart-final.json", final); + } ev.checks.push( ...storyLifecycleChecks({ issues: ev.issues as StoryIssue[], @@ -1590,6 +1653,8 @@ export async function runEverydayFlow(input: Input) { parentId: parent!.id, leadId: fixtures.agent.id, allowedInterruptedRuns: ev.allowedInterruptedRuns, + requireRemoteRestart, + remoteRestart: ev.remoteRestart, }), ); check( diff --git a/tests/runner-e2e/everyday-observations.ts b/tests/runner-e2e/everyday-observations.ts index 15935a9c091..40ab35e4a44 100644 --- a/tests/runner-e2e/everyday-observations.ts +++ b/tests/runner-e2e/everyday-observations.ts @@ -1,4 +1,5 @@ import type { IssueComment } from "../../packages/shared/src/types/issue.js"; +import { gradeRemoteRestart, type RemoteRestartEvidence } from "./everyday-remote-restart.js"; export interface StoryCheck { id: string; @@ -140,6 +141,7 @@ export interface StoryRun { lastOutputSeq?: number | null; errorCode?: string | null; error?: string | null; + nativePhase?: string | null; } export type StoryArtifactGradeMode = "base" | "separator" | "max-length"; @@ -272,6 +274,8 @@ export function storyLifecycleChecks(input: { parentId: string; leadId: string; allowedInterruptedRuns?: string[]; + requireRemoteRestart?: boolean; + remoteRestart?: RemoteRestartEvidence; }): StoryCheck[] { const executed = input.runs.filter((r) => !isStoryWorkspaceDeferral(r)); const allowed = input.allowedInterruptedRuns ?? []; @@ -281,6 +285,7 @@ export function storyLifecycleChecks(input: { detail, }); return [ + ...(input.requireRemoteRestart ? gradeRemoteRestart(input.remoteRestart) : []), check( "tasks-done", input.issues.length > 0 && input.issues.every((i) => i.status === "done"), diff --git a/tests/runner-e2e/everyday-remote-restart.test.ts b/tests/runner-e2e/everyday-remote-restart.test.ts new file mode 100644 index 00000000000..938c9b30bf6 --- /dev/null +++ b/tests/runner-e2e/everyday-remote-restart.test.ts @@ -0,0 +1,152 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { storyLifecycleChecks, type StoryRun } from "./everyday-observations.js"; +import { gradeRemoteRestart, observeRemoteRestartBoundary, projectRemoteRestartEvents, projectRemoteRestartRun, waitForRemoteRestartBoundary } from "./everyday-remote-restart.js"; + +const run: StoryRun = { + id: "run", companyId: "company", agentId: "lead", nativeIssueId: "task", + status: "succeeded", runtimeMode: "native", runnerInstanceId: "runner", nativeSessionId: "session", +}; + +function fixture() { + const active = { ...run, status: "running", processPid: 42, processStartedAt: "2026-10-14T00:00:00.000Z" }; + const scope = { companyId: "company", agentId: "lead", issueId: "task", environmentId: "environment" }; + const leases = [{ id: "lease", ...scope, heartbeatRunId: "run", provider: "daytona", providerLeaseId: "private-sandbox", + status: "active", releasedAt: null, executionWorkspaceId: "workspace" }]; + const event = (seq: number, eventType: string, payload: Record = {}, sourceKind = "runner", sourceSeq = seq) => { + const sourceInstanceId = sourceKind === "runner" ? "runner" : "runner:control"; + return { seq, runId: "run", companyId: "company", eventType, protocolSchemaVersion: 1, payload: { prpEvent: { + schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: "run", normalizedSessionId: "session", sourceKind, + sourceInstanceId, sourceSeq, sourceEventId: `${sourceInstanceId}:run:${sourceSeq}`, eventType, + turnId: eventType === "turn.submitted" ? null : "turn", payload, + } } }; + }; + const rows = [event(1, "turn.submitted"), event(2, "turn.started"), + event(3, "tool.execution.started", { executionId: "tool", status: "running" }), + event(4, "tool.execution.completed", { executionId: "tool", status: "completed", output: "private tool output" })]; + const before = observeRemoteRestartBoundary(active, rows, leases, scope); + const afterRows = [...rows, event(5, "item.delta", { text: "private provider text" })]; + const after = observeRemoteRestartBoundary(active, afterRows, leases, scope); + const finished: StoryRun = { ...run, nativePhase: "committed", resultJson: { + finalizationPhase: "committed", workspaceFinalizeStatus: "succeeded", nextAttemptAt: null, failureCode: null, + } }; + const finalRows = [...afterRows, event(6, "run.result.proposed"), event(7, "turn.completed"), + event(8, "run.result.accepted", {}, "control_plane", 1), event(9, "run.terminal", {}, "control_plane", 2)]; + const final = { run: finished, events: projectRemoteRestartEvents(finished, finalRows), + operations: [{ companyId: "company", heartbeatRunId: "run", issueId: "task", executionWorkspaceId: "workspace", + phase: "workspace_finalize", status: "succeeded" }], scheduledRetry: null as unknown, recoveryActions: [] as Record[] }; + return { active, scope, leases, event, rows, afterRows, finalRows, evidence: { before, after, final } }; +} + +describe("remote restart boundary polling", () => { + afterEach(() => vi.useRealTimers()); + + it.each([undefined, "succeeded", "failed", "cancelled", "timed_out"] as const)("stops immediately when the original run is %s", async status => { + vi.useFakeTimers(); + const now = Date.now(); + const load = vi.fn(async () => ({ run: status === undefined ? undefined : { ...run, status } })); + const outcome = waitForRemoteRestartBoundary({ deadlineAt: now + 60_000, load }).then( + () => undefined, error => error as Error, + ); + await vi.runAllTimersAsync(); + expect((await outcome)?.message).toBe("Stopped waiting for active remote native turn with completed tool receipt: Remote restart boundary unexercised: original run ended before fault injection"); + expect(load).toHaveBeenCalledTimes(1); + expect(Date.now()).toBe(now); + }); + + it("waits for useful work while the original run remains active", async () => { + vi.useFakeTimers(); + const { active, evidence } = fixture(); + const load = vi.fn().mockResolvedValueOnce({ run: active }) + .mockResolvedValueOnce({ run: active, boundary: evidence.before }); + const waiting = waitForRemoteRestartBoundary({ deadlineAt: Date.now() + 60_000, load }); + await vi.runAllTimersAsync(); + expect(await waiting).toEqual(evidence.before); + expect(load).toHaveBeenCalledTimes(2); + }); +}); + +describe("remote restart lifecycle evidence", () => { + it("does not accept a successful replacement as proof that the original remote runner recovered", () => { + const { evidence } = fixture(); + const replacement = { ...evidence.final.run, id: "replacement", runnerInstanceId: "replacement-runner" }; + evidence.final.run = replacement; + const checks = storyLifecycleChecks({ issues: [{ id: "task", companyId: "company", title: "task", status: "done" }], + runs: [replacement], parentId: "task", leadId: "lead", requireRemoteRestart: true, remoteRestart: evidence, + } as Parameters[0]); + expect(checks.every(check => check.passed)).toBe(false); + }); + + it("does not accept a terminal native result without its workspace finalization receipt", () => { + const { evidence } = fixture(); + evidence.final.operations = []; + const checks = storyLifecycleChecks({ issues: [{ id: "task", companyId: "company", title: "task", status: "done" }], + runs: [evidence.final.run], parentId: "task", leadId: "lead", requireRemoteRestart: true, remoteRestart: evidence, + } as Parameters[0]); + expect(checks.every(check => check.passed)).toBe(false); + }); + + it("accepts the original live process and lease followed by committed native and workspace receipts", () => { + const { evidence } = fixture(); + expect(storyLifecycleChecks({ issues: [{ id: "task", companyId: "company", title: "task", status: "done" }], + runs: [evidence.final.run], parentId: "task", leadId: "lead", requireRemoteRestart: true, remoteRestart: evidence, + }).every(check => check.passed)).toBe(true); + }); + + it.each(["runId", "runnerInstanceId", "nativeSessionId", "processPid", "processStartedAt"] as const)("rejects a changed %s after restart", field => { + const { evidence } = fixture(); + const identity = evidence.after.identity as Record; + identity[field] = field === "processPid" ? 43 : "different"; + expect(gradeRemoteRestart(evidence)[0]!.passed).toBe(false); + }); + + it("rejects a replacement sandbox and changed or missing receipt prefixes", () => { + const changedLease = fixture().evidence; + changedLease.after.lease.providerLeaseSha256 = "0".repeat(64); + expect(gradeRemoteRestart(changedLease)[0]!.passed).toBe(false); + const changedPrefix = fixture().evidence; + changedPrefix.after.events[0]!.sha256 = "0".repeat(64); + expect(gradeRemoteRestart(changedPrefix)[0]!.passed).toBe(false); + const missingPrefix = fixture().evidence; + missingPrefix.final.events.shift(); + expect(gradeRemoteRestart(missingPrefix)[1]!.passed).toBe(false); + }); + + it.each(["replacement", "second submission", "duplicate tool receipt", "missing terminal"])("rejects %s at completion", fault => { + const { evidence } = fixture(); + if (fault === "replacement") evidence.final.run.id = "replacement"; + if (fault === "second submission") evidence.final.events.push({ ...evidence.final.events[0]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" }); + if (fault === "duplicate tool receipt") evidence.final.events.push({ ...evidence.final.events[3]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" }); + if (fault === "missing terminal") evidence.final.events.pop(); + expect(gradeRemoteRestart(evidence)[1]!.passed).toBe(false); + }); + + it.each(["no receipt", "foreign receipt", "wrong phase", "pending operation", "failed copyback", "retry", "recovery"])("rejects %s after useful work", fault => { + const { evidence } = fixture(); + if (fault === "no receipt") evidence.final.operations = []; + if (fault === "foreign receipt") evidence.final.operations[0]!.heartbeatRunId = "other"; + if (fault === "wrong phase") evidence.final.operations[0]!.phase = "workspace_prepare"; + if (fault === "pending operation") evidence.final.operations.push({ ...evidence.final.operations[0]!, status: "running" }); + if (fault === "failed copyback") evidence.final.run.resultJson!.workspaceFinalizeStatus = "failed"; + if (fault === "retry") evidence.final.scheduledRetry = { runId: "run" }; + if (fault === "recovery") evidence.final.recoveryActions.push({ status: "active" }); + expect(gradeRemoteRestart(evidence)[2]!.passed).toBe(false); + }); + + it("rejects unexercised or foreign boundaries before any restart", () => { + const f = fixture(); + expect(() => observeRemoteRestartBoundary({ ...f.active, status: "succeeded" }, f.rows, f.leases, f.scope)).toThrow("live native process"); + expect(() => observeRemoteRestartBoundary(f.active, f.rows.slice(0, 3), f.leases, f.scope)).toThrow("completed tool"); + expect(() => observeRemoteRestartBoundary(f.active, f.rows, [{ ...f.leases[0]!, companyId: "foreign" }], f.scope)).toThrow("exact active sandbox"); + const foreign = structuredClone(f.rows); + foreign[0]!.payload.prpEvent.normalizedSessionId = "foreign"; + expect(() => observeRemoteRestartBoundary(f.active, foreign, f.leases, f.scope)).toThrow("foreign, duplicate or invalid"); + }); + + it("exports hashes instead of provider text, tool output or sandbox IDs", () => { + const { evidence } = fixture(); + const privateRun = { ...evidence.final.run, error: "private provider failure", runnerProfileJson: { provider: "private-provider" }, + resultJson: { ...evidence.final.run.resultJson, nextAttemptAt: "private-time", failureCode: "private-failure", summary: "private summary" } }; + expect(JSON.stringify(projectRemoteRestartRun(privateRun))).not.toContain("private"); + expect(JSON.stringify(evidence)).not.toContain("private"); + }); +}); diff --git a/tests/runner-e2e/everyday-remote-restart.ts b/tests/runner-e2e/everyday-remote-restart.ts new file mode 100644 index 00000000000..59c1db429e8 --- /dev/null +++ b/tests/runner-e2e/everyday-remote-restart.ts @@ -0,0 +1,155 @@ +import { createHash } from "node:crypto"; +import { canonicalJson } from "../../packages/shared/src/portability-hash.js"; +import { isValidNativePrpEnvelope } from "./native-event-envelope.js"; +import type { StoryCheck, StoryRun } from "./everyday-observations.js"; +import { pollUntil } from "./api.js"; + +type Row = Record; +const record = (value: unknown): Row => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {}; +const id = (value: unknown): value is string => typeof value === "string" && value.length > 0 && value.length <= 256 && !/[\u0000-\u001f\u007f]/u.test(value); +const digest = (value: unknown) => createHash("sha256").update(canonicalJson(value)).digest("hex"); + +export interface RemoteRestartEvent { + seq: number; sourceSeq: number; sourceEventId: string; eventType: string; + sourceKind: "runner" | "control_plane"; turnId: string | null; + toolExecutionId: string | null; toolCompleted: boolean; sha256: string; +} +export interface RemoteRestartBoundary { + identity: { runId: string; companyId: string; agentId: string; issueId: string; + runnerInstanceId: string; nativeSessionId: string; processPid: number; processStartedAt: string }; + lease: { id: string; environmentId: string; executionWorkspaceId: string | null; providerLeaseSha256: string }; + events: RemoteRestartEvent[]; +} +export interface RemoteRestartEvidence { + before?: RemoteRestartBoundary; + after?: RemoteRestartBoundary; + final?: { run: StoryRun; events: RemoteRestartEvent[]; operations: Row[]; + scheduledRetry: unknown; recoveryActions: Row[] }; +} + +export async function waitForRemoteRestartBoundary(input: { + deadlineAt: number; + load: () => Promise<{ run: StoryRun | undefined; boundary?: RemoteRestartBoundary }>; +}): Promise { + const observed = await pollUntil({ + label: "active remote native turn with completed tool receipt", + deadlineAt: input.deadlineAt, + load: input.load, + accept: state => state.run?.status === "running" && state.boundary !== undefined, + reject: state => !state.run || state.run.status !== "running" + ? "Remote restart boundary unexercised: original run ended before fault injection" : undefined, + }); + return observed.boundary!; +} + +export function projectRemoteRestartRun(run: StoryRun): StoryRun { + return { id: run.id, companyId: run.companyId, agentId: run.agentId, nativeIssueId: run.nativeIssueId, + status: run.status, runtimeMode: run.runtimeMode, runnerInstanceId: run.runnerInstanceId, nativeSessionId: run.nativeSessionId, + nativePhase: run.nativePhase === "committed" ? "committed" : null, + resultJson: { finalizationPhase: run.resultJson?.finalizationPhase === "committed" ? "committed" : null, + workspaceFinalizeStatus: run.resultJson?.workspaceFinalizeStatus === "succeeded" ? "succeeded" : null, + nextAttemptAt: run.resultJson?.nextAttemptAt == null ? null : "scheduled", + failureCode: run.resultJson?.failureCode == null ? null : "present" } }; +} + +/** Keep only receipt identities and hashes; never retain provider text or tool arguments. */ +export function projectRemoteRestartEvents(run: StoryRun, rows: readonly Row[]): RemoteRestartEvent[] { + const seen = new Set(); + const cursors = new Map(); + let lastSeq = 0; + return rows.filter(row => record(row.payload).prpEvent !== undefined).map(row => { + const event = record(record(row.payload).prpEvent); + const source = event.sourceInstanceId; + const runnerSource = event.sourceKind === "runner" && source === run.runnerInstanceId; + const controllerSource = event.sourceKind === "control_plane" && source === `${run.runnerInstanceId}:control` + && ["run.result.accepted", "run.terminal"].includes(String(event.eventType)); + if (row.companyId !== run.companyId || row.runId !== run.id || !isValidNativePrpEnvelope(event, row.protocolSchemaVersion) + || event.runId !== run.id || event.normalizedSessionId !== run.nativeSessionId || event.eventType !== row.eventType + || (!runnerSource && !controllerSource) || !id(source) || !id(event.sourceEventId) + || !Number.isSafeInteger(row.seq) || Number(row.seq) <= lastSeq || !Number.isSafeInteger(event.sourceSeq) + || Number(event.sourceSeq) <= (cursors.get(source) ?? 0) || seen.has(event.sourceEventId) + || event.sourceEventId !== `${source}:${run.id}:${event.sourceSeq}`) { + throw new Error("Remote restart has foreign, duplicate or invalid native receipts"); + } + seen.add(event.sourceEventId); cursors.set(source, Number(event.sourceSeq)); lastSeq = Number(row.seq); + const payload = record(event.payload); + return { seq: lastSeq, sourceSeq: Number(event.sourceSeq), sourceEventId: event.sourceEventId, + eventType: String(event.eventType), sourceKind: event.sourceKind as RemoteRestartEvent["sourceKind"], + turnId: id(event.turnId) ? event.turnId : null, + toolExecutionId: id(payload.executionId) ? payload.executionId : null, + toolCompleted: event.eventType === "tool.execution.completed" && payload.status === "completed", + sha256: digest(event) }; + }); +} + +export function observeRemoteRestartBoundary(run: StoryRun, rows: readonly Row[], leases: readonly Row[], scope: { + companyId: string; agentId: string; issueId: string; environmentId: string; +}): RemoteRestartBoundary { + if (run.companyId !== scope.companyId || run.agentId !== scope.agentId || run.nativeIssueId !== scope.issueId + || run.status !== "running" || run.runtimeMode !== "native" || !id(run.id) || !id(run.runnerInstanceId) + || !id(run.nativeSessionId) || !Number.isSafeInteger(run.processPid) || Number(run.processPid) <= 0 + || !id(run.processStartedAt) || !Number.isFinite(Date.parse(run.processStartedAt))) { + throw new Error("Remote restart boundary requires the original live native process"); + } + const owned = leases.filter(lease => lease.companyId === scope.companyId && lease.environmentId === scope.environmentId + && lease.issueId === scope.issueId && lease.heartbeatRunId === run.id && lease.provider === "daytona" + && lease.status === "active" && lease.releasedAt == null && id(lease.id) && id(lease.providerLeaseId)); + if (owned.length !== 1) throw new Error("Remote restart boundary requires one exact active sandbox lease"); + const lease = owned[0]!; + const events = projectRemoteRestartEvents(run, rows); + const starts = events.filter(event => event.eventType === "turn.started" && event.sourceKind === "runner"); + const completed = events.filter(event => event.toolCompleted && event.sourceKind === "runner" && event.toolExecutionId); + if (starts.length !== 1 || !id(starts[0]!.turnId) || completed.length === 0 + || completed.some(event => event.turnId !== starts[0]!.turnId) + || events.some(event => ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted", "run.terminal"].includes(event.eventType))) { + throw new Error("Remote restart boundary has no completed tool in an active original provider turn"); + } + return { + identity: { runId: run.id, companyId: scope.companyId, agentId: scope.agentId, issueId: scope.issueId, + runnerInstanceId: run.runnerInstanceId, nativeSessionId: run.nativeSessionId, + processPid: Number(run.processPid), processStartedAt: run.processStartedAt }, + lease: { id: String(lease.id), environmentId: scope.environmentId, + executionWorkspaceId: typeof lease.executionWorkspaceId === "string" ? lease.executionWorkspaceId : null, + providerLeaseSha256: digest(lease.providerLeaseId) }, + events, + }; +} + +function retainsPrefix(before: readonly RemoteRestartEvent[], after: readonly RemoteRestartEvent[]): boolean { + return before.length > 0 && after.length >= before.length && before.every((event, index) => digest(event) === digest(after[index])); +} + +export function gradeRemoteRestart(evidence: RemoteRestartEvidence | undefined): StoryCheck[] { + const { before, after, final } = evidence ?? {}; + const check = (id: string, passed: boolean, detail: string) => ({ id, passed, detail }); + const sameIdentity = Boolean(before && after && digest(before.identity) === digest(after.identity) + && digest(before.lease) === digest(after.lease) && retainsPrefix(before.events, after.events)); + const run = final?.run; + const identity = before?.identity; + const finalIdentity = Boolean(identity && run && run.id === identity.runId && run.companyId === identity.companyId + && run.agentId === identity.agentId && run.nativeIssueId === identity.issueId && run.runtimeMode === "native" + && run.runnerInstanceId === identity.runnerInstanceId && run.nativeSessionId === identity.nativeSessionId); + const events = final?.events ?? []; + const turnId = before?.events.find(event => event.eventType === "turn.started")?.turnId; + const once = (kind: string, source: RemoteRestartEvent["sourceKind"]) => events.filter(event => event.eventType === kind && event.sourceKind === source).length === 1; + const completedTools = events.filter(event => event.toolCompleted).map(event => event.toolExecutionId); + const nativeCompletion = finalIdentity && Boolean(before && after && retainsPrefix(after.events, events)) + && once("turn.submitted", "runner") && once("turn.started", "runner") && once("turn.completed", "runner") + && once("run.result.accepted", "control_plane") && once("run.terminal", "control_plane") + && events.filter(event => event.eventType.startsWith("turn.")).every(event => event.turnId === turnId || (event.eventType === "turn.submitted" && event.turnId === null)) + && new Set(completedTools).size === completedTools.length; + const operations = final?.operations ?? []; + const saved = finalIdentity && run?.status === "succeeded" && run.nativePhase === "committed" + && run.resultJson?.finalizationPhase === "committed" && run.resultJson.workspaceFinalizeStatus === "succeeded" + && run.resultJson.nextAttemptAt == null && run.resultJson.failureCode == null + && operations.some(operation => operation.companyId === identity?.companyId && operation.heartbeatRunId === identity?.runId + && operation.issueId === identity?.issueId && operation.phase === "workspace_finalize" && operation.status === "succeeded" + && operation.executionWorkspaceId === before?.lease.executionWorkspaceId) + && !operations.some(operation => ["pending", "queued", "running"].includes(String(operation.status))) + && final?.scheduledRetry == null && !final?.recoveryActions.some(action => ["active", "pending"].includes(String(action.status))); + return [ + check("remote-restart-same-process-and-lease", sameIdentity, "The same run, native session, Runner process and sandbox lease remain live after restart."), + check("remote-restart-native-completion-once", nativeCompletion, "The original provider turn completes once and retains its native receipt prefix."), + check("remote-restart-files-finalized", Boolean(saved), "The original run commits its workspace receipt and leaves no active operation or recovery."), + ]; +}