diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 36ffe3f25ac..1fdea9c7337 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -246,6 +246,36 @@ runner instance, PID, and process-start identity. Each turn is bounded to ten minutes, the cell to thirty minutes, and cleanup explicitly deletes the sandbox rather than waiting for Daytona's idle timeout. +`daytona-git-streaming` is an explicit-only native Codex Daytona cell for +large Git filename snapshots. Run +`pnpm test:e2e:runner -- --id daytona-git-streaming.runner-codex.daytona.large-path-three-turn`. +This heavy-file cell explicitly configures the environment's 20-minute native +idle timeout and a 25-minute Daytona auto-stop interval. It checks the admitted +runtime policy before each continuation; the environment policy takes precedence +over the agent setting. Large copyback plus the next preparation +can exceed the normal five-minute idle window; the PID and process-fingerprint +continuity checks remain strict. The ordinary warm-continuity cell keeps its +existing five-minute policy. +This cell uses a fixed external instruction bundle. Managed agent folders +intentionally checkpoint and stop the provider after every turn for file +collection; external instructions allow this cell to test retained-process +continuity without changing that collection policy. +It seeds an empty local Git project, creates 60,000 small untracked files through +the real provider, then performs the same three browser-driven review turns. +Each later turn updates all 60,000 generated files to distinct turn-specific +contents. Before each follow-up and after the last turn an independent host oracle reads every copied-back file +and proves the generated NUL-delimited filename list exceeds 32 MiB +(39,828,890 bytes). It also checks whitespace, newline, option-like, Unicode, +and glob-like filenames. Each turn is bounded to fifteen minutes and the cell +to fifty minutes, including five minutes for setup, host verification, and cleanup +outside the turns. Preparing and copying back this many files exceeded the +ordinary warm fixture's ten-minute turn limit on CI. It keeps the warm suite's +billing scope, screenshots, and explicit sandbox cleanup; `--all` excludes it. +Before each follow-up and after the last turn, public durable run records must +show committed native finalization, successful workspace receipts, no active +workspace operation (including cleanup without a run ID), and no scheduled native recovery. A succeeded run or +correct host bytes alone cannot hide an overlapping finalizer retry. + `agent-chat` (**Persistent Agent Chat**) has eight workflows on `legacy-codex`, `legacy-claude`, `runner-codex`, and `runner-acpx-claude`: **28 local cells**. They cover continuity across server restart, fresh context after `/new`, diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 6cca6f70a67..70aa2d7366d 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; import { contextIntegrityTasks } from "./context-integrity-cases.js"; import { normalizePrpResultSignals } from "../../packages/paperclip-runner/src/protocol/result-normalization.js"; import { @@ -30,6 +31,26 @@ import { } from "./selectors.js"; describe("runner E2E catalog", () => { + it("keeps the large Git filename workload explicit-only with three real turns", () => { + const suite = runnerSuites.find(suite => suite.id === "daytona-git-streaming")!; + expect(suite.manualOnly).toBe(true); + expect(runnerMatrix.filter(entry => entry.suite.id === suite.id)).toHaveLength(1); + expect(suite.tasks[0]).toMatchObject({ expectedRunCount: 3, flow: "warm_three_turn", turnTimeoutMs: 15 * 60_000, attemptTimeoutMs: { daytona: 50 * 60_000 } }); + expect(daytonaWarmContinuityTask).toMatchObject({ turnTimeoutMs: 10 * 60_000, attemptTimeoutMs: { daytona: 30 * 60_000 } }); + expect(suite.definitionMetadata).toMatchObject({ generatedFileCount: 60_000, filenameBytes: 39_828_890, nativeIdleTimeoutMs: 1_200_000, autoStopIntervalMinutes: 25 }); + const secretRefs = { + OPENAI_API_KEY: { type: "secret_ref" as const, secretId: "22222222-2222-4222-8222-222222222222", version: "latest" as const }, + DAYTONA_API_KEY: { type: "secret_ref" as const, secretId: "33333333-3333-4333-8333-333333333333", version: "latest" as const }, + }; + const buildInput = { executionId: "heavy-git", environmentId: "env-1", environmentFixtureId: "daytona" as const, workspacePath: "/workspace", secretRefs }; + const adapterConfig = suite.profiles[0]!.buildAgent(buildInput).adapterConfig as Record; + expect(adapterConfig).toMatchObject({ idleTimeoutMs: 1_200_000, instructionsBundleMode: "external", instructionsEntryFile: "AGENTS.md" }); + expect(readFileSync(String(adapterConfig.instructionsFilePath), "utf8").trim()).not.toBe(""); + expect(runnerProfiles.find(profile => profile.id === "runner-codex")!.buildAgent(buildInput).adapterConfig).toMatchObject({ idleTimeoutMs: 300_000 }); + const environmentInput = { executionId: "heavy-git", secretRefs, daytonaImage: `fixture@sha256:${"a".repeat(64)}` }; + expect(suite.environments[0]!.buildEnvironment(environmentInput).config).toMatchObject({ runnerIdleTimeoutMs: 1_200_000, autoStopInterval: 25, autoArchiveInterval: 30 }); + expect(daytonaWarmEnvironment.buildEnvironment(environmentInput).config).toMatchObject({ autoStopInterval: 5 }); + }); it("validates context-integrity task IDs and selectable groups", () => { const task = contextIntegrityTasks[0]!; const originalId = task.id; @@ -91,10 +112,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2, + 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2, 1, ]); - expect(validateRunnerCatalog()).toHaveLength(407); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(407); + expect(validateRunnerCatalog()).toHaveLength(408); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(408); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 0659b509d3d..696add96c39 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -9,6 +9,7 @@ import { everydayTasks, productionStoryProfile } from "./everyday-cases.js"; import { firstTaskTasks } from "./first-task-cases.js"; import { chatTasks, chatHardeningTasks, chatStoryTasks, chatQualificationTasks, chatCompletionTasks } from "./chat-cases.js"; import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; import { createAgentSchema } from "../../packages/shared/src/validators/agent.js"; import { createEnvironmentSchema } from "../../packages/shared/src/validators/environment.js"; import { DEFAULT_CODEX_LOCAL_MODEL } from "../../packages/adapters/codex-local/src/index.js"; @@ -18,6 +19,7 @@ import { DEFAULT_GROK_LOCAL_MODEL } from "../../packages/adapters/grok-local/src import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; import { QUALIFIED_OPENCODE_MODEL } from "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.js"; import { CREDENTIAL_NAMES } from "./types.js"; +import { createGitStreamingTask } from "./daytona-git-streaming.js"; import { PENDING_PROFILE_PREREQUISITES } from "./prerequisites.js"; import { openRouterProfileId, @@ -972,6 +974,8 @@ export const daytonaWarmContinuityTask: RunnerTaskFixture = { }, }; +export const daytonaGitStreamingTask = createGitStreamingTask(daytonaWarmContinuityTask); + const codexContinuityProfiles = runnerProfiles.filter((profile) => ["legacy-codex", "runner-codex"].includes(profile.id), ); @@ -1257,6 +1261,42 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [daytonaWarmContinuityTask], expectedMatrixSize: 2, }, + { + id: "daytona-git-streaming", + label: "Daytona Git Streaming", + manualOnly: true, + description: "Copy back 60,000 real untracked files and continue twice with a Git filename manifest above 32 MiB.", + groups: ["daytona", "warm"], + // Copyback plus the next preparation can exceed the ordinary five-minute + // idle window for this 60,000-file workload. Keep the PID oracle strict + // while explicitly retaining both runner and sandbox for the workload. + profiles: codexContinuityProfiles.filter(profile => profile.id === "runner-codex").map(profile => ({ + ...profile, + buildAgent(input: AgentFixtureBuildInput) { + const agent = profile.buildAgent(input); + // Managed agent-folder collection intentionally stops the provider at + // every turn. Fixed external instructions exercise retained processes. + return { ...agent, adapterConfig: { + ...(agent.adapterConfig as Record), + idleTimeoutMs: 1_200_000, + instructionsBundleMode: "external", + instructionsRootPath: fileURLToPath(new URL("./fixtures/git-streaming/", import.meta.url)), + instructionsEntryFile: "AGENTS.md", + instructionsFilePath: fileURLToPath(new URL("./fixtures/git-streaming/AGENTS.md", import.meta.url)), + } }; + }, + })), + environments: [{ + ...daytonaWarmEnvironment, + buildEnvironment(input: EnvironmentFixtureBuildInput) { + const environment = daytonaWarmEnvironment.buildEnvironment(input); + return { ...environment, config: { ...(environment.config as Record), runnerIdleTimeoutMs: 1_200_000, autoStopInterval: 25, autoArchiveInterval: 30 } }; + }, + }], + tasks: [daytonaGitStreamingTask], + expectedMatrixSize: 1, + definitionMetadata: { version: 8, instructions: "fixed-external", nativeIdleTimeoutMs: 1_200_000, autoStopIntervalMinutes: 25, generatedFileCount: 60_000, filenameBytes: 39_828_890, scheduling: "explicit-only", finalization: "committed-without-active-sync-or-retry", copyback: "all-generated-file-contents-change-each-turn" }, + }, ] as const; export function suiteDefinitionHash(suite: RunnerSuiteFixture) { @@ -1375,6 +1415,7 @@ export function validateRunnerCatalog(): MatrixExecution[] { ...localIntegrityTasks, ...openRouterBreadthTasks, daytonaWarmContinuityTask, + daytonaGitStreamingTask, instructionPersistenceTask, ]; for (const [label, values] of [ diff --git a/tests/runner-e2e/daytona-git-streaming.test.ts b/tests/runner-e2e/daytona-git-streaming.test.ts new file mode 100644 index 00000000000..d8be7a50e32 --- /dev/null +++ b/tests/runner-e2e/daytona-git-streaming.test.ts @@ -0,0 +1,80 @@ +import { readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { GIT_STREAMING_FILE_COUNT, GIT_STREAMING_PARENT, gitStreamingEvidence, gitStreamingFilename, gradeGitFinalization, gradeGitStreamingInventory } from "./daytona-git-streaming.js"; + +vi.mock("node:fs/promises", () => ({ readFile: vi.fn(), readdir: vi.fn() })); + +describe("Daytona Git filename boundary oracle", () => { + const names = Array.from({ length: GIT_STREAMING_FILE_COUNT }, (_, index) => gitStreamingFilename(index)); + it("independently proves the generated manifest exceeds the old 32 MiB bound", () => { + expect(gradeGitStreamingInventory(names)).toEqual({ generatedFiles: 60_000, filenameBytes: 39_828_890 }); + }); + it("rejects a missing file and a plausible duplicate replacement", () => { + expect(() => gradeGitStreamingInventory(names.slice(1))).toThrow("missing 1"); + expect(() => gradeGitStreamingInventory([names[1]!, ...names.slice(1)])).toThrow("duplicate"); + }); + it("rejects stale copies from an earlier turn, even with the complete inventory", async () => { + vi.mocked(readdir).mockResolvedValue(names as never); + let contents = "base"; + vi.mocked(readFile).mockImplementation(async filename => String(filename).includes(GIT_STREAMING_PARENT) ? contents : path.basename(String(filename))); + await expect(gitStreamingEvidence("/fixture", 1)).resolves.toMatchObject({ verifiedContents: 60_000 }); + await expect(gitStreamingEvidence("/fixture", 2)).rejects.toThrow("changed generated file contents"); + contents = "base-turn-2"; + await expect(gitStreamingEvidence("/fixture", 2)).resolves.toMatchObject({ verifiedContents: 60_000 }); + await expect(gitStreamingEvidence("/fixture", 3)).rejects.toThrow("changed generated file contents"); + contents = "base-turn-3"; + await expect(gitStreamingEvidence("/fixture", 3)).resolves.toMatchObject({ verifiedContents: 60_000 }); + }); +}); + +describe("Git copyback finalization oracle", () => { + const clean = () => ({ + runs: [{ id: "run-1", status: "succeeded", nativePhase: "committed", resultJson: { + finalizationPhase: "committed", workspaceFinalizeStatus: "succeeded", nextAttemptAt: null as string | null, failureCode: null as string | null, + }, runnerProfileJson: { nativeExecutionInput: { session: { lifecyclePolicy: { mode: "warm", idleTimeoutMs: 1_200_000 } } } } }], + operations: [{ heartbeatRunId: "run-1" as string | null, status: "succeeded" }], + recovery: { active: null, actions: [] as Array<{ status: string; wakePolicy: { kind: string } }> }, + scheduledRetry: null, + }); + it("accepts matching durable committed receipts", () => { + expect(gradeGitFinalization(clean())).toEqual({ passed: true, failures: [] }); + }); + it("rejects the environment's default idle policy overriding the heavy fixture", () => { + const observed = clean(); + observed.runs[0]!.runnerProfileJson.nativeExecutionInput.session.lifecyclePolicy.idleTimeoutMs = 300_000; + expect(gradeGitFinalization(observed).failures).toEqual(["Run run-1 did not admit the required 20-minute warm idle policy"]); + }); + it("rejects the observed success/finalization retry contradiction", () => { + const observed = clean(); + observed.runs[0]!.nativePhase = "retryable_failure"; + observed.runs[0]!.resultJson.finalizationPhase = "retryable_failure"; + observed.runs[0]!.resultJson.nextAttemptAt = "2026-09-27T22:04:26.756Z"; + observed.operations.push({ heartbeatRunId: "run-1", status: "running" }); + observed.recovery.actions.push({ status: "active", wakePolicy: { kind: "resume_native_run" } }); + expect(gradeGitFinalization(observed).failures).toHaveLength(4); + }); + it("rejects a stale active sync even when the run is committed", () => { + const observed = clean(); + observed.operations.push({ heartbeatRunId: "run-1", status: "running" }); + expect(gradeGitFinalization(observed).passed).toBe(false); + }); + it("rejects active workspace cleanup receipts without a run ID", () => { + const observed = clean(); + observed.operations.push({ heartbeatRunId: null, status: "running" }); + expect(gradeGitFinalization(observed).failures).toContain("The workspace still has an active operation"); + observed.operations[1]!.status = "succeeded"; + expect(gradeGitFinalization(observed).passed).toBe(true); + }); + it("rejects an active failure summary left on a committed run", () => { + const observed = clean(); + observed.runs[0]!.resultJson.failureCode = "side_effect_planning_failed"; + expect(gradeGitFinalization(observed).failures).toEqual(["Run run-1 still projects an active finalization failure"]); + }); + it("rejects missing finalization evidence", () => { + const observed = clean(); + observed.operations = []; + expect(gradeGitFinalization(observed).passed).toBe(false); + expect(gradeGitFinalization({ ...observed, runs: [] }).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/daytona-git-streaming.ts b/tests/runner-e2e/daytona-git-streaming.ts new file mode 100644 index 00000000000..764c820c33f --- /dev/null +++ b/tests/runner-e2e/daytona-git-streaming.ts @@ -0,0 +1,157 @@ +import { execFile } from "node:child_process"; +import { readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import type { RunnerTaskFixture } from "./types.js"; +import type { RunnerApi } from "./api.js"; + +const exec = promisify(execFile); +export const GIT_STREAMING_FILE_COUNT = 60_000; +export const GIT_STREAMING_PARENT = [...Array.from({ length: 2 }, () => "nested-".repeat(30)), "storybook-output"].join("/"); +export const gitStreamingFilename = (index: number) => `${"asset-".repeat(36)}${index}.js`; +const unusualNames = [" space ", "line\nbreak", "-option", "雪-💾", "wild[?]*"]; + +/** Seed an ordinary empty project repository before its first public task. */ +export async function setupGitStreamingWorkspace(workspacePath: string): Promise { + await exec("git", ["init", "-q"], { cwd: workspacePath }); + await exec("git", ["-c", "user.name=Runner E2E", "-c", "user.email=runner-e2e@example.test", "commit", "--allow-empty", "-qm", "Empty E2E workspace"], { cwd: workspacePath }); +} + +export function createGitStreamingTask(base: RunnerTaskFixture): RunnerTaskFixture { + const setup = [ + "python3 - <<'PY'", + "from pathlib import Path", + `parent = Path(${JSON.stringify(GIT_STREAMING_PARENT)})`, + "parent.mkdir(parents=True, exist_ok=True)", + `for index in range(${GIT_STREAMING_FILE_COUNT}):`, + " (parent / ('asset-' * 36 + str(index) + '.js')).write_text('base')", + `for name in ${JSON.stringify(unusualNames)}:`, + " Path(name).write_text(name)", + `print('generated', ${GIT_STREAMING_FILE_COUNT}, 'files')`, + "PY", + ].join("\n"); + const verify = (previousTurn: number, nextTurn: number) => [ + "Before continuing, run this command and confirm it succeeds:", + "python3 - <<'PY'", + "from pathlib import Path", + `parent = Path(${JSON.stringify(GIT_STREAMING_PARENT)})`, + `assert len(list(parent.iterdir())) == ${GIT_STREAMING_FILE_COUNT}`, + `for index in range(${GIT_STREAMING_FILE_COUNT}):`, + ` assert (parent / ('asset-' * 36 + str(index) + '.js')).read_text() == ${JSON.stringify(previousTurn === 1 ? "base" : `base-turn-${previousTurn}`)}`, + `for name in ${JSON.stringify(unusualNames)}:`, + " assert Path(name).read_text() == name", + `for index in range(${GIT_STREAMING_FILE_COUNT}):`, + ` (parent / ('asset-' * 36 + str(index) + '.js')).write_text('base-turn-${nextTurn}')`, + "print('all generated files preserved')", + "PY", + "Keep these generated files untracked. Do not rename, remove, add, commit, or ignore them.", + ].join("\n"); + return { + ...base, + id: "large-path-three-turn", + label: "Large Git filename manifest across three Daytona turns", + // CI needs more than the ordinary warm fixture's ten minutes to prepare + // and copy back 60,000 files (a measured continuation took eleven minutes). + turnTimeoutMs: 15 * 60_000, + // Reserve another five minutes for setup, host checks, and cleanup outside + // the three per-turn deadlines. + attemptTimeoutMs: { ...base.attemptTimeoutMs, daytona: 50 * 60_000 }, + buildTitle: nonce => `Runner E2E Git streaming ${nonce}`, + buildPrompt: nonce => [ + "Execute this bounded fixture command once in the current execution workspace. It creates 60,000 small untracked files whose Git filename list exceeds 32 MiB. Keep all files untracked; do not add, commit, rename, delete, or ignore them. Do not print the filenames or file contents.", + setup, + base.buildPrompt(nonce), + ].join("\n"), + buildFollowupMessages: nonce => { + const [second, third] = base.buildFollowupMessages!(nonce); + return [`${verify(1, 2)}\n${second}`, `${verify(2, 3)}\n${third}`]; + }, + }; +} + +export function gradeGitStreamingInventory(names: string[]): { generatedFiles: number; filenameBytes: number } { + const expected = new Set(Array.from({ length: GIT_STREAMING_FILE_COUNT }, (_, index) => gitStreamingFilename(index))); + let filenameBytes = 0; + for (const name of names) { + if (!expected.delete(name)) throw new Error("Git copyback contains an unexpected or duplicate generated filename"); + filenameBytes += Buffer.byteLength(`${GIT_STREAMING_PARENT}/${name}`) + 1; + } + if (expected.size) throw new Error(`Git copyback is missing ${expected.size} generated files`); + if (filenameBytes <= 32 * 1024 * 1024) throw new Error("Git filename fixture did not cross the 32 MiB boundary"); + return { generatedFiles: names.length, filenameBytes }; +} + +/** Read every copied-back file independently of the agent's verification. */ +export async function gitStreamingEvidence(workspacePath: string, completedTurn: number) { + if (![1, 2, 3].includes(completedTurn)) throw new Error("Git copyback requires an exact fixture turn"); + const expectedContents = completedTurn === 1 ? "base" : `base-turn-${completedTurn}`; + const parent = path.join(workspacePath, GIT_STREAMING_PARENT); + const names = await readdir(parent); + const inventory = gradeGitStreamingInventory(names); + for (let start = 0; start < names.length; start += 100) { + await Promise.all(names.slice(start, start + 100).map(async name => { + if (await readFile(path.join(parent, name), "utf8") !== expectedContents) throw new Error("Git copyback changed generated file contents or retained an earlier turn"); + })); + } + for (const name of unusualNames) { + if (await readFile(path.join(workspacePath, name), "utf8") !== name) throw new Error("Git copyback changed an unusual filename or its contents"); + } + return { ...inventory, verifiedContents: names.length, verifiedTurn: completedTurn, unusualNamesPreserved: unusualNames.length }; +} + +interface FinalizedRun { + id: string; + status: string; + nativePhase?: string | null; + resultJson?: { finalizationPhase?: string; workspaceFinalizeStatus?: string; nextAttemptAt?: string | null; failureCode?: string | null } | null; + runnerProfileJson?: { nativeExecutionInput?: { session?: { lifecyclePolicy?: { mode?: string; idleTimeoutMs?: number | null } } } } | null; +} +interface WorkspaceOperation { heartbeatRunId: string | null; status: string } +interface RecoveryAction { status: string; wakePolicy?: { kind?: string } | null } +interface GitFinalizationObservation { + runs: FinalizedRun[]; + operations: WorkspaceOperation[]; + recovery: { active: RecoveryAction | null; actions: RecoveryAction[] }; + scheduledRetry: unknown; +} + +export function gradeGitFinalization(observation: GitFinalizationObservation) { + const failures: string[] = []; + if (observation.runs.length === 0) failures.push("No completed native run evidence"); + for (const run of observation.runs) { + const policy = run.runnerProfileJson?.nativeExecutionInput?.session?.lifecyclePolicy; + if (policy?.mode !== "warm" || policy.idleTimeoutMs !== 1_200_000) { + failures.push(`Run ${run.id} did not admit the required 20-minute warm idle policy`); + } + if (run.status !== "succeeded" || run.nativePhase !== "committed" || + run.resultJson?.finalizationPhase !== "committed" || run.resultJson.workspaceFinalizeStatus !== "succeeded") { + failures.push(`Run ${run.id} is not durably committed after workspace finalization`); + } + if (run.resultJson?.nextAttemptAt != null) failures.push(`Run ${run.id} still schedules a finalization retry`); + if (run.resultJson?.failureCode != null) failures.push(`Run ${run.id} still projects an active finalization failure`); + const operations = observation.operations.filter(operation => operation.heartbeatRunId === run.id); + if (!operations.some(operation => operation.status === "succeeded")) failures.push(`Run ${run.id} has no successful workspace receipt`); + } + // The scoped endpoint also includes workspace cleanup with no run ID. + if (observation.operations.some(operation => ["pending", "queued", "running"].includes(operation.status))) { + failures.push("The workspace still has an active operation"); + } + const actions = [observation.recovery.active, ...observation.recovery.actions].filter(action => action !== null); + if (actions.some(action => ["active", "pending"].includes(action.status) && action.wakePolicy?.kind === "resume_native_run")) { + failures.push("A recovery action still schedules a native run retry"); + } + if (observation.scheduledRetry != null) failures.push("The task still has a scheduled retry"); + return { passed: failures.length === 0, failures }; +} + +/** Public durable receipts must agree before the next browser turn is sent. */ +export async function gitFinalizationEvidence(api: RunnerApi, issueId: string, runIds: string[]) { + const [runs, operationLists, recovery, issue] = await Promise.all([ + Promise.all(runIds.map(id => api.get(`/api/heartbeat-runs/${id}`))), + Promise.all(runIds.map(id => api.get(`/api/heartbeat-runs/${id}/workspace-operations`))), + api.get(`/api/issues/${issueId}/recovery-actions`), + api.get<{ scheduledRetry?: unknown }>(`/api/issues/${issueId}`), + ]); + const observation: GitFinalizationObservation = { runs, operations: operationLists.flat(), recovery, scheduledRetry: issue.scheduledRetry }; + return { ...observation, ...gradeGitFinalization(observation) }; +} diff --git a/tests/runner-e2e/fixtures/git-streaming/AGENTS.md b/tests/runner-e2e/fixtures/git-streaming/AGENTS.md new file mode 100644 index 00000000000..c964f86f88d --- /dev/null +++ b/tests/runner-e2e/fixtures/git-streaming/AGENTS.md @@ -0,0 +1,6 @@ +# Git streaming fixture agent + +Complete the current task and its authorized review revisions using the supplied +task context. Preserve the workspace files across turns and follow the task's +verification and completion instructions. These external instructions are fixed; +do not edit this instruction file or create a personal agent folder. diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index b7b432fabf9..fde941160d9 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,4 @@ +import { gitFinalizationEvidence, gitStreamingEvidence, setupGitStreamingWorkspace } from "./daytona-git-streaming.js"; import { completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js"; import { completionDelivery, type CompletionObservation } from "./completion-updates.js"; import { runInstructionPersistenceFlow } from "./instruction-persistence.js"; @@ -734,6 +735,9 @@ for (const execution of executions) { const runEvidence = await Promise.all( detailedRuns.map(async (candidate) => ({ runId: candidate.id, + workspaceOperations: await capture(() => + api.get(`/api/heartbeat-runs/${candidate.id}/workspace-operations`), + ), log: await capture(() => api.get( `/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`, @@ -819,6 +823,9 @@ for (const execution of executions) { }); expect(experimental.enableNativeRunner).toBe(true); + if (execution.suite.id === "daytona-git-streaming") { + await setupGitStreamingWorkspace(workspacePath); + } fixtures = execution.task.flow === "first_task" ? await setupFirstTaskFixtures({ page, api, execution, nonce, credentials, observe: value => { fixtures = value; } }) : await setupLiveFixtures({ @@ -1514,6 +1521,12 @@ for (const execution of executions) { ? `warm turn ${completedTurn} dispatched duplicate runs` : undefined), }); + if (execution.suite.id === "daytona-git-streaming") { + await writeSanitizedJson(snapshotsDir, `git-copyback-turn-${completedTurn}.json`, await gitStreamingEvidence(workspacePath, completedTurn), secrets); + const finalization = await gitFinalizationEvidence(api, issue.id, waitingState.taskRuns.map(run => run.id)); + await writeSanitizedJson(snapshotsDir, `git-finalization-turn-${completedTurn}.json`, finalization, secrets); + expect(finalization.passed, finalization.failures.join("; ")).toBe(true); + } const expectedPrefix = `${Array.from( { length: completedTurn }, (_, index) => `T${index + 1}-${nonce}`, @@ -1728,6 +1741,12 @@ for (const execution of executions) { ), ); selectedRuns = sortRunsChronologically(selectedRuns); + if (execution.suite.id === "daytona-git-streaming") { + await writeSanitizedJson(snapshotsDir, "git-copyback-turn-3.json", await gitStreamingEvidence(workspacePath, 3), secrets); + const finalization = await gitFinalizationEvidence(api, issue.id, selectedRuns.map(run => run.id)); + await writeSanitizedJson(snapshotsDir, "git-finalization-turn-3.json", finalization, secrets); + expect(finalization.passed, finalization.failures.join("; ")).toBe(true); + } const lifecycleProbe = execution.suite.id === "lifecycle-baseline" ? lifecycleLiveCase(execution.task.id) : undefined; if (lifecycleProbe?.family === "repair") { const grade = gradeLifecycleRepair({ runs: selectedRuns, comments: terminal.comments,