diff --git a/doc/evals.md b/doc/evals.md index 499d29278ce..0ba2b13d203 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -60,6 +60,11 @@ harness are under `tests/runner-e2e`; the package scripts are `test:e2e:runner`, the server and embedded database, creates the company/agent/task through the real APIs, drives Chromium, and invokes the selected local or Daytona runner. +The explicit-only `daytona-journal-continuity` Product E2E cell exercises +continuation after ordinary tool output grows the durable control-plane journal +past two MiB. Its size oracle is read-only and retains only byte counts; see the +[Product E2E fixture](../tests/runner-e2e/README.md). + The explicit-only `agent-chat-hardening` Product E2E suite covers native chat recovery, hiring, status evidence, and review handoff on local and selected warm Daytona paths. Its [fixture contract](../tests/runner-e2e/README.md) distinguishes diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 893e4b3a8a5..9e7168dda25 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -83,7 +83,7 @@ pnpm test:e2e:runner -- --suite daytona-warm-continuity pnpm test:e2e:runner -- --all ``` -The catalog contains thirteen suites, including the explicit-only everyday and +The catalog contains fourteen suites, including the explicit-only everyday and [lifecycle baseline](LIFECYCLE-BASELINE.md) suites. The latter adds 46 real-provider cells pairing narrative variants and exercising durable lifecycle boundaries; it is excluded from `--all`. `core-compatibility` (**Core Runner @@ -134,6 +134,16 @@ runner instance, PID, and process-start identity. Each turn is bounded to ten minutes, the cell to thirty minutes, and cleanup explicitly deletes the sandbox rather than waiting for Daytona's idle timeout. +`daytona-journal-continuity` is one explicit-only native Codex cell. Select +`daytona-journal-continuity.runner-codex.daytona.large-journal-three-turn`. +It reuses the three-turn warm workflow after streaming four MiB of synthetic +command output through the real provider. Before the first browser follow-up, +a read-only controller journal oracle requires the exact completed run's journal +to exceed two MiB. Only byte counts enter evidence. No runner state or database +is injected or modified. The usual workspace, sandbox, provider, process, +three-run, screenshot, timeout, billing, and cleanup assertions remain required; +`--all` excludes this stress case. + `agent-chat` (**Persistent Agent Chat**) has eight workflows on `legacy-codex`, `legacy-claude`, `runner-codex`, and `runner-acpx-claude`: **28 local cells**. They cover continuity across server restart, fresh context after `/new`, diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 6d3504082e8..a3bad4bdf87 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -13,6 +13,7 @@ import { runnerSuites, runnerTasks, daytonaWarmContinuityTask, + daytonaLargeJournalTask, daytonaWarmEnvironment, isImmutableDaytonaImage, suiteDefinitionHash, @@ -71,10 +72,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 8, 46, 23, 47, 52, 28, 18, 6, 6, 42, 14, 10, 2, + 8, 46, 23, 47, 52, 28, 18, 6, 6, 42, 14, 10, 2, 1, ]); - expect(validateRunnerCatalog()).toHaveLength(302); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(302); + expect(validateRunnerCatalog()).toHaveLength(303); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(303); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); @@ -186,6 +187,18 @@ describe("runner E2E catalog", () => { ).not.toBe(suiteDefinitionHash(suite)); }); + it("keeps large-journal stress explicit-only and uses the ordinary warm workflow", () => { + const suite = runnerSuites.find((candidate) => candidate.id === "daytona-journal-continuity")!; + expect(suite.manualOnly).toBe(true); + expect(suite.expectedMatrixSize).toBe(1); + expect(suite.profiles.map((profile) => profile.id)).toEqual(["runner-codex"]); + expect(daytonaLargeJournalTask.flow).toBe("warm_three_turn"); + expect(daytonaLargeJournalTask.buildPrompt("nonce")).toContain("range(64)"); + expect(daytonaLargeJournalTask.buildFollowupMessages!("nonce")).toEqual(daytonaWarmContinuityTask.buildFollowupMessages!("nonce")); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])) + .some((cell) => cell.suite.id === suite.id)).toBe(false); + }); + it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => { expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]); expect(openRouterBreadthExcludedExecutionIds).toEqual([ diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index f22fd9b8bf2..e64ca6ec0dc 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -874,6 +874,18 @@ export const daytonaWarmContinuityTask: RunnerTaskFixture = { }, }; +export const daytonaLargeJournalTask: RunnerTaskFixture = { + ...daytonaWarmContinuityTask, + id: "large-journal-three-turn", + label: "Large journal three-turn workspace continuity", + buildTitle: (nonce) => `Runner E2E large journal continuity ${nonce}`, + buildPrompt: (nonce) => [ + "First exercise ordinary command-output streaming with this bounded diagnostic. Execute this Python command exactly once, allowing it to print all 64 chunks. Do not redirect, suppress, pipe, or summarize its stdout; the transport must receive the output. The repeated text is synthetic fixture data and needs no analysis. Wait for the command to exit successfully before continuing.", + `python3 -c 'import sys,time; [(sys.stdout.write("journal-continuity-" + "x" * 65500 + "\\n"), sys.stdout.flush(), time.sleep(0.05)) for _ in range(64)]'`, + warmTurnInstructions(1, nonce), + ].join("\n"), +}; + const codexContinuityProfiles = runnerProfiles.filter((profile) => ["legacy-codex", "runner-codex"].includes(profile.id), ); @@ -1053,6 +1065,18 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [daytonaWarmContinuityTask], expectedMatrixSize: 2, }, + { + id: "daytona-journal-continuity", + label: "Daytona Large Journal Continuity", + manualOnly: true, + description: "Continue the same native session after real tool output grows its durable journal beyond 2 MiB.", + groups: ["daytona", "warm"], + profiles: codexContinuityProfiles.filter((profile) => profile.id === "runner-codex"), + environments: [daytonaWarmEnvironment], + tasks: [daytonaLargeJournalTask], + expectedMatrixSize: 1, + definitionMetadata: { version: 1, journalMinimumBytes: 2 * 1024 * 1024, outputChunks: 64, scheduling: "explicit-only" }, + }, ] as const; export function suiteDefinitionHash(suite: RunnerSuiteFixture) { @@ -1169,6 +1193,7 @@ export function validateRunnerCatalog(): MatrixExecution[] { ...localIntegrityTasks, ...openRouterBreadthTasks, daytonaWarmContinuityTask, + daytonaLargeJournalTask, ]; for (const [label, values] of [ ["suite", runnerSuites], diff --git a/tests/runner-e2e/journal-evidence.test.ts b/tests/runner-e2e/journal-evidence.test.ts new file mode 100644 index 00000000000..38e56f85d18 --- /dev/null +++ b/tests/runner-e2e/journal-evidence.test.ts @@ -0,0 +1,26 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { largeJournalEvidence } from "./journal-evidence.js"; + +describe("large journal boundary evidence", () => { + it("requires one correctly owned journal over the boundary and returns only sizes", async () => { + const root = await mkdtemp(path.join(tmpdir(), "journal-evidence-")); + const directory = path.join(root, "a".repeat(64), "control-plane"); + await mkdir(directory, { recursive: true }); + const filename = path.join(directory, "control-plane-state.json"); + const state = { schema: "paperclip.runner.durable.control-plane-state.v1", identity: { runId: "expected-run" }, commands: [] }; + try { + await writeFile(filename, JSON.stringify(state)); + await expect(largeJournalEvidence({ stateRoot: root, runId: "expected-run", minimumBytes: 1024 })).rejects.toThrow("boundary not reached"); + await writeFile(filename, JSON.stringify({ ...state, commands: [{ payload: "x".repeat(2048) }] })); + const evidence = await largeJournalEvidence({ stateRoot: root, runId: "expected-run", minimumBytes: 1024 }); + expect(Object.keys(evidence)).toEqual(["journalBytes", "minimumBytes"]); + expect(evidence.journalBytes).toBeGreaterThan(2048); + await expect(largeJournalEvidence({ stateRoot: root, runId: "another-run", minimumBytes: 1024 })).rejects.toThrow("boundary not reached"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/runner-e2e/journal-evidence.ts b/tests/runner-e2e/journal-evidence.ts new file mode 100644 index 00000000000..770045489a0 --- /dev/null +++ b/tests/runner-e2e/journal-evidence.ts @@ -0,0 +1,30 @@ +import { lstat, readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +// The fixture reads bounded private evidence independently of production admission. +const MAX_JOURNAL_EVIDENCE_BYTES = 192 * 1024 * 1024; + +/** Read-only size oracle; retained journal contents never enter public evidence. */ +export async function largeJournalEvidence(input: { + stateRoot: string; + runId: string; + minimumBytes: number; +}): Promise<{ journalBytes: number; minimumBytes: number }> { + const matches: number[] = []; + for (const entry of await readdir(input.stateRoot, { withFileTypes: true })) { + if (!entry.isDirectory() || !/^[a-f0-9]{64}$/.test(entry.name)) continue; + const filename = path.join(input.stateRoot, entry.name, "control-plane", "control-plane-state.json"); + const metadata = await lstat(filename).catch(() => null); + if (!metadata?.isFile() || metadata.size > MAX_JOURNAL_EVIDENCE_BYTES) continue; + const state = JSON.parse(await readFile(filename, "utf8")) as { + schema?: string; + identity?: { runId?: string }; + }; + if (state.schema === "paperclip.runner.durable.control-plane-state.v1" && state.identity?.runId === input.runId) { + matches.push(metadata.size); + } + } + if (matches.length !== 1 || matches[0]! <= input.minimumBytes) { + throw new Error(`Large-journal boundary not reached: expected one matching journal above ${input.minimumBytes} bytes; observed ${JSON.stringify(matches)}`); + } + return { journalBytes: matches[0]!, minimumBytes: input.minimumBytes }; +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 19923ae6f3a..1a6179e1666 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,4 @@ +import { largeJournalEvidence } from "./journal-evidence.js"; import { observeBrowserBootstrap } from "./browser-bootstrap-diagnostics.js"; import { runAccountingFlow } from "./accounting-flow.js"; import type { Issue } from "../../packages/shared/src/types/issue.js"; @@ -1545,7 +1546,18 @@ for (const execution of executions) { `Warm turn ${completedTurn} replaced its Daytona sandbox`, ); } + const journalEvidence = execution.suite.id === "daytona-journal-continuity" && completedTurn === 1 + ? await largeJournalEvidence({ + stateRoot: path.join(process.env.PAPERCLIP_HOME!, "instances", process.env.PAPERCLIP_INSTANCE_ID!, "runtime", "paperclip-runner", "durable-sessions"), + runId: chronologicalRuns.at(-1)!.id, + minimumBytes: 2 * 1024 * 1024, + }) + : undefined; + if (journalEvidence) { + await writeSanitizedJson(snapshotsDir, "large-journal-boundary.json", journalEvidence, secrets); + } turnEvidence.push({ + ...(journalEvidence ? { journalEvidence } : {}), turn: completedTurn, issue: waitingState.currentIssue, run: chronologicalRuns.at(-1),