diff --git a/doc/evals.md b/doc/evals.md index a61e4d6f6e3..6cb2ce4b3da 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -539,5 +539,7 @@ API and download oracles; no live qualification is implied by fixture registrati The explicit-only [feedback persistence case](../tests/runner-e2e/README.md#feedback-persistence-explicit-only) checks real native complaint/suggestion submissions and retained bodies with -run-attributed tool receipts. It is manual, uses the normal Product E2E report, +run-attributed tool receipts on local and Daytona execution. Its four native +Claude/Codex cells are explicit-only. Remote registrations remain unqualified +until retained live attempts prove them. It uses the normal Product E2E report and keeps synthetic feedback distinct from actionable product findings. diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 377b12913a3..87a2ad25b25 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -1021,14 +1021,18 @@ The oracle rejects a missing or pending terminal intent. ## Feedback persistence (explicit only) -`--suite feedback-persistence` selects two local native Claude/Codex cells. +`--suite feedback-persistence` selects four native Claude/Codex cells across local and Daytona execution. Each creates one standard task through the browser, asks for two distinct synthetic feedback items, and requires the real `submit_complaint` and -`submit_suggestion` tools. The five-minute case expects one task work run and -disables automatic retries. Company and agent accounted-work budgets are $10. +`submit_suggestion` tools. The case expects one task work run and disables +automatic retries. Local attempts have five minutes; Daytona inherits the +15-minute remote deadline. Company and agent accounted-work budgets are $10. Setup probes and any background work must also pass through the separately budgeted gateway when using an operations manifest. Local compute is unmetered. -Cost and duration forecasts remain unmeasured until a retained live attempt. +Daytona cells require an immutable image and the existing provider credential. +Their ephemeral sandboxes carry the 60-minute provider lifetime. Historical local +measurements remain separate; Daytona cost and time are unknown until measured. +These registrations do not qualify remote behavior or change daily/weekly selections. The runner E2E maintainers own this manual case; `--all` excludes it. After collection and fixture teardown, the oracle reads the retained evidence @@ -1043,5 +1047,6 @@ quality or survival after an abrupt worker kill. ```sh pnpm test:e2e:runner -- --list --suite feedback-persistence +pnpm test:e2e:runner -- --list --suite feedback-persistence --environment daytona pnpm test:e2e:runner -- --id feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion --max-automatic-retries 0 --max-parallel 1 ``` diff --git a/tests/runner-e2e/OPERATIONS-COVERAGE.md b/tests/runner-e2e/OPERATIONS-COVERAGE.md index 1b75a563435..6cc961daca6 100644 --- a/tests/runner-e2e/OPERATIONS-COVERAGE.md +++ b/tests/runner-e2e/OPERATIONS-COVERAGE.md @@ -61,6 +61,11 @@ current `pnpm test:e2e:runner -- --list` before choosing work. In particular, Everyday Workflows, provider connections, feedback persistence, interruption, and case handoffs must not be inferred from a passing `--all` campaign. +The explicit feedback suite also registers native Claude/Codex cells on Daytona. +Their model usage, duration, feedback persistence and cleanup remain unqualified +until retained live attempts prove them. These cells do not enter the daily or +weekly selections, and `--all` continues to exclude the suite. + Keep the broader rotating lane disabled until its exact cells have a metered route, complete retained attempts, a measured cost/time distribution, and a budget that includes failures. Prefer targeted continuation, approval, and diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 5d45ece6627..1a860869e37 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -11,11 +11,13 @@ Eval audit, and remaining activation gaps. ## Supported admission -The initial path admits local `core-compatibility` and `feedback-persistence` cells for `runner-codex`, -`runner-acpx-claude`, `legacy-codex`, and `legacy-claude`. Each cell must have one -provider credential and an exact priced model. Other suites, remote environments, -subscription profiles, hosted tools, multimodal inputs, premium service modes, -and separate judges fail admission. This whitelist is implementation scope, not +The bounded path admits local `core-compatibility` and `feedback-persistence` cells +for `runner-codex`, `runner-acpx-claude`, `legacy-codex`, and `legacy-claude`. +Daytona admits the two native profiles in those suites when the manifest declares +`remoteGateway: "cloudflare"`. Each cell needs one exact priced model and its +provider credential; Daytona also needs its provider key. Other suites and remote +routes, subscription profiles, hosted tools, multimodal inputs, premium service +modes, and separate judges fail admission. This whitelist is implementation scope, not evidence that every admitted harness has passed live qualification. Before creating the journal or requesting credentials, manifest validation checks @@ -899,8 +901,9 @@ launch manifests must use the adopted cap. The migration cannot extend an expire deadline, change an existing cap, or replenish an allowance. An interrupted append remains an accounting failure and must be recovered before more paid work. -The explicit local `feedback-persistence` native Claude/Codex cells also accept -an operations manifest. They use the same metered connection, prices, campaign +The explicit `feedback-persistence` native Claude/Codex cells on local and Daytona +execution also accept an operations manifest. Remote cells require the Cloudflare +gateway declaration and immutable image. They use the same metered connection, prices, campaign journal, setup boundary, and durable attempt allowance as core compatibility. Collection and the independent saved-evidence oracle require no additional model call. The synthetic items need review as test data, not product incidents. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index a97715ed1ed..2d1545a339f 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -2985,14 +2985,18 @@ from live qualification and lists remaining billing/reporting coverage gaps. ## Feedback persistence (explicit only) -`--suite feedback-persistence` selects two local native Claude/Codex cells. +`--suite feedback-persistence` selects four native Claude/Codex cells across local and Daytona execution. Each creates one standard task through the browser, asks for two distinct synthetic feedback items, and requires the real `submit_complaint` and -`submit_suggestion` tools. The five-minute case expects one task work run and -disables automatic retries. Company and agent accounted-work budgets are $10. +`submit_suggestion` tools. The case expects one task work run and disables +automatic retries. Local attempts have five minutes; Daytona inherits the +15-minute remote deadline. Company and agent accounted-work budgets are $10. Setup probes and any background work must also pass through the separately budgeted gateway when using an operations manifest. Local compute is unmetered. -Cost and duration forecasts remain unmeasured until a retained live attempt. +Daytona cells require an immutable image and the existing provider credential. +Their ephemeral sandboxes carry the 60-minute provider lifetime. Historical local +measurements remain separate; Daytona cost and time are unknown until measured. +These registrations do not qualify remote behavior or change daily/weekly selections. The runner E2E maintainers own this manual case; `--all` excludes it. After collection and fixture teardown, the oracle reads the retained evidence @@ -3007,6 +3011,7 @@ quality or survival after an abrupt worker kill. ```sh pnpm test:e2e:runner -- --list --suite feedback-persistence +pnpm test:e2e:runner -- --list --suite feedback-persistence --environment daytona pnpm test:e2e:runner -- --id feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion --max-automatic-retries 0 --max-parallel 1 ``` diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index bb499b6970c..009445d2723 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -219,10 +219,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 2, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2, + 2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 2, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(769); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(769); + expect(validateRunnerCatalog()).toHaveLength(771); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(771); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); @@ -259,12 +259,13 @@ describe("runner E2E catalog", () => { executionId: "bounded-lifetime", daytonaImage: `runner@sha256:${"a".repeat(64)}`, secretRefs: { DAYTONA_API_KEY: { type: "secret_ref", secretId: "22222222-2222-4222-8222-222222222222", version: "latest" } }, }); - const lifetimeMs = Number(payload.config?.ttlMinutes) * 60_000; + const config = payload.config as Record; + const lifetimeMs = Number(config.ttlMinutes) * 60_000; expect(Number.isFinite(lifetimeMs)).toBe(true); const cells = runnerMatrix.filter(entry => entry.environment === environment); expect(cells.length).toBeGreaterThan(0); for (const cell of cells) { - expect(lifetimeMs, cell.id).toBeGreaterThan(cell.task.attemptTimeoutMs.daytona + Number(payload.config?.timeoutMs)); + expect(lifetimeMs, cell.id).toBeGreaterThan(cell.task.attemptTimeoutMs.daytona + Number(config.timeoutMs)); } }); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 24691ef102c..d5f83321734 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1698,8 +1698,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Real complaint and suggestion submissions, durable bodies, and retained tool receipts.", groups: ["native"], profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)), - environments: [localEnvironment], tasks: [feedbackPersistenceTask(runnerTasks.find(task => task.id === "message-marker")!)], expectedMatrixSize: 2, - definitionMetadata: { version: 1, owner: "runner-e2e-maintainers", scheduling: "explicit-only", + environments: runnerEnvironments, tasks: [feedbackPersistenceTask(runnerTasks.find(task => task.id === "message-marker")!)], expectedMatrixSize: 4, + definitionMetadata: { version: 2, owner: "runner-e2e-maintainers", scheduling: "explicit-only", sourceDigest: feedbackPersistenceDigest(runnerTasks.find(task => task.id === "message-marker")!), expectedWorkRuns: 1, automaticRetryPolicy: "single_attempt", budgetCents: 1_000, forecast: "unmeasured; gateway manifest required for bounded qualification", grading: "retained-exact-bodies-and-run-attributed-completed-tools", feedback: "synthetic-disposable-items" }, diff --git a/tests/runner-e2e/feedback-persistence.test.ts b/tests/runner-e2e/feedback-persistence.test.ts index ca178d36a78..9a81b4b0cb0 100644 --- a/tests/runner-e2e/feedback-persistence.test.ts +++ b/tests/runner-e2e/feedback-persistence.test.ts @@ -55,13 +55,26 @@ describe("feedback persistence oracle", () => { expect(gradeFeedbackPersistence({ ...fixture(), evidence }).some(check => !check.passed)).toBe(true); } }); - it("registers only two explicit local native cases with no automatic retry", () => { + it("registers explicit local and Daytona native cases with no automatic retry", () => { const selected = selectRunnerExecutions(parseRunnerSelectors(["--suite", "feedback-persistence"])); expect(selected.map(execution => execution.id)).toEqual([ "feedback-persistence.runner-codex.local.complaint-and-suggestion", + "feedback-persistence.runner-codex.daytona.complaint-and-suggestion", "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion", + "feedback-persistence.runner-acpx-claude.daytona.complaint-and-suggestion", ]); expect(selected.every(execution => execution.task.automaticRetryPolicy === "single_attempt")).toBe(true); + expect(selected.every(execution => execution.task.attemptTimeoutMs.local === 5 * 60_000)).toBe(true); + for (const execution of selected.filter(entry => entry.environment.id === "daytona")) { + expect(execution.requiredCredentials).toEqual(expect.arrayContaining([execution.profile.credential, "DAYTONA_API_KEY"])); + expect(execution.requiredCredentials).toHaveLength(2); + expect(execution.task.expectedRunCount).toBe(1); + expect(execution.task.attemptTimeoutMs.daytona).toBe(15 * 60_000); + const secretRefs = { DAYTONA_API_KEY: { type: "secret_ref" as const, secretId: "22222222-2222-4222-8222-222222222222", version: "latest" as const } }; + expect(() => execution.environment.buildEnvironment({ executionId: execution.id, secretRefs, daytonaImage: "runner:latest" })).toThrow("immutable"); + const payload = execution.environment.buildEnvironment({ executionId: execution.id, secretRefs, daytonaImage: `runner@sha256:${"a".repeat(64)}` }); + expect(payload).toMatchObject({ driver: "sandbox", config: { provider: "daytona", reuseLease: false, ttlMinutes: 60 } }); + } expect(runnerMatrix).toEqual(expect.arrayContaining(selected)); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "feedback-persistence")).toBe(false); }); diff --git a/tests/runner-e2e/feedback-persistence.ts b/tests/runner-e2e/feedback-persistence.ts index fd4faf1ee76..af2c5c9febc 100644 --- a/tests/runner-e2e/feedback-persistence.ts +++ b/tests/runner-e2e/feedback-persistence.ts @@ -26,7 +26,7 @@ export function expectedFixtureFeedback(nonce: string) { export function feedbackPersistenceTask(base: RunnerTaskFixture): RunnerTaskFixture { return { ...base, id: "complaint-and-suggestion", label: "Retain submitted feedback", automaticRetryPolicy: "single_attempt", - attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 }, + attemptTimeoutMs: { local: 5 * 60_000, daytona: base.attemptTimeoutMs.daytona }, buildPrompt(nonce) { const bodies = expectedFixtureFeedback(nonce); return [ diff --git a/tests/runner-e2e/operations.test.ts b/tests/runner-e2e/operations.test.ts index 74cff67c76f..7c2ef3f6abd 100644 --- a/tests/runner-e2e/operations.test.ts +++ b/tests/runner-e2e/operations.test.ts @@ -164,6 +164,8 @@ it.each([ cell.id, "feedback-persistence.runner-codex.local.complaint-and-suggestion", "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion", + "feedback-persistence.runner-codex.daytona.complaint-and-suggestion", + "feedback-persistence.runner-acpx-claude.daytona.complaint-and-suggestion", "core-compatibility.runner-acpx-claude.daytona.message-marker", ])("routes %s through a shared connection without ambient provider bindings", async executionId => { const selected = runnerMatrix.find(execution => execution.id === executionId)!;