diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 2e94cf78c4a..9f1d82597b1 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -111,6 +111,19 @@ registry therefore discovers that row through the public environments API. This still provides full isolation because every cell starts a new Paperclip instance and database. + +Native-generation Codex on Daytona qualifies its immutable image through one +existing saved-environment probe before agent creation. Automatic Codex requests +also inspect the scoped server discovery default. Legacy and other provider +fixtures keep their existing setup. The probe uses no provider turn, permits one +setup lease per cell, has a 330-second HTTP deadline and a 10-second platform +command budget, and performs no retry. It uses the existing archive/release and +provider-expiry cleanup policy. The returned owned probe lease joins task leases +in the existing runtime-cost estimator; unrelated lease IDs do not gain task +attribution. Failed setup before fixture completion retains unavailable/unknown +cost coverage and its holds. A connection-only probe or name-only snapshot cannot +qualify native defaults. + Daytona creates sandbox environments through the public API. The core fixture keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated warm-continuity fixture uses `reuseLease:true` and diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index d3ac65d6d79..a9722e956d7 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it } from "vitest"; import type { RunnerApi } from "./api.js"; +import { buildRuntimeUsage } from "./billing.js"; import { runnerMatrix } from "./catalog.js"; -import { setupLiveFixtures } from "./live-fixtures.js"; +import { setupLiveFixtures, selectFixtureRuntimeLeases } from "./live-fixtures.js"; describe("live runner fixtures", () => { it.each(["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile => @@ -163,7 +164,7 @@ describe("live runner fixtures", () => { }, ); - it("installs the Daytona provider through the public API before creating its environment", async () => { + it.each(["legacy-codex", "runner-acpx-claude", "runner-opencode"])("installs Daytona before its %s environment without adding a Codex qualification probe", async profile => { const calls: string[] = []; const api = { async post(path: string, data?: Record) { @@ -186,7 +187,7 @@ describe("live runner fixtures", () => { } if (path.endsWith("/environments")) { expect(calls).toContain("POST /api/plugins/install"); - return { id: "environment-1", driver: "sandbox" }; + return { id: "environment-1", driver: "sandbox", config: data?.config }; } if (path.endsWith("/agents")) { return { id: "agent-1", name: "Agent", companyId: "company-1" }; @@ -204,7 +205,7 @@ describe("live runner fixtures", () => { const execution = runnerMatrix.find( (candidate) => candidate.id === - "core-compatibility.legacy-codex.daytona.message-marker", + `core-compatibility.${profile}.daytona.message-marker`, ); expect(execution).toBeDefined(); @@ -214,7 +215,7 @@ describe("live runner fixtures", () => { executionNonce: "nonce", workspacePath: "/tmp/workspace", credentials: { - OPENAI_API_KEY: "openai-test-value", + [execution!.profile.credential]: "provider-test-value", DAYTONA_API_KEY: "daytona-test-value", }, daytonaImage: @@ -224,12 +225,99 @@ describe("live runner fixtures", () => { expect(calls.indexOf("POST /api/plugins/install")).toBeLessThan( calls.indexOf("POST /api/companies/company-1/environments"), ); + expect(calls.some(call => call.includes("/probe"))).toBe(false); await fixtures.teardown(); expect(calls).toContain( "DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true", ); }); + it.each(["explicit", "auto"])("qualifies the immutable Codex image once before %s agent creation", async mode => { + const original = runnerMatrix.find(row => row.id === "core-compatibility.runner-codex.daytona.message-marker")!; + const execution = { ...original, profile: { ...original.profile, buildAgent(input: Parameters[0]) { + const payload = original.profile.buildAgent(input); + return { ...payload, adapterType: mode === "auto" ? "codex_local" : "paperclip_runner", + runner: mode === "auto" ? "auto" : "paperclip" }; + } } }; + const calls: string[] = []; + const api = { + async get(path: string) { + calls.push(`GET ${path}`); + return [{ type: "codex_local", defaultRunner: "paperclip", supportedRunners: ["paperclip", "legacy"] }]; + }, + async post(path: string, data: any, options: any) { + calls.push(`POST ${path}`); + if (path === "/api/plugins/install") return { id: "plugin", status: "ready" }; + if (path === "/api/companies") return { id: "company", name: "Fixture" }; + if (path.endsWith("/environments")) return { id: "environment", driver: "sandbox", config: data.config }; + if (path === "/api/environments/environment/probe?companyId=company") { + expect(options).toEqual({ timeout: 330_000 }); + return { ok: true, details: { platformQualificationRecorded: true, leaseId: "setup-probe" } }; + } + if (path.endsWith("/agents")) { + expect(calls).toContain("POST /api/environments/environment/probe?companyId=company"); + expect(data.adapterType).toBe(mode === "auto" ? "codex_local" : "paperclip_runner"); + expect(data.runner).toBe(mode === "auto" ? "auto" : "paperclip"); + if (mode === "auto") expect(calls).toContain("GET /api/adapters?companyId=company&environmentId=environment"); + return { id: "agent", companyId: "company", ...data }; + } + throw new Error(`Unexpected POST ${path}`); + }, + async postSensitive() { return { id: "secret" }; }, + async delete() {}, + } as unknown as RunnerApi; + const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture", + credentials: { OPENAI_API_KEY: "fixture-key", DAYTONA_API_KEY: "fixture-provider-key" }, + daytonaImage: `registry.example.test/runtime@sha256:${"a".repeat(64)}` }); + expect(fixtures.setupProbe).toMatchObject({ leaseId: "setup-probe", ...(mode === "auto" ? { defaultRunner: "paperclip" } : {}) }); + expect(calls.filter(call => call.includes("/probe?"))).toHaveLength(1); + expect(calls.filter(call => call.startsWith("GET"))).toHaveLength(mode === "auto" ? 1 : 0); + await fixtures.teardown(); + }); + + it.each(["probe-failed", "not-recorded", "wrong-default"])("stops before automatic Codex creation when image qualification is %s", async failure => { + const original = runnerMatrix.find(row => row.id === "core-compatibility.runner-codex.daytona.message-marker")!; + const execution = { ...original, profile: { ...original.profile, buildAgent(input: Parameters[0]) { + return { ...original.profile.buildAgent(input), adapterType: "codex_local", runner: "auto" }; + } } }; + const calls: string[] = []; + const api = { + async get() { return [{ type: "codex_local", defaultRunner: "legacy", supportedRunners: ["legacy"] }]; }, + async post(path: string, data: any) { + calls.push(path); + if (path === "/api/plugins/install") return { id: "plugin", status: "ready" }; + if (path === "/api/companies") return { id: "company", name: "Fixture" }; + if (path.endsWith("/environments")) return { id: "environment", driver: "sandbox", config: data.config }; + if (path.includes("/probe?")) return { ok: failure !== "probe-failed", + details: { platformQualificationRecorded: failure !== "not-recorded", leaseId: "setup-probe" } }; + throw new Error(`Unexpected POST ${path}`); + }, + async postSensitive() { return { id: "secret" }; }, + async delete(path: string) { calls.push(`DELETE ${path}`); }, + } as unknown as RunnerApi; + await expect(setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture", + credentials: { OPENAI_API_KEY: "fixture-key", DAYTONA_API_KEY: "fixture-provider-key" }, + daytonaImage: `registry.example.test/runtime@sha256:${"a".repeat(64)}` })).rejects.toThrow(failure === "wrong-default" ? "server-owned default" : "not recorded"); + expect(calls.some(call => call.endsWith("/agents"))).toBe(false); + expect(calls.filter(call => call.includes("/probe?"))).toHaveLength(1); + expect(calls).toContain("DELETE /api/environments/environment?destroyReusableSandboxLeases=true"); + }); + + it("includes the owned setup probe in existing runtime estimates while excluding unrelated leases", () => { + const lease = (id: string, duration: number, heartbeatRunId?: string) => ({ id, provider: "daytona", heartbeatRunId, + acquiredAt: "2026-10-12T00:00:00Z", releasedAt: new Date(Date.parse("2026-10-12T00:00:00Z") + duration).toISOString(), + metadata: { cpu: 4, memory: 4, disk: 10 } }); + const leases = [lease("task", 20_000, "selected-run"), lease("setup-probe", 10_000), lease("unrelated", 600_000, "other-run")]; + const selected = selectFixtureRuntimeLeases({ leases, runIds: ["selected-run"], setupProbeLeaseId: "setup-probe" }); + expect(selected.map(value => value.id)).toEqual(["task", "setup-probe"]); + const usage = buildRuntimeUsage({ environmentId: "daytona", runs: [], leases: selected }); + expect(usage.leaseCount).toBe(2); + expect(usage.leaseDurationMs).toBe(30_000); + expect(usage.costStatus).toBe("estimated"); + expect(usage.estimatedListCostUsd).toBeGreaterThan(0); + expect(selectFixtureRuntimeLeases({ leases, runIds: ["selected-run"] }).map(value => value.id)).toEqual(["task"]); + }); + it("creates a primary project workspace for reusable Daytona scope", async () => { const calls: string[] = []; const api = { @@ -246,11 +334,14 @@ describe("live runner fixtures", () => { return { id: "company-1", name: "Runner E2E" }; } if (path.endsWith("/environments")) { - return { id: "environment-1", driver: "sandbox" }; + return { id: "environment-1", driver: "sandbox", config: data?.config }; } if (path.endsWith("/agents")) { return { id: "agent-1", name: "Agent", companyId: "company-1" }; } + if (path.startsWith("/api/environments/environment-1/probe?")) { + return { ok: true, details: { platformQualificationRecorded: true, leaseId: "setup-probe" } }; + } if (path.endsWith("/projects")) { expect(data).toMatchObject({ executionWorkspacePolicy: { @@ -275,6 +366,7 @@ describe("live runner fixtures", () => { } throw new Error(`Unexpected POST ${path}`); }, + async get() { return [{ type: "codex_local", defaultRunner: "paperclip", supportedRunners: ["paperclip", "legacy"] }]; }, async postSensitive(_path: string, data?: Record) { return { id: `secret-${String(data?.key).toLowerCase()}` }; }, @@ -292,7 +384,7 @@ describe("live runner fixtures", () => { executionNonce: "nonce", workspacePath: "/tmp/workspace", credentials: { - OPENAI_API_KEY: "openai-test-value", + [execution!.profile.credential]: "provider-test-value", DAYTONA_API_KEY: "daytona-test-value", }, daytonaImage: diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 1a8bd126d28..0faf1a9d58a 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { installedReleaseDaytonaPlugin } from "./installed-release.js"; import { isManagedHiringCase } from "./chat-cases.js"; import { FixtureRegistry } from "./fixture-registry.js"; +import { isImmutableDaytonaImage } from "./catalog.js"; import { TASK_TITLE_BUDGET_CENTS } from "./task-titles.js"; import { stageGrokSubscriptionFixture } from "./grok-subscription-fixture.js"; import type { RunnerApi } from "./api.js"; @@ -63,6 +64,8 @@ export interface LiveFixtureValues { agent: AgentRecord; project?: ProjectRecord; aiConnection?: ManagedAccountFixture; + /** Private setup receipt; its lease is included in existing runtime cost accounting. */ + setupProbe?: { leaseId: string; durationMs: number; defaultRunner?: string }; onboardingRuntime?: { mode: "production-wizard" | "post-onboarding-runtime-switch"; @@ -79,6 +82,19 @@ function value(resolved: ReadonlyMap, id: string): T { return result as T; } +/** Preserve task/run attribution and include only this fixture's explicit setup probe. */ +export function selectFixtureRuntimeLeases(input: { leases: T[]; runIds: string[]; issueId?: string; setupProbeLeaseId?: string }): T[] { + const selectedRunIds = new Set(input.runIds); + const relevant = input.leases.filter(lease => lease.provider === "daytona" && ( + lease.id === input.setupProbeLeaseId || selectedRunIds.size === 0 + || (lease.heartbeatRunId && selectedRunIds.has(lease.heartbeatRunId)) + || (input.issueId && lease.issueId === input.issueId) + )); + return relevant.length > 0 ? relevant : input.leases; +} + async function deleteDaytonaEnvironment(api: RunnerApi, environmentId: string) { const deadlineAt = Date.now() + 120_000; let lastError: unknown; @@ -111,6 +127,7 @@ export async function setupLiveFixtures(input: { }): Promise { const { api, execution } = input; const registry = new FixtureRegistry(); + let setupProbe: LiveFixtureValues["setupProbe"]; if (execution.environment.id === "daytona") { registry.register({ @@ -295,6 +312,35 @@ export async function setupLiveFixtures(input: { secretRefs, executionId: input.executionNonce, }); + const autoCodex = agent.adapterType === "codex_local" && (agent.runner === undefined || agent.runner === "auto"); + const nativeCodex = agent.adapterType === "paperclip_runner" && (agent.adapterConfig as Record).provider === "codex"; + if (environment.driver === "sandbox" && execution.profile.generation === "native" + && execution.profile.provider === "codex" && agent.runner !== "legacy" && (autoCodex || nativeCodex)) { + if (!isImmutableDaytonaImage(String(environment.config?.image ?? "")) || environment.config?.snapshot) { + throw new Error("Native Codex qualification requires this fixture's immutable image-backed environment."); + } + const startedAt = Date.now(); + // One existing explicit connection probe: a fresh lease, bounded platform read, + // verified release, and server-owned image evidence before agent creation. + const probe = await api.post<{ ok: boolean; details?: { platformQualificationRecorded?: boolean; leaseId?: string } }>( + `/api/environments/${environment.id}/probe?companyId=${encodeURIComponent(company.id)}`, + {}, { timeout: 330_000 }, + ); + if (probe.ok !== true || probe.details?.platformQualificationRecorded !== true || !probe.details.leaseId) { + throw new Error("Native Codex image qualification was not recorded; agent creation was not attempted."); + } + setupProbe = { leaseId: probe.details.leaseId, durationMs: Date.now() - startedAt }; + if (autoCodex) { + const adapters = await api.get>( + `/api/adapters?companyId=${encodeURIComponent(company.id)}&environmentId=${encodeURIComponent(environment.id)}`, + ); + const selected = adapters.find(adapter => adapter.type === "codex_local"); + if (selected?.defaultRunner !== "paperclip" || !selected.supportedRunners?.includes("paperclip")) { + throw new Error("The qualified image does not advertise native Codex as its server-owned default."); + } + setupProbe.defaultRunner = selected.defaultRunner; + } + } if (["stock-harness", "native-connection-guidance"].includes(execution.suite.id) || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) { agent.budgetMonthlyCents = 1_000; @@ -362,6 +408,7 @@ export async function setupLiveFixtures(input: { secretRefs: value(setup.values, "secrets"), environment: value(setup.values, "environment"), agent: value(setup.values, "agent"), + ...(setupProbe ? { setupProbe } : {}), ...(setup.values.has("project") ? { project: value(setup.values, "project") } : {}), diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 826ed82b4e1..e3cd1b78d80 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -51,7 +51,7 @@ import { runnerExecutionById } from "./catalog.js"; import { classifyFailure } from "./failure-classifier.js"; import { runnerE2EServerControlPaths } from "./harness-env.js"; import { setupConnectionReview } from "./connection-reviews.js"; -import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js"; +import { setupLiveFixtures, selectFixtureRuntimeLeases, type LiveFixtureValues } from "./live-fixtures.js"; import { evaluateMatcher, persistedFinalRunMessage, type MatcherResult } from "./matchers.js"; import { readRegisteredArtifacts } from "./registered-artifact.js"; import { @@ -695,16 +695,9 @@ for (const execution of executions) { const listed = await api.get( `/api/environments/${fixtures.environment.id}/leases`, ); - const selectedRunIds = new Set(selectedRuns.map((run) => run.id)); - const relevant = listed.filter( - (lease) => - lease.provider === "daytona" && - (selectedRunIds.size === 0 || - (lease.heartbeatRunId && - selectedRunIds.has(lease.heartbeatRunId)) || - (issue && lease.issueId === issue.id)), - ); - runtimeLeases = relevant.length > 0 ? relevant : listed; + runtimeLeases = selectFixtureRuntimeLeases({ leases: listed, + runIds: selectedRuns.map(run => run.id), issueId: issue?.id, + setupProbeLeaseId: fixtures.setupProbe?.leaseId }); }; const cancelActiveRunsForCleanup = async () => { @@ -967,6 +960,8 @@ for (const execution of executions) { companyId: fixtures.company.id, environmentId: fixtures.environment.id, agentId: fixtures.agent.id, + ...(fixtures.setupProbe ? { setupProbe: { durationMs: fixtures.setupProbe.durationMs, + ...(fixtures.setupProbe.defaultRunner ? { defaultRunner: fixtures.setupProbe.defaultRunner } : {}) } } : {}), secretIds: Object.fromEntries( Object.entries(fixtures.secretRefs).map(([name, ref]) => [ name,