test(e2e): qualify managed Codex images in existing fixtures

Use the existing environment probe before native-generation Codex creation and inspect automatic server defaults. Include only the owned setup probe lease in the existing runtime estimator. Preserve explicit and Legacy cases, cleanup, unknown failure costs and reporting.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/FIXTURES.md

@@ -111,6 +111,19 @@ registry therefore discovers that row through the public environments API.
 This still provides full isolation because every cell starts a new Paperclip
 instance and database.
 
+
+Native-generation Codex on Daytona qualifies its immutable image through one
+existing saved-environment probe before agent creation. Automatic Codex requests
+also inspect the scoped server discovery default. Legacy and other provider
+fixtures keep their existing setup. The probe uses no provider turn, permits one
+setup lease per cell, has a 330-second HTTP deadline and a 10-second platform
+command budget, and performs no retry. It uses the existing archive/release and
+provider-expiry cleanup policy. The returned owned probe lease joins task leases
+in the existing runtime-cost estimator; unrelated lease IDs do not gain task
+attribution. Failed setup before fixture completion retains unavailable/unknown
+cost coverage and its holds. A connection-only probe or name-only snapshot cannot
+qualify native defaults.
+
 Daytona creates sandbox environments through the public API. The core fixture
 keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated
 warm-continuity fixture uses `reuseLease:true` and

tests/runner-e2e/live-fixtures.test.ts

@@ -1,7 +1,8 @@
 import { describe, expect, it } from "vitest";
 import type { RunnerApi } from "./api.js";
+import { buildRuntimeUsage } from "./billing.js";
 import { runnerMatrix } from "./catalog.js";
-import { setupLiveFixtures } from "./live-fixtures.js";
+import { setupLiveFixtures, selectFixtureRuntimeLeases } from "./live-fixtures.js";
 
 describe("live runner fixtures", () => {
   it.each(["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile =>
@@ -163,7 +164,7 @@ describe("live runner fixtures", () => {
     },
   );
 
-  it("installs the Daytona provider through the public API before creating its environment", async () => {
+  it.each(["legacy-codex", "runner-acpx-claude", "runner-opencode"])("installs Daytona before its %s environment without adding a Codex qualification probe", async profile => {
     const calls: string[] = [];
     const api = {
       async post(path: string, data?: Record<string, unknown>) {
@@ -186,7 +187,7 @@ describe("live runner fixtures", () => {
         }
         if (path.endsWith("/environments")) {
           expect(calls).toContain("POST /api/plugins/install");
-          return { id: "environment-1", driver: "sandbox" };
+          return { id: "environment-1", driver: "sandbox", config: data?.config };
         }
         if (path.endsWith("/agents")) {
           return { id: "agent-1", name: "Agent", companyId: "company-1" };
@@ -204,7 +205,7 @@ describe("live runner fixtures", () => {
     const execution = runnerMatrix.find(
       (candidate) =>
         candidate.id ===
-        "core-compatibility.legacy-codex.daytona.message-marker",
+        `core-compatibility.${profile}.daytona.message-marker`,
     );
     expect(execution).toBeDefined();
 
@@ -214,7 +215,7 @@ describe("live runner fixtures", () => {
       executionNonce: "nonce",
       workspacePath: "/tmp/workspace",
       credentials: {
-        OPENAI_API_KEY: "openai-test-value",
+        [execution!.profile.credential]: "provider-test-value",
         DAYTONA_API_KEY: "daytona-test-value",
       },
       daytonaImage:
@@ -224,12 +225,99 @@ describe("live runner fixtures", () => {
     expect(calls.indexOf("POST /api/plugins/install")).toBeLessThan(
       calls.indexOf("POST /api/companies/company-1/environments"),
     );
+    expect(calls.some(call => call.includes("/probe"))).toBe(false);
     await fixtures.teardown();
     expect(calls).toContain(
       "DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true",
     );
   });
 
+  it.each(["explicit", "auto"])("qualifies the immutable Codex image once before %s agent creation", async mode => {
+    const original = runnerMatrix.find(row => row.id === "core-compatibility.runner-codex.daytona.message-marker")!;
+    const execution = { ...original, profile: { ...original.profile, buildAgent(input: Parameters<typeof original.profile.buildAgent>[0]) {
+      const payload = original.profile.buildAgent(input);
+      return { ...payload, adapterType: mode === "auto" ? "codex_local" : "paperclip_runner",
+        runner: mode === "auto" ? "auto" : "paperclip" };
+    } } };
+    const calls: string[] = [];
+    const api = {
+      async get(path: string) {
+        calls.push(`GET ${path}`);
+        return [{ type: "codex_local", defaultRunner: "paperclip", supportedRunners: ["paperclip", "legacy"] }];
+      },
+      async post(path: string, data: any, options: any) {
+        calls.push(`POST ${path}`);
+        if (path === "/api/plugins/install") return { id: "plugin", status: "ready" };
+        if (path === "/api/companies") return { id: "company", name: "Fixture" };
+        if (path.endsWith("/environments")) return { id: "environment", driver: "sandbox", config: data.config };
+        if (path === "/api/environments/environment/probe?companyId=company") {
+          expect(options).toEqual({ timeout: 330_000 });
+          return { ok: true, details: { platformQualificationRecorded: true, leaseId: "setup-probe" } };
+        }
+        if (path.endsWith("/agents")) {
+          expect(calls).toContain("POST /api/environments/environment/probe?companyId=company");
+          expect(data.adapterType).toBe(mode === "auto" ? "codex_local" : "paperclip_runner");
+          expect(data.runner).toBe(mode === "auto" ? "auto" : "paperclip");
+          if (mode === "auto") expect(calls).toContain("GET /api/adapters?companyId=company&environmentId=environment");
+          return { id: "agent", companyId: "company", ...data };
+        }
+        throw new Error(`Unexpected POST ${path}`);
+      },
+      async postSensitive() { return { id: "secret" }; },
+      async delete() {},
+    } as unknown as RunnerApi;
+    const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture",
+      credentials: { OPENAI_API_KEY: "fixture-key", DAYTONA_API_KEY: "fixture-provider-key" },
+      daytonaImage: `registry.example.test/runtime@sha256:${"a".repeat(64)}` });
+    expect(fixtures.setupProbe).toMatchObject({ leaseId: "setup-probe", ...(mode === "auto" ? { defaultRunner: "paperclip" } : {}) });
+    expect(calls.filter(call => call.includes("/probe?"))).toHaveLength(1);
+    expect(calls.filter(call => call.startsWith("GET"))).toHaveLength(mode === "auto" ? 1 : 0);
+    await fixtures.teardown();
+  });
+
+  it.each(["probe-failed", "not-recorded", "wrong-default"])("stops before automatic Codex creation when image qualification is %s", async failure => {
+    const original = runnerMatrix.find(row => row.id === "core-compatibility.runner-codex.daytona.message-marker")!;
+    const execution = { ...original, profile: { ...original.profile, buildAgent(input: Parameters<typeof original.profile.buildAgent>[0]) {
+      return { ...original.profile.buildAgent(input), adapterType: "codex_local", runner: "auto" };
+    } } };
+    const calls: string[] = [];
+    const api = {
+      async get() { return [{ type: "codex_local", defaultRunner: "legacy", supportedRunners: ["legacy"] }]; },
+      async post(path: string, data: any) {
+        calls.push(path);
+        if (path === "/api/plugins/install") return { id: "plugin", status: "ready" };
+        if (path === "/api/companies") return { id: "company", name: "Fixture" };
+        if (path.endsWith("/environments")) return { id: "environment", driver: "sandbox", config: data.config };
+        if (path.includes("/probe?")) return { ok: failure !== "probe-failed",
+          details: { platformQualificationRecorded: failure !== "not-recorded", leaseId: "setup-probe" } };
+        throw new Error(`Unexpected POST ${path}`);
+      },
+      async postSensitive() { return { id: "secret" }; },
+      async delete(path: string) { calls.push(`DELETE ${path}`); },
+    } as unknown as RunnerApi;
+    await expect(setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture",
+      credentials: { OPENAI_API_KEY: "fixture-key", DAYTONA_API_KEY: "fixture-provider-key" },
+      daytonaImage: `registry.example.test/runtime@sha256:${"a".repeat(64)}` })).rejects.toThrow(failure === "wrong-default" ? "server-owned default" : "not recorded");
+    expect(calls.some(call => call.endsWith("/agents"))).toBe(false);
+    expect(calls.filter(call => call.includes("/probe?"))).toHaveLength(1);
+    expect(calls).toContain("DELETE /api/environments/environment?destroyReusableSandboxLeases=true");
+  });
+
+  it("includes the owned setup probe in existing runtime estimates while excluding unrelated leases", () => {
+    const lease = (id: string, duration: number, heartbeatRunId?: string) => ({ id, provider: "daytona", heartbeatRunId,
+      acquiredAt: "2026-10-12T00:00:00Z", releasedAt: new Date(Date.parse("2026-10-12T00:00:00Z") + duration).toISOString(),
+      metadata: { cpu: 4, memory: 4, disk: 10 } });
+    const leases = [lease("task", 20_000, "selected-run"), lease("setup-probe", 10_000), lease("unrelated", 600_000, "other-run")];
+    const selected = selectFixtureRuntimeLeases({ leases, runIds: ["selected-run"], setupProbeLeaseId: "setup-probe" });
+    expect(selected.map(value => value.id)).toEqual(["task", "setup-probe"]);
+    const usage = buildRuntimeUsage({ environmentId: "daytona", runs: [], leases: selected });
+    expect(usage.leaseCount).toBe(2);
+    expect(usage.leaseDurationMs).toBe(30_000);
+    expect(usage.costStatus).toBe("estimated");
+    expect(usage.estimatedListCostUsd).toBeGreaterThan(0);
+    expect(selectFixtureRuntimeLeases({ leases, runIds: ["selected-run"] }).map(value => value.id)).toEqual(["task"]);
+  });
+
   it("creates a primary project workspace for reusable Daytona scope", async () => {
     const calls: string[] = [];
     const api = {
@@ -246,11 +334,14 @@ describe("live runner fixtures", () => {
           return { id: "company-1", name: "Runner E2E" };
         }
         if (path.endsWith("/environments")) {
-          return { id: "environment-1", driver: "sandbox" };
+          return { id: "environment-1", driver: "sandbox", config: data?.config };
         }
         if (path.endsWith("/agents")) {
           return { id: "agent-1", name: "Agent", companyId: "company-1" };
         }
+        if (path.startsWith("/api/environments/environment-1/probe?")) {
+          return { ok: true, details: { platformQualificationRecorded: true, leaseId: "setup-probe" } };
+        }
         if (path.endsWith("/projects")) {
           expect(data).toMatchObject({
             executionWorkspacePolicy: {
@@ -275,6 +366,7 @@ describe("live runner fixtures", () => {
         }
         throw new Error(`Unexpected POST ${path}`);
       },
+      async get() { return [{ type: "codex_local", defaultRunner: "paperclip", supportedRunners: ["paperclip", "legacy"] }]; },
       async postSensitive(_path: string, data?: Record<string, unknown>) {
         return { id: `secret-${String(data?.key).toLowerCase()}` };
       },
@@ -292,7 +384,7 @@ describe("live runner fixtures", () => {
       executionNonce: "nonce",
       workspacePath: "/tmp/workspace",
       credentials: {
-        OPENAI_API_KEY: "openai-test-value",
+        [execution!.profile.credential]: "provider-test-value",
         DAYTONA_API_KEY: "daytona-test-value",
       },
       daytonaImage:

tests/runner-e2e/live-fixtures.ts

@@ -4,6 +4,7 @@ import path from "node:path";
 import { installedReleaseDaytonaPlugin } from "./installed-release.js";
 import { isManagedHiringCase } from "./chat-cases.js";
 import { FixtureRegistry } from "./fixture-registry.js";
+import { isImmutableDaytonaImage } from "./catalog.js";
 import { TASK_TITLE_BUDGET_CENTS } from "./task-titles.js";
 import { stageGrokSubscriptionFixture } from "./grok-subscription-fixture.js";
 import type { RunnerApi } from "./api.js";
@@ -63,6 +64,8 @@ export interface LiveFixtureValues {
   agent: AgentRecord;
   project?: ProjectRecord;
   aiConnection?: ManagedAccountFixture;
+  /** Private setup receipt; its lease is included in existing runtime cost accounting. */
+  setupProbe?: { leaseId: string; durationMs: number; defaultRunner?: string };
 
   onboardingRuntime?: {
     mode: "production-wizard" | "post-onboarding-runtime-switch";
@@ -79,6 +82,19 @@ function value<T>(resolved: ReadonlyMap<string, unknown>, id: string): T {
   return result as T;
 }
 
+/** Preserve task/run attribution and include only this fixture's explicit setup probe. */
+export function selectFixtureRuntimeLeases<T extends {
+  id: string; provider?: string | null; heartbeatRunId?: string | null; issueId?: string | null;
+}>(input: { leases: T[]; runIds: string[]; issueId?: string; setupProbeLeaseId?: string }): T[] {
+  const selectedRunIds = new Set(input.runIds);
+  const relevant = input.leases.filter(lease => lease.provider === "daytona" && (
+    lease.id === input.setupProbeLeaseId || selectedRunIds.size === 0
+    || (lease.heartbeatRunId && selectedRunIds.has(lease.heartbeatRunId))
+    || (input.issueId && lease.issueId === input.issueId)
+  ));
+  return relevant.length > 0 ? relevant : input.leases;
+}
+
 async function deleteDaytonaEnvironment(api: RunnerApi, environmentId: string) {
   const deadlineAt = Date.now() + 120_000;
   let lastError: unknown;
@@ -111,6 +127,7 @@ export async function setupLiveFixtures(input: {
 }): Promise<LiveFixtureValues> {
   const { api, execution } = input;
   const registry = new FixtureRegistry();
+  let setupProbe: LiveFixtureValues["setupProbe"];
 
   if (execution.environment.id === "daytona") {
     registry.register<PluginRecord>({
@@ -295,6 +312,35 @@ export async function setupLiveFixtures(input: {
         secretRefs,
         executionId: input.executionNonce,
       });
+      const autoCodex = agent.adapterType === "codex_local" && (agent.runner === undefined || agent.runner === "auto");
+      const nativeCodex = agent.adapterType === "paperclip_runner" && (agent.adapterConfig as Record<string, unknown>).provider === "codex";
+      if (environment.driver === "sandbox" && execution.profile.generation === "native"
+        && execution.profile.provider === "codex" && agent.runner !== "legacy" && (autoCodex || nativeCodex)) {
+        if (!isImmutableDaytonaImage(String(environment.config?.image ?? "")) || environment.config?.snapshot) {
+          throw new Error("Native Codex qualification requires this fixture's immutable image-backed environment.");
+        }
+        const startedAt = Date.now();
+        // One existing explicit connection probe: a fresh lease, bounded platform read,
+        // verified release, and server-owned image evidence before agent creation.
+        const probe = await api.post<{ ok: boolean; details?: { platformQualificationRecorded?: boolean; leaseId?: string } }>(
+          `/api/environments/${environment.id}/probe?companyId=${encodeURIComponent(company.id)}`,
+          {}, { timeout: 330_000 },
+        );
+        if (probe.ok !== true || probe.details?.platformQualificationRecorded !== true || !probe.details.leaseId) {
+          throw new Error("Native Codex image qualification was not recorded; agent creation was not attempted.");
+        }
+        setupProbe = { leaseId: probe.details.leaseId, durationMs: Date.now() - startedAt };
+        if (autoCodex) {
+          const adapters = await api.get<Array<{ type: string; defaultRunner?: string; supportedRunners?: string[] }>>(
+            `/api/adapters?companyId=${encodeURIComponent(company.id)}&environmentId=${encodeURIComponent(environment.id)}`,
+          );
+          const selected = adapters.find(adapter => adapter.type === "codex_local");
+          if (selected?.defaultRunner !== "paperclip" || !selected.supportedRunners?.includes("paperclip")) {
+            throw new Error("The qualified image does not advertise native Codex as its server-owned default.");
+          }
+          setupProbe.defaultRunner = selected.defaultRunner;
+        }
+      }
       if (["stock-harness", "native-connection-guidance"].includes(execution.suite.id)
         || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) {
         agent.budgetMonthlyCents = 1_000;
@@ -362,6 +408,7 @@ export async function setupLiveFixtures(input: {
     secretRefs: value<SecretReferenceMap>(setup.values, "secrets"),
     environment: value<EnvironmentRecord>(setup.values, "environment"),
     agent: value<AgentRecord>(setup.values, "agent"),
+    ...(setupProbe ? { setupProbe } : {}),
     ...(setup.values.has("project")
       ? { project: value<ProjectRecord>(setup.values, "project") }
       : {}),

tests/runner-e2e/runner.spec.ts

@@ -51,7 +51,7 @@ import { runnerExecutionById } from "./catalog.js";
 import { classifyFailure } from "./failure-classifier.js";
 import { runnerE2EServerControlPaths } from "./harness-env.js";
 import { setupConnectionReview } from "./connection-reviews.js";
-import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js";
+import { setupLiveFixtures, selectFixtureRuntimeLeases, type LiveFixtureValues } from "./live-fixtures.js";
 import { evaluateMatcher, persistedFinalRunMessage, type MatcherResult } from "./matchers.js";
 import { readRegisteredArtifacts } from "./registered-artifact.js";
 import {
@@ -695,16 +695,9 @@ for (const execution of executions) {
       const listed = await api.get<EnvironmentLeaseRecord[]>(
         `/api/environments/${fixtures.environment.id}/leases`,
       );
-      const selectedRunIds = new Set(selectedRuns.map((run) => run.id));
-      const relevant = listed.filter(
-        (lease) =>
-          lease.provider === "daytona" &&
-          (selectedRunIds.size === 0 ||
-            (lease.heartbeatRunId &&
-              selectedRunIds.has(lease.heartbeatRunId)) ||
-            (issue && lease.issueId === issue.id)),
-      );
-      runtimeLeases = relevant.length > 0 ? relevant : listed;
+      runtimeLeases = selectFixtureRuntimeLeases({ leases: listed,
+        runIds: selectedRuns.map(run => run.id), issueId: issue?.id,
+        setupProbeLeaseId: fixtures.setupProbe?.leaseId });
     };
 
     const cancelActiveRunsForCleanup = async () => {
@@ -967,6 +960,8 @@ for (const execution of executions) {
           companyId: fixtures.company.id,
           environmentId: fixtures.environment.id,
           agentId: fixtures.agent.id,
+          ...(fixtures.setupProbe ? { setupProbe: { durationMs: fixtures.setupProbe.durationMs,
+            ...(fixtures.setupProbe.defaultRunner ? { defaultRunner: fixtures.setupProbe.defaultRunner } : {}) } } : {}),
           secretIds: Object.fromEntries(
             Object.entries(fixtures.secretRefs).map(([name, ref]) => [
               name,