merge: reconcile measured sandbox qualification with Claude platforms

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/FIXTURES.md

@@ -111,6 +111,19 @@ registry therefore discovers that row through the public environments API.
 This still provides full isolation because every cell starts a new Paperclip
 instance and database.
 
+
+Native-generation Codex on Daytona qualifies its immutable image through one
+existing saved-environment probe before agent creation. Automatic Codex requests
+also inspect the scoped server discovery default. Legacy and other provider
+fixtures keep their existing setup. The probe uses no provider turn, permits one
+setup lease per cell, has a 330-second HTTP deadline and a 10-second platform
+command budget, and performs no retry. It uses the existing archive/release and
+provider-expiry cleanup policy. The returned owned probe lease joins task leases
+in the existing runtime-cost estimator; unrelated lease IDs do not gain task
+attribution. Failed setup before fixture completion retains unavailable/unknown
+cost coverage and its holds. A connection-only probe or name-only snapshot cannot
+qualify native defaults.
+
 Daytona creates sandbox environments through the public API. The core fixture
 keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated
 warm-continuity fixture uses `reuseLease:true` and

tests/runner-e2e/live-fixtures.test.ts

@@ -1,7 +1,8 @@
 import { describe, expect, it } from "vitest";
 import type { RunnerApi } from "./api.js";
+import { buildRuntimeUsage } from "./billing.js";
 import { runnerMatrix } from "./catalog.js";
-import { setupLiveFixtures } from "./live-fixtures.js";
+import { setupLiveFixtures, selectFixtureRuntimeLeases } from "./live-fixtures.js";
 
 describe("live runner fixtures", () => {
   it.each(["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile =>
@@ -163,7 +164,7 @@ describe("live runner fixtures", () => {
     },
   );
 
-  it("installs the Daytona provider through the public API before creating its environment", async () => {
+  it.each(["legacy-codex", "runner-acpx-claude", "runner-opencode"])("installs Daytona before its %s environment without adding a Codex qualification probe", async profile => {
     const calls: string[] = [];
     const api = {
       async post(path: string, data?: Record<string, unknown>) {
@@ -186,7 +187,7 @@ describe("live runner fixtures", () => {
         }
         if (path.endsWith("/environments")) {
           expect(calls).toContain("POST /api/plugins/install");
-          return { id: "environment-1", driver: "sandbox" };
+          return { id: "environment-1", driver: "sandbox", config: data?.config };
         }
         if (path.endsWith("/agents")) {
           return { id: "agent-1", name: "Agent", companyId: "company-1" };
@@ -204,7 +205,7 @@ describe("live runner fixtures", () => {
     const execution = runnerMatrix.find(
       (candidate) =>
         candidate.id ===
-        "core-compatibility.legacy-codex.daytona.message-marker",
+        `core-compatibility.${profile}.daytona.message-marker`,
     );
     expect(execution).toBeDefined();
 
@@ -214,7 +215,7 @@ describe("live runner fixtures", () => {
       executionNonce: "nonce",
       workspacePath: "/tmp/workspace",
       credentials: {
-        OPENAI_API_KEY: "openai-test-value",
+        [execution!.profile.credential]: "provider-test-value",
         DAYTONA_API_KEY: "daytona-test-value",
       },
       daytonaImage:
@@ -224,12 +225,99 @@ describe("live runner fixtures", () => {
     expect(calls.indexOf("POST /api/plugins/install")).toBeLessThan(
       calls.indexOf("POST /api/companies/company-1/environments"),
     );
+    expect(calls.some(call => call.includes("/probe"))).toBe(false);
     await fixtures.teardown();
     expect(calls).toContain(
       "DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true",
     );
   });
 
+  it.each(["explicit", "auto"])("qualifies the immutable Codex image once before %s agent creation", async mode => {
+    const original = runnerMatrix.find(row => row.id === "core-compatibility.runner-codex.daytona.message-marker")!;
+    const execution = { ...original, profile: { ...original.profile, buildAgent(input: Parameters<typeof original.profile.buildAgent>[0]) {
+      const payload = original.profile.buildAgent(input);
+      return { ...payload, adapterType: mode === "auto" ? "codex_local" : "paperclip_runner",
+        runner: mode === "auto" ? "auto" : "paperclip" };
+    } } };
+    const calls: string[] = [];
+    const api = {
+      async get(path: string) {
+        calls.push(`GET ${path}`);
+        return [{ type: "codex_local", defaultRunner: "paperclip", supportedRunners: ["paperclip", "legacy"] }];
+      },
+      async post(path: string, data: any, options: any) {
+        calls.push(`POST ${path}`);
+        if (path === "/api/plugins/install") return { id: "plugin", status: "ready" };
+        if (path === "/api/companies") return { id: "company", name: "Fixture" };
+        if (path.endsWith("/environments")) return { id: "environment", driver: "sandbox", config: data.config };
+        if (path === "/api/environments/environment/probe?companyId=company") {
+          expect(options).toEqual({ timeout: 330_000 });
+          return { ok: true, details: { platformQualificationRecorded: true, leaseId: "setup-probe" } };
+        }
+        if (path.endsWith("/agents")) {
+          expect(calls).toContain("POST /api/environments/environment/probe?companyId=company");
+          expect(data.adapterType).toBe(mode === "auto" ? "codex_local" : "paperclip_runner");
+          expect(data.runner).toBe(mode === "auto" ? "auto" : "paperclip");
+          if (mode === "auto") expect(calls).toContain("GET /api/adapters?companyId=company&environmentId=environment");
+          return { id: "agent", companyId: "company", ...data };
+        }
+        throw new Error(`Unexpected POST ${path}`);
+      },
+      async postSensitive() { return { id: "secret" }; },
+      async delete() {},
+    } as unknown as RunnerApi;
+    const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture",
+      credentials: { OPENAI_API_KEY: "fixture-key", DAYTONA_API_KEY: "fixture-provider-key" },
+      daytonaImage: `registry.example.test/runtime@sha256:${"a".repeat(64)}` });
+    expect(fixtures.setupProbe).toMatchObject({ leaseId: "setup-probe", ...(mode === "auto" ? { defaultRunner: "paperclip" } : {}) });
+    expect(calls.filter(call => call.includes("/probe?"))).toHaveLength(1);
+    expect(calls.filter(call => call.startsWith("GET"))).toHaveLength(mode === "auto" ? 1 : 0);
+    await fixtures.teardown();
+  });
+
+  it.each(["probe-failed", "not-recorded", "wrong-default"])("stops before automatic Codex creation when image qualification is %s", async failure => {
+    const original = runnerMatrix.find(row => row.id === "core-compatibility.runner-codex.daytona.message-marker")!;
+    const execution = { ...original, profile: { ...original.profile, buildAgent(input: Parameters<typeof original.profile.buildAgent>[0]) {
+      return { ...original.profile.buildAgent(input), adapterType: "codex_local", runner: "auto" };
+    } } };
+    const calls: string[] = [];
+    const api = {
+      async get() { return [{ type: "codex_local", defaultRunner: "legacy", supportedRunners: ["legacy"] }]; },
+      async post(path: string, data: any) {
+        calls.push(path);
+        if (path === "/api/plugins/install") return { id: "plugin", status: "ready" };
+        if (path === "/api/companies") return { id: "company", name: "Fixture" };
+        if (path.endsWith("/environments")) return { id: "environment", driver: "sandbox", config: data.config };
+        if (path.includes("/probe?")) return { ok: failure !== "probe-failed",
+          details: { platformQualificationRecorded: failure !== "not-recorded", leaseId: "setup-probe" } };
+        throw new Error(`Unexpected POST ${path}`);
+      },
+      async postSensitive() { return { id: "secret" }; },
+      async delete(path: string) { calls.push(`DELETE ${path}`); },
+    } as unknown as RunnerApi;
+    await expect(setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture",
+      credentials: { OPENAI_API_KEY: "fixture-key", DAYTONA_API_KEY: "fixture-provider-key" },
+      daytonaImage: `registry.example.test/runtime@sha256:${"a".repeat(64)}` })).rejects.toThrow(failure === "wrong-default" ? "server-owned default" : "not recorded");
+    expect(calls.some(call => call.endsWith("/agents"))).toBe(false);
+    expect(calls.filter(call => call.includes("/probe?"))).toHaveLength(1);
+    expect(calls).toContain("DELETE /api/environments/environment?destroyReusableSandboxLeases=true");
+  });
+
+  it("includes the owned setup probe in existing runtime estimates while excluding unrelated leases", () => {
+    const lease = (id: string, duration: number, heartbeatRunId?: string) => ({ id, provider: "daytona", heartbeatRunId,
+      acquiredAt: "2026-10-12T00:00:00Z", releasedAt: new Date(Date.parse("2026-10-12T00:00:00Z") + duration).toISOString(),
+      metadata: { cpu: 4, memory: 4, disk: 10 } });
+    const leases = [lease("task", 20_000, "selected-run"), lease("setup-probe", 10_000), lease("unrelated", 600_000, "other-run")];
+    const selected = selectFixtureRuntimeLeases({ leases, runIds: ["selected-run"], setupProbeLeaseId: "setup-probe" });
+    expect(selected.map(value => value.id)).toEqual(["task", "setup-probe"]);
+    const usage = buildRuntimeUsage({ environmentId: "daytona", runs: [], leases: selected });
+    expect(usage.leaseCount).toBe(2);
+    expect(usage.leaseDurationMs).toBe(30_000);
+    expect(usage.costStatus).toBe("estimated");
+    expect(usage.estimatedListCostUsd).toBeGreaterThan(0);
+    expect(selectFixtureRuntimeLeases({ leases, runIds: ["selected-run"] }).map(value => value.id)).toEqual(["task"]);
+  });
+
   it("creates a primary project workspace for reusable Daytona scope", async () => {
     const calls: string[] = [];
     const api = {
@@ -246,11 +334,14 @@ describe("live runner fixtures", () => {
           return { id: "company-1", name: "Runner E2E" };
         }
         if (path.endsWith("/environments")) {
-          return { id: "environment-1", driver: "sandbox" };
+          return { id: "environment-1", driver: "sandbox", config: data?.config };
         }
         if (path.endsWith("/agents")) {
           return { id: "agent-1", name: "Agent", companyId: "company-1" };
         }
+        if (path.startsWith("/api/environments/environment-1/probe?")) {
+          return { ok: true, details: { platformQualificationRecorded: true, leaseId: "setup-probe" } };
+        }
         if (path.endsWith("/projects")) {
           expect(data).toMatchObject({
             executionWorkspacePolicy: {
@@ -275,6 +366,7 @@ describe("live runner fixtures", () => {
         }
         throw new Error(`Unexpected POST ${path}`);
       },
+      async get() { return [{ type: "codex_local", defaultRunner: "paperclip", supportedRunners: ["paperclip", "legacy"] }]; },
       async postSensitive(_path: string, data?: Record<string, unknown>) {
         return { id: `secret-${String(data?.key).toLowerCase()}` };
       },
@@ -292,7 +384,7 @@ describe("live runner fixtures", () => {
       executionNonce: "nonce",
       workspacePath: "/tmp/workspace",
       credentials: {
-        OPENAI_API_KEY: "openai-test-value",
+        [execution!.profile.credential]: "provider-test-value",
         DAYTONA_API_KEY: "daytona-test-value",
       },
       daytonaImage:

tests/runner-e2e/live-fixtures.ts

@@ -4,6 +4,7 @@ import path from "node:path";
 import { installedReleaseDaytonaPlugin } from "./installed-release.js";
 import { isManagedHiringCase } from "./chat-cases.js";
 import { FixtureRegistry } from "./fixture-registry.js";
+import { isImmutableDaytonaImage } from "./catalog.js";
 import { TASK_TITLE_BUDGET_CENTS } from "./task-titles.js";
 import { stageGrokSubscriptionFixture } from "./grok-subscription-fixture.js";
 import type { RunnerApi } from "./api.js";
@@ -63,6 +64,8 @@ export interface LiveFixtureValues {
   agent: AgentRecord;
   project?: ProjectRecord;
   aiConnection?: ManagedAccountFixture;
+  /** Private setup receipt; its lease is included in existing runtime cost accounting. */
+  setupProbe?: { leaseId: string; durationMs: number; defaultRunner?: string };
 
   onboardingRuntime?: {
     mode: "production-wizard" | "post-onboarding-runtime-switch";
@@ -79,6 +82,19 @@ function value<T>(resolved: ReadonlyMap<string, unknown>, id: string): T {
   return result as T;
 }
 
+/** Preserve task/run attribution and include only this fixture's explicit setup probe. */
+export function selectFixtureRuntimeLeases<T extends {
+  id: string; provider?: string | null; heartbeatRunId?: string | null; issueId?: string | null;
+}>(input: { leases: T[]; runIds: string[]; issueId?: string; setupProbeLeaseId?: string }): T[] {
+  const selectedRunIds = new Set(input.runIds);
+  const relevant = input.leases.filter(lease => lease.provider === "daytona" && (
+    lease.id === input.setupProbeLeaseId || selectedRunIds.size === 0
+    || (lease.heartbeatRunId && selectedRunIds.has(lease.heartbeatRunId))
+    || (input.issueId && lease.issueId === input.issueId)
+  ));
+  return relevant.length > 0 ? relevant : input.leases;
+}
+
 async function deleteDaytonaEnvironment(api: RunnerApi, environmentId: string) {
   const deadlineAt = Date.now() + 120_000;
   let lastError: unknown;
@@ -111,6 +127,7 @@ export async function setupLiveFixtures(input: {
 }): Promise<LiveFixtureValues> {
   const { api, execution } = input;
   const registry = new FixtureRegistry();
+  let setupProbe: LiveFixtureValues["setupProbe"];
 
   if (execution.environment.id === "daytona") {
     registry.register<PluginRecord>({
@@ -295,6 +312,35 @@ export async function setupLiveFixtures(input: {
         secretRefs,
         executionId: input.executionNonce,
       });
+      const autoCodex = agent.adapterType === "codex_local" && (agent.runner === undefined || agent.runner === "auto");
+      const nativeCodex = agent.adapterType === "paperclip_runner" && (agent.adapterConfig as Record<string, unknown>).provider === "codex";
+      if (environment.driver === "sandbox" && execution.profile.generation === "native"
+        && execution.profile.provider === "codex" && agent.runner !== "legacy" && (autoCodex || nativeCodex)) {
+        if (!isImmutableDaytonaImage(String(environment.config?.image ?? "")) || environment.config?.snapshot) {
+          throw new Error("Native Codex qualification requires this fixture's immutable image-backed environment.");
+        }
+        const startedAt = Date.now();
+        // One existing explicit connection probe: a fresh lease, bounded platform read,
+        // verified release, and server-owned image evidence before agent creation.
+        const probe = await api.post<{ ok: boolean; details?: { platformQualificationRecorded?: boolean; leaseId?: string } }>(
+          `/api/environments/${environment.id}/probe?companyId=${encodeURIComponent(company.id)}`,
+          {}, { timeout: 330_000 },
+        );
+        if (probe.ok !== true || probe.details?.platformQualificationRecorded !== true || !probe.details.leaseId) {
+          throw new Error("Native Codex image qualification was not recorded; agent creation was not attempted.");
+        }
+        setupProbe = { leaseId: probe.details.leaseId, durationMs: Date.now() - startedAt };
+        if (autoCodex) {
+          const adapters = await api.get<Array<{ type: string; defaultRunner?: string; supportedRunners?: string[] }>>(
+            `/api/adapters?companyId=${encodeURIComponent(company.id)}&environmentId=${encodeURIComponent(environment.id)}`,
+          );
+          const selected = adapters.find(adapter => adapter.type === "codex_local");
+          if (selected?.defaultRunner !== "paperclip" || !selected.supportedRunners?.includes("paperclip")) {
+            throw new Error("The qualified image does not advertise native Codex as its server-owned default.");
+          }
+          setupProbe.defaultRunner = selected.defaultRunner;
+        }
+      }
       if (["stock-harness", "native-connection-guidance"].includes(execution.suite.id)
         || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) {
         agent.budgetMonthlyCents = 1_000;
@@ -362,6 +408,7 @@ export async function setupLiveFixtures(input: {
     secretRefs: value<SecretReferenceMap>(setup.values, "secrets"),
     environment: value<EnvironmentRecord>(setup.values, "environment"),
     agent: value<AgentRecord>(setup.values, "agent"),
+    ...(setupProbe ? { setupProbe } : {}),
     ...(setup.values.has("project")
       ? { project: value<ProjectRecord>(setup.values, "project") }
       : {}),

tests/runner-e2e/runner.spec.ts

@@ -51,7 +51,7 @@ import { runnerExecutionById } from "./catalog.js";
 import { classifyFailure } from "./failure-classifier.js";
 import { runnerE2EServerControlPaths } from "./harness-env.js";
 import { setupConnectionReview } from "./connection-reviews.js";
-import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js";
+import { setupLiveFixtures, selectFixtureRuntimeLeases, type LiveFixtureValues } from "./live-fixtures.js";
 import { evaluateMatcher, persistedFinalRunMessage, type MatcherResult } from "./matchers.js";
 import { readRegisteredArtifacts } from "./registered-artifact.js";
 import {
@@ -698,16 +698,9 @@ for (const execution of executions) {
       const listed = await api.get<EnvironmentLeaseRecord[]>(
         `/api/environments/${fixtures.environment.id}/leases`,
       );
-      const selectedRunIds = new Set(selectedRuns.map((run) => run.id));
-      const relevant = listed.filter(
-        (lease) =>
-          lease.provider === "daytona" &&
-          (selectedRunIds.size === 0 ||
-            (lease.heartbeatRunId &&
-              selectedRunIds.has(lease.heartbeatRunId)) ||
-            (issue && lease.issueId === issue.id)),
-      );
-      runtimeLeases = relevant.length > 0 ? relevant : listed;
+      runtimeLeases = selectFixtureRuntimeLeases({ leases: listed,
+        runIds: selectedRuns.map(run => run.id), issueId: issue?.id,
+        setupProbeLeaseId: fixtures.setupProbe?.leaseId });
     };
 
     const cancelActiveRunsForCleanup = async () => {
@@ -972,6 +965,8 @@ for (const execution of executions) {
           companyId: fixtures.company.id,
           environmentId: fixtures.environment.id,
           agentId: fixtures.agent.id,
+          ...(fixtures.setupProbe ? { setupProbe: { durationMs: fixtures.setupProbe.durationMs,
+            ...(fixtures.setupProbe.defaultRunner ? { defaultRunner: fixtures.setupProbe.defaultRunner } : {}) } } : {}),
           secretIds: Object.fromEntries(
             Object.entries(fixtures.secretRefs).map(([name, ref]) => [
               name,