From 16922a1832ca8d6c2dafdba25f56d530e23e432b Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 21 Oct 2026 08:06:11 -0500 Subject: [PATCH] test: qualify conversion prerequisites before runner selection Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 5 ++ .../repository-conversion-setup.test.ts | 58 +++++++++++++++++++ .../runner-e2e/repository-conversion-setup.ts | 20 +++++++ .../repository-verification-flow.ts | 12 +--- tests/runner-e2e/repository-verification.ts | 7 ++- 5 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 tests/runner-e2e/repository-conversion-setup.test.ts create mode 100644 tests/runner-e2e/repository-conversion-setup.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index dd87e0ac11c..82c43d88aa8 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -35,6 +35,11 @@ hard stops. The conversion journey creates a real legacy agent and selects Paperclip Runner through the public agent update API. It checks retained configuration, encrypted secret references, environment, instructions, and corresponding permission policy before submitting work through the browser. +The disposable legacy profile starts without optional CLI-only extra arguments; +the journey does not remove settings from an existing user agent. For either +provider, Daytona's public environment probe must save platform qualification +before the same agent selects Runner. Probe leases and uncertain probe accounting +remain included in attempt evidence and cleanup. The observer seeds a random reference in the fixture-owned Git checkout before task creation. It checks the downloaded JSON against the original file bytes, diff --git a/tests/runner-e2e/repository-conversion-setup.test.ts b/tests/runner-e2e/repository-conversion-setup.test.ts new file mode 100644 index 00000000000..ef265e0fbef --- /dev/null +++ b/tests/runner-e2e/repository-conversion-setup.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, vi } from "vitest"; +import { runnerSuites } from "./catalog.js"; +import { convertRepositoryProofAgent } from "./repository-conversion-setup.js"; + +function fixtures(driver = "sandbox"): Parameters[1] { + return { agent: { id: "agent", name: "Fixture Agent", companyId: "company" }, company: { id: "company", name: "Fixture Company" }, environment: { id: "environment", driver, + config: { image: `image@sha256:${"a".repeat(64)}` } } }; +} + +describe("real legacy conversion prerequisites", () => { + it("constructs compatible legacy settings before recording a retention baseline", () => { + const suite = runnerSuites.find(suite => suite.id === "legacy-conversion")!; + for (const profile of suite.profiles) { + const agent = profile.buildAgent({ environmentId: "environment", environmentFixtureId: "local", workspacePath: "/owned/workspace", + executionId: "fixture", secretRefs: { OPENAI_API_KEY: { type: "secret_ref", secretId: "codex-reference", version: "latest" }, + ANTHROPIC_API_KEY: { type: "secret_ref", secretId: "claude-reference", version: "latest" } } }); + expect(agent.runner).toBe("legacy"); + expect(agent.adapterType).toBe(profile.provider === "codex" ? "codex_local" : "claude_local"); + expect(agent.adapterConfig).toMatchObject({ model: profile.model, timeoutSec: 360, cwd: "/owned/workspace" }); + expect(agent.adapterConfig).not.toHaveProperty("extraArgs"); + expect((agent.adapterConfig as Record).env).toMatchObject(profile.provider === "codex" + ? { OPENAI_API_KEY: { type: "secret_ref", secretId: "codex-reference", version: "latest" } } + : { ANTHROPIC_API_KEY: { type: "secret_ref", secretId: "claude-reference", version: "latest" } }); + } + }); + it("qualifies the sandbox before selecting Runner for either provider", async () => { + const fixture = fixtures(); let qualified = false; + const api = { post: vi.fn(async () => { qualified = true; return { ok: true, details: { platformQualificationRecorded: true, leaseId: "lease" } }; }), + patch: vi.fn(async () => { if (!qualified) throw new Error("agent_runner_unavailable"); return {}; }) }; + await expect(convertRepositoryProofAgent(api as any, fixture)).resolves.toBeUndefined(); + expect(api.post).toHaveBeenCalledWith("/api/environments/environment/probe?companyId=company", {}, { timeout: 330_000 }); + expect(fixture.setupProbe).toMatchObject({ leaseId: "lease" }); + expect(api.patch).toHaveBeenCalledExactlyOnceWith("/api/agents/agent", { runner: "paperclip" }); + }); + it("does not attempt conversion when platform qualification is absent", async () => { + const fixture = fixtures(); const api = { post: vi.fn(async () => ({ ok: true, details: { leaseId: "lease" } })), patch: vi.fn() }; + await expect(convertRepositoryProofAgent(api as any, fixture)).rejects.toThrow("qualification"); + expect(api.patch).not.toHaveBeenCalled(); + expect(fixture.setupProbe?.leaseId).toBe("lease"); + }); + it("retains uncertain probe accounting when the response is lost", async () => { + const fixture = fixtures(); const api = { post: vi.fn(async () => { throw new Error("connection lost"); }), patch: vi.fn() }; + await expect(convertRepositoryProofAgent(api as any, fixture)).rejects.toThrow("connection lost"); + expect(api.patch).not.toHaveBeenCalled(); + expect(fixture.setupLeaseAccountingIncomplete).toBe(true); + }); + it.each([{ image: "image:latest" }, { image: `image@sha256:${"a".repeat(64)}`, snapshot: "snapshot" }])("refuses an unqualified image before creating a sandbox", async config => { + const fixture = fixtures(); fixture.environment.config = config; + const api = { post: vi.fn(), patch: vi.fn() }; + await expect(convertRepositoryProofAgent(api as any, fixture)).rejects.toThrow("immutable"); + expect(api.post).not.toHaveBeenCalled(); expect(api.patch).not.toHaveBeenCalled(); + }); + it("converts a local agent without creating a remote sandbox", async () => { + const api = { post: vi.fn(), patch: vi.fn(async () => ({})) }; + await convertRepositoryProofAgent(api as any, fixtures("local")); + expect(api.post).not.toHaveBeenCalled(); expect(api.patch).toHaveBeenCalledOnce(); + }); +}); diff --git a/tests/runner-e2e/repository-conversion-setup.ts b/tests/runner-e2e/repository-conversion-setup.ts new file mode 100644 index 00000000000..2f35c46a5d3 --- /dev/null +++ b/tests/runner-e2e/repository-conversion-setup.ts @@ -0,0 +1,20 @@ +import type { RunnerApi } from "./api.js"; +import type { LiveFixtureValues } from "./live-fixtures.js"; + +export async function convertRepositoryProofAgent(api: Pick, fixtures: Pick) { + if (fixtures.environment.driver === "sandbox") { + if (!/^.+@sha256:[a-f0-9]{64}$/i.test(String(fixtures.environment.config?.image ?? "")) || fixtures.environment.config?.snapshot) + throw new Error("Legacy conversion requires the immutable fixture image"); + const startedAt = Date.now(), previousAccounting = fixtures.setupLeaseAccountingIncomplete; + fixtures.setupLeaseAccountingIncomplete = true; + const probe = await api.post<{ ok: boolean; details?: { platformQualificationRecorded?: boolean; leaseId?: string } }>( + `/api/environments/${fixtures.environment.id}/probe?companyId=${encodeURIComponent(fixtures.company.id)}`, {}, { timeout: 330_000 }); + if (probe.details?.leaseId) { + fixtures.setupProbe = { leaseId: probe.details.leaseId, durationMs: Date.now() - startedAt }; + fixtures.setupLeaseAccountingIncomplete = previousAccounting; + } + if (!probe.ok || !probe.details?.platformQualificationRecorded || !probe.details.leaseId) + throw new Error("Legacy conversion platform qualification was not recorded"); + } + await api.patch(`/api/agents/${fixtures.agent.id}`, { runner: "paperclip" }); +} diff --git a/tests/runner-e2e/repository-verification-flow.ts b/tests/runner-e2e/repository-verification-flow.ts index 8dd7ac74a0d..b3db9c7da32 100644 --- a/tests/runner-e2e/repository-verification-flow.ts +++ b/tests/runner-e2e/repository-verification-flow.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { createTaskThroughUi } from "./user-actions.js"; +import { convertRepositoryProofAgent } from "./repository-conversion-setup.js"; import { readRegisteredArtifacts } from "./registered-artifact.js"; import { gradeRepositoryProof, repositoryConfigurationDigest, repositoryProofFiles, type RepositoryProofObservation } from "./repository-verification.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; @@ -85,7 +86,7 @@ export async function runRepositoryVerificationFlow(input: { observation.originalAdapter = before.adapterType; observation.configurationBefore = repositoryConfigurationDigest(before); observation.instructionsBefore = await instructionsDigest(); - if (conversion) await api.patch(`/api/agents/${fixtures.agent.id}`, { runner: "paperclip" }); + if (conversion) await convertRepositoryProofAgent(api, fixtures); const after = await api.get(`/api/agents/${fixtures.agent.id}`); observation.finalAdapter = after.adapterType; observation.configurationAfter = repositoryConfigurationDigest(after); @@ -95,15 +96,6 @@ export async function runRepositoryVerificationFlow(input: { if (after.defaultEnvironmentId !== fixtures.environment.id || observation.configurationBefore !== observation.configurationAfter || observation.instructionsBefore !== observation.instructionsAfter || !observation.permissionsPreserved || after.adapterType !== "paperclip_runner") throw new Error("Production runner selection did not retain the configured agent context"); - if (conversion && execution.environment.id === "daytona" && execution.profile.provider === "codex") { - if (!/^.+@sha256:[a-f0-9]{64}$/i.test(String(fixtures.environment.config?.image ?? "")) || fixtures.environment.config?.snapshot) - throw new Error("Converted Codex requires the immutable fixture image"); - const startedAt = Date.now(); - const probe = await api.post<{ ok: boolean; details?: { platformQualificationRecorded?: boolean; leaseId?: string } }>( - `/api/environments/${fixtures.environment.id}/probe?companyId=${encodeURIComponent(fixtures.company.id)}`, {}, { timeout: 330_000 }); - if (!probe.ok || !probe.details?.platformQualificationRecorded || !probe.details.leaseId) throw new Error("Converted Codex platform qualification was not recorded"); - fixtures.setupProbe = { leaseId: probe.details.leaseId, durationMs: Date.now() - startedAt }; - } await save("configured"); const created = await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, projectName: fixtures.project.name, title: execution.task.buildTitle(nonce), prompt: execution.task.buildPrompt(nonce), workMode: "standard", requireExplicitTitle: true }); diff --git a/tests/runner-e2e/repository-verification.ts b/tests/runner-e2e/repository-verification.ts index 60897837595..d6a13daa1e5 100644 --- a/tests/runner-e2e/repository-verification.ts +++ b/tests/runner-e2e/repository-verification.ts @@ -9,8 +9,9 @@ export function repositoryProofFiles(nonce: string) { export function repositoryProofProfile(native: RunnerProfileFixture, legacy?: RunnerProfileFixture): RunnerProfileFixture { return { ...native, buildAgent(input) { const { instructionsBundle: _manual, ...agent } = (legacy ?? native).buildAgent(input); - return { ...agent, ...(legacy ? { runner: "legacy" } : {}), budgetMonthlyCents: 1000, - adapterConfig: { ...(agent.adapterConfig as Record), model: native.model } }; + const config = { ...(agent.adapterConfig as Record), model: native.model }; + if (legacy) delete (config as Record).extraArgs; + return { ...agent, ...(legacy ? { runner: "legacy" } : {}), budgetMonthlyCents: 1000, adapterConfig: config }; } }; } @@ -95,7 +96,7 @@ export function repositoryConfigurationDigest(agent: Record) { export function repositoryProofDigest() { const hash = createHash("sha256"); - for (const file of ["repository-verification.ts", "repository-verification-flow.ts", "registered-artifact.ts", "live-fixtures.ts", "runner.spec.ts", "catalog.ts", "stock-harness.ts"]) + for (const file of ["repository-verification.ts", "repository-verification-flow.ts", "repository-conversion-setup.ts", "registered-artifact.ts", "live-fixtures.ts", "runner.spec.ts", "catalog.ts", "stock-harness.ts"]) hash.update(file).update(readFileSync(new URL(file, import.meta.url))); return hash.digest("hex"); }