From 9cbec30a4f905e5bc6589fea2082e719c2761be9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 01:53:15 -0600 Subject: [PATCH] Use released provider admission in deployment qualification Co-Authored-By: Paperclip --- tests/runner-e2e/catalog.test.ts | 6 +++--- tests/runner-e2e/catalog.ts | 5 +++-- tests/runner-e2e/controller-rolling-deployment.test.ts | 10 +++++++--- tests/runner-e2e/installed-cli.test.ts | 5 +++-- tests/runner-e2e/installed-daytona-plugin.test.ts | 3 ++- tests/runner-e2e/pi-file-evidence.test.ts | 3 ++- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 3729583453d..32c66c5450c 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -312,11 +312,11 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 1, 4, 2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 5, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, + 1, 9, 9, 4, 2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 5, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(802); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(802); + expect(validateRunnerCatalog()).toHaveLength(819); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(819); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 7d5b470d0ca..ecf24df00fd 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1241,9 +1241,10 @@ export const kimiRunnerProfile = nativeProfile({ }); const rollingDeploymentProfiles = [...runnerProfiles, ...extendedHarnessProfiles, kimiRunnerProfile] - .filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona")) + .filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona") + && (!profile.qualificationCandidate || QUALIFIED_ACPX_PROFILES[profile.qualificationCandidate].qualificationStatus !== "pending")) .map(original => { - const profile = productionStoryProfile(original); + const { qualificationCandidate: _candidate, ...profile } = productionStoryProfile(original); return { ...profile, buildAgent(input: AgentFixtureBuildInput) { const agent = profile.buildAgent(input); const adapterConfig = { ...(agent.adapterConfig as Record) }; diff --git a/tests/runner-e2e/controller-rolling-deployment.test.ts b/tests/runner-e2e/controller-rolling-deployment.test.ts index d7310512412..2484abb0212 100644 --- a/tests/runner-e2e/controller-rolling-deployment.test.ts +++ b/tests/runner-e2e/controller-rolling-deployment.test.ts @@ -1,3 +1,4 @@ +import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; import { describe, expect, it } from "vitest"; import { rollController, validControllerRollingDeploymentReceipt } from "./controller-rolling-deployment.js"; import { validControllerReplacementReceipt } from "./controller-replacement.js"; @@ -56,11 +57,12 @@ describe("rolling controller qualification", () => { const [outage] = selectRunnerExecutions(parseRunnerSelectors(["--suite", "remote-workspace-storage"])); expect(controllerOutageForExecution(outage!, {})).toEqual({ signal: "SIGKILL", holdMs: 300_000 }); }); - it("requires both lifecycle modes for every registered native sandbox profile", () => { + it("requires both lifecycle modes for every production-qualified native sandbox profile", () => { const executions = ["rolling-web-deployment", "rolling-web-deployment-warm"] .flatMap(suite => selectRunnerExecutions(parseRunnerSelectors(["--suite", suite]))); const native = [...runnerProfiles, ...extendedHarnessProfiles, kimiRunnerProfile] - .filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona")); + .filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona") + && (!profile.qualificationCandidate || QUALIFIED_ACPX_PROFILES[profile.qualificationCandidate].qualificationStatus !== "pending")); const kind = (profile: (typeof native)[number]) => { const provider = profile.expectedRuntimeMetadata.provider; const harness = "acpxAgent" in profile ? profile.acpxAgent @@ -68,12 +70,14 @@ describe("rolling controller qualification", () => { return `${provider}:${harness}`; }; const registeredKinds = new Set(runnerSuites.flatMap(suite => suite.profiles) - .filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona")) + .filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona") + && (!profile.qualificationCandidate || QUALIFIED_ACPX_PROFILES[profile.qualificationCandidate].qualificationStatus !== "pending")) .map(kind)); expect(new Set(native.map(kind))).toEqual(registeredKinds); for (const profile of native) { const cells = executions.filter(execution => execution.profile.id === profile.id); expect(cells).toHaveLength(2); + expect(cells.every(cell => !cell.profile.qualificationCandidate)).toBe(true); expect(new Set(cells.map(cell => cell.environment.configurationKey ?? "per_turn"))) .toEqual(new Set(["per_turn", "warm-controller-outage-v1"])); } diff --git a/tests/runner-e2e/installed-cli.test.ts b/tests/runner-e2e/installed-cli.test.ts index 7f3074e0fd8..a2ef8190188 100644 --- a/tests/runner-e2e/installed-cli.test.ts +++ b/tests/runner-e2e/installed-cli.test.ts @@ -1,3 +1,4 @@ +import { isRollingWebDeployment } from "./controller-rolling-deployment.js"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { chmod, lstat, mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises"; @@ -7,7 +8,7 @@ import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { runnerMatrix } from "./catalog.js"; import { assertInstalledCliSelection, installedCliKeys, probeInstalledCodexStartup, verifyInstalledCli } from "./installed-cli.js"; import { buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment } from "./harness-env.js"; -const cells = runnerMatrix.filter(e => e.profile.id === "runner-acpx-pi"); +const cells = runnerMatrix.filter(e => !isRollingWebDeployment(e) && e.profile.id === "runner-acpx-pi"); const roots: string[] = []; // pnpm's source test wrapper sets NODE_PATH; the disposable public-install fixture must start closed. beforeEach(() => { vi.stubEnv("NODE_PATH", undefined); vi.stubEnv("NODE_OPTIONS", undefined); }); @@ -46,7 +47,7 @@ it("admits existing Codex onboarding, useful-file revision and restart journeys }); it("admits only the 20 agreed Copilot cells through normal installed resolution with no qualification override", async () => { const { env, server } = await fixture(); - const selected = runnerMatrix.filter(e => e.profile.id === "runner-acpx-copilot"); + const selected = runnerMatrix.filter(e => !isRollingWebDeployment(e) && e.profile.id === "runner-acpx-copilot"); expect(selected).toHaveLength(20); expect(selected.filter(e => e.environment.id === "local")).toHaveLength(10); expect(selected.filter(e => e.environment.id === "daytona")).toHaveLength(10); diff --git a/tests/runner-e2e/installed-daytona-plugin.test.ts b/tests/runner-e2e/installed-daytona-plugin.test.ts index 1f2ed5a29a8..953a314c905 100644 --- a/tests/runner-e2e/installed-daytona-plugin.test.ts +++ b/tests/runner-e2e/installed-daytona-plugin.test.ts @@ -1,3 +1,4 @@ +import { isRollingWebDeployment } from "./controller-rolling-deployment.js"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; @@ -36,7 +37,7 @@ it("admits pinned published plugin and resolved SDK graph without claiming a ful }); it("admits all ten Copilot Daytona cells only with the pinned normal plugin graph", async () => { const f = await fixture(); - const selected = runnerMatrix.filter(e => e.profile.id === "runner-acpx-copilot" && e.environment.id === "daytona"); + const selected = runnerMatrix.filter(e => !isRollingWebDeployment(e) && e.profile.id === "runner-acpx-copilot" && e.environment.id === "daytona"); expect(selected).toHaveLength(10); for (const selectedCell of selected) { expect(await verifyInstalledDaytonaPlugin(f.env, [selectedCell])).toMatchObject({ packageRoot: f.authority.plugin.root, sourceFallback: false, fullGraphVerified: false, completeGraphAuthorityRequired: true }); diff --git a/tests/runner-e2e/pi-file-evidence.test.ts b/tests/runner-e2e/pi-file-evidence.test.ts index 0eda6cd77a4..a66111a3786 100644 --- a/tests/runner-e2e/pi-file-evidence.test.ts +++ b/tests/runner-e2e/pi-file-evidence.test.ts @@ -1,3 +1,4 @@ +import { isRollingWebDeployment } from "./controller-rolling-deployment.js"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { createHash } from "node:crypto"; @@ -206,7 +207,7 @@ describe("Pi edit, validation and public artifact oracle", () => { expect(prompt).toContain(`title ${artifact.name},`); } } - expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi")).toHaveLength(26); + expect(runnerMatrix.filter(c => !isRollingWebDeployment(c) && c.profile.qualificationCandidate === "pi")).toHaveLength(26); }); it.each(["local", "daytona"] as const)("seeds once and collects public %s evidence", async environment => { const root = await mkdtemp(join(tmpdir(), "pi-file-evidence-"));