From 9079e0f20d49d903f9df4b592b1a2c2e6fd946ce Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 08:55:33 -0500 Subject: [PATCH] evals: select OpenCode DeepSeek for remote chat catalog --- tests/runner-e2e/README.md | 19 ++++++++++++++ tests/runner-e2e/catalog.ts | 6 +++-- .../chat-archive-fault-control.test.ts | 3 ++- .../runner-e2e/chat-archive-fault-control.ts | 15 ++++++----- .../external-team-configure.ts | 15 +++++++---- .../runtime-native-failure.test.ts | 12 ++++++++- .../runtime-native-failure.ts | 19 +++++++++----- .../runtime-runs-adapters.test.ts | 7 +++++ .../runtime-runs-adapters.ts | 10 ++++--- .../chat-catalog-fingerprint.test.ts | 9 +++++++ tests/runner-e2e/chat-catalog-fingerprint.ts | 5 ++-- tests/runner-e2e/chat-feature-cases.ts | 2 +- tests/runner-e2e/chat-feature-scoring.test.ts | 5 ++-- tests/runner-e2e/chat-installed-lane.test.ts | 4 +++ tests/runner-e2e/chat-installed-lane.ts | 3 ++- tests/runner-e2e/chat-runtime-profile.test.ts | 26 +++++++++++++++++++ tests/runner-e2e/chat-runtime-profile.ts | 11 ++++++++ .../chat-vault-private-server.test.ts | 9 +++++++ tests/runner-e2e/chat-vault-private-server.ts | 8 +++--- 19 files changed, 154 insertions(+), 34 deletions(-) create mode 100644 tests/runner-e2e/chat-runtime-profile.test.ts create mode 100644 tests/runner-e2e/chat-runtime-profile.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index c83fc66455a..9aa0500b740 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -3002,3 +3002,22 @@ PAPERCLIP_RUNNER_E2E_SOURCE_SHA= node --import ./cli/node_modules/ Its receipt is retained under `results/chat-catalog-campaign/`; it is never a scenario measurement or feature acceptance. Missing/changed ownership or an unknown inspection result is not proof that a resource is absent. + +### Chat catalog remote OpenCode campaign + +The explicit `chat-features` matrix now selects `runner-opencode` with native +`paperclip_runner`, provider `opencode`, and exact model +`openrouter/deepseek/deepseek-v4.1-flash-1017`. Its credential is an encrypted +`OPENROUTER_API_KEY` reference. OpenCode remains pinned to the runner-owned +1.18.34 executable; this candidate does not change the previously qualified +OpenCode model for other suites. `local` means the selected execution host, +including an isolated AWS worker, rather than a requirement to use a developer laptop. + +This is a new 244-cell behavioral campaign. Historical Codex results and passes +remain separate and do not qualify the new provider/model. Named-provider feature +requests retain their original meaning. Unavailable-model history requires an +actually admitted failed OpenCode run with the exact configured provider/model; +a preflight rejection does not establish that prerequisite. Provider/account +qualification, paid-job approval, and owned AWS Spot capacity must be established +before remote admission. Discovery and credential-free controls establish none +of those live outcomes. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index ceb804079a2..417fbf07848 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,3 +1,4 @@ +import { CHAT_OPENCODE_MODEL, CHAT_OPENCODE_PROFILE } from "./chat-runtime-profile.js"; import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js"; import { followupStopTask, followupStopDigest } from "./followup-stop-resume.js"; import { finalizationRecoveryTask, finalizationRecoveryDigest } from "./finalization-recovery.js"; @@ -1608,8 +1609,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ id: CHAT_FEATURE_SUITE, label: "Chat feature journeys", manualOnly: true, description: "Executable catalog requests through Agent Chat, with independently graded persisted state.", groups: ["chat", "native"], - profiles: [chatFeatureProfile(nativeProfile({ id: "runner-codex-subscription", label: "Native Codex subscription", - provider: "codex", model: DEFAULT_CODEX_LOCAL_MODEL, credential: "CODEX_AUTH_JSON", supportedEnvironments: ["local"] }))], + profiles: [chatFeatureProfile(nativeProfile({ id: CHAT_OPENCODE_PROFILE, label: "Native OpenCode OpenRouter DeepSeek V4.1 Flash 1017", + provider: "opencode", model: CHAT_OPENCODE_MODEL, credential: "OPENROUTER_API_KEY", supportedEnvironments: ["local"], + modelQualification: { source: "candidate_runner_profile", qualificationId: "chat-catalog:opencode-1.18.34:deepseek-v4.1-flash-1017:remote-initial" } }))], environments: [localEnvironment], tasks: [...chatFeatureTasks, ...chatCatalogTasks], expectedMatrixSize: 2 + chatCatalogTasks.length, definitionMetadata: { version: 2, definitionDigest: chatFeatureDefinitionDigest, inventoryDigest: chatCatalogInventoryDigest, frozenScenarioCount: 244, scheduling: "explicit-only", instructions: "production-role-defaults", companyAndAgentBudgetCents: CHAT_FEATURE_BUDGET_CENTS, diff --git a/tests/runner-e2e/chat-archive-fault-control.test.ts b/tests/runner-e2e/chat-archive-fault-control.test.ts index 0b9fb18db4f..affa357acef 100644 --- a/tests/runner-e2e/chat-archive-fault-control.test.ts +++ b/tests/runner-e2e/chat-archive-fault-control.test.ts @@ -11,7 +11,8 @@ import { ARCHIVE_CONTROL_ENV, ARCHIVE_SCENARIO, archiveFaultClient, archiveServe import { buildPaperclipServerEnvironment } from "./harness-env.js"; import { reserveRunnerE2EServerPort } from "./ports.js"; const id = `chat-features.runner-codex-subscription.local.${ARCHIVE_SCENARIO}`; -it("starts before backend and gives only the test child a private authenticated control handoff", async () => { +it.each(["runner-codex-subscription", "runner-opencode"])("binds %s archive control to its own selected execution and private handoff", async profile => { + const id = `chat-features.${profile}.local.${ARCHIVE_SCENARIO}`; const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "paperclip-runner-e2e-"))); await mkdir(path.join(temporaryRoot, "selected-private-inputs"), { mode: 0o700 }); const port = await reserveRunnerE2EServerPort(), backendOrigin = `http://127.0.0.1:${port}`, secrets: string[] = []; const control = await startArchiveFaultControl({ selected: [id], temporaryRoot, backendOrigin, privateSecrets: secrets }); diff --git a/tests/runner-e2e/chat-archive-fault-control.ts b/tests/runner-e2e/chat-archive-fault-control.ts index aeceab44d36..164bdef04cf 100644 --- a/tests/runner-e2e/chat-archive-fault-control.ts +++ b/tests/runner-e2e/chat-archive-fault-control.ts @@ -1,3 +1,4 @@ +import { reviewedChatExecutionIds } from "./chat-runtime-profile.js"; import { createServer } from "node:http"; import { randomBytes, timingSafeEqual } from "node:crypto"; import { lstat, readFile, realpath, rm, writeFile } from "node:fs/promises"; @@ -6,7 +7,7 @@ import { listenOnFetchAllowedPort } from "../e2e/fetch-allowed-port.js"; import { startArchiveFaultProxy, validArchiveBinding, type ArchiveBinding, type ArchiveFaultTransport, type ArchiveTransportReceipt } from "./chat-catalog-domains/internal-archive-transport.js"; export const ARCHIVE_CONTROL_ENV = "PAPERCLIP_RUNNER_E2E_ARCHIVE_CONTROL"; export const ARCHIVE_SCENARIO = "inbox-search.archive-partial-failure"; -const executionId = `chat-features.runner-codex-subscription.local.${ARCHIVE_SCENARIO}`; +const executionIds = reviewedChatExecutionIds(ARCHIVE_SCENARIO); interface Binding { version: 1; executionId: string; temporaryRoot: string; backendOrigin: string; origin: string; controlOrigin: string; token: string } const origin = (value: unknown): value is string => { try { const u = new URL(String(value)); return u.protocol === "http:" && u.hostname === "127.0.0.1" && !!u.port && u.origin === value; } catch { return false; } }; async function privateDirectory(root: string) { @@ -17,14 +18,14 @@ async function privateDirectory(root: string) { } catch { throw new Error("Private archive root is unavailable or unsafe"); } } async function privateBinding(filename: string, root: string, selected: readonly string[]) { - if (selected.length !== 1 || selected[0] !== executionId || filename !== path.join(root, "selected-private-inputs", "archive-fault.json")) throw new Error("Archive transport requires its exact selected owned case"); + if (selected.length !== 1 || !executionIds.includes(selected[0]!) || filename !== path.join(root, "selected-private-inputs", "archive-fault.json")) throw new Error("Archive transport requires its exact selected owned case"); try { await privateDirectory(root); const directory = path.dirname(filename), dir = await lstat(directory), file = await lstat(filename); if (await realpath(root) !== root || await realpath(directory) !== directory || !dir.isDirectory() || dir.mode & 0o077 || file.isSymbolicLink() || !file.isFile() || (file.mode & 0o777) !== 0o600 || file.size > 4096 || typeof process.getuid === "function" && (dir.uid !== process.getuid() || file.uid !== process.getuid())) throw new Error(); const value = JSON.parse(await readFile(filename, "utf8")) as Binding; - if (Object.keys(value).sort().join() !== "backendOrigin,controlOrigin,executionId,origin,temporaryRoot,token,version" || value.version !== 1 || value.executionId !== executionId || value.temporaryRoot !== root + if (Object.keys(value).sort().join() !== "backendOrigin,controlOrigin,executionId,origin,temporaryRoot,token,version" || value.version !== 1 || value.executionId !== selected[0] || value.temporaryRoot !== root || !origin(value.backendOrigin) || !origin(value.origin) || !origin(value.controlOrigin) || new Set([value.backendOrigin, value.origin, value.controlOrigin]).size !== 3 || !/^[a-f0-9]{64}$/.test(value.token)) throw new Error(); return value; } catch { throw new Error("Private archive transport binding is unavailable or unsafe"); } @@ -40,7 +41,7 @@ function validReceipt(value: unknown): value is ArchiveTransportReceipt { && (row.status === undefined || Number.isInteger(row.status) && Number(row.status) >= 100 && Number(row.status) <= 599); } export async function startArchiveFaultControl(input: { selected: readonly string[]; temporaryRoot: string; backendOrigin: string; privateSecrets: string[] }) { - if (input.selected.length !== 1 || input.selected[0] !== executionId || !origin(input.backendOrigin)) throw new Error("Archive fault is allowlisted only for its owned local execution"); + if (input.selected.length !== 1 || !executionIds.includes(input.selected[0]!) || !origin(input.backendOrigin)) throw new Error("Archive fault is allowlisted only for its owned local execution"); await privateDirectory(input.temporaryRoot); if (/(?:^|[\s,])(?:http|net|tls|undici|\*)(?:$|[\s,])/i.test(process.env.NODE_DEBUG ?? "")) throw new Error("Archive private control refuses raw transport debug output"); const proxy = await startArchiveFaultProxy(input.backendOrigin), token = randomBytes(32).toString("hex"); @@ -59,7 +60,7 @@ export async function startArchiveFaultControl(input: { selected: readonly strin const port = await listenOnFetchAllowedPort(control), controlOrigin = `http://127.0.0.1:${port}`; filename = path.join(input.temporaryRoot, "selected-private-inputs", "archive-fault.json"); input.privateSecrets.push(token, controlOrigin, filename); - const binding: Binding = { version: 1, executionId, temporaryRoot: input.temporaryRoot, backendOrigin: input.backendOrigin, origin: proxy.origin, controlOrigin, token }; + const binding: Binding = { version: 1, executionId: input.selected[0]!, temporaryRoot: input.temporaryRoot, backendOrigin: input.backendOrigin, origin: proxy.origin, controlOrigin, token }; await writeFile(filename, `${JSON.stringify(binding)}\n`, { mode: 0o600, flag: "wx" }); created = true; await privateBinding(filename, input.temporaryRoot, input.selected); let closed = false; @@ -70,14 +71,14 @@ export async function startArchiveFaultControl(input: { selected: readonly strin * normal API-origin override; never a control filename, URL or bearer. */ export async function archiveServerEnvironment(input: { selected: readonly string[]; temporaryRoot: string; backendOrigin: string; env: NodeJS.ProcessEnv }) { const source = { ...input.env }, filename = source[ARCHIVE_CONTROL_ENV]; delete source[ARCHIVE_CONTROL_ENV]; - if (!filename) { if (input.selected.includes(executionId)) throw new Error("Selected archive case is missing pre-server fault transport"); return { source, overrides: {} }; } + if (!filename) { if (input.selected.some(id => executionIds.includes(id))) throw new Error("Selected archive case is missing pre-server fault transport"); return { source, overrides: {} }; } const binding = await privateBinding(filename, input.temporaryRoot, input.selected); if (binding.backendOrigin !== input.backendOrigin) throw new Error("Archive backend origin differs from owned server"); return { source, overrides: { PAPERCLIP_API_URL: binding.origin } }; } export async function archiveFaultClient(input: { selected: string; backendOrigin: string; privateSecrets?: string[]; env?: NodeJS.ProcessEnv }): Promise { const env = input.env ?? process.env, filename = env[ARCHIVE_CONTROL_ENV]; - if (!filename) { if (input.selected === executionId) throw new Error("Selected archive case is missing private transport handoff"); return; } + if (!filename) { if (executionIds.includes(input.selected)) throw new Error("Selected archive case is missing private transport handoff"); return; } const binding = await privateBinding(filename, env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "", [input.selected]); if (binding.backendOrigin !== input.backendOrigin) throw new Error("Archive client belongs to another backend"); input.privateSecrets?.push(binding.token, binding.controlOrigin, filename); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts index 421d979a42c..89af0eb3cf9 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts @@ -49,26 +49,27 @@ export function gradeTeamConfigure(value: unknown) { return c.result(); } export const teamConfigureExternalCases = [definition("teams.configure-team", async ctx => { - if (ctx.input.execution.profile.credential !== "CODEX_AUTH_JSON") throw new Error("Setup blocker: this disposable team fixture requires the selected Codex subscription account, not an inferred personal login"); + if (!["CODEX_AUTH_JSON", "OPENROUTER_API_KEY"].includes(ctx.input.execution.profile.credential)) throw new Error("Setup blocker: this disposable team fixture requires the selected campaign account, not an inferred personal login"); const team = (await ctx.input.api.get("/api/teams/catalog")).find(row => row.slug === "product-engineering" && row.compatibility === "compatible"); if (!team) throw new Error("Setup blocker: compatible real Product Engineering package is unavailable"); const built = ctx.input.execution.profile.buildAgent({ environmentId: ctx.input.fixtures.environment.id, environmentFixtureId: ctx.input.execution.environment.id, workspacePath: ctx.input.workspacePath, secretRefs: ctx.input.fixtures.secretRefs, executionId: ctx.input.nonce }); if (typeof built.adapterType !== "string") throw new Error("Setup blocker: selected profile does not declare a supported runtime adapter"); - const configured = codexSubscriptionAgentConfig(built, process.env, ctx.input.fixtures.company.id); - const override: CompanyPortabilityAdapterOverride = { adapterType: built.adapterType, adapterConfig: configured.adapterConfig }; + const subscription = ctx.input.execution.profile.credential === "CODEX_AUTH_JSON"; + const configured = subscription ? codexSubscriptionAgentConfig(built, process.env, ctx.input.fixtures.company.id) : built; + const override: CompanyPortabilityAdapterOverride = { adapterType: built.adapterType, adapterConfig: configured.adapterConfig as Record }; const emptyHome = await mkdtemp(path.join(tmpdir(), "paperclip-missing-team-login-")); fixtures.set(ctx, { emptyHome, override }); try { const install = await ctx.input.api.postSensitive(`${root(ctx)}/teams/catalog/${encodeURIComponent(team.id)}/install`, { targetManagerAgentId: ctx.input.fixtures.agent.id, adapterOverrides: Object.fromEntries(team.agentSlugs.map(slug => [slug, override])) }); const missingId = install.portabilityImport.agents.find(row => row.slug === "qa")?.id, workingId = install.portabilityImport.agents.find(row => row.slug === "senior-coder")?.id; if (!missingId || !workingId) throw new Error("Setup blocker: installer did not produce the exact QA and senior-coder members"); - await ctx.input.api.patchSensitive(`/api/agents/${missingId}`, { adapterConfig: { ...override.adapterConfig, env: { ...(override.adapterConfig?.env as Record ?? {}), CODEX_HOME: emptyHome, OPENAI_API_KEY: "", CODEX_API_KEY: "" } } }); + await ctx.input.api.patchSensitive(`/api/agents/${missingId}`, { adapterConfig: { ...override.adapterConfig, env: { ...(override.adapterConfig?.env as Record ?? {}), ...missingTeamAccountEnvironment(subscription, emptyHome) } } }); const members = catalogTeamMembers(await observeTeamResources(ctx), team.id); const initialProbes = await Promise.all(members.map(member => probe(ctx, member))); if (!initialProbes.some(row => row.agentId === missingId && row.status === "fail" && row.checks.some(check => check.level === "error" && /auth|credential|login/i.test(check.code))) || !initialProbes.some(row => row.agentId === workingId && row.status === "pass")) throw new Error("Setup blocker: actual per-member runtime probes do not establish the required missing-versus-ready account split"); const working = await ctx.input.api.get(`/api/agents/${workingId}`); const workingRevisionIdsBefore = (await ctx.input.api.get>(`/api/agents/${workingId}/config-revisions`)).map(row => row.id).sort(); await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue(); - await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The website team is the installed ${team.name} package. Its QA member (${missingId}) has not selected the eligible company Codex subscription account. The senior coder (${workingId}) is already configured. The existing company subscription is authorized for the QA member too; its value must not enter chat.` }); + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The website team is the installed ${team.name} package. Its QA member (${missingId}) has not selected the eligible company ${subscription ? "Codex subscription" : "OpenRouter"} account. The senior coder (${workingId}) is already configured. The existing company account is authorized for the QA member too; its value must not enter chat.` }); return { teamId: team.id, installedFromCatalog: true, memberIdsBefore: members.map(row => row.id).sort(), expectedMembers: members.length, missingId, missingName: members.find(row => row.id === missingId)!.name, workingId, workingHashBefore: configHash(working), workingRevisionIdsBefore, initialProbes }; } catch (error) { fixtures.delete(ctx); await rm(emptyHome, { recursive: true, force: true }); throw error; } @@ -105,3 +106,7 @@ export const teamConfigureExternalCases = [definition("teams.configure-team", as async verifyResult(ctx, state, value) { await verifyRenderedReply(ctx, value, String(state.initialRunId)); }, async cleanup(ctx) { const fixture = fixtures.get(ctx); fixtures.delete(ctx); if (fixture) await rm(fixture.emptyHome, { recursive: true, force: true }); }, })]; + +export function missingTeamAccountEnvironment(subscription: boolean, emptyHome: string) { + return subscription ? { CODEX_HOME: emptyHome, OPENAI_API_KEY: "", CODEX_API_KEY: "" } : { OPENROUTER_API_KEY: "" }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.test.ts index 97eb42a3f79..d43a7db1785 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.test.ts @@ -1,5 +1,5 @@ import { it, expect } from "vitest"; -import { admittedModelFailure } from "./runtime-native-failure.js"; +import { admittedModelFailure, unavailableNativeModel } from "./runtime-native-failure.js"; function valid() { return { companyId: "company", agentId: "morgan", issueId: "website", model: "unavailable", run: { id: "actual", companyId: "company", agentId: "morgan", runtimeMode: "native", status: "failed", startedAt: "2026-10-26T00:00:00Z", errorCode: "native_provider_model_rejected", error: "The selected model was rejected", nativeIssueId: "website", @@ -12,3 +12,13 @@ it("rejects configuration rejection before native admission", () => { const o = it("rejects an unrelated prior failure", () => { const o = valid(); o.run.nativeIssueId = "other"; expect(admittedModelFailure(o)).toBe(false); }); it("rejects a changed or substituted provider model", () => { const o = valid(); o.run.runnerProfileJson.nativeExecutionInput.provider.model = "supported"; expect(admittedModelFailure(o)).toBe(false); }); it("rejects absent durable terminal events", () => { const o = valid(); o.events = []; expect(admittedModelFailure(o)).toBe(false); }); + +it("uses an explicit unavailable OpenRouter ID and requires actual OpenCode admission, never a Codex failure", () => { + const model = unavailableNativeModel("opencode", "owned-nonce"); + expect(model).toBe("openrouter/paperclip-eval/paperclip-eval-unavailable-owned-nonce"); + const o = { ...valid(), provider: "opencode" as const, model }; + o.run.runnerProfileJson.nativeExecutionInput.provider = { kind: "opencode", model }; + expect(admittedModelFailure(o)).toBe(true); + o.run.runnerProfileJson.nativeExecutionInput.provider.kind = "codex"; + expect(admittedModelFailure(o)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts index 173b505a361..5e54dcf6feb 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-native-failure.ts @@ -9,12 +9,12 @@ interface NativeFailureRun extends ChatRun { runnerProfileJson?: { nativeExecutionInput?: { provider?: { kind?: string; model?: string | null } } }; } interface NativeEvent { seq?: number; eventType?: string; payload?: { prpEvent?: { eventType?: string } } } -export function admittedModelFailure(input: { run?: NativeFailureRun; events: NativeEvent[]; companyId: string; agentId: string; issueId: string; model: string }) { +export function admittedModelFailure(input: { run?: NativeFailureRun; events: NativeEvent[]; companyId: string; agentId: string; issueId: string; model: string; provider?: "codex" | "opencode" }) { const r = input.run, provider = r?.runnerProfileJson?.nativeExecutionInput?.provider; return !!r && r.runtimeMode === "native" && r.status === "failed" && !!r.startedAt && !!r.errorCode && !!r.error && r.companyId === input.companyId && r.agentId === input.agentId && [r.nativeIssueId, r.contextSnapshot?.issueId, r.contextSnapshot?.taskId].includes(input.issueId) - && provider?.kind === "codex" && provider.model === input.model + && provider?.kind === (input.provider ?? "codex") && provider.model === input.model && input.events.some(event => [event.eventType, event.payload?.prpEvent?.eventType].some(type => type === "run.terminal" || type === "turn.failed")); } /** A real public configuration/invocation fixture, never a synthetic failure. @@ -28,9 +28,10 @@ export async function unavailableNativeModelFailure(ctx: ChatCatalogContext, wor const endpoint = `/api/agents/${worker.id}`; const original = await ctx.input.api.get(endpoint); if (original.lifecycleState !== "ready" || !["active", "idle"].includes(original.status ?? "") - || original.adapterType !== "paperclip_runner" || original.adapterConfig?.provider !== "codex") - throw new Error("Native failure prerequisite requires an actually ready native Codex worker"); - const before = await ctx.allRuns(); const model = `paperclip-eval-unavailable-${ctx.input.nonce}`; + || original.adapterType !== "paperclip_runner" || !["codex", "opencode"].includes(String(original.adapterConfig?.provider))) + throw new Error("Native failure prerequisite requires an actually ready selected native Codex or OpenCode worker"); + const provider = original.adapterConfig!.provider as "codex" | "opencode"; + const before = await ctx.allRuns(); const model = unavailableNativeModel(provider, ctx.input.nonce); for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }]) await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); @@ -48,7 +49,7 @@ export async function unavailableNativeModelFailure(ctx: ChatCatalogContext, wor : rows.some(run => ["succeeded", "cancelled", "timed_out"].includes(run.status)) ? "Invocation did not produce genuine native failed history" : undefined }); failed = await ctx.input.api.get(`/api/heartbeat-runs/${settled[0]!.id}`); const events = await collectRunEvents((afterSeq, limit) => ctx.input.api.get(`/api/heartbeat-runs/${failed!.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); - if (!admittedModelFailure({ run: failed, events, companyId: ctx.input.fixtures.company.id, agentId: worker.id, issueId: receipt.issueId, model })) + if (!admittedModelFailure({ run: failed, events, companyId: ctx.input.fixtures.company.id, agentId: worker.id, issueId: receipt.issueId, model, provider })) throw new Error("Actual invocation lacks admitted model-bound native failure evidence; no failure coverage claimed"); return { failedRun: failed, failedRunEvents: events, failedIssueId: receipt.issueId, failedModel: model, failureKind: "native-failure-under-explicit-unavailable-model-configuration", originalConfiguration: original }; @@ -60,3 +61,9 @@ export async function unavailableNativeModelFailure(ctx: ChatCatalogContext, wor if (!isDeepStrictEqual(restored.adapterConfig, original.adapterConfig)) throw new Error("Original admitted model configuration was not restored"); } } + +/** OpenCode requires a provider-qualified ID; actual admission and persisted failure remain mandatory. */ +export function unavailableNativeModel(provider: "codex" | "opencode", nonce: string) { + const id = `paperclip-eval-unavailable-${nonce}`; + return provider === "opencode" ? `openrouter/paperclip-eval/${id}` : id; +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts index 2aaeb9c86ee..6a49390c4b5 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts @@ -1,3 +1,4 @@ +import { modelDiscoveryEndpoint } from "./runtime-runs-adapters.js"; import { it, expect } from "vitest"; import { gradeModelOptions } from "./runtime-runs-adapters.js"; function valid() { @@ -19,3 +20,9 @@ it("rejects silently changing Morgan's selected model", () => { it("rejects leaked private account credentials", () => { const o = valid(); o.state.secretLeaked = true; expect(gradeModelOptions(o).passed).toBe(false); }); + +it("discovers the actual selected native provider/environment rather than a Codex alias", () => { + const worker = { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode" }, defaultEnvironmentId: "owned-environment" }; + expect(modelDiscoveryEndpoint("company", worker as never)).toBe("/api/companies/company/adapters/paperclip_runner/models?provider=opencode&environmentId=owned-environment"); + expect(() => modelDiscoveryEndpoint("company", { ...worker, adapterConfig: { provider: "unknown" } } as never)).toThrow(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts index 0b6f4653973..6f9bf5f1111 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts @@ -27,9 +27,7 @@ export function gradeModelOptions(value: unknown) { } export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-model-options", async ctx => { const worker = await runtimeWorker(ctx); - if (worker.adapterConfig?.provider !== "codex" && worker.adapterType !== "codex_local") - throw new Error("Current runtime model-discovery fixture requires a genuinely qualified Codex account"); - const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/adapters/codex_local/models`; + const endpoint = modelDiscoveryEndpoint(ctx.input.fixtures.company.id, worker); const discovered = await readAgentResource(ctx, endpoint); const restricted = await readAgentResource(ctx, `/api/agents/${worker.id}/runtime-state`); if (discovered.status !== 200 || !Array.isArray(discovered.data)) throw new Error("Scenario requires admitted real coding model discovery"); @@ -51,3 +49,9 @@ export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-m await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").filter({ hasText: match.id }).last()).toBeVisible(); }, })]; + +export function modelDiscoveryEndpoint(companyId: string, worker: RuntimeAgent) { + if (worker.adapterType !== "paperclip_runner" || !["codex", "opencode"].includes(String(worker.adapterConfig?.provider)) || !worker.defaultEnvironmentId) + throw new Error("Model discovery requires the actually selected native provider and environment"); + return `/api/companies/${encodeURIComponent(companyId)}/adapters/paperclip_runner/models?provider=${worker.adapterConfig!.provider}&environmentId=${encodeURIComponent(worker.defaultEnvironmentId)}`; +} diff --git a/tests/runner-e2e/chat-catalog-fingerprint.test.ts b/tests/runner-e2e/chat-catalog-fingerprint.test.ts index ec021c60381..a15a6d9dcaf 100644 --- a/tests/runner-e2e/chat-catalog-fingerprint.test.ts +++ b/tests/runner-e2e/chat-catalog-fingerprint.test.ts @@ -1,3 +1,4 @@ +import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js"; import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -118,3 +119,11 @@ it("rejects a missing extensionless component instead of binding an unrelated TS f.write("OtherComponent.tsx", 'export const value = "unrelated";'); expect(f.fingerprint).toThrow("Unresolved fingerprint dependency: ./MissingComponent"); }); + +it("admits installed OpenCode only for its exact reviewed model, case and source-local lane", () => { + const lane = { suiteId: "chat-features", flow: "agent_chat", environment: "local", scenarioId: "instance-operations.update-instance", installedSource: "published-global-npm", + profile: { id: "runner-opencode", provider: "opencode", model: CHAT_OPENCODE_MODEL, adapterType: "paperclip_runner", credential: "OPENROUTER_API_KEY" } }; + expect(assertChatCatalogFingerprintLane(lane, {}, []).installedLane).toBe(true); + expect(() => assertChatCatalogFingerprintLane({ ...lane, profile: { ...lane.profile, provider: "codex" } }, {}, [])).toThrow(); + expect(() => assertChatCatalogFingerprintLane({ ...lane, scenarioId: "tasks.create-backlog" }, { PAPERCLIP_RUNNER_E2E_INSTALLED_CLI: "/candidate" }, [])).toThrow(); +}); diff --git a/tests/runner-e2e/chat-catalog-fingerprint.ts b/tests/runner-e2e/chat-catalog-fingerprint.ts index 39cee9a349d..96c2987aac2 100644 --- a/tests/runner-e2e/chat-catalog-fingerprint.ts +++ b/tests/runner-e2e/chat-catalog-fingerprint.ts @@ -1,3 +1,4 @@ +import { reviewedChatProfile } from "./chat-runtime-profile.js"; import { publicOriginFingerprintIdentity } from "./chat-public-origin.js"; import { databaseFingerprintIdentity } from "./docker-postgres.js"; import { verifyRetainedChatOracleIdentity } from "./chat-private-inputs.js"; @@ -80,14 +81,14 @@ export function fingerprintChatDependencyClosure(input: { } export function assertChatCatalogFingerprintLane(selected: unknown, env: NodeJS.ProcessEnv = process.env, args: readonly string[] = process.argv) { - const execution = selected as { suiteId?: string; flow?: string; environment?: string; executionId?: string; scenarioId?: string; installedSource?: string; databaseLane?: unknown; profile?: { id?: string } }; + const execution = selected as { suiteId?: string; flow?: string; environment?: string; executionId?: string; scenarioId?: string; installedSource?: string; databaseLane?: unknown; profile?: { id?: string; provider?: string; model?: string; adapterType?: string; credential?: string } }; const database = databaseFingerprintIdentity(env); if (execution.databaseLane !== undefined && JSON.stringify(execution.databaseLane) !== JSON.stringify(database)) throw new Error("Selected database lane differs from fingerprint environment"); const installed = execution.installedSource === "published-global-npm" && (installedChatScenarioIds as readonly string[]).includes(String(execution.scenarioId)); const installedKeys = ["PAPERCLIP_RUNNER_E2E_INSTALLED_CLI", "PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256", "PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY"]; const forbiddenArguments = ["--target-config", "--github-review-config", "--connection-config", "--installed-startup-only", "product-journeys"]; - if (installed && execution.profile?.id !== "runner-codex-subscription" || execution.suiteId !== "chat-features" || execution.flow !== "agent_chat" || execution.environment !== "local" + if (installed && !reviewedChatProfile(execution.profile) || execution.suiteId !== "chat-features" || execution.flow !== "agent_chat" || execution.environment !== "local" || !installed && installedKeys.some(key => env[key] && env[key] !== "0") || args.some(argument => forbiddenArguments.includes(argument))) throw new Error("Reviewed dynamic resolutions require source-local Agent Chat without external dispatch overrides"); return { suiteId: execution.suiteId, flow: execution.flow, environment: execution.environment, diff --git a/tests/runner-e2e/chat-feature-cases.ts b/tests/runner-e2e/chat-feature-cases.ts index 42328efba05..4c3505c163d 100644 --- a/tests/runner-e2e/chat-feature-cases.ts +++ b/tests/runner-e2e/chat-feature-cases.ts @@ -61,7 +61,7 @@ export function chatFeatureProfile(profile: RunnerProfileFixture): RunnerProfile return { ...payload, name: "CEO", title: "CEO", role: "ceo", capabilities: "Coordinates company work and delegates tasks to the team.", budgetMonthlyCents: CHAT_FEATURE_BUDGET_CENTS, - adapterConfig: { ...adapterConfig, timeoutSec: 120, modelReasoningEffort: "low" }, + adapterConfig: { ...adapterConfig, timeoutSec: 120, ...(profile.provider === "codex" ? { modelReasoningEffort: "low" } : {}) }, runtimeConfig: { ...(payload.runtimeConfig as Record), heartbeat: { enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 3, maxDailyCostCents: CHAT_FEATURE_BUDGET_CENTS } }, }; diff --git a/tests/runner-e2e/chat-feature-scoring.test.ts b/tests/runner-e2e/chat-feature-scoring.test.ts index ecf3040748b..72285b775ee 100644 --- a/tests/runner-e2e/chat-feature-scoring.test.ts +++ b/tests/runner-e2e/chat-feature-scoring.test.ts @@ -76,12 +76,13 @@ describe("chat feature saved-state oracle", () => { }); describe("chat feature selection", () => { - it("preserves the two original queries as explicit local subscription cells", () => { + it("preserves the two original queries as explicit host-local OpenCode cells", () => { const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features" && ["create-backlog", "delegate-announcement"].includes(cell.task.id)); expect(cells).toHaveLength(2); for (const cell of cells) { expect(cell.environment.id).toBe("local"); - expect(cell.requiredCredentials).toEqual(["CODEX_AUTH_JSON"]); + expect(cell.requiredCredentials).toEqual(["OPENROUTER_API_KEY"]); + expect(cell.profile).toMatchObject({ id: "runner-opencode", provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017" }); expect(cell.task.automaticRetryPolicy).toBe("single_attempt"); expect(cell.task.buildPrompt("nonce")).toBe(chatFeatureScenarios.find(row => row.id === `tasks.${cell.task.id}`)!.query); expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id])).map(row => row.id)).toEqual([cell.id]); diff --git a/tests/runner-e2e/chat-installed-lane.test.ts b/tests/runner-e2e/chat-installed-lane.test.ts index edd4605d2a1..afed8ddb972 100644 --- a/tests/runner-e2e/chat-installed-lane.test.ts +++ b/tests/runner-e2e/chat-installed-lane.test.ts @@ -1,3 +1,4 @@ +import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js"; import { mkdtempSync, mkdirSync, writeFileSync, symlinkSync, rmSync, readFileSync } from "node:fs"; import { createHash } from "node:crypto"; import path from "node:path"; @@ -15,6 +16,9 @@ it("admits only exact published pins and the two selected source-local Codex ope expect(() => validateChatInstallConfiguration({ schemaVersion: 1, disposable: true, source })).toThrow(); const execution = { suite: { id: "chat-features", manualOnly: true }, task: { id: "instance-operations.update-instance", flow: "agent_chat" }, profile: { id: "runner-codex-subscription" }, environment: { id: "local" } } as MatrixExecution; expect(admitsInstalledChat(execution)).toBe(true); + const remote = { ...execution, profile: { ...execution.profile, id: "runner-opencode", provider: "opencode", model: CHAT_OPENCODE_MODEL, adapterType: "paperclip_runner", credential: "OPENROUTER_API_KEY" as const } }; + expect(admitsInstalledChat(remote)).toBe(true); + expect(admitsInstalledChat({ ...remote, profile: { ...remote.profile, model: "other" } })).toBe(false); expect(admitsInstalledChat({ ...execution, task: { ...execution.task, id: "tasks.create-backlog" } })).toBe(false); expect(admitsInstalledChat({ ...execution, environment: { ...execution.environment, id: "daytona" } })).toBe(false); }); diff --git a/tests/runner-e2e/chat-installed-lane.ts b/tests/runner-e2e/chat-installed-lane.ts index 47756cd10ba..a4b4fb37e43 100644 --- a/tests/runner-e2e/chat-installed-lane.ts +++ b/tests/runner-e2e/chat-installed-lane.ts @@ -1,3 +1,4 @@ +import { reviewedChatProfile } from "./chat-runtime-profile.js"; import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js"; import { spawn, execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; @@ -71,7 +72,7 @@ export function installedChatTreeIdentity(prefix: string) { } export function admitsInstalledChat(execution: Pick) { return execution.suite.id === "chat-features" && execution.suite.manualOnly && execution.task.flow === "agent_chat" - && execution.profile.id === "runner-codex-subscription" && execution.environment.id === "local" + && reviewedChatProfile(execution.profile) && execution.environment.id === "local" && (installedChatScenarioIds as readonly string[]).includes(execution.task.id); } /** Runs only inside an explicitly selected governed attempt; never during discovery. */ diff --git a/tests/runner-e2e/chat-runtime-profile.test.ts b/tests/runner-e2e/chat-runtime-profile.test.ts new file mode 100644 index 00000000000..8893c738067 --- /dev/null +++ b/tests/runner-e2e/chat-runtime-profile.test.ts @@ -0,0 +1,26 @@ +import { expect, it } from "vitest"; +import { CHAT_OPENCODE_MODEL, reviewedChatProfile, reviewedChatExecutionIds } from "./chat-runtime-profile.js"; +import { runnerMatrix, runnerProfiles } from "./catalog.js"; +import { missingTeamAccountEnvironment } from "./chat-catalog-domains/external-team-configure.js"; +const profile = { id: "runner-opencode", provider: "opencode", model: CHAT_OPENCODE_MODEL, adapterType: "paperclip_runner", credential: "OPENROUTER_API_KEY" }; +it("pins the new controller identity and rejects model/provider/credential substitution", () => { + expect(reviewedChatProfile(profile)).toBe(true); + for (const patch of [{ id: "runner-unknown" }, { provider: "codex" }, { model: "openrouter/other/model" }, { credential: "CODEX_AUTH_JSON" }, { adapterType: "opencode_local" }]) + expect(reviewedChatProfile({ ...profile, ...patch })).toBe(false); + expect(reviewedChatExecutionIds("secrets.vault-reference")).toEqual(["chat-features.runner-codex-subscription.local.secrets.vault-reference", "chat-features.runner-opencode.local.secrets.vault-reference"]); +}); +it("changes the complete chat matrix without relabeling the existing global qualification model", () => { + const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features"); + expect(cells.length).toBe(244); + expect(cells.every(cell => cell.profile.model === CHAT_OPENCODE_MODEL && reviewedChatProfile(cell.profile))).toBe(true); + expect(runnerProfiles.find(row => row.id === "runner-opencode")!.model).not.toBe(CHAT_OPENCODE_MODEL); + const cell = cells[0]!; + const payload = cell.profile.buildAgent({ environmentId: "environment", environmentFixtureId: "local", workspacePath: "/owned", secretRefs: { OPENROUTER_API_KEY: { type: "secret_ref", secretId: "22222222-2222-4222-8222-222222222222", version: "latest" } }, executionId: "owned" }); + expect(payload.adapterConfig).toMatchObject({ provider: "opencode", model: CHAT_OPENCODE_MODEL }); + expect(payload.adapterConfig).not.toHaveProperty("modelReasoningEffort"); + expect(payload.adapterConfig).not.toHaveProperty("codexPermissionMode"); +}); +it("withholds only the selected account from the missing OpenRouter member", () => { + expect(missingTeamAccountEnvironment(false, "/owned/empty")).toEqual({ OPENROUTER_API_KEY: "" }); + expect(missingTeamAccountEnvironment(true, "/owned/empty")).toEqual({ CODEX_HOME: "/owned/empty", OPENAI_API_KEY: "", CODEX_API_KEY: "" }); +}); diff --git a/tests/runner-e2e/chat-runtime-profile.ts b/tests/runner-e2e/chat-runtime-profile.ts new file mode 100644 index 00000000000..b485cf9ecd6 --- /dev/null +++ b/tests/runner-e2e/chat-runtime-profile.ts @@ -0,0 +1,11 @@ +/** Exact controller identity for the new remote campaign; historical Codex cells remain distinct. */ +export const CHAT_OPENCODE_PROFILE = "runner-opencode"; +export const CHAT_OPENCODE_MODEL = "openrouter/deepseek/deepseek-v4.1-flash-1017"; +export function reviewedChatProfile(profile: { id?: string; provider?: string; model?: string; adapterType?: string; credential?: string } | undefined) { + return profile?.id === "runner-codex-subscription" || !!profile && profile.id === CHAT_OPENCODE_PROFILE + && profile.provider === "opencode" && profile.model === CHAT_OPENCODE_MODEL + && profile.adapterType === "paperclip_runner" && profile.credential === "OPENROUTER_API_KEY"; +} +export function reviewedChatExecutionIds(scenarioId: string) { + return ["runner-codex-subscription", CHAT_OPENCODE_PROFILE].map(profile => `chat-features.${profile}.local.${scenarioId}`); +} diff --git a/tests/runner-e2e/chat-vault-private-server.test.ts b/tests/runner-e2e/chat-vault-private-server.test.ts index ff654c6bdfb..0687118026d 100644 --- a/tests/runner-e2e/chat-vault-private-server.test.ts +++ b/tests/runner-e2e/chat-vault-private-server.test.ts @@ -13,6 +13,7 @@ import { validVaultBinding, vaultPrivateFields, vaultExecutionId, + vaultExecutionIds, } from "./chat-vault-private-server.js"; import type { MatrixExecution } from "./types.js"; @@ -216,3 +217,11 @@ it("scrubs ambient AWS and vault SDK config while preserving explicit unrelated ]); expect(other.AWS_ACCESS_KEY_ID).toBe("ambient"); }); + +it("binds the OpenCode vault envelope to the exact selected new execution, never the prior campaign", async () => { + const f = await fixture(); + f.input.executionIds = [vaultExecutionIds[1]!]; f.envelope.executionId = vaultExecutionIds[1]!; await f.save(); + expect(await readSelectedVaultServerEnvironment(f.input)).toHaveProperty("AWS_ACCESS_KEY_ID", f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_ACCESS); + f.envelope.executionId = vaultExecutionId; await f.save(); + await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow(); +}); diff --git a/tests/runner-e2e/chat-vault-private-server.ts b/tests/runner-e2e/chat-vault-private-server.ts index d0554e7b4cd..07c2d6f66f0 100644 --- a/tests/runner-e2e/chat-vault-private-server.ts +++ b/tests/runner-e2e/chat-vault-private-server.ts @@ -1,10 +1,12 @@ +import { reviewedChatExecutionIds } from "./chat-runtime-profile.js"; import { constants } from "node:fs"; import { lstat, realpath, open, writeFile } from "node:fs/promises"; import path from "node:path"; import { CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js"; export const vaultScenarioId = "secrets.vault-reference"; -export const vaultExecutionId = `chat-features.runner-codex-subscription.local.${vaultScenarioId}`; +export const vaultExecutionId = reviewedChatExecutionIds(vaultScenarioId)[0]!; +export const vaultExecutionIds = reviewedChatExecutionIds(vaultScenarioId); export const vaultPrivateFields = [ "provider", "region", @@ -102,7 +104,7 @@ export async function readSelectedVaultServerEnvironment(input: { backendOrigin: string; env: NodeJS.ProcessEnv; }): Promise { - if (!input.executionIds.includes(vaultExecutionId)) return {}; + if (!input.executionIds.some(id => vaultExecutionIds.includes(id))) return {}; if (input.executionIds.length !== 1) throw new Error( "Vault runtime inputs require the exact single selected source execution", @@ -169,7 +171,7 @@ export async function readSelectedVaultServerEnvironment(input: { } finally { await handle.close(); } - if (envelope.version !== 1 || envelope.executionId !== vaultExecutionId) + if (envelope.version !== 1 || envelope.executionId !== input.executionIds[0]) throw new Error("Vault runtime input belongs to another execution"); // Ordinary reserved setup handles missing selected provider inputs. It never // receives an ambient fallback merely because the private binding is absent.