diff --git a/.env.runner-e2e.example b/.env.runner-e2e.example index 85c99a7836..95d29ad32e 100644 --- a/.env.runner-e2e.example +++ b/.env.runner-e2e.example @@ -5,6 +5,10 @@ ANTHROPIC_API_KEY= OPENROUTER_API_KEY= DAYTONA_API_KEY= +# Explicit-only extended-harnesses candidates. +CURSOR_AUTH_TOKEN= +COPILOT_GITHUB_TOKEN= + # Required only for Daytona cells. Use an immutable, anonymously pullable # digest from the Runner Full-Stack E2E image job or a locally published image. PAPERCLIP_E2E_DAYTONA_IMAGE=ghcr.io/paperclipai/paperclip-daytona-runner@sha256:REPLACE_ME diff --git a/doc/evals.md b/doc/evals.md index f343f07c01..90b37a4fa8 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -266,3 +266,9 @@ Continuation accounting has an explicit-only eight-cell Product E2E [baseline su The explicit-only Product E2E `api-response-reading` suite verifies retrieval of large saved API responses on local and Daytona native Codex runs. See the [Runner E2E guide](../tests/runner-e2e/README.md#bounded-api-response-reading). + +The explicit-only Product E2E `extended-harnesses` suite covers pending Cursor, +Copilot and Pi ACP profiles on local and Daytona. See the +[fixture admission, credentials and budget contract](../tests/runner-e2e/README.md#extended-acp-harnesses-explicit-only). +The private Runner Evals campaign of the same name provides complementary +semantic protocol cases; catalog membership is not live qualification. diff --git a/server/src/services/native-runtime/acpx-qualification.test.ts b/server/src/services/native-runtime/acpx-qualification.test.ts new file mode 100644 index 0000000000..19c4162ccc --- /dev/null +++ b/server/src/services/native-runtime/acpx-qualification.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { ACPX_QUALIFICATION_ENV, resolveAcpxQualification } from "./acpx-qualification.js"; +import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; + +const provider = { kind: "acpx", agent: "cursor", model: "exact-model", permissionMode: "approve-all" } as NativeExecutionInput["provider"]; +const authorize = (value: unknown) => ({ [ACPX_QUALIFICATION_ENV]: JSON.stringify(value) }); +describe("host ACPX qualification admission", () => { + it("keeps normal candidate execution closed and admits only the exact operator pair", () => { + expect(resolveAcpxQualification(provider, {})).toBeUndefined(); + expect(resolveAcpxQualification(provider, authorize([{ agent: "cursor", model: "exact-model" }]))).toBe("cursor"); + for (const entries of [[{ agent: "copilot", model: "exact-model" }], [{ agent: "cursor", model: "other-model" }]]) { + expect(() => resolveAcpxQualification(provider, authorize(entries))).toThrow("exact model"); + } + }); + it.each([[], {}, [{ agent: "cursor", model: "" }], [{ agent: "cursor", model: " exact-model" }], + [{ agent: "cursor", model: "exact-model", allowAll: true }], [{ agent: "claude", model: "exact-model" }], + [{ agent: "cursor", model: "exact-model" }, { agent: "cursor", model: "exact-model" }], + ])("rejects malformed or broad authorization %j", entries => { + expect(() => resolveAcpxQualification(provider, authorize(entries))).toThrow("Invalid ACPX"); + }); + it("does not alter existing qualified providers", () => { + expect(resolveAcpxQualification({ ...provider, agent: "codex" } as typeof provider, authorize([]))).toBeUndefined(); + }); + it("binds the executor to host process environment rather than agent configuration", () => { + const source = readFileSync(new URL("./native-session-executor.ts", import.meta.url), "utf8"); + expect(source).toContain("resolveAcpxQualification(input.execution.provider, process.env)"); + expect(source).not.toContain("resolveAcpxQualification(input.execution.provider, effectiveRunnerEnvironment)"); + }); +}); diff --git a/server/src/services/native-runtime/acpx-qualification.ts b/server/src/services/native-runtime/acpx-qualification.ts new file mode 100644 index 0000000000..9295d41384 --- /dev/null +++ b/server/src/services/native-runtime/acpx-qualification.ts @@ -0,0 +1,32 @@ +import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; + +/** Operator-only admission for an exact candidate/model during qualification. */ +export const ACPX_QUALIFICATION_ENV = "PAPERCLIP_RUNNER_ACPX_QUALIFICATION"; +type Candidate = "cursor" | "copilot" | "pi"; + +export function resolveAcpxQualification( + provider: NativeExecutionInput["provider"], + hostEnvironment: NodeJS.ProcessEnv, +): Candidate | undefined { + if (provider.kind !== "acpx" || !["cursor", "copilot", "pi"].includes(provider.agent)) return undefined; + const encoded = hostEnvironment[ACPX_QUALIFICATION_ENV]; + if (!encoded) return undefined; + let entries: unknown; + try { entries = JSON.parse(encoded); } catch { throw new Error("Invalid ACPX qualification authorization"); } + if (!Array.isArray(entries) || entries.length < 1 || entries.length > 3) { + throw new Error("Invalid ACPX qualification authorization"); + } + const agents = new Set(); + for (const entry of entries) { + if (!entry || typeof entry !== "object" || Array.isArray(entry) + || Object.keys(entry).sort().join(",") !== "agent,model" + || !["cursor", "copilot", "pi"].includes(entry.agent) + || typeof entry.model !== "string" || !entry.model.trim() || entry.model !== entry.model.trim() + || agents.has(entry.agent)) throw new Error("Invalid ACPX qualification authorization"); + agents.add(entry.agent); + } + if (!entries.some(entry => entry.agent === provider.agent && entry.model === provider.model)) { + throw new Error("ACPX qualification requires the explicitly authorized candidate and exact model"); + } + return provider.agent as Candidate; +} diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 0c80ed1065..be680b6eea 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { resolveAcpxQualification } from "./acpx-qualification.js"; import { isSupportedRemoteCodexVersion, parseCodexCliVersion, @@ -12258,6 +12259,8 @@ async function createRunnerdBackendWithinSessionClaim( ...(input.execution.provider.kind === "acpx" ? { acpxAgent: input.execution.provider.agent, + // Read only the server operator environment, never agent/runtime env. + acpxCandidateProfile: resolveAcpxQualification(input.execution.provider, process.env), acpxPermissionMode: input.execution.provider.permissionMode, acpxPermissionModePinned: input.execution.schema === diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 885b3e724c..9f86d56bc1 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -225,3 +225,11 @@ first provider turn intentionally omits task disposition, and their second turn must be an automatic, causally bound repair that records completion. They use public task comments/status APIs and run-detail evidence; no private runtime hooks or database mutations are used by the fixture. + +The explicit-only `extended-harnesses` suite uses five bounded journeys for each +pending ACP candidate on local and Daytona. Candidate profile metadata includes +the exact authenticated discovery choice without promoting it to a product +default. Its file case anchors the task to a public project workspace, validates +the model's claimed result by reading the actual final bytes, and also exercises +remote copy-back. Keep candidate admission scoped to the selected model and the +isolated operator environment; ordinary agent configuration must not enable it. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 04dfe551e2..18b55e5d27 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -93,6 +93,8 @@ Shell variables take precedence over the local file. The recognized names are: - `ANTHROPIC_API_KEY` - `OPENROUTER_API_KEY` - `DAYTONA_API_KEY` +- `CURSOR_AUTH_TOKEN` (extended Cursor candidate) +- `COPILOT_GITHUB_TOKEN` (extended Copilot candidate) - `PAPERCLIP_E2E_DAYTONA_IMAGE` (Daytona only) The image must be an immutable `image@sha256:...` reference. The launcher @@ -1106,3 +1108,46 @@ missing events or a narrative completion cannot pass. Existing run, copyback, screenshot, billing and environment cleanup checks apply. Use `--id api-response-reading.runner-codex.daytona.saved-text-pages` with an immutable Daytona image; no private hooks or fixture database writes are used. + +## Extended ACP harnesses (explicit only) + +`--suite extended-harnesses` declares 30 Product E2E cells: Cursor, Copilot, +and Pi on local and Daytona, each exercising authenticated completion, +question/answer continuation, revision-bound semantic plan approval, restart +with pending input, and file edit plus independent byte validation. The file +case uses a public project workspace so Daytona copy-back is graded too. +These are candidate definitions, not a claim of provider qualification. Native +provider-specific questions, plan decisions, restrictive permissions and steering +need their separate conformance/qualification evidence. + +```sh +pnpm test:e2e:runner -- --list --suite extended-harnesses +pnpm test:e2e:runner -- --id extended-harnesses.runner-acpx-pi.local.hello-complete +``` + +The suite is excluded from `--all`, and candidate cells never automatically +retry. Select one cell first, reserve its spend and reconcile provider billing +before another attempt. Existing subscriptions/credits and incremental cash +charges are separate; unavailable receipts do not mean zero cost. The September +28 qualification budget is $100 total including retries and Daytona resources: +$25 per provider and $25 coordinated infrastructure/diagnosis. + +The launcher binds only the selected candidate and exact discovered model in +`PAPERCLIP_RUNNER_ACPX_QUALIFICATION`, a JSON array of `{agent,model}` pairs. +The server reads this operator environment at its normal runnerd construction +boundary; agent config/environment cannot enable qualification. Normal hosts +have no admission override. It does not bypass profile, executable, credential, +company, tool or permission checks. Keep this variable confined to isolated +qualification instances. Model catalog discovery alone does not prove inference +entitlement; all three profiles remain pending until the required live evidence +passes. Cursor and Copilot models are the explicit September 28 authenticated +discovery choices; Pi imports its production profile's fixed OpenRouter model. + +Run each candidate from its provider branch, with its verified candidate assets +materialized under the runner package, and build the TypeScript sidecar before +local execution. Daytona additionally requires that branch's immutable Linux +candidate image and the matching controller-owned provider pack described in +[`docker/daytona-runner/README.md`](../../docker/daytona-runner/README.md). +The separate Runner Evals `extended-harnesses` campaign lives in the private +`paperclip-evals` repository and grades semantic protocol behavior against the +mock control plane. Neither suite substitutes for the other. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 2653cec086..57be4a20d5 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -260,3 +260,17 @@ specific process signal, never a name-based or machine-wide process kill. The bounded fixture command is released even on failure; normal instance cleanup still owns all disposable processes and files. No credentials enter the prompt, fault metadata, or structured grading fixtures. + +### Candidate ACP qualification + +The explicit-only `extended-harnesses` launcher removes ambient candidate +admission, then authorizes only selected candidate/model pairs in its isolated +server process. The server never reads admission from agent configuration, +resolved credential bindings, or provider environment. Invalid, duplicate and +mismatched authorization fails closed. Verified packaging and normal company +and tool governance still apply. `CURSOR_AUTH_TOKEN` and +`COPILOT_GITHUB_TOKEN` enter via encrypted company secret references, and all +Cursor, Copilot, GitHub and GH environment variables are stripped from the +server environment. GitHub PAT shapes are included in retained-evidence scans. +Candidates have no automatic infrastructure retries; spending must be reconciled +before a deliberate repeat. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index ac64f1db0e..90e7ef82f4 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -71,10 +71,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 2, 8, 46, 23, 47, 52, 28, 18, 6, 6, 4, 42, 14, 10, 2, + 30, 2, 8, 46, 23, 47, 52, 28, 18, 6, 6, 4, 42, 14, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(308); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(308); + expect(validateRunnerCatalog()).toHaveLength(338); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(338); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index e1a613cb2f..ab2ccc3d74 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -145,7 +145,8 @@ function nativeProfile(input: { provider: "codex" | "opencode" | "acpx"; model: string; credential: RunnerProfileFixture["credential"]; - acpxAgent?: "claude" | "codex"; + acpxAgent?: "claude" | "codex" | "cursor" | "copilot" | "pi"; + qualificationCandidate?: RunnerProfileFixture["qualificationCandidate"]; supportedEnvironments?: readonly (typeof ENVIRONMENT_IDS)[number][]; modelQualification?: RunnerProfileFixture["modelQualification"]; ranking?: RunnerProfileFixture["ranking"]; @@ -294,6 +295,27 @@ export const runnerProfiles: readonly RunnerProfileFixture[] = [ }), ] as const; +// Explicit qualification choices from authenticated model discovery, not shipped +// defaults or claims that these candidates have passed inference qualification. +export const extendedHarnessProfiles: readonly RunnerProfileFixture[] = [ + nativeProfile({ + id: "runner-acpx-cursor", label: "Runner Cursor (candidate)", provider: "acpx", acpxAgent: "cursor", + qualificationCandidate: "cursor", credential: "CURSOR_AUTH_TOKEN", + model: "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]", + modelQualification: { source: "candidate_runner_profile", qualificationId: "cursor:2026.09.26-dd393fe:discovery-2026-09-28" }, + }), + nativeProfile({ + id: "runner-acpx-copilot", label: "Runner Copilot (candidate)", provider: "acpx", acpxAgent: "copilot", + qualificationCandidate: "copilot", credential: "COPILOT_GITHUB_TOKEN", model: "gpt-5.6-luna", + modelQualification: { source: "candidate_runner_profile", qualificationId: "copilot:1.0.88:discovery-2026-09-28" }, + }), + nativeProfile({ + id: "runner-acpx-pi", label: "Runner Pi (candidate)", provider: "acpx", acpxAgent: "pi", + qualificationCandidate: "pi", credential: "OPENROUTER_API_KEY", model: QUALIFIED_ACPX_PROFILES.pi.qualificationModel, + modelQualification: { source: "candidate_runner_profile", qualificationId: "pi:0.0.33:0.84.2:openrouter" }, + }), +]; + export const openRouterBreadthExcludedModelIds = ["xiaomi/mimo-v2.5"] as const; export const openRouterBreadthExcludedExecutionIds = [ "openrouter-model-breadth.openrouter-deepseek-deepseek-v4-flash-0731.local.plan-approve-complete", @@ -911,7 +933,34 @@ const everydayProfiles = [ nativeProfile({ id: "runner-codex-mini", label: "Runner Codex Mini", provider: "codex", model: "gpt-5.4-mini", modelQualification: {source:"qualified_runner_profile",qualificationId:"everyday-codex-mini-pilot"}, credential: "OPENAI_API_KEY", supportedEnvironments: ["local"] }), ].map(productionStoryProfile); +export const extendedHarnessFileTask: RunnerTaskFixture = { + id: "file-edit-validate", label: "Edit a file and validate its contents", groups: [], + workMode: "standard", flow: "single_turn", expectedRunCount: 1, + attemptTimeoutMs: { local: 180_000, daytona: 300_000 }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: nonce => `Extended harness file validation ${nonce}`, + buildVisibleMarker: nonce => `EXTENDED-FILE-${nonce}`, + buildPrompt: nonce => [ + `Create extended-${nonce}.txt in the current execution workspace with exactly ready-${nonce} followed by a newline.`, + `Edit that file to contain exactly verified-${nonce} followed by a newline, then run a command that reads it and fails unless those exact bytes match.`, + `After successful validation, call paperclip_finish with reportedWorkDisposition done, summary EXTENDED-FILE-${nonce}, the current completion contract revision and satisfied objective criterion, no remaining work, and the actual validation command and result.`, + `Wait for paperclip_finish to succeed, then emit exactly EXTENDED-FILE-${nonce} as your final response. Do not create unrelated files or work.`, + ].join("\n"), + buildMatchers: (nonce, execution) => [ + ...terminalMatchers(`EXTENDED-FILE-${nonce}`, execution), + { kind: "file_exact", path: `extended-${nonce}.txt`, expected: `verified-${nonce}\n` }, + ], +}; + export const runnerSuites: readonly RunnerSuiteFixture[] = [ + { + id: "extended-harnesses", label: "Extended ACP harnesses", manualOnly: true, + description: "Explicit candidate qualification through real Paperclip tools, browser interactions, file edits and restart recovery.", + groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments, + tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask], + expectedMatrixSize: 30, + definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28" }, + }, { id: "api-response-reading", label: "Bounded API response reading", manualOnly: true, description: "Retrieve evidence beyond a saved API preview through authorized bounded text windows.", @@ -1179,8 +1228,9 @@ function assertNoRawSecretValues(value: unknown, label: string) { } export function validateRunnerCatalog(): MatrixExecution[] { - const allProfiles = [...runnerProfiles, ...openRouterBreadthProfiles, ...everydayProfiles.filter(p => !runnerProfiles.some(existing => existing.id === p.id))]; + const allProfiles = [...extendedHarnessProfiles, ...runnerProfiles, ...openRouterBreadthProfiles, ...everydayProfiles.filter(p => !runnerProfiles.some(existing => existing.id === p.id))]; const allTasks = [ + extendedHarnessFileTask, ...accountingTasks, ...lifecycleLiveTasks, ...continuationTasks, @@ -1222,12 +1272,7 @@ export function validateRunnerCatalog(): MatrixExecution[] { } const sampleRefs = Object.fromEntries( - [ - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - "OPENROUTER_API_KEY", - "DAYTONA_API_KEY", - ].map((name, index) => [ + CREDENTIAL_NAMES.map((name, index) => [ name, { type: "secret_ref" as const, diff --git a/tests/runner-e2e/extended-harnesses.test.ts b/tests/runner-e2e/extended-harnesses.test.ts new file mode 100644 index 0000000000..b668a93c48 --- /dev/null +++ b/tests/runner-e2e/extended-harnesses.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { runnerMatrix, runnerSuites, extendedHarnessProfiles, extendedHarnessFileTask } from "./catalog.js"; +import { buildRunnerE2EProcessEnvironment, buildPaperclipServerEnvironment } from "./harness-env.js"; +import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import { findSecretLeak, redactText } from "./redaction.js"; + +const selected = runnerMatrix.filter(cell => cell.suite.id === "extended-harnesses"); +describe("extended ACP harness qualification", () => { + it("has exactly five real product journeys per provider in both environments, excluded from all", () => { + expect(selected).toHaveLength(30); + expect(new Set(selected.map(cell => cell.environment.id))).toEqual(new Set(["local", "daytona"])); + expect(new Set(selected.map(cell => cell.task.id))).toEqual(new Set([ + "hello-complete", "question-resume-complete", "plan-approve-complete", "structured-question-restart-resume", "file-edit-validate", + ])); + const all = selectRunnerExecutions(parseRunnerSelectors(["--all"])); + expect(all.some(cell => cell.suite.id === "extended-harnesses")).toBe(false); + expect(runnerSuites.find(suite => suite.id === "extended-harnesses")?.manualOnly).toBe(true); + }); + it("uses exact discovered models, encrypted credential references and pending qualification metadata", () => { + expect(extendedHarnessProfiles.map(profile => profile.model)).toEqual([ + "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]", "gpt-5.6-luna", "openrouter/deepseek/deepseek-v4-flash-0731", + ]); + for (const profile of extendedHarnessProfiles) { + expect(profile.modelQualification.source).toBe("candidate_runner_profile"); + const secretRef = { type: "secret_ref" as const, secretId: "11111111-1111-4111-8111-111111111111", version: "latest" as const }; + const payload = profile.buildAgent({ executionId: "fixture", environmentId: "local", environmentFixtureId: "local", workspacePath: "/tmp/workspace", secretRefs: { [profile.credential]: secretRef } }); + expect(payload.adapterConfig).toMatchObject({ provider: "acpx", model: profile.model, acpxAgent: profile.qualificationCandidate, env: { [profile.credential]: secretRef } }); + } + }); + it("replaces ambient admission with the selected exact pairs and strips raw provider credentials from server", () => { + const cell = selected[0]!; + const ambient = { PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient", CURSOR_AUTH_TOKEN: "cursor", CURSOR_API_KEY: "cursor-key", COPILOT_GITHUB_TOKEN: "copilot", GH_TOKEN: "github", GITHUB_TOKEN: "github" }; + const env = buildRunnerE2EProcessEnvironment(ambient, [cell]); + expect(JSON.parse(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "cursor", model: cell.profile.model }]); + const server = buildPaperclipServerEnvironment(env); + for (const name of Object.keys(ambient).filter(name => name !== "PAPERCLIP_RUNNER_ACPX_QUALIFICATION")) expect(server[name]).toBeUndefined(); + expect(buildRunnerE2EProcessEnvironment(ambient, []).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); + expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, manualOnly: false } }])).toThrow("explicit"); + }); + it("grades the actual file independently of the model's validation claim", () => { + const cell = selected.find(cell => cell.task.id === "file-edit-validate")!; + expect(extendedHarnessFileTask.buildMatchers("nonce", cell)).toContainEqual({ kind: "file_exact", path: "extended-nonce.txt", expected: "verified-nonce\n" }); + }); + it("redacts and detects GitHub credential shapes even without the bound value", () => { + const token = `github_pat_${"x".repeat(40)}`; + expect(findSecretLeak(token, [])).toBe("secret-shaped value"); + expect(redactText(token, [])).not.toContain(token); + }); +}); diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index aea1808633..3089433b10 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -25,7 +25,7 @@ const AMBIENT_EXTERNAL_STATE_KEYS = [ "PAPERCLIP_STORAGE_S3_PREFIX", "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", ] as const; -const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)(?:_|$)/; +const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/; export function runnerE2EServerControlPaths(temporaryRoot: string) { const controlDirectory = path.join(temporaryRoot, "control"); @@ -108,6 +108,25 @@ export function buildRunnerE2EProcessEnvironment( // Announcements are unrelated to the scenarios and obscure screenshot evidence. result.PAPERCLIP_ANNOUNCEMENTS_ENABLED = "false"; delete result.OPENCODE_ALLOW_ALL_MODELS; + // Discard ambient admission. Only explicit candidate cells authorize the + // exact model in their isolated server; credentials still use company secrets. + delete result.PAPERCLIP_RUNNER_ACPX_QUALIFICATION; + const candidates = new Map(); + for (const execution of executions) { + const agent = execution.profile.qualificationCandidate; + if (!agent) continue; + if (execution.suite.id !== "extended-harnesses" || !execution.suite.manualOnly) { + throw new Error("Candidate qualification requires the explicit extended-harnesses suite"); + } + const prior = candidates.get(agent); + if (prior !== undefined && prior !== execution.profile.model) throw new Error("Conflicting candidate models"); + candidates.set(agent, execution.profile.model); + } + if (candidates.size > 0) { + result.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = JSON.stringify( + [...candidates].map(([agent, model]) => ({ agent, model })), + ); + } // These stories explicitly require the native API surface. Other suites // retain the server default or any supplied operator restriction. if (executions.some((e) => isManagedHiringCase(e.suite.id, e.task.id) || chatNeedsApiTools(e.suite.id, e.task.id))) { diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index c159246c88..b69f57d110 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1021,6 +1021,7 @@ async function runExecutionWithRetry(input: { }); if (!firstResult) throw new Error(`No result produced for ${execution.id}`); if ( + execution.profile.qualificationCandidate !== undefined || options.ui || options.debug || firstResult.status !== "failed" || diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index 78cc912d61..17422768c9 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -4,6 +4,24 @@ import { runnerMatrix } from "./catalog.js"; import { setupLiveFixtures } from "./live-fixtures.js"; describe("live runner fixtures", () => { + it("anchors extended file validation to a public project workspace for local and remote copy-back", async () => { + const execution = runnerMatrix.find(e => e.id === "extended-harnesses.runner-acpx-pi.local.file-edit-validate")!; + let projectBody: any; + const api = { + async get() { return [{ id: "local", driver: "local" }]; }, + async postSensitive() { return { id: "secret" }; }, + async post(url: string, data: any) { + if (url === "/api/companies") return { id: "company", name: "Test" }; + if (url.endsWith("/agents")) return { id: "agent", ...data }; + if (url.endsWith("/projects")) { projectBody = data; return { id: "project", ...data }; } + throw new Error(`Unexpected POST ${url}`); + }, + } as unknown as RunnerApi; + const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture-workspace", credentials: { OPENROUTER_API_KEY: "fixture-key" } }); + expect(fixtures.project?.id).toBe("project"); + expect(projectBody).toMatchObject({ executionWorkspacePolicy: { environmentId: "local", workspaceStrategy: { type: "project_primary" } }, workspace: { cwd: "/tmp/fixture-workspace", sourceType: "local_path" } }); + }); + it.each([ ["runner-codex", "everyday-workflows", "hire-reuse"], ["runner-acpx-claude", "everyday-workflows", "hire-reuse"], diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index edb960b851..0fb01b2412 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -285,7 +285,8 @@ export async function setupLiveFixtures(input: { }, }); - if (execution.environment.configurationKey === "warm-reuse-v1") { + if (execution.environment.configurationKey === "warm-reuse-v1" + || (execution.suite.id === "extended-harnesses" && execution.task.id === "file-edit-validate")) { registry.register({ id: "project", dependencies: ["company", "environment"], @@ -295,9 +296,9 @@ export async function setupLiveFixtures(input: { return api.post( `/api/companies/${company.id}/projects`, { - name: `Runner E2E warm project ${input.executionNonce}`, + name: `Runner E2E workspace project ${input.executionNonce}`, description: - "Ephemeral project anchoring a reusable Daytona execution workspace", + "Ephemeral project anchoring the fixture execution workspace and file copy-back", executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", diff --git a/tests/runner-e2e/redaction.ts b/tests/runner-e2e/redaction.ts index d6316a6f59..5710df1443 100644 --- a/tests/runner-e2e/redaction.ts +++ b/tests/runner-e2e/redaction.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { redactDiagnosticText } from "../../packages/adapter-utils/src/command-redaction.js"; const SECRET_SHAPES = [ + /\b(?:github_pat_|ghp_)[A-Za-z0-9_]{16,}\b/g, /\bsk-ant-[A-Za-z0-9_-]{16,}\b/g, /\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}\b/g, /\b(?:openrouter|daytona)[-_]?(?:api)?[-_]?key["'=:\s]+[A-Za-z0-9._-]{12,}\b/gi, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index d2704db83e..f8c4c5caeb 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -3,6 +3,8 @@ export const CREDENTIAL_NAMES = [ "ANTHROPIC_API_KEY", "OPENROUTER_API_KEY", "DAYTONA_API_KEY", + "CURSOR_AUTH_TOKEN", + "COPILOT_GITHUB_TOKEN", ] as const; export type CredentialName = (typeof CREDENTIAL_NAMES)[number]; @@ -59,9 +61,11 @@ export interface RunnerProfileFixture { source: | "adapter_constant" | "qualified_runner_profile" + | "candidate_runner_profile" | "openrouter_rankings_snapshot"; qualificationId: string; }; + qualificationCandidate?: "cursor" | "copilot" | "pi"; ranking?: { rank: number; canonicalModelId: string;