diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index b7f8efad0c2..a7114ed7c31 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -104,6 +104,61 @@ Claude Managed uses its Agent version, while AgentCore uses its qualification revision. Transport API or beta versions remain separate protocol metadata and must not replace that runtime identity in a report. +## Controller-assigned gateway route + +The Runner eval CLI accepts `--gateway-config /absolute/path/gateway.json` for +native Codex and ACPX Claude. This is an explicit route for a controller +that owns a local budget proxy. Normal invocations keep the direct path. +ACPX Codex is rejected before service creation because its runtime prohibits +preconfigured homes to protect Runner state. +The descriptor must bind the exact model and attempt ID from `request.json`: + +```json +{ + "schema": "paperclip.runner-eval.gateway/v1", + "provider": "openai", + "model": "gpt-5.4-mini", + "baseUrl": "http://127.0.0.1:4567", + "campaignId": "campaign-id", + "executionId": "cell-id", + "attemptId": "attempt-id", + "upstreamRoute": "direct" +} +``` + +Use `anthropic` for ACPX Claude. `upstreamRoute` is the controller's declaration: +`direct` or `braintrust`. It is not an independent verification of the upstream. +The base URL must be the proxy's explicit IPv4 loopback address and port, with +no path, query, fragment, or embedded credential. The controller supplies its +32-byte hexadecimal token through `PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN`. +Do not put provider or Braintrust credentials in this child environment or file. + +```sh +node /path/to/runner/dist/cli/eval-session.js \ + --request /path/to/request.json --output /path/to/artifact.json \ + --gateway-config /path/to/gateway.json +``` + +The CLI validates the descriptor before creating a provider service. It creates +an owned temporary home, pins Codex's Responses endpoint or Claude's Messages +endpoint and exact model, and passes only the assigned proxy credential. +It does not copy ambient provider keys, subscription logins, proxy settings, +startup hooks, or user homes into this route. Unsupported harnesses and +mismatched descriptor identities fail before provider creation. The temporary +home is removed on success and failure. + +Success and failure artifacts retain `requestRouting`: controller gateway mode, +provider, declared upstream route, campaign/execution/attempt IDs, and the +SHA-256 of the descriptor bytes. The token and endpoint are excluded from this +metadata. Existing artifact usage and provider outcomes remain authoritative +for their respective fields; the route declaration is not a billing receipt. +The controller must retain its request ledger and link these identities to it. + +This entry point does not itself allocate a campaign budget, run the proxy, or +change the hosted matrix. The controller must enforce request admission, +deadlines, and limits before forwarding traffic. A local fake-API probe only +proves routing; it does not qualify live provider or Braintrust behavior. + ## Hosted campaign Use the `Runner Direct Live Protocol Evals` workflow. Dispatch the workflow diff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts new file mode 100644 index 00000000000..cd1cb1e03e1 --- /dev/null +++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts @@ -0,0 +1,156 @@ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { chmod, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { createSanitizedCodexEnvironment } from "../drivers/codex/app-server-transport.js"; +import { createSanitizedAcpxSpawnInput } from "../drivers/acpx/environment.js"; +import { createCapabilityFixtureState } from "../mock-core/capability-control-plane-types.js"; +import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js"; +import { parseEvalSessionRequest } from "./eval-session-contract.js"; +import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js"; +import { runEvalSessionCli } from "./eval-session.js"; + +const roots: string[] = []; +afterEach(async () => { + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) { + for (const entry of await readdir(root).catch(() => [])) { + if (entry.startsWith(".paperclip-eval-runtime-context-")) await chmod(join(root, entry, "instructions"), 0o700).catch(() => undefined); + } + await rm(root, { recursive: true, force: true }); + } +}); +const token = "a".repeat(64); +async function fixture(provider = "codex", acpxAgent?: string) { + const root = await mkdtemp(join(tmpdir(), "runner-gateway-test-")); roots.push(root); + const model = acpxAgent === "claude" ? "claude-sonnet-5" : "gpt-5.6-sol"; + const binary = join(root, "runnerd"); + await writeFile(binary, "fake runnerd"); + const request = parseEvalSessionRequest({ + schema: "paperclip-runner/eval-session-request/v1", attemptId: "attempt-1", prompt: "Read the governed task.", + provider, ...(acpxAgent ? { acpxAgent } : {}), model, + runnerd: { path: binary, sha256: createHash("sha256").update("fake runnerd").digest("hex") }, + limits: { turnTimeoutMs: 120000, maxAgentTurns: 1, maxEstimatedCostNanodollars: 100000000 }, + session: { workingDirectory: root }, + }); + const descriptor = { schema: "paperclip.runner-eval.gateway/v1", provider: acpxAgent === "claude" ? "anthropic" : "openai", + model, baseUrl: "http://127.0.0.1:4567", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1", upstreamRoute: "braintrust" }; + const descriptorPath = join(root, "gateway.json"); + await writeFile(descriptorPath, JSON.stringify(descriptor)); + return { root, request, descriptor, descriptorPath }; +} + +it.each([["codex", undefined], ["acpx", "claude"]])("routes %s/%s with only its proxy credential and an owned home", async (provider, agent) => { + const f = await fixture(provider, agent); + const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { + PATH: process.env.PATH, HOME: "/ambient/home", CODEX_HOME: "/ambient/codex", CLAUDE_CONFIG_DIR: "/ambient/claude", + PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token, OPENAI_API_KEY: "ambient-openai", CODEX_API_KEY: "ambient-codex", + ANTHROPIC_API_KEY: "ambient-anthropic", ANTHROPIC_AUTH_TOKEN: "ambient-oauth", CLAUDE_CODE_OAUTH_TOKEN: "ambient-claude", + BRAINTRUST_API_KEY: "ambient-braintrust", AWS_SECRET_ACCESS_KEY: "ambient-aws", GH_TOKEN: "ambient-github", + HTTPS_PROXY: "https://ambient-proxy", NODE_OPTIONS: "--require ambient-hook", PAPERCLIP_CONFIGURED_ENVIRONMENT_KEYS: '["GH_TOKEN"]', + }); + const env = gateway.transportOptions.environment; + try { + expect(gateway.transportOptions.homeMode).toBe("preconfigured"); + expect(JSON.stringify(env)).not.toContain("ambient"); + expect(env.OPENAI_API_KEY).toBeUndefined(); + expect(env.CODEX_API_KEY).toBeUndefined(); + expect(env.BRAINTRUST_API_KEY).toBeUndefined(); + expect(env.PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN).toBeUndefined(); + const child = provider === "codex" ? createSanitizedCodexEnvironment(env) : createSanitizedAcpxSpawnInput(env, agent as "codex" | "claude").env; + expect(child.HOME).toBe(env.HOME); + if (agent === "claude") { + expect(child.ANTHROPIC_API_KEY).toBe(token); + expect(child.ANTHROPIC_BASE_URL).toBe(f.descriptor.baseUrl); + expect(child.CLAUDE_CODE_SUBAGENT_MODEL).toBe(f.request.model); + expect(JSON.parse(await readFile(join(env.CLAUDE_CONFIG_DIR!, "settings.json"), "utf8"))).toEqual({ model: f.request.model, availableModels: [f.request.model] }); + } else { + expect(child.PAPERCLIP_AI_PROVIDER_KEY).toBe(token); + const config = await readFile(join(env.CODEX_HOME!, "config.toml"), "utf8"); + expect(config).toContain('base_url = "http://127.0.0.1:4567/v1"'); + expect(config).toContain('env_key = "PAPERCLIP_AI_PROVIDER_KEY"'); + expect(config).toContain("requires_openai_auth = false"); + expect(config).toContain("shell_snapshot = false"); + expect(config).not.toContain(token); + expect((await stat(join(env.CODEX_HOME!, "config.toml"))).mode & 0o777).toBe(0o600); + } + expect(JSON.stringify(gateway.evidence)).not.toContain(token); + expect(gateway.evidence).toMatchObject({ mode: "controller_gateway", attemptId: f.request.attemptId, declaredUpstreamRoute: "braintrust" }); + } finally { await gateway.close(); } + expect(existsSync(env.HOME!)).toBe(false); +}); + +it.each([ + { baseUrl: "https://api.openai.com" }, { baseUrl: "http://localhost:4567" }, + { baseUrl: "http://127.0.0.1:4567/v1" }, { baseUrl: "http://127.0.0.1:99999" }, + { baseUrl: "http://127.0.0.1:4567?token=secret" }, { attemptId: "another-attempt" }, + { model: "another-model" }, { campaignId: "bad identity" }, { unexpectedSecret: "must-not-print" }, +])("rejects invalid descriptor binding %j", async (change) => { + const f = await fixture(); + await writeFile(f.descriptorPath, JSON.stringify({ ...f.descriptor, ...change })); + await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })) + .rejects.toThrow("Invalid controller gateway descriptor"); +}); + +it("rejects missing tokens and unsupported provider routes before runtime creation", async () => { + const f = await fixture(); + await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, {})).rejects.toThrow("gateway token"); + await expect(prepareEvalGatewayRuntime(f.descriptorPath, { ...f.request, provider: "opencode" }, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })).rejects.toThrow("does not support"); + await writeFile(f.descriptorPath, JSON.stringify({ ...f.descriptor, provider: "anthropic" })); + await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })).rejects.toThrow("does not support"); +}); + +it.each([false, true])("the real CLI retains route identity and removes the owned home after failure=%s", async (fail) => { + const f = await fixture(); + vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token); + vi.stubEnv("OPENAI_API_KEY", "upstream-must-not-reach-provider"); + const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json"); + await writeFile(requestPath, JSON.stringify(f.request)); + const state = JSON.stringify(createCapabilityFixtureState({})); + const snapshot = { + sessionId: "session-1", revision: 1, providerThreadId: "provider-1", providerSessionId: "provider-1", + providerModel: { id: f.request.model, provider: "openai" }, status: "idle", activeTurnId: null, + createdAt: "2026-10-24T00:00:00Z", updatedAt: "2026-10-24T00:00:01Z", + authority: { companyId: "company-1", actorId: "actor-1", taskId: "task-1", runId: "run-1", sessionId: "session-1", scenarioId: "gateway-test" }, + config: { provider: "codex", driver: "codex_app_server", seedState: state, workingDirectory: f.root, scenario: { id: "gateway-test" }, capabilities: [], explicitClaims: [], turnTimeoutMs: 1000 }, + mockState: state, process: null, networkEvidence: { realPaperclipRequests: 0, childPaperclipEnvironmentKeys: [] }, + transcript: [], evidence: [], authorizationRecords: [], attempts: [], terminalTurns: [{ turnId: "turn-1", status: "completed" }], + usageLedger: [{ receiptId: "receipt-1", turnId: "turn-1", providerCalls: 1, providerRequests: 1, inputTokens: 100, outputTokens: 10, cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0 }], usageUnavailable: [], + } as unknown as CapabilityLiveSessionSnapshot; + let home: string | undefined; + const factory = vi.fn((_binary, options) => { + expect(options.homeMode).toBe("preconfigured"); + expect(options.environment.OPENAI_API_KEY).toBeUndefined(); + expect(options.environment.PAPERCLIP_AI_PROVIDER_KEY).toBe(token); + home = options.environment.HOME; + expect(existsSync(home!)).toBe(true); + return { create: async () => ({ + sendMessage: async () => { if (fail) throw new Error("simulated provider failure"); return { turnId: "turn-1", status: "completed", snapshot }; }, + completeAttempt: async () => undefined, shutdown: async () => { snapshot.status = "closed"; }, snapshot: () => snapshot, + }) } as never; + }); + const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath], { serviceFactory: factory }); + expect(result).toBe(fail ? 2 : 0); + expect(factory).toHaveBeenCalledTimes(1); + expect(existsSync(home!)).toBe(false); + const raw = await readFile(outputPath, "utf8"), artifact = JSON.parse(raw); + expect(artifact.requestRouting).toMatchObject({ mode: "controller_gateway", provider: "openai", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1" }); + expect(raw).not.toContain(token); + expect(raw).not.toContain("upstream-must-not-reach-provider"); + if (fail) expect(artifact.infrastructureError).toBe("simulated provider failure"); +}); + + +it.each(["codex", undefined])("rejects ACPX Codex (%s) before creating any provider service", async (agent) => { + const f = await fixture("acpx", agent); + vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token); + const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json"); + await writeFile(requestPath, JSON.stringify(f.request)); + const factory = vi.fn(() => { throw new Error("must not create a provider service"); }); + await expect(runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath], { serviceFactory: factory })) + .rejects.toThrow("Controller gateway does not support this eval provider"); + expect(factory).not.toHaveBeenCalled(); + expect(existsSync(outputPath)).toBe(false); +}); diff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts new file mode 100644 index 00000000000..0d32670c3a6 --- /dev/null +++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts @@ -0,0 +1,104 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { EvalSessionRequest } from "./eval-session-contract.js"; + +export interface EvalGatewayDescriptor { + schema: "paperclip.runner-eval.gateway/v1"; + provider: "openai" | "anthropic"; + model: string; + baseUrl: string; + campaignId: string; + executionId: string; + attemptId: string; + upstreamRoute: "direct" | "braintrust"; +} + +const TOKEN_ENV = "PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN"; +const safeId = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,511}$/.test(value); + +export async function prepareEvalGatewayRuntime( + descriptorPath: string, + request: EvalSessionRequest, + source: NodeJS.ProcessEnv = process.env, +) { + const raw = await readFile(descriptorPath, "utf8"); + let descriptor: EvalGatewayDescriptor; + try { + if (Buffer.byteLength(raw) > 8192) throw new Error(); + descriptor = JSON.parse(raw) as EvalGatewayDescriptor; + const fields = ["schema", "provider", "model", "baseUrl", "campaignId", "executionId", "attemptId", "upstreamRoute"]; + if (!descriptor || typeof descriptor !== "object" || Array.isArray(descriptor) + || Object.keys(descriptor).length !== fields.length || Object.keys(descriptor).some(key => !fields.includes(key)) + || descriptor.schema !== "paperclip.runner-eval.gateway/v1" + || !["openai", "anthropic"].includes(descriptor.provider) + || !["direct", "braintrust"].includes(descriptor.upstreamRoute) + || ![descriptor.campaignId, descriptor.executionId, descriptor.attemptId].every(safeId) + || descriptor.attemptId !== request.attemptId || descriptor.model !== request.model + || typeof descriptor.baseUrl !== "string" || !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}$/.test(descriptor.baseUrl)) throw new Error(); + const url = new URL(descriptor.baseUrl); + if (!url.port || Number(url.port) > 65535) throw new Error(); + } catch { + throw new Error("Invalid controller gateway descriptor"); + } + const provider = request.provider ?? "codex"; + const codex = provider === "codex"; + const claude = provider === "acpx" && request.acpxAgent === "claude"; + if (!codex && !claude || descriptor.provider !== (codex ? "openai" : "anthropic")) { + throw new Error("Controller gateway does not support this eval provider"); + } + const token = source[TOKEN_ENV]; + if (typeof token !== "string" || !/^[a-f0-9]{64}$/.test(token)) throw new Error("Missing or invalid controller gateway token"); + const root = await mkdtemp(join(tmpdir(), "paperclip-runner-eval-gateway-")); + try { + const home = join(root, "home"), codexHome = join(root, "codex"), claudeHome = join(root, "claude"); + const environment: NodeJS.ProcessEnv = Object.fromEntries( + ["PATH", "LANG", "LANGUAGE", "LC_ALL", "TZ", "SystemRoot", "WINDIR", "PATHEXT"] + .flatMap(key => source[key] === undefined ? [] : [[key, source[key]!]]), + ); + Object.assign(environment, { + HOME: home, CODEX_HOME: codexHome, CLAUDE_CONFIG_DIR: claudeHome, + XDG_CONFIG_HOME: join(root, "config"), XDG_DATA_HOME: join(root, "data"), XDG_CACHE_HOME: join(root, "cache"), + TMPDIR: join(root, "tmp"), TMP: join(root, "tmp"), TEMP: join(root, "tmp"), + PAPERCLIP_RUNNER_HOME_MODE: "preconfigured", + }); + for (const directory of [home, codexHome, claudeHome, environment.XDG_CONFIG_HOME!, environment.XDG_DATA_HOME!, environment.XDG_CACHE_HOME!, environment.TMPDIR!]) { + await mkdir(directory, { mode: 0o700 }); + } + if (codex) { + environment.PAPERCLIP_AI_PROVIDER_KEY = token; + await writeFile(join(codexHome, "config.toml"), [ + 'model_provider = "paperclip_eval"', + `model = ${JSON.stringify(request.model)}`, + '[model_providers.paperclip_eval]', + 'name = "Paperclip eval controller"', + `base_url = ${JSON.stringify(descriptor.baseUrl + "/v1")}`, + 'wire_api = "responses"', 'requires_openai_auth = false', 'env_key = "PAPERCLIP_AI_PROVIDER_KEY"', + '[features]', 'shell_snapshot = false', '', + ].join("\n"), { mode: 0o600, flag: "wx" }); + } else { + Object.assign(environment, { + ANTHROPIC_API_KEY: token, ANTHROPIC_BASE_URL: descriptor.baseUrl, + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", ACPX_CLAUDE_INCLUDE_USER_SETTINGS: "1", + ANTHROPIC_MODEL: request.model, ANTHROPIC_DEFAULT_OPUS_MODEL: request.model, + ANTHROPIC_DEFAULT_SONNET_MODEL: request.model, ANTHROPIC_DEFAULT_HAIKU_MODEL: request.model, + CLAUDE_CODE_SUBAGENT_MODEL: request.model, + }); + await writeFile(join(claudeHome, "settings.json"), JSON.stringify({ model: request.model, availableModels: [request.model] }) + "\n", { mode: 0o600, flag: "wx" }); + } + return { + transportOptions: { homeMode: "preconfigured" as const, environment }, + evidence: { + mode: "controller_gateway" as const, provider: descriptor.provider, + declaredUpstreamRoute: descriptor.upstreamRoute, + campaignId: descriptor.campaignId, executionId: descriptor.executionId, attemptId: descriptor.attemptId, + descriptorSha256: createHash("sha256").update(raw).digest("hex"), + }, + close: () => rm(root, { recursive: true, force: true }), + }; + } catch (error) { + await rm(root, { recursive: true, force: true }); + throw error; + } +} diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index 788dc61602b..c42a22ec933 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -23,6 +23,7 @@ import { CapabilityLiveSessionService, type CapabilityLiveSession, type CapabilityLiveSessionSnapshot, + type CapabilityLiveSessionServiceOptions, type CapabilityLiveTurnResult, type CreateCapabilityLiveSessionInput, } from "../live/live-session.js"; @@ -35,6 +36,7 @@ import { type EvalSessionUsage, } from "./eval-session-contract.js"; import { evalProviderTransportOptions } from "./eval-provider-runtime.js"; +import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js"; import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; interface EvalSessionCliOptions { @@ -42,6 +44,7 @@ interface EvalSessionCliOptions { outputPath: string; candidateProfile?: EvalCandidateProfile; expectedAcpxProfile?: Record; + gatewayConfigPath?: string; } function argument(args: string[], name: string): string { @@ -52,7 +55,7 @@ function argument(args: string[], name: string): string { } export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions { - const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile"]); + const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile", "--gateway-config"]); const seen = new Set(); for (let index = 0; index < args.length; index += 2) { if (!allowed.has(args[index] ?? "")) { @@ -89,6 +92,7 @@ export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions { return { ...(candidateProfile === undefined ? {} : { candidateProfile }), ...(expectedAcpxProfile === undefined ? {} : { expectedAcpxProfile }), + ...(seen.has("--gateway-config") ? { gatewayConfigPath: argument(args, "--gateway-config") } : {}), requestPath: argument(args, "--request"), outputPath: argument(args, "--output"), }; @@ -362,6 +366,7 @@ export async function runEvalSessionCli( options: { serviceFactory?: ( runnerBinary: string, + transportOptions: NonNullable, ) => CapabilityLiveSessionService; } = {}, ): Promise { @@ -398,32 +403,31 @@ export async function runEvalSessionCli( const requestedDriver = request.driver ?? expectedEvalSessionDriver(requestedProvider); const requestedProviderVersion = evalSessionProviderVersion(request); - const runtimeContext = await prepareEvalRuntimeContext( - resolve(request.session.workingDirectory ?? process.cwd()), - ); - const service = options.serviceFactory?.(runnerdPath) ?? - new CapabilityLiveSessionService({ - transportOptions: { - ...evalProviderTransportOptions(requestedProvider, request.limits.turnTimeoutMs, request.session.workingDirectory), - runnerBinary: runnerdPath, - ...(cli.candidateProfile === undefined ? {} : { acpxCandidateProfile: cli.candidateProfile }), - runtimeContext, - baseInstructions: evalRuntimeSystemInstructions(runtimeContext), - // The transport performs the provider-specific allowlisting. Supplying - // the source environment here is still required: without it the - // isolated Codex home has no credential source and runnerd receives no - // executable PATH from the Evalbook CLI process. - environment: process.env, - onDiagnostic: (message) => { - process.stderr.write(`[eval-session runnerd] ${message}\n`); - }, - }, - }); + const gateway = cli.gatewayConfigPath + ? await prepareEvalGatewayRuntime(cli.gatewayConfigPath, request) + : undefined; let session: CapabilityLiveSession | null = null; let turn: CapabilityLiveTurnResult | null = null; let snapshot: CapabilityLiveSessionSnapshot | null = null; try { + const runtimeContext = await prepareEvalRuntimeContext( + resolve(request.session.workingDirectory ?? process.cwd()), + ); + const transportOptions: NonNullable = { + ...evalProviderTransportOptions(requestedProvider, request.limits.turnTimeoutMs, request.session.workingDirectory), + runnerBinary: runnerdPath, + ...(cli.candidateProfile === undefined ? {} : { acpxCandidateProfile: cli.candidateProfile }), + runtimeContext, + baseInstructions: evalRuntimeSystemInstructions(runtimeContext), + environment: process.env, + ...gateway?.transportOptions, + onDiagnostic: (message) => { + process.stderr.write(`[eval-session runnerd] ${message}\n`); + }, + }; + const service = options.serviceFactory?.(runnerdPath, transportOptions) + ?? new CapabilityLiveSessionService({ transportOptions }); // PR3 expands this same service input with the two qualified remote // profiles. The cast keeps this isolated PR typecheckable before PR3 lands; // the runtime fields and their fail-closed validation are already present. @@ -470,6 +474,7 @@ export async function runEvalSessionCli( build: PAPERCLIP_RUNNER_BUILD_METADATA, runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` }, requestedModel: request.model, + ...(gateway ? { requestRouting: gateway.evidence } : {}), ...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }), provider: requestedProvider, driver: requestedDriver, @@ -549,6 +554,7 @@ export async function runEvalSessionCli( build: PAPERCLIP_RUNNER_BUILD_METADATA, runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` }, requestedModel: request.model, + ...(gateway ? { requestRouting: gateway.evidence } : {}), ...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }), provider: requestedProvider, driver: requestedDriver, @@ -575,6 +581,8 @@ export async function runEvalSessionCli( }, }, null, 2)}\n`); return 2; + } finally { + await gateway?.close(); } }