diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 1481b4451b1..be61693fb07 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -909,3 +909,42 @@ reservations and model-readiness status separately. It does not release any historical hold or certify a full-lifecycle vendor cost bound. Missing model receipts, conflicts, active requests, excess commitments, and the existing product, cleanup, feedback and trace checks still fail qualification. + + +### Explicit bounded Daytona gateway qualification + +A manual manifest can set `remoteGateway: "cloudflare"` for the native Codex +or Claude marker/feedback Daytona cells. Install `cloudflared` on the controller +PATH, or set `PAPERCLIP_EVAL_CLOUDFLARED` to its executable path. Empty overrides +use the PATH default. The parent starts its existing model +gateway and native OTLP relay, then exposes only their authenticated endpoints +through one owned HTTPS quick tunnel. It verifies a fresh health nonce before +fixture setup, waits up to 120 seconds for registration and 120 seconds for +public readiness, and does not retry tunnel creation or provider work. + +Only attempt-scoped tokens leave the controller. Provider and Braintrust keys +stay in the parent; the fixture/server retains the existing Daytona provisioning +credential separately. The model gateway still enforces its campaign allowance, +request count, concurrency, deadline and receipt journal. The bridge streams +responses and preserves terminal-usage draining after a client disconnect. +The tunnel receives no ambient provider credentials or Cloudflare config. Its +exposure closes at the campaign deadline or four hours after startup, whichever +comes first, and on attempt cleanup. Owned gateway, +relay and tunnel cleanup failures remain failures. A configured remote route +is not live acceptance; retain the actual Daytona result, cleanup, feedback, +native traces and complete model receipts. + +The Daytona image runs Linux/x64. ARM or macOS controllers use the image's +verified daemon by default; only an explicit cross-compiled remote override +replaces it. + +Daily and weekly selections remain local. This explicit remote option adds no +scheduled coverage or public report publication. + +The worker generates the attempt's trace parent. The relay validates the project +in both the header and payload, but does not pin that worker-generated parent. +Treat the attempt OTLP token as authority to submit trace attribution within +this eval project. Keep it limited to the trusted harness/provider runtime; +never expose it to test prompts or unrelated clients. The project check is not +an attempt-isolation guarantee. Indexing verification is a separate retained +or scheduled check; the manual launcher alone does not perform it. diff --git a/tests/runner-e2e/braintrust.test.ts b/tests/runner-e2e/braintrust.test.ts index 118621d3ffe..da60c202173 100644 --- a/tests/runner-e2e/braintrust.test.ts +++ b/tests/runner-e2e/braintrust.test.ts @@ -211,3 +211,13 @@ it("retains finalization export failures without changing the recorded product r }, transport)).toMatchObject({ status: "failed", errorCode: "invalid_finalization_context" }); expect(transport).not.toHaveBeenCalled(); }); + + +it("binds remote OTLP to the explicitly assigned eval gateway origin", () => { + const origin = "https://synthetic-eval.trycloudflare.com"; + const env = { PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: projectId, BRAINTRUST_API_KEY: "synthetic-token", + PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN: origin, PAPERCLIP_E2E_OTLP_RELAY_URL: `${origin}/otel` }; + expect(evalBraintrustConfig(env)).toMatchObject({ remoteOrigin: origin, endpointBaseUrl: `${origin}/otel` }); + expect(() => evalBraintrustConfig({ ...env, PAPERCLIP_E2E_OTLP_RELAY_URL: "https://different-eval.trycloudflare.com/otel" })).toThrow(); + expect(buildPaperclipServerEnvironment(env).PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN).toBeUndefined(); +}); diff --git a/tests/runner-e2e/braintrust.ts b/tests/runner-e2e/braintrust.ts index adb5d94e4b9..83b7c2091f5 100644 --- a/tests/runner-e2e/braintrust.ts +++ b/tests/runner-e2e/braintrust.ts @@ -1,13 +1,15 @@ import { randomBytes } from "node:crypto"; import { createRequire } from "node:module"; +import { isEvalRemoteOrigin } from "./eval-remote-gateway.js"; import type { RunnerApi } from "./api.js"; import type { FailureClass, RunnerE2EResult } from "./types.js"; export const EVAL_OBSERVABILITY_SECRET_NAMES = ["BRAINTRUST_API_KEY"] as const; const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -export interface EvalBraintrustConfig { projectId: string; apiKey: string; endpointBaseUrl?: string } +export interface EvalBraintrustConfig { projectId: string; apiKey: string; endpointBaseUrl?: string; remoteOrigin?: string } function evalOtlpEndpoint(config: EvalBraintrustConfig) { - if (config.endpointBaseUrl !== undefined && !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}\/otel$/.test(config.endpointBaseUrl)) { + if (config.remoteOrigin !== undefined && !isEvalRemoteOrigin(config.remoteOrigin)) throw new Error("Invalid eval remote OTLP origin"); + if (config.endpointBaseUrl !== undefined && !(config.remoteOrigin && config.endpointBaseUrl === `${config.remoteOrigin}/otel`) && !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}\/otel$/.test(config.endpointBaseUrl)) { throw new Error("Eval OTLP relay must use the parent-owned loopback endpoint"); } if (config.endpointBaseUrl && Number(new URL(config.endpointBaseUrl).port) > 65535) throw new Error("Invalid eval OTLP relay port"); @@ -20,6 +22,7 @@ export function evalBraintrustConfig(env: NodeJS.ProcessEnv): EvalBraintrustConf throw new Error("Eval Braintrust export requires an explicit project UUID and API key"); } const config = { projectId, apiKey: env.BRAINTRUST_API_KEY.trim(), + ...(env.PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN ? { remoteOrigin: env.PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN } : {}), ...(env.PAPERCLIP_E2E_OTLP_RELAY_URL ? { endpointBaseUrl: env.PAPERCLIP_E2E_OTLP_RELAY_URL } : {}) }; evalOtlpEndpoint(config); return config; diff --git a/tests/runner-e2e/eval-gateway-drain.test.ts b/tests/runner-e2e/eval-gateway-drain.test.ts index 24e6f43e776..9c5a73a06cc 100644 --- a/tests/runner-e2e/eval-gateway-drain.test.ts +++ b/tests/runner-e2e/eval-gateway-drain.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, expect, it, vi } from "vitest"; import { openEvalCostLedger, readEvalCostLedger } from "./cost-ledger.js"; +import { startEvalRemoteBridge } from "./eval-remote-gateway.js"; import { forwardEvalRequest, startEvalGateway, type EvalGatewayReceipt, type EvalGatewayRoute } from "./eval-gateway.js"; const cleanups: Array<() => Promise> = []; @@ -181,3 +182,19 @@ it.each(["persist", "observe", "finish"] as const)("records a disconnect during expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000070000", heldUsd: "0.000000000", unknownRequests: 0 }); expect(transport).toHaveBeenCalledOnce(); }); + +it("retains terminal usage when a remote bridge client disconnects during shutdown", async () => { + const f = await fixture(), started = deferred(), release = deferred(); + const transport = vi.fn(async () => { started.resolve(); await release.promise; return finalResponse("openai"); }); + const gateway = await startEvalGateway({ ...f, transport }); cleanups.push(() => gateway.close()); + const bridge = await startEvalRemoteBridge({ gateway, deadlineAt: f.deadlineAt }); cleanups.push(() => bridge.close()); + const client = new AbortController(); + const result = fetch(bridge.baseUrl + "/responses", { method: "POST", signal: client.signal, + headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify({ model: "fixture", input: "hello" }) }).catch(error => error); + await started.promise; client.abort(); await bridge.close(); + const closing = gateway.close(); release.resolve(); await closing; + expect(await result).not.toBeInstanceOf(Response); + expect(transport).toHaveBeenCalledOnce(); + expect(f.receipts.at(-1)).toMatchObject({ clientDisconnected: true, final: true, usageComplete: true, costUsd: "0.000070000" }); + expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000070000", heldUsd: "0.000000000", unknownRequests: 0, inflight: 0 }); +}); diff --git a/tests/runner-e2e/eval-remote-gateway.test.ts b/tests/runner-e2e/eval-remote-gateway.test.ts new file mode 100644 index 00000000000..befe8d1e366 --- /dev/null +++ b/tests/runner-e2e/eval-remote-gateway.test.ts @@ -0,0 +1,101 @@ +import { mkdtemp, readFile, writeFile, rm, readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createServer } from "node:http"; +import { once } from "node:events"; +import { expect, it, vi } from "vitest"; +import { isEvalRemoteOrigin, startEvalRemoteBridge, startEvalRemoteGateway } from "./eval-remote-gateway.js"; + +it("forwards only authenticated model/OTLP routes and closes owned exposure", async () => { + const calls: Array<{ url: string; auth?: string; body: string }> = []; + const upstream = createServer(async (request, response) => { + let body = ""; for await (const chunk of request) body += chunk; + calls.push({ url: request.url!, auth: request.headers.authorization, body }); + response.writeHead(200, { "content-type": "text/event-stream" }); response.end("data: synthetic\n\n"); + }); + upstream.listen(0, "127.0.0.1"); await once(upstream, "listening"); + const address = upstream.address(); if (!address || typeof address === "string") throw new Error("Fixture did not bind"); + const url = `http://127.0.0.1:${address.port}`, modelToken = "a".repeat(64), otlpToken = "b".repeat(64); + const bridge = await startEvalRemoteBridge({ gateway: { baseUrl: url, token: modelToken }, + observability: { baseUrl: url + "/otel", token: otlpToken }, deadlineAt: Date.now() + 60_000 }); + try { + expect(await (await fetch(bridge.baseUrl + "/health")).json()).toEqual({ nonce: bridge.nonce }); + for (const path of ["/v1/responses", "/v1/messages?beta=true", "/otel/v1/traces", "/otel/v1/logs"]) { + expect((await fetch(bridge.baseUrl + path, { method: "POST" })).status).toBe(401); + const token = path.startsWith("/otel") ? otlpToken : modelToken; + const response = await fetch(bridge.baseUrl + path, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: "{}" }); + expect(response.status).toBe(200); expect(await response.text()).toBe("data: synthetic\n\n"); + } + expect(calls).toHaveLength(4); + expect(calls.map(call => call.url)).toEqual(["/v1/responses", "/v1/messages?beta=true", "/otel/v1/traces", "/otel/v1/logs"]); + expect(calls.map(call => call.auth)).toEqual([`Bearer ${modelToken}`, `Bearer ${modelToken}`, `Bearer ${otlpToken}`, `Bearer ${otlpToken}`]); + expect((await fetch(bridge.baseUrl + "/admin", { method: "POST", headers: { authorization: `Bearer ${modelToken}` } })).status).toBe(404); + expect((await fetch(bridge.baseUrl + "/otel/v1/traces", { method: "POST", headers: { authorization: `Bearer ${modelToken}` } })).status).toBe(401); + await bridge.close(); await expect(fetch(bridge.baseUrl + "/health")).rejects.toThrow(); + } finally { await bridge.close(); upstream.closeAllConnections(); await new Promise(resolve => upstream.close(() => resolve())); } +}); + +it("admits only HTTPS quick-tunnel origins and bounded loopback destinations", async () => { + expect(isEvalRemoteOrigin("https://synthetic-fixture.trycloudflare.com")).toBe(true); + for (const value of ["https://foreign.invalid", "http://synthetic.trycloudflare.com", "https://synthetic.trycloudflare.com/path", "https://synthetic.trycloudflare.com?token=private"]) expect(isEvalRemoteOrigin(value)).toBe(false); + await expect(startEvalRemoteBridge({ gateway: { baseUrl: "https://foreign.invalid", token: "a".repeat(64) }, deadlineAt: Date.now() + 60_000 })).rejects.toThrow("policy"); +}); + + +it("fails before remote admission when the tunnel executable is unavailable", async () => { + await expect(startEvalRemoteGateway({ gateway: { baseUrl: "http://127.0.0.1:1", token: "a".repeat(64) }, + deadlineAt: Date.now() + 5000 }, { PAPERCLIP_EVAL_CLOUDFLARED: "/missing/synthetic-cloudflared", PATH: "" })) + .rejects.toThrow("Remote eval tunnel executable unavailable"); +}); + +it("closes remote exposure at its original campaign deadline", async () => { + const bridge = await startEvalRemoteBridge({ gateway: { baseUrl: "http://127.0.0.1:1", token: "a".repeat(64) }, deadlineAt: Date.now() + 50 }); + try { + await new Promise(resolve => setTimeout(resolve, 100)); + await expect(fetch(bridge.baseUrl + "/health")).rejects.toThrow(); + } finally { await bridge.close(); } +}); + + +it.each(["explicit", "empty-override"])("waits for registration and the correct public nonce, then terminates the owned tunnel: %s", async mode => { + const root = await mkdtemp(path.join(tmpdir(), "eval-fake-tunnel-")); + const executable = path.join(root, "cloudflared"), state = path.join(root, "state.json"), stopped = path.join(root, "stopped"); + const origin = "https://synthetic-owned.trycloudflare.com"; + const script = `#!${process.execPath} +const fs = require("node:fs"); +fs.writeFileSync(${JSON.stringify(state)}, JSON.stringify({baseUrl:process.argv.at(-1),pid:process.pid,env:process.env})); +process.stderr.write("${origin}\\n"); +setTimeout(() => process.stderr.write("Registered tunnel connection\\n"), 20); +process.on("SIGTERM", () => { fs.writeFileSync(${JSON.stringify(stopped)}, "stopped"); process.exit(0); }); +setInterval(() => {}, 1000); +`; + await writeFile(executable, script, { mode: 0o700 }); + const realFetch = globalThis.fetch; let healthChecks = 0; + const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation(async url => { + expect(String(url)).toBe(origin + "/health"); + if (++healthChecks === 1) return new Response(JSON.stringify({ nonce: "foreign" }), { status: 200 }); + const { baseUrl } = JSON.parse(await readFile(state, "utf8")); + return realFetch(baseUrl + "/health"); + }); + let remote: Awaited> | undefined; + try { + remote = await startEvalRemoteGateway({ gateway: { baseUrl: "http://127.0.0.1:1", token: "a".repeat(64) }, + deadlineAt: Date.now() + 10_000 }, { PAPERCLIP_EVAL_CLOUDFLARED: mode === "explicit" ? executable : "", PATH: root, OPENAI_API_KEY: "must-not-leave" }); + expect(remote.baseUrl).toBe(origin); expect(healthChecks).toBe(2); + const { baseUrl, pid, env } = JSON.parse(await readFile(state, "utf8")); + expect(env.OPENAI_API_KEY).toBeUndefined(); expect(env.HOME).not.toBe(process.env.HOME); + await remote.close(); + expect(await readFile(stopped, "utf8")).toBe("stopped"); + expect(() => process.kill(pid, 0)).toThrow(); + await expect(realFetch(baseUrl + "/health")).rejects.toThrow(); + await expect(readFile(path.join(env.HOME, "anything"))).rejects.toThrow(); + } finally { fetchSpy.mockRestore(); await remote?.close(); await rm(root, { recursive: true, force: true }); } +}, 15_000); + + +it("cleans the private tunnel home after synchronous spawn rejection", async () => { + const before = new Set((await readdir(tmpdir())).filter(name => name.startsWith("eval-cloudflare-"))); + await expect(startEvalRemoteGateway({ gateway: { baseUrl: "http://127.0.0.1:1", token: "a".repeat(64) }, + deadlineAt: Date.now() + 60_000 }, { PAPERCLIP_EVAL_CLOUDFLARED: "invalid\0executable", PATH: "" })).rejects.toThrow(); + expect((await readdir(tmpdir())).filter(name => name.startsWith("eval-cloudflare-") && !before.has(name))).toEqual([]); +}); diff --git a/tests/runner-e2e/eval-remote-gateway.ts b/tests/runner-e2e/eval-remote-gateway.ts new file mode 100644 index 00000000000..2d5d1d85054 --- /dev/null +++ b/tests/runner-e2e/eval-remote-gateway.ts @@ -0,0 +1,119 @@ +import { spawn } from "node:child_process"; +import { randomBytes, timingSafeEqual } from "node:crypto"; +import { once } from "node:events"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createServer, request as httpRequest } from "node:http"; + +export const isEvalRemoteOrigin = (value: string) => /^https:\/\/[a-z0-9-]+\.trycloudflare\.com$/.test(value); + +export async function startEvalRemoteBridge(input: { + gateway: { baseUrl: string; token: string }; + observability?: { baseUrl: string; token: string }; + deadlineAt: number; +}) { + if (!Number.isFinite(input.deadlineAt) || input.deadlineAt <= Date.now() || input.deadlineAt > Date.now() + 4 * 3_600_000 + || !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}$/.test(input.gateway.baseUrl) + || input.observability && !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}\/otel$/.test(input.observability.baseUrl) + || !/^[a-f0-9]{64}$/.test(input.gateway.token) + || input.observability && !/^[a-f0-9]{64}$/.test(input.observability.token)) throw new Error("Invalid remote eval bridge policy"); + try { new URL(input.gateway.baseUrl); if (input.observability) new URL(input.observability.baseUrl); } + catch { throw new Error("Invalid remote eval bridge policy"); } + const nonce = randomBytes(32).toString("hex"); + let closed = false; + const server = createServer((incoming, outgoing) => { + if (closed || Date.now() >= input.deadlineAt) { outgoing.writeHead(503).end(); return; } + if (incoming.method === "GET" && incoming.url === "/health") { + outgoing.writeHead(200, { "content-type": "application/json", "cache-control": "no-store" }).end(JSON.stringify({ nonce })); return; + } + const model = ["/responses", "/v1/responses", "/messages", "/messages?beta=true", "/v1/messages", "/v1/messages?beta=true"].includes(incoming.url ?? ""); + const otlp = ["/otel/v1/traces", "/otel/v1/logs"].includes(incoming.url ?? ""); + const destination = model ? input.gateway : otlp ? input.observability : undefined; + if (incoming.method !== "POST" || !destination) { outgoing.writeHead(404).end(); return; } + const auth = String(incoming.headers.authorization ?? "").replace(/^Bearer /, "") || String(incoming.headers["x-api-key"] ?? ""); + const presented = Buffer.from(auth), expected = Buffer.from(destination.token); + if (presented.length !== expected.length || !timingSafeEqual(presented, expected)) { outgoing.writeHead(401).end(); return; } + const suffix = otlp ? incoming.url!.slice("/otel".length) : incoming.url!; + const headers: Record = { authorization: `Bearer ${destination.token}` }; + for (const name of ["content-type", "content-encoding", "anthropic-beta", "anthropic-version", "x-bt-parent"]) { + const value = incoming.headers[name]; if (typeof value === "string") headers[name] = value; + } + const upstream = httpRequest(destination.baseUrl + suffix, { method: "POST", headers }, response => { + if (outgoing.destroyed) { response.destroy(); return; } + outgoing.writeHead(response.statusCode ?? 502, response.headers); response.pipe(outgoing); + response.on("error", () => outgoing.destroy()); + }); + upstream.on("error", () => { if (!outgoing.headersSent) outgoing.writeHead(502).end(); else outgoing.destroy(); }); + incoming.on("aborted", () => upstream.destroy()); + incoming.on("error", () => upstream.destroy()); + outgoing.on("error", () => upstream.destroy()); + outgoing.on("close", () => { if (!outgoing.writableEnded) upstream.destroy(); }); + incoming.pipe(upstream); + }); + server.requestTimeout = 15_000; server.headersTimeout = 10_000; + server.listen(0, "127.0.0.1"); await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Remote eval bridge did not bind loopback"); + let timer: ReturnType; + let closing: Promise | undefined; + const close = () => closing ??= (async () => { + closed = true; clearTimeout(timer); + server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + })(); + timer = setTimeout(() => { void close().catch(() => {}); }, input.deadlineAt - Date.now()); timer.unref(); + return { baseUrl: `http://127.0.0.1:${address.port}`, nonce, close }; +} + +export async function startEvalRemoteGateway(input: Parameters[0], env: NodeJS.ProcessEnv) { + const bridge = await startEvalRemoteBridge(input); + let home: string; + try { home = await mkdtemp(path.join(tmpdir(), "eval-cloudflare-")); } + catch (error) { await bridge.close(); throw error; } + let tunnel: ReturnType | undefined; + let closing: Promise | undefined, deadlineTimer: ReturnType | undefined; + const close = () => closing ??= (async () => { + clearTimeout(deadlineTimer); + const errors: unknown[] = []; + try { await bridge.close(); } catch (error) { errors.push(error); } + try { + if (tunnel?.pid && tunnel.exitCode === null && tunnel.signalCode === null) { + const exited = once(tunnel, "exit").catch(() => {}); + tunnel.kill("SIGTERM"); const timer = setTimeout(() => tunnel?.kill("SIGKILL"), 5000); + try { await exited; } finally { clearTimeout(timer); } + } + } catch (error) { errors.push(error); } + try { await rm(home, { recursive: true, force: true }); } catch (error) { errors.push(error); } + if (errors.length) throw new AggregateError(errors, "Remote eval exposure cleanup failed"); + })(); + try { + tunnel = spawn(env.PAPERCLIP_EVAL_CLOUDFLARED?.trim() || "cloudflared", + ["tunnel", "--no-autoupdate", "--protocol", "http2", "--url", bridge.baseUrl], + { stdio: ["ignore", "pipe", "pipe"], env: { PATH: env.PATH, HOME: home, TMPDIR: home } }); + deadlineTimer = setTimeout(() => { void close().catch(() => {}); }, input.deadlineAt - Date.now()); deadlineTimer.unref(); + const child = tunnel; + const origin = await new Promise((resolve, reject) => { + let output = ""; + const timer = setTimeout(() => finish(new Error("Remote eval tunnel startup timed out")), Math.min(120_000, input.deadlineAt - Date.now())); + const finish = (error?: Error, value?: string) => { clearTimeout(timer); if (error) reject(error); else resolve(value!); }; + child.once("error", () => finish(new Error("Remote eval tunnel executable unavailable"))); + child.once("exit", () => finish(new Error("Remote eval tunnel exited"))); + const read = (chunk: Buffer) => { + output = (output + chunk.toString()).slice(-32768); + const value = output.match(/https:\/\/[a-z0-9-]+\.trycloudflare\.com/)?.[0]; + if (value && output.includes("Registered tunnel connection")) finish(undefined, value); + }; + child.stdout!.on("data", read); child.stderr!.on("data", read); + }); + const readyBy = Math.min(Date.now() + 120_000, input.deadlineAt); + while (Date.now() < readyBy) { + try { + const response = await fetch(origin + "/health", { redirect: "error", signal: AbortSignal.timeout(Math.min(5000, readyBy - Date.now())) }); + if (response.ok && (await response.json() as { nonce?: string }).nonce === bridge.nonce) return { baseUrl: origin, close }; + await response.body?.cancel(); + } catch {} + await new Promise(resolve => setTimeout(resolve, 1000)); + } + throw new Error("Remote eval tunnel not publicly reachable before admission"); + } catch (error) { await close(); throw error; } +} diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index 2ea3cd91a40..9aa865b0c1f 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -2,6 +2,7 @@ import { installedDaytonaPluginKeys } from "./installed-daytona-plugin.js"; import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; import { assertInstalledCliSelection, installedCliKeys, usesInstalledCli } from "./installed-cli.js"; import path from "node:path"; +import { isEvalRemoteOrigin } from "./eval-remote-gateway.js"; import { chatNeedsApiTools, isManagedHiringCase } from "./chat-cases.js"; import { CREDENTIAL_NAMES } from "./types.js"; import type { MatrixExecution } from "./types.js"; @@ -14,6 +15,7 @@ const AMBIENT_PAPERCLIP_CREDENTIAL_KEYS = [ "PAPERCLIP_E2E_GATEWAY_TOKEN", "PAPERCLIP_E2E_GATEWAY_PHASE_PATH", "PAPERCLIP_E2E_OTLP_RELAY_URL", + "PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN", "PAPERCLIP_SETUP_TOKEN", "PAPERCLIP_SECRETS_MASTER_KEY", "PAPERCLIP_SECRETS_MASTER_KEY_FILE", @@ -41,6 +43,8 @@ export function applyEvalGatewayEnvironment(env: NodeJS.ProcessEnv, credential: || PROVIDER_SECRET_KEY.test(name) && /KEY|TOKEN|SECRET|PASSWORD/.test(name)) delete env[name]; } delete env.PAPERCLIP_E2E_OTLP_RELAY_URL; + delete env.PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN; + if (isEvalRemoteOrigin(gateway.baseUrl)) env.PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN = gateway.baseUrl; env[credential] = gateway.token; env.PAPERCLIP_E2E_GATEWAY_BASE_URL = gateway.baseUrl; env.PAPERCLIP_E2E_GATEWAY_TOKEN = gateway.token; @@ -103,13 +107,13 @@ export function resolvePaperclipRemoteRunnerBinaryForHarness( runnerBinary: string | undefined, configuredPath = process.env.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH, platform: NodeJS.Platform = process.platform, + architecture: NodeJS.Architecture = process.arch, ): string | undefined { if (configuredPath?.trim()) return configuredPath; if (!runnerBinary) return undefined; - // Daytona runs Linux. A default debug binary built by a macOS developer is - // Mach-O and cannot be staged into that sandbox. Leave the remote override - // unset so the pinned Daytona image's verified runnerd is discovered instead. - if (platform !== "linux") return undefined; + // The pinned Daytona runtime is Linux/x64. Other controller binaries cannot + // run there; discover its verified runnerd unless an explicit cross-build is set. + if (platform !== "linux" || architecture !== "x64") return undefined; return executions.some( (execution) => execution.profile.generation === "native" && @@ -133,6 +137,7 @@ export function buildRunnerE2EProcessEnvironment( const result = { ...source }; delete result.PAPERCLIP_E2E_GATEWAY_PHASE_PATH; delete result.PAPERCLIP_E2E_OTLP_RELAY_URL; + delete result.PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN; // Announcements are unrelated to the scenarios and obscure screenshot evidence. result.PAPERCLIP_ANNOUNCEMENTS_ENABLED = "false"; delete result.OPENCODE_ALLOW_ALL_MODELS; diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index b65d4f9fecb..74ec9179866 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -595,6 +595,9 @@ async function runAttempt(input: { }; if (gateway) { applyEvalGatewayEnvironment(childEnv, execution.profile.credential, gateway); + // The fixture/server controller needs Daytona provisioning access. The + // remote provider still receives only the scoped model and OTLP tokens. + if (execution.environment.id === "daytona") childEnv.DAYTONA_API_KEY = process.env.DAYTONA_API_KEY; } // The database URLs are stripped here and again at the Playwright web-server // boundary so a developer's shell can never redirect this paid test. diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 348db2d08af..517b25b3d90 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,3 +1,4 @@ +import { isEvalRemoteOrigin } from "./eval-remote-gateway.js"; import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js"; import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; import { NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js"; @@ -280,8 +281,9 @@ export async function setupLiveFixtures(input: { const gatewayUrl = process.env.PAPERCLIP_E2E_GATEWAY_BASE_URL; const gatewayToken = process.env.PAPERCLIP_E2E_GATEWAY_TOKEN; const budgetedGateway = Boolean(gatewayUrl || gatewayToken); - if (budgetedGateway && (!gatewayUrl || !gatewayToken || !/^http:\/\/127\.0\.0\.1:[0-9]+$/.test(gatewayUrl) - || execution.environment.id !== "local" || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id))) throw new Error("Unqualified fixture gateway route"); + if (budgetedGateway && (!gatewayUrl || !gatewayToken || !(execution.environment.id === "local" ? /^http:\/\/127\.0\.0\.1:[0-9]+$/.test(gatewayUrl) + : isEvalRemoteOrigin(gatewayUrl) && gatewayUrl === process.env.PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN) + || !["local", "daytona"].includes(execution.environment.id) || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id))) throw new Error("Unqualified fixture gateway route"); if (managedHiring || savedCopilot || budgetedGateway) { registry.register({ id: "ai-connection", diff --git a/tests/runner-e2e/operations.test.ts b/tests/runner-e2e/operations.test.ts index 452403765fd..8e03148dfdc 100644 --- a/tests/runner-e2e/operations.test.ts +++ b/tests/runner-e2e/operations.test.ts @@ -3,6 +3,7 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promise import { createHash } from "node:crypto"; import path from "node:path"; import { tmpdir } from "node:os"; +import * as remoteGateway from "./eval-remote-gateway.js"; import { runnerMatrix } from "./catalog.js"; import { validateEvalOperationsManifest, openEvalOperations, type EvalOperationsManifest } from "./operations.js"; import { setupLiveFixtures } from "./live-fixtures.js"; @@ -17,7 +18,7 @@ function manifest(): EvalOperationsManifest { models: [{ provider: "openai", maximumInputTokens: 1000, maximumOutputTokens: 100, timeoutMs: 1000, price: { provider: "openai", model: cell.profile.model, version: "fixture-only", sourceUrl: "https://provider.invalid/fixture", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" } }] }; } -afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); +afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); it("admits only the exact bounded local selection and explicitly priced models", () => { expect(validateEvalOperationsManifest(manifest(), [cell], 1)).toMatchObject({ route: "direct" }); expect(() => validateEvalOperationsManifest({ ...manifest(), executionIds: [] }, [cell], 1)).toThrow("selection"); @@ -191,3 +192,44 @@ it.each([ expect(agent!.budgetMonthlyCents).toBe(1_000); } }); + + +it("requires explicit remote gateway admission and infrastructure credentials before opening a journal", async () => { + const remote = runnerMatrix.find(e => e.suite.id === cell.suite.id && e.profile.id === cell.profile.id && e.environment.id === "daytona")!; + const value = { ...manifest(), executionIds: [remote.id], remoteGateway: "cloudflare" as const }; + expect(validateEvalOperationsManifest(value, [remote], 0)).toBe(value); + const root = await mkdtemp(path.join(tmpdir(), "eval-remote-preflight-")); + try { + value.ledgerPath = path.join(root, "ledger.jsonl"); + const file = path.join(root, "manifest.json"); await writeFile(file, JSON.stringify(value)); + await expect(openEvalOperations(file, [remote], 0, { OPENAI_API_KEY: "synthetic" })).rejects.toThrow("credential"); + await expect(access(value.ledgerPath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("gives remote workers scoped gateway/relay URLs and closes both local listeners and the tunnel", async () => { + const remote = runnerMatrix.find(e => e.suite.id === cell.suite.id && e.profile.id === cell.profile.id && e.environment.id === "daytona")!; + const root = await mkdtemp(path.join(tmpdir(), "eval-remote-operation-")); + const close = vi.fn(async () => {}), origin = "https://synthetic-owned.trycloudflare.com"; + const start = vi.spyOn(remoteGateway, "startEvalRemoteGateway").mockResolvedValue({ baseUrl: origin, close }); + const value = { ...manifest(), executionIds: [remote.id], remoteGateway: "cloudflare" as const, ledgerPath: path.join(root, "ledger.jsonl") }; + value.limits.deadlineAt = new Date(Date.now() + 24 * 3_600_000).toISOString(); + const file = path.join(root, "manifest.json"), env = { OPENAI_API_KEY: "provider-private", DAYTONA_API_KEY: "infra-private", + BRAINTRUST_API_KEY: "trace-private", PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "11111111-1111-4111-8111-111111111111" }; + let operations: Awaited> | undefined; + try { + await writeFile(file, JSON.stringify(value)); operations = await openEvalOperations(file, [remote], 0, env); + const endpoint = await operations.start(remote, "remote-one", 1); + const worker: NodeJS.ProcessEnv = { ...env }; applyEvalGatewayEnvironment(worker, remote.profile.credential, endpoint); + expect(worker).toMatchObject({ PAPERCLIP_E2E_GATEWAY_BASE_URL: origin, PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN: origin, + PAPERCLIP_E2E_OTLP_RELAY_URL: `${origin}/otel`, OPENAI_API_KEY: endpoint.token, BRAINTRUST_API_KEY: endpoint.observability!.token }); + expect(JSON.stringify(worker)).not.toMatch(/provider-private|infra-private|trace-private/); + const local = start.mock.calls[0]![0]; + expect(local.gateway.baseUrl).toMatch(/^http:\/\/127\.0\.0\.1:/); + expect(local.deadlineAt).toBeLessThanOrEqual(Date.now() + 4 * 3_600_000); + expect(local.deadlineAt).toBeLessThan(Date.parse(value.limits.deadlineAt)); + await operations.close(); operations = undefined; expect(close).toHaveBeenCalledOnce(); + await expect(fetch(local.gateway.baseUrl)).rejects.toThrow(); + await expect(fetch(local.observability!.baseUrl)).rejects.toThrow(); + } finally { await operations?.close(); await rm(root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/operations.ts b/tests/runner-e2e/operations.ts index ed4e82d1cee..831520dd9e2 100644 --- a/tests/runner-e2e/operations.ts +++ b/tests/runner-e2e/operations.ts @@ -4,6 +4,7 @@ import { createHash } from "node:crypto"; import { createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js"; import { EvalSpendState, openEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js"; import { evalRequestReservation, startEvalGateway, type EvalGatewayRoute, type EvalGatewayReceipt } from "./eval-gateway.js"; +import { startEvalRemoteGateway } from "./eval-remote-gateway.js"; import type { MatrixExecution } from "./types.js"; import { evalBraintrustConfig } from "./braintrust.js"; import { startEvalOtlpRelay } from "./eval-otlp-relay.js"; @@ -17,13 +18,15 @@ export interface EvalOperationsManifest { maxAttemptsPerExecution: number; route: "direct" | "braintrust"; projectId?: string; + remoteGateway?: "cloudflare"; models: Array>; } const fields = (value: object, names: string[]) => { if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(name => !names.includes(name))) throw new Error("Invalid eval operations manifest fields"); }; export function validateEvalOperationsManifest(value: EvalOperationsManifest, executions: readonly MatrixExecution[], maxRetries: number) { - fields(value, ["version", "limits", "ledgerPath", "executionIds", "maxAttemptsPerExecution", "route", "projectId", "models"]); + fields(value, ["version", "limits", "ledgerPath", "executionIds", "maxAttemptsPerExecution", "route", "projectId", "remoteGateway", "models"]); + if (value.remoteGateway !== undefined && value.remoteGateway !== "cloudflare") throw new Error("Invalid remote eval gateway policy"); const initial = new EvalSpendState(value.limits).snapshot(); const initialModelAllowance = usdToUnits(value.limits.maxUsd) - usdToUnits(initial.committedUsd); if (value.version !== 1 || !["direct", "braintrust"].includes(value.route) || !path.isAbsolute(value.ledgerPath) @@ -46,9 +49,11 @@ export function validateEvalOperationsManifest(value: EvalOperationsManifest, ex models.set(key, usdToUnits(evalRequestReservation({ ...model, mode: value.route, apiKey: "validation" }))); } for (const execution of executions) { - if (execution.environment.id !== "local" || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id) + if (!(execution.environment.id === "local" || execution.environment.id === "daytona" && value.remoteGateway === "cloudflare" + && ["runner-codex", "runner-acpx-claude"].includes(execution.profile.id)) || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id) || !["runner-codex", "runner-acpx-claude", "legacy-codex", "legacy-claude"].includes(execution.profile.id) - || execution.requiredCredentials.length !== 1) throw new Error(`Unqualified budgeted eval route: ${execution.id}`); + || execution.requiredCredentials.length !== (execution.environment.id === "daytona" ? 2 : 1) + || execution.requiredCredentials.filter(name => name !== "DAYTONA_API_KEY").length !== 1) throw new Error(`Unqualified budgeted eval route: ${execution.id}`); const provider = execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai"; if (!models.has(`${provider}:${execution.profile.model}`)) throw new Error(`Missing exact model pricing: ${execution.id}`); if (models.get(`${provider}:${execution.profile.model}`)! > initialModelAllowance) { @@ -65,7 +70,8 @@ export async function openEvalOperations(file: string, executions: readonly Matr || Date.parse(manifest.limits.deadlineAt) >= Date.parse(manifest.limits.overhead.expiresAt))) { throw new Error("Operating overhead allowance does not cover the campaign window"); } - const required = manifest.route === "braintrust" ? ["BRAINTRUST_API_KEY"] : executions.map(execution => execution.profile.credential); + const required = [...(manifest.route === "braintrust" ? ["BRAINTRUST_API_KEY"] : executions.map(execution => execution.profile.credential)), + ...(executions.some(execution => execution.environment.id === "daytona") ? ["DAYTONA_API_KEY"] : [])]; if (required.some(name => !env[name]?.trim())) throw new Error("Missing eval gateway upstream credential"); const observability = evalBraintrustConfig({ ...env, PAPERCLIP_E2E_OTLP_RELAY_URL: undefined }); if (observability && manifest.route === "braintrust" && observability.projectId !== manifest.projectId) throw new Error("Eval gateway and OTLP projects differ"); @@ -99,14 +105,19 @@ export async function openEvalOperations(file: string, executions: readonly Matr persist: (receipt: EvalGatewayReceipt) => atomicJson(path.join(receiptDirectory, `${receipt.requestId}.json`), receipt), }); let relay: Awaited> | undefined; + let remote: Awaited> | undefined; try { if (observability) relay = await startEvalOtlpRelay({ projectId: observability.projectId, apiKey: observability.apiKey, campaignId: manifest.limits.campaignId, attemptId, deadlineAt: Math.min(Date.parse(manifest.limits.deadlineAt), Date.now() + 4 * 3_600_000), receiptPath: path.join(`${manifest.ledgerPath}.otlp`, `${createHash("sha256").update(attemptId).digest("hex")}.jsonl`) }); - } catch (error) { await gateway.close(); throw error; } + if (execution.environment.id === "daytona") remote = await startEvalRemoteGateway({ gateway, observability: relay, + deadlineAt: Math.min(Date.parse(manifest.limits.deadlineAt), Date.now() + 4 * 3_600_000) }, env); + } catch (error) { await Promise.allSettled([gateway.close(), relay?.close()]); throw error; } let closing: Promise | undefined; - const endpoint = { ...gateway, phasePath, observability: relay, close: () => closing ??= (async () => { - const results = await Promise.allSettled([gateway.close(), relay?.close()]); + const endpoint = { ...gateway, baseUrl: remote?.baseUrl ?? gateway.baseUrl, phasePath, + observability: relay ? { ...relay, baseUrl: remote ? `${remote.baseUrl}/otel` : relay.baseUrl } : undefined, close: () => closing ??= (async () => { + const remoteResults = await Promise.allSettled([remote?.close()]); + const results = [...remoteResults, ...await Promise.allSettled([gateway.close(), relay?.close()])]; active.delete(endpoint); if (results.some(result => result.status === "rejected")) throw new Error("Eval gateway or OTLP relay cleanup failed"); })() }; diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts index 7de63dd9068..07c22457fe6 100644 --- a/tests/runner-e2e/support.test.ts +++ b/tests/runner-e2e/support.test.ts @@ -175,6 +175,7 @@ describe("runner E2E local binary resolution", () => { runnerBinary, undefined, "linux", + "x64", ), ).toBe(runnerBinary); expect( @@ -183,6 +184,7 @@ describe("runner E2E local binary resolution", () => { runnerBinary, undefined, "linux", + "x64", ), ).toBeUndefined(); expect( @@ -201,7 +203,14 @@ describe("runner E2E local binary resolution", () => { "darwin", ), ).toBe("/cross-compiled/paperclip-runnerd"); + expect(resolvePaperclipRemoteRunnerBinaryForHarness( + [remoteNativeExecution], runnerBinary, undefined, "linux", "arm64", + )).toBeUndefined(); + expect(resolvePaperclipRemoteRunnerBinaryForHarness( + [remoteNativeExecution], runnerBinary, "/cross-compiled/linux-x64-runnerd", "linux", "arm64", + )).toBe("/cross-compiled/linux-x64-runnerd"); }); + }); describe("runner E2E provider environment", () => {