diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 6b374dfea81..2de69815d93 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -6,6 +6,7 @@ import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; import { canonical, costRuntime, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; import { attemptDirectory, runControlledEval } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; @@ -13,6 +14,8 @@ import { readRegularFile, replaceEvidenceFile } from './evidence-files.mjs'; import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } from './archive.mjs'; import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs'; +const { build } = createRequire(new URL('../../../../package.json', import.meta.url))('esbuild'); + const childScript = options => ` import assert from 'node:assert/strict'; import {createHash} from 'node:crypto'; @@ -49,10 +52,7 @@ if(args.has('--observability-config')) { } nativeObservability={mode:'controller_otlp',configured:true,verification:'unverified',projectId:config.projectId,traceId:config.traceId,spanId:config.spanId,descriptorSha256:createHash('sha256').update(raw).digest('hex')}; if(${Boolean(options.realObservabilityValidator)}) { - const {createRequire}=await import('node:module'); - const require=createRequire(${JSON.stringify(fileURLToPath(new URL('../../../../server/package.json', import.meta.url)))}); - const {tsImport}=await import(require.resolve('tsx/esm/api')); - const {prepareEvalObservabilityRuntime}=await tsImport(${JSON.stringify(fileURLToPath(new URL('../../src/cli/eval-observability-runtime.ts', import.meta.url)))},import.meta.url); + const {prepareEvalObservabilityRuntime}=await import(${JSON.stringify(options.observabilityValidatorModule ?? '')}); await mkdir(process.env.HOME+'/codex',{recursive:true}); const prepared=await prepareEvalObservabilityRuntime(args.get('--observability-config'),request, {evidence:{campaignId:descriptor.campaignId,executionId:descriptor.executionId},transportOptions:{environment:{CODEX_HOME:process.env.HOME+'/codex'}}},process.env); @@ -85,6 +85,11 @@ async function fixture(t, { count = 1, allocation = '0.01', providerProfile = 'c const root = await mkdtemp(join(tmpdir(), 'runner-controller-test-')); t.after(() => rm(root, { recursive: true, force: true })); const cli = join(root, 'fake-cli.mjs'), binary = join(root, 'runnerd'), archive = join(root, 'runner.tgz'); + if (options.realObservabilityValidator) { + options.observabilityValidatorModule = join(root, 'eval-observability-runtime.mjs'); + await build({ entryPoints: [fileURLToPath(new URL('../../src/cli/eval-observability-runtime.ts', import.meta.url))], + outfile: options.observabilityValidatorModule, bundle: true, platform: 'node', format: 'esm', target: 'node24', logLevel: 'silent' }); + } await writeFile(cli, childScript(options)); await writeFile(binary, 'synthetic daemon'); await writeFile(archive, 'synthetic package'); const provenance = async path => ({ path, sha256: sha256(await readFile(path)) }); const now = Date.now(), verifiedAt = new Date(now - 1000).toISOString(), expiresAt = new Date(now + 86_400_000).toISOString(); diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index c9ad6c6d91d..72dc383a7a9 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -506,6 +506,17 @@ select arbitrary destinations. Gateway response caching and fallback are refused Request journal and usage receipts contain no prompt/response bodies or keys. The manifest, journal, and receipts are not public report projections. +`boundedEvalPayload` is the shared, credential-free request validator and +normalizer. Qualification callers must apply it before recording budget-denial +proof. The forwarding path applies the same policy before a ledger reservation. +OpenAI tools may use one namespace level around local function or custom tools. +Function and custom leaves require nonblank string names. +Namespace wrappers contain only `type`, `name`, optional `description`, and +`tools`. Nested namespaces, hosted tools, and tool lists attached to leaves fail +closed. The request may contain at most 1,024 tool definitions, counting namespace +wrappers and their children. Anthropic tool policy stays separate. Output limits, +billing-mode restrictions, and multimodal exclusions apply in both paths. + ## Configured repository and conversion proof The repository proof flow seeds only the exact `workspace` child of its resolved diff --git a/tests/runner-e2e/eval-gateway.test.ts b/tests/runner-e2e/eval-gateway.test.ts index c4c9e785b8d..150fac62f5b 100644 --- a/tests/runner-e2e/eval-gateway.test.ts +++ b/tests/runner-e2e/eval-gateway.test.ts @@ -6,7 +6,7 @@ import { connect } from "node:net"; import { afterEach, expect, it, vi } from "vitest"; import { openEvalCostLedger, readEvalCostLedger } from "./cost-ledger.js"; import { beginEvalAgentPhase, createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js"; -import { evalRequestReservation, EvalUsageStream, forwardEvalRequest, startEvalGateway, type EvalGatewayRoute, type EvalGatewayReceipt } from "./eval-gateway.js"; +import { boundedEvalPayload, evalRequestReservation, EvalUsageStream, forwardEvalRequest, startEvalGateway, type EvalGatewayRoute, type EvalGatewayReceipt } from "./eval-gateway.js"; const directories: string[] = [], closes: Array<() => Promise> = []; afterEach(async () => { for (const close of closes.splice(0).reverse()) await close(); for (const dir of directories.splice(0)) await rm(dir, { recursive: true, force: true }); }); const identity = { executionId: "hello", attemptId: "attempt", phase: "agent" as const }; @@ -135,6 +135,71 @@ it("admits local function and custom tools inside a Codex namespace", async () = expect(result.outcome).toBe("succeeded"); expect(transport).toHaveBeenCalledOnce(); }); +it("validates the native daemon's namespace envelope before reserving and forwarding", async () => { + const f = await fixture(); + const localTool = (name: string) => ({ type: "function", name, description: "Synthetic local tool", + parameters: { type: "object", properties: {}, additionalProperties: false }, strict: true }); + const tools = [...Array.from({ length: 7 }, (_, index) => localTool(`local_${index}`)), + { type: "namespace", name: "local_tools", description: "Synthetic local namespace", + tools: Array.from({ length: 5 }, (_, index) => localTool(`operation_${index}`)) }]; + const payload = Object.freeze({ ...f.payload, tools }); + const normalized = boundedEvalPayload(payload, { provider: route.provider, price: route.price, maximumOutputTokens: 100 }); + expect(normalized).toEqual({ ...payload, max_output_tokens: 100, service_tier: "default" }); + expect(payload).not.toHaveProperty("service_tier"); + expect(payload.max_output_tokens).toBe(500); + expect(f.ledger.snapshot().requests).toHaveLength(0); + const transport = vi.fn(async (_url, init) => { + expect(JSON.parse(init!.body as string)).toEqual(normalized); + expect((await readEvalCostLedger(f.file)).state.snapshot().inflight).toBe(1); + return response(); + }); + const receipt = await forwardEvalRequest({ ...f, payload, transport }); + expect(receipt).toMatchObject({ outcome: "succeeded", costUsd: "0.000068000", final: true }); + expect(transport).toHaveBeenCalledOnce(); +}); +it.each([ + { type: "namespace", name: " ", tools: [{ type: "function", name: "local" }] }, + { type: "namespace", name: "local", description: {}, tools: [{ type: "function", name: "local" }] }, + { type: "namespace", name: "local", tools: [] }, + { type: "namespace", name: "local", tools: null }, + { type: "namespace", name: "local", tools: [null] }, + { type: ["function"], name: "local" }, + { type: ["custom"], name: "local" }, + { type: "namespace", name: "local", tools: [{ type: ["function"], name: "local" }] }, + { type: "namespace", name: "local", tools: [{ type: ["custom"], name: "local" }] }, + ...[undefined, null, 7, "", " "].flatMap(name => ["function", "custom"].flatMap(type => + [{ type, name }, { type: "namespace", name: "local", tools: [{ type, name }] }])), + { type: "namespace", name: "local", tools: [{ type: "function", name: "local" }], future_hosted_tool: true }, + { type: "namespace", name: "local", tools: [{ type: "namespace", name: "nested", tools: [{ type: "function", name: "local" }] }] }, + ...["web_search", "file_search", "image_generation", "code_interpreter", "computer", "mcp", "unknown"].map(type => + ({ type: "namespace", name: "local", tools: [{ type }] })), +])("rejects malformed, nested, or hosted namespace tools before any side effects", async tool => { + const f = await fixture(), transport = vi.fn(), persist = vi.fn(); + const payload = { ...f.payload, tools: [tool] }; + expect(() => boundedEvalPayload(payload, f.route)).toThrow("unqualified_hosted_tool"); + await expect(forwardEvalRequest({ ...f, payload, transport, persist })).rejects.toThrow("unqualified_hosted_tool"); + expect(transport).not.toHaveBeenCalled(); expect(persist).not.toHaveBeenCalled(); + expect(f.ledger.snapshot().requests).toHaveLength(0); +}); +it.each(["flat", "namespace"] as const)("bounds the total %s tool definitions including namespace wrappers", async shape => { + const f = await fixture(), transport = vi.fn(), persist = vi.fn(); + const tool = { type: "function", name: "local_tool" }; + const tools = shape === "flat" ? Array.from({ length: 1024 }, () => tool) + : [{ type: "namespace", name: "local", tools: Array.from({ length: 1023 }, () => tool) }]; + expect(boundedEvalPayload({ ...f.payload, tools }, f.route).tools).toEqual(tools); + const payload = { ...f.payload, tools: [...tools, tool] }; + await expect(forwardEvalRequest({ ...f, payload, transport, persist })).rejects.toThrow("unqualified_hosted_tool"); + expect(transport).not.toHaveBeenCalled(); expect(persist).not.toHaveBeenCalled(); + expect(f.ledger.snapshot().requests).toHaveLength(0); +}); +it("keeps Anthropic custom tools and output limits separate from OpenAI namespaces", () => { + const anthropic = { provider: "anthropic" as const, price: { ...route.price, provider: "anthropic" as const }, maximumOutputTokens: 100 }; + const tools = [{ name: "local_tool", input_schema: { type: "object" } }, { type: "custom", name: "other_tool" }]; + expect(boundedEvalPayload({ model: "fixture", tools, max_tokens: 500 }, anthropic)) + .toEqual({ model: "fixture", tools, max_tokens: 100, service_tier: "standard_only" }); + expect(() => boundedEvalPayload({ model: "fixture", tools: [{ type: "namespace", name: "local", tools }] }, anthropic)) + .toThrow("unqualified_hosted_tool"); +}); it.each([ { type: "namespace", name: "local", tools: [{ type: "web_search" }] }, { type: "namespace", name: "local", tools: [{ type: "namespace", name: "nested", tools: [{ type: "web_search" }] }] }, diff --git a/tests/runner-e2e/eval-gateway.ts b/tests/runner-e2e/eval-gateway.ts index 6d42ce680e5..a90cd28047f 100644 --- a/tests/runner-e2e/eval-gateway.ts +++ b/tests/runner-e2e/eval-gateway.ts @@ -65,19 +65,34 @@ function modelReservationBound(route: EvalGatewayRoute, reservation: string) { }) : undefined; } -function boundedPayload(value: unknown, route: EvalGatewayRoute) { +function qualifiedOpenAiTools(tools: unknown): boolean { + if (!Array.isArray(tools)) return false; + let remaining = 1024; + const leaf = (tool: unknown): boolean => { + const definition = row(tool); + return --remaining >= 0 && (definition.type === "function" || definition.type === "custom") + && typeof definition.name === "string" && Boolean(definition.name.trim()) && !Object.hasOwn(definition, "tools"); + }; + return tools.every(tool => { + const definition = row(tool); + if (definition.type !== "namespace") return leaf(tool); + return --remaining >= 0 && typeof definition.name === "string" && Boolean(definition.name.trim()) + && (definition.description === undefined || typeof definition.description === "string") + && Object.keys(definition).every(key => ["type", "name", "description", "tools"].includes(key)) + && Array.isArray(definition.tools) && definition.tools.length > 0 && definition.tools.every(leaf); + }); +} + +export function boundedEvalPayload(value: unknown, route: Pick): Record { const body = row(value); if (body.model !== route.price.model) throw new EvalRequestError("unqualified_model"); if (body.background === true || body.mcp_servers !== undefined || body.service_tier !== undefined && !["auto", "default", "standard_only"].includes(String(body.service_tier))) throw new EvalRequestError("unqualified_billing_mode"); if (body.speed !== undefined && body.speed !== "standard" || body.inference_geo !== undefined) throw new EvalRequestError("unqualified_billing_mode"); - if (body.tools !== undefined && (!Array.isArray(body.tools) || body.tools.some(tool => { - const definition = row(tool), type = definition.type; - if (route.provider !== "openai") return type !== undefined && type !== "custom" || Object.hasOwn(definition, "tools"); - if (type === "namespace") return typeof definition.name !== "string" || !definition.name - || !Array.isArray(definition.tools) || !definition.tools.length - || definition.tools.some(nested => !["function", "custom"].includes(String(row(nested).type)) || Object.hasOwn(row(nested), "tools")); - return !["function", "custom"].includes(String(type)) || Object.hasOwn(definition, "tools"); - }))) throw new EvalRequestError("unqualified_hosted_tool"); + if (body.tools !== undefined && (route.provider === "openai" ? !qualifiedOpenAiTools(body.tools) + : !Array.isArray(body.tools) || body.tools.some(tool => { + const definition = row(tool), type = definition.type; + return type !== undefined && type !== "custom" || Object.hasOwn(definition, "tools"); + }))) throw new EvalRequestError("unqualified_hosted_tool"); const inspect = (item: unknown): void => { if (Array.isArray(item)) { item.forEach(inspect); return; } if (!item || typeof item !== "object") return; @@ -119,8 +134,8 @@ export async function forwardEvalRequest(input: { transport?: typeof fetch; }): Promise { const { route, ledger } = input; - let body: ReturnType; - try { body = boundedPayload(input.payload, route); } + let body: ReturnType; + try { body = boundedEvalPayload(input.payload, route); } catch (error) { throw error instanceof EvalRequestError ? error : new EvalRequestError("invalid_request_payload"); } const reservation = evalRequestReservation(route); if (!route.apiKey || /[\r\n]/.test(route.apiKey)) throw new Error("missing_gateway_credential");