From 7b75c0349f582e7052df54c82d2ebc74f070d050 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 13:32:56 -0700 Subject: [PATCH] Add explicit native observability route to Runner eval CLI Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 39 +++++ .../src/cli/eval-gateway-runtime.test.ts | 112 ++++++++++++++- .../src/cli/eval-observability-runtime.ts | 135 ++++++++++++++++++ .../paperclip-runner/src/cli/eval-session.ts | 23 ++- 4 files changed, 297 insertions(+), 12 deletions(-) create mode 100644 packages/paperclip-runner/src/cli/eval-observability-runtime.ts diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index a7114ed7c31..6d3cf586e0b 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -159,6 +159,45 @@ change the hosted matrix. The controller must enforce request admission, deadlines, and limits before forwarding traffic. A local fake-API probe only proves routing; it does not qualify live provider or Braintrust behavior. +## Controller-assigned native observability + +The same CLI accepts `--observability-config /absolute/path/observability.json` +only with `--gateway-config`. This opt-in route supports native Codex and ACPX +Claude. Without that argument, the gateway route exports no native traces. + +The descriptor uses schema `paperclip.runner-eval.observability/v1`. It contains +`campaignId`, `executionId`, and `attemptId` matching the gateway and request; +`projectId` (UUID), `traceId` (32 lowercase hex characters), and `spanId` (16 +lowercase hex characters); `baseUrl` in the form +`http://127.0.0.1:PORT/otel`; and `launch`, the secret-free result from the existing +`buildProviderObservabilityLaunch` in `packages/adapter-utils`. The CLI accepts +only its known Codex/Claude exporter settings and the mock session's exact +company and agent attributes. Bind `revision` to `compatibilityKey`. Use a +compatibility key of at most 128 letters, digits, underscores, or hyphens. + +Build the launch with `/v1/logs` and `/v1/traces` under that base URL, a timeout +of at most 3 seconds, and only the `x-bt-parent` header from +`exportBraintrustAttemptParent` for those project/trace/span IDs. Do not include +upstream authorization in the file. Supply the attempt's separate 32-byte +hexadecimal relay credential through `PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN`. The CLI +adds authorization to the provider environment and redacts it from diagnostics +and artifacts. Codex configuration is written only into the disposable gateway +home; the existing model route is preserved. The descriptor must be a regular +file, not a symlink, and fit within 128 KiB. + +Success and failure artifacts record `nativeObservability` with the declared +project/trace/span IDs, descriptor SHA-256, `configured: true`, and +`verification: "unverified"`. Configuration does not prove export, ingestion, +indexing, complete capture, or billing coverage. Relay receipts and destination +readback must establish those separate facts. Provider errors keep their +original outcome even when traces arrive. + +The controller owns the relay, credentials, ingestion quotas, deadline, drain, +and private receipt retention. This CLI option does not start a relay, set an +operating allowance, or enable a paid campaign. The existing Product E2E relay +and launch generator remain the integration owners; the hosted Runner workflow +still needs its own qualified controller integration. + ## Hosted campaign Use the `Runner Direct Live Protocol Evals` workflow. Dispatch the workflow diff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts index cd1cb1e03e1..9e468c9f484 100644 --- a/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts +++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; -import { chmod, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { chmod, mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; @@ -10,7 +10,11 @@ import { createCapabilityFixtureState } from "../mock-core/capability-control-pl import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js"; import { parseEvalSessionRequest } from "./eval-session-contract.js"; import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js"; -import { runEvalSessionCli } from "./eval-session.js"; +import { prepareEvalObservabilityRuntime } from "./eval-observability-runtime.js"; +import { buildProviderObservabilityLaunch } from "../../../adapter-utils/src/agent-observability.js"; +import { exportBraintrustAttemptParent } from "../../../../tests/runner-e2e/braintrust.js"; +import { parse } from "smol-toml"; +import { parseEvalSessionCliArgs, runEvalSessionCli } from "./eval-session.js"; const roots: string[] = []; afterEach(async () => { @@ -42,6 +46,82 @@ async function fixture(provider = "codex", acpxAgent?: string) { return { root, request, descriptor, descriptorPath }; } +const otlpToken = "b".repeat(64); +async function observabilityFixture(f: Awaited>) { + const identity = { projectId: "11111111-1111-4111-8111-111111111111", traceId: "1".repeat(32), spanId: "2".repeat(16) }; + const baseUrl = "http://127.0.0.1:4568/otel"; + const descriptor = { + schema: "paperclip.runner-eval.observability/v1", campaignId: f.descriptor.campaignId, + executionId: f.descriptor.executionId, attemptId: f.request.attemptId, ...identity, baseUrl, + launch: buildProviderObservabilityLaunch({ kind: f.request.provider === "codex" ? "codex" : "claude", + companyId: "company-1", agentId: "actor-1", revision: "revision-1", compatibilityKey: "revision-1", enabled: true, + capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true }, + logsEndpoint: `${baseUrl}/v1/logs`, tracesEndpoint: `${baseUrl}/v1/traces`, + headers: { "x-bt-parent": exportBraintrustAttemptParent(identity) }, + }), + }; + const file = join(f.root, "observability.json"); + await writeFile(file, JSON.stringify(descriptor)); + return { descriptor, file }; +} + +it.each([["codex", undefined], ["acpx", "claude"]])("applies the existing %s/%s launch contract only to its owned gateway home", async (provider, agent) => { + const f = await fixture(provider, agent), observation = await observabilityFixture(f); + const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); + try { + const result = await prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken, BRAINTRUST_API_KEY: "upstream-secret" }); + const launch = result.transportOptions.observability; + expect(result.evidence).toMatchObject({ mode: "controller_otlp", configured: true, verification: "unverified", projectId: observation.descriptor.projectId }); + expect(JSON.stringify(result.evidence)).not.toContain(otlpToken); + expect(JSON.stringify(result)).not.toContain("upstream-secret"); + expect(launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toContain(`authorization=Bearer%20${otlpToken}`); + expect(result.transportOptions.environment.BRAINTRUST_API_KEY).toBeUndefined(); + if (provider === "codex") { + const raw = await readFile(join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"), "utf8"), config = parse(raw); + expect(config.model_provider).toBe("paperclip_eval"); + expect(config.otel).toEqual(observation.descriptor.launch.otelConfig); + expect(raw).not.toContain(otlpToken); + expect((await stat(join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"))).mode & 0o777).toBe(0o600); + } else { + const sanitized = createSanitizedAcpxSpawnInput(result.transportOptions.environment, "claude").env; + expect(sanitized.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT).toBe(observation.descriptor.baseUrl + "/v1/traces"); + expect(sanitized.OTEL_EXPORTER_OTLP_TRACES_HEADERS).toContain(otlpToken); + expect(sanitized.ANTHROPIC_API_KEY).toBe(token); + } + } finally { await gateway.close(); } +}); + +it.each(["identity", "endpoint", "parent", "credential", "company", "rawBodies", "modelEnvironment", "extraField", "unknownExporter"])("rejects observability %s drift before changing the gateway home", async fault => { + const f = await fixture(), observation = await observabilityFixture(f); + const descriptor = observation.descriptor as unknown as Record; + if (fault === "identity") descriptor.attemptId = "other"; + if (fault === "endpoint") descriptor.launch.otelConfig.exporter["otlp-http"].endpoint = "https://api.braintrust.dev/otel/v1/logs"; + if (fault === "parent") descriptor.traceId = "3".repeat(32); + if (fault === "credential") descriptor.launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS += ",authorization=Bearer%20upstream-secret"; + if (fault === "company") descriptor.launch.otelConfig.span_attributes["paperclip.company.id"] = "other"; + if (fault === "rawBodies") descriptor.launch.otelConfig.log_guardian_assessments = true; + if (fault === "modelEnvironment") descriptor.launch.environment.OPENAI_API_KEY = "upstream-secret"; + if (fault === "extraField") descriptor.extra = "unknown"; + if (fault === "unknownExporter") descriptor.launch.otelConfig.extra_exporter = "console"; + await writeFile(observation.file, JSON.stringify(descriptor)); + const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); + const config = join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"), before = await readFile(config); + try { + await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow("Invalid controlled eval observability"); + expect(await readFile(config)).toEqual(before); + } finally { await gateway.close(); } +}); + +it("refuses missing relay credentials and oversized observability files before launch", async () => { + const f = await fixture(), observation = await observabilityFixture(f); + const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); + try { + await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, {})).rejects.toThrow("OTLP token"); + await writeFile(observation.file, " ".repeat(131073)); + await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow("Invalid controlled eval observability"); + } finally { await gateway.close(); } +}); + it.each([["codex", undefined], ["acpx", "claude"]])("routes %s/%s with only its proxy credential and an owned home", async (provider, agent) => { const f = await fixture(provider, agent); const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { @@ -102,8 +182,10 @@ it("rejects missing tokens and unsupported provider routes before runtime creati await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })).rejects.toThrow("does not support"); }); -it.each([false, true])("the real CLI retains route identity and removes the owned home after failure=%s", async (fail) => { +it.each([[false, false], [true, false], [false, true], [true, true]])("the real CLI retains route identity and removes its home after failure=%s, observability=%s", async (fail, withObservability) => { const f = await fixture(); + const observation = withObservability ? await observabilityFixture(f) : undefined; + vi.stubEnv("PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN", otlpToken); vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token); vi.stubEnv("OPENAI_API_KEY", "upstream-must-not-reach-provider"); const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json"); @@ -122,24 +204,27 @@ it.each([false, true])("the real CLI retains route identity and removes the owne let home: string | undefined; const factory = vi.fn((_binary, options) => { expect(options.homeMode).toBe("preconfigured"); + expect(Boolean(options.observability)).toBe(Boolean(withObservability)); expect(options.environment.OPENAI_API_KEY).toBeUndefined(); expect(options.environment.PAPERCLIP_AI_PROVIDER_KEY).toBe(token); home = options.environment.HOME; expect(existsSync(home!)).toBe(true); return { create: async () => ({ - sendMessage: async () => { if (fail) throw new Error("simulated provider failure"); return { turnId: "turn-1", status: "completed", snapshot }; }, + sendMessage: async () => { if (fail) throw new Error(`simulated provider failure${withObservability ? ` Bearer ${otlpToken}` : ""}`); return { turnId: "turn-1", status: "completed", snapshot }; }, completeAttempt: async () => undefined, shutdown: async () => { snapshot.status = "closed"; }, snapshot: () => snapshot, }) } as never; }); - const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath], { serviceFactory: factory }); + const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath, ...(observation ? ["--observability-config", observation.file] : [])], { serviceFactory: factory }); expect(result).toBe(fail ? 2 : 0); expect(factory).toHaveBeenCalledTimes(1); expect(existsSync(home!)).toBe(false); const raw = await readFile(outputPath, "utf8"), artifact = JSON.parse(raw); expect(artifact.requestRouting).toMatchObject({ mode: "controller_gateway", provider: "openai", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1" }); expect(raw).not.toContain(token); + expect(raw).not.toContain(otlpToken); + expect(artifact.nativeObservability).toEqual(withObservability ? expect.objectContaining({ configured: true, verification: "unverified" }) : undefined); expect(raw).not.toContain("upstream-must-not-reach-provider"); - if (fail) expect(artifact.infrastructureError).toBe("simulated provider failure"); + if (fail) expect(artifact.infrastructureError).toBe(`simulated provider failure${withObservability ? " [REDACTED]" : ""}`); }); @@ -154,3 +239,18 @@ it.each(["codex", undefined])("rejects ACPX Codex (%s) before creating any provi expect(factory).not.toHaveBeenCalled(); expect(existsSync(outputPath)).toBe(false); }); + +it("requires gateway ownership before accepting observability", () => { + expect(() => parseEvalSessionCliArgs(["--request", "request.json", "--output", "artifact.json", "--observability-config", "otel.json"])) + .toThrow("requires an explicit controller gateway"); +}); + +it("rejects a symlinked observability descriptor", async () => { + const f = await fixture(), observation = await observabilityFixture(f); + const linked = join(f.root, "linked.json"); + await symlink(observation.file, linked); + const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); + try { + await expect(prepareEvalObservabilityRuntime(linked, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow(); + } finally { await gateway.close(); } +}); diff --git a/packages/paperclip-runner/src/cli/eval-observability-runtime.ts b/packages/paperclip-runner/src/cli/eval-observability-runtime.ts new file mode 100644 index 00000000000..5bbe66b82d3 --- /dev/null +++ b/packages/paperclip-runner/src/cli/eval-observability-runtime.ts @@ -0,0 +1,135 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; +import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js"; +import { + applyProviderObservabilityEnvironment, + writeManagedCodexObservability, + type ProviderObservabilityLaunch, +} from "../drivers/agent-observability.js"; +import type { EvalSessionRequest } from "./eval-session-contract.js"; +import type { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js"; + +type Gateway = Awaited>; +const invalid = () => new Error("Invalid controlled eval observability configuration"); +function object(value: unknown, names: string[]): Record { + if (!value || typeof value !== "object" || Array.isArray(value) + || Object.keys(value).length !== names.length || Object.keys(value).some(key => !names.includes(key))) throw invalid(); + return value as Record; +} +function text(value: unknown): string { + if (typeof value !== "string" || !value || value.length > 4096 || /[\r\n\0]/.test(value)) throw invalid(); + return value; +} +function decodedMap(value: unknown): Record { + const result: Record = {}; + for (const item of text(value).split(",")) { + const index = item.indexOf("="); + if (index < 1) throw invalid(); + const key = item.slice(0, index); + if (Object.hasOwn(result, key)) throw invalid(); + result[key] = decodeURIComponent(item.slice(index + 1)); + } + return result; +} +function same(value: unknown, expected: Record) { + const actual = object(value, Object.keys(expected)); + if (Object.entries(expected).some(([key, item]) => actual[key] !== item)) throw invalid(); +} +function parentHeader(value: unknown, projectId: string, traceId: string, spanId: string): string { + const headers = decodedMap(value), parent = text(headers["x-bt-parent"]); + object(headers, ["x-bt-parent"]); + const bytes = Buffer.from(parent, "base64"); + if (bytes.toString("base64") !== parent || !bytes.subarray(0, 3).equals(Buffer.from([4, 2, 0]))) throw invalid(); + same(JSON.parse(bytes.subarray(3).toString("utf8")), { object_id: projectId, row_id: spanId, span_id: spanId, root_span_id: traceId }); + return parent; +} + +export async function prepareEvalObservabilityRuntime( + file: string, + request: EvalSessionRequest, + gateway: Gateway, + environment: NodeJS.ProcessEnv = process.env, +) { + let raw: Buffer; + const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > 131_072) throw invalid(); + raw = Buffer.alloc(stat.size + 1); + const { bytesRead } = await handle.read(raw, 0, raw.length, 0); + if (bytesRead !== stat.size) throw invalid(); + raw = raw.subarray(0, bytesRead); + } finally { await handle.close(); } + let launch: ProviderObservabilityLaunch; + let projectId: string, traceId: string, spanId: string; + try { + const config = object(JSON.parse(raw.toString("utf8")), ["schema", "campaignId", "executionId", "attemptId", "projectId", "traceId", "spanId", "baseUrl", "launch"]); + projectId = text(config.projectId); traceId = text(config.traceId); spanId = text(config.spanId); + if (config.schema !== "paperclip.runner-eval.observability/v1" || config.attemptId !== request.attemptId + || config.campaignId !== gateway.evidence.campaignId || config.executionId !== gateway.evidence.executionId + || !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(projectId) + || !/^[a-f0-9]{32}$/.test(traceId) || /^0+$/.test(traceId) || !/^[a-f0-9]{16}$/.test(spanId) || /^0+$/.test(spanId) + || !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}\/otel$/.test(text(config.baseUrl)) + || Number(new URL(config.baseUrl as string).port) > 65535) throw invalid(); + const kind = request.provider === "codex" ? "codex" : request.provider === "acpx" && request.acpxAgent === "claude" ? "claude" : null; + if (!kind) throw invalid(); + const source = object(config.launch, ["kind", "compatibilityKey", "environment", ...(kind === "codex" ? ["otelConfig"] : [])]); + if (source.kind !== kind || !/^[A-Za-z0-9_-]{1,128}$/.test(text(source.compatibilityKey))) throw invalid(); + const state = new CapabilityMockControlPlaneAdapter(request.session.seed).snapshot(); + const attributes = { + "paperclip.company.id": request.session.companyId ?? state.company.id, + "paperclip.agent.id": request.session.actorId ?? state.actors[0]!.id, + "paperclip.observability.revision": source.compatibilityKey, + }; + const common = ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"]; + const env = object(source.environment, kind === "codex" ? common : [...common, + "CLAUDE_CODE_ENABLE_TELEMETRY", "CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", "OTEL_LOGS_EXPORTER", "OTEL_TRACES_EXPORTER", "OTEL_METRICS_EXPORTER", + "OTEL_LOG_USER_PROMPTS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_TOOL_CONTENT", "OTEL_LOG_RAW_API_BODIES", + "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_RESOURCE_ATTRIBUTES", + ]); + for (const value of Object.values(env)) text(value); + for (const signal of ["LOGS", "TRACES"]) { + if (!/^[1-9][0-9]{0,3}$/.test(env[`OTEL_EXPORTER_OTLP_${signal}_TIMEOUT`] as string) || Number(env[`OTEL_EXPORTER_OTLP_${signal}_TIMEOUT`]) > 3000) throw invalid(); + parentHeader(env[`OTEL_EXPORTER_OTLP_${signal}_HEADERS`], projectId, traceId, spanId); + } + if (kind === "codex") { + const otel = object(source.otelConfig, ["exporter", "trace_exporter", "metrics_exporter", "log_user_prompt", "log_agent_responses", "log_guardian_assessments", "tool_result", "span_attributes"]); + for (const [key, signal] of [["exporter", "logs"], ["trace_exporter", "traces"]]) { + const exporter = object(otel[key!], ["otlp-http"]); + const http = object(exporter["otlp-http"], ["endpoint", "protocol", "headers"]); + if (http.endpoint !== `${config.baseUrl}/v1/${signal}` || http.protocol !== "binary") throw invalid(); + object(http.headers, []); + } + const tool = object(otel.tool_result, ["max_bytes"]); + if (otel.metrics_exporter !== "none" || otel.log_guardian_assessments !== false + || typeof otel.log_user_prompt !== "boolean" || typeof otel.log_agent_responses !== "boolean" + || ![0, 65536].includes(tool.max_bytes as number)) throw invalid(); + same(otel.span_attributes, attributes); + } else { + const fixed: Record = { CLAUDE_CODE_ENABLE_TELEMETRY: "1", CLAUDE_CODE_ENHANCED_TELEMETRY_BETA: "1", + OTEL_LOGS_EXPORTER: "otlp", OTEL_TRACES_EXPORTER: "otlp", OTEL_METRICS_EXPORTER: "none", OTEL_LOG_RAW_API_BODIES: "0", + OTEL_EXPORTER_OTLP_LOGS_PROTOCOL: "http/protobuf", OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: "http/protobuf", + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: `${config.baseUrl}/v1/logs`, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: `${config.baseUrl}/v1/traces` }; + if (Object.entries(fixed).some(([key, value]) => env[key] !== value) + || ["OTEL_LOG_USER_PROMPTS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_TOOL_CONTENT"].some(key => !["0", "1"].includes(env[key] as string))) throw invalid(); + same(decodedMap(env.OTEL_RESOURCE_ATTRIBUTES), attributes); + } + launch = structuredClone(source) as unknown as ProviderObservabilityLaunch; + } catch { throw invalid(); } + const token = environment.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN; + if (typeof token !== "string" || !/^[a-f0-9]{64}$/.test(token)) throw new Error("Missing or invalid controlled eval OTLP token"); + const launchEnvironment = { ...launch.environment }; + for (const signal of ["LOGS", "TRACES"]) { + const key = `OTEL_EXPORTER_OTLP_${signal}_HEADERS`; + launchEnvironment[key] += `,authorization=${encodeURIComponent(`Bearer ${token}`)}`; + } + launch = { ...launch, environment: launchEnvironment }; + if (launch.kind === "codex") writeManagedCodexObservability(gateway.transportOptions.environment.CODEX_HOME!, launch.otelConfig!); + return { + transportOptions: { ...gateway.transportOptions, observability: launch, + environment: applyProviderObservabilityEnvironment(gateway.transportOptions.environment, launch) }, + evidence: { mode: "controller_otlp" as const, configured: true, verification: "unverified" as const, + projectId, traceId, spanId, descriptorSha256: createHash("sha256").update(raw).digest("hex") }, + }; +} diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index c42a22ec933..cae0b83a61b 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -37,6 +37,8 @@ import { } from "./eval-session-contract.js"; import { evalProviderTransportOptions } from "./eval-provider-runtime.js"; import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js"; +import { prepareEvalObservabilityRuntime } from "./eval-observability-runtime.js"; +import { redactObservabilityDiagnostic } from "../drivers/agent-observability.js"; import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; interface EvalSessionCliOptions { @@ -45,6 +47,7 @@ interface EvalSessionCliOptions { candidateProfile?: EvalCandidateProfile; expectedAcpxProfile?: Record; gatewayConfigPath?: string; + observabilityConfigPath?: string; } function argument(args: string[], name: string): string { @@ -55,7 +58,7 @@ function argument(args: string[], name: string): string { } export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions { - const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile", "--gateway-config"]); + const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile", "--gateway-config", "--observability-config"]); const seen = new Set(); for (let index = 0; index < args.length; index += 2) { if (!allowed.has(args[index] ?? "")) { @@ -65,6 +68,7 @@ export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions { seen.add(args[index]!); if (args[index + 1] === undefined || args[index + 1]!.startsWith("--")) throw new Error(`missing ${args[index]}`); } + if (seen.has("--observability-config") && !seen.has("--gateway-config")) throw new Error("Eval observability requires an explicit controller gateway"); const candidateIndex = args.indexOf("--candidate-profile"); const candidateProfile = candidateIndex < 0 ? undefined : args[candidateIndex + 1]; if (candidateProfile !== undefined && candidateProfile !== "pi" && candidateProfile !== "cursor" && candidateProfile !== "copilot") { @@ -93,6 +97,7 @@ export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions { ...(candidateProfile === undefined ? {} : { candidateProfile }), ...(expectedAcpxProfile === undefined ? {} : { expectedAcpxProfile }), ...(seen.has("--gateway-config") ? { gatewayConfigPath: argument(args, "--gateway-config") } : {}), + ...(seen.has("--observability-config") ? { observabilityConfigPath: argument(args, "--observability-config") } : {}), requestPath: argument(args, "--request"), outputPath: argument(args, "--output"), }; @@ -406,11 +411,14 @@ export async function runEvalSessionCli( const gateway = cli.gatewayConfigPath ? await prepareEvalGatewayRuntime(cli.gatewayConfigPath, request) : undefined; + let observability: Awaited> | undefined; + const redact = (message: string) => redactObservabilityDiagnostic(message, observability?.transportOptions.environment, observability?.transportOptions.observability.otelConfig); let session: CapabilityLiveSession | null = null; let turn: CapabilityLiveTurnResult | null = null; let snapshot: CapabilityLiveSessionSnapshot | null = null; try { + if (cli.observabilityConfigPath) observability = await prepareEvalObservabilityRuntime(cli.observabilityConfigPath, request, gateway!); const runtimeContext = await prepareEvalRuntimeContext( resolve(request.session.workingDirectory ?? process.cwd()), ); @@ -422,8 +430,9 @@ export async function runEvalSessionCli( baseInstructions: evalRuntimeSystemInstructions(runtimeContext), environment: process.env, ...gateway?.transportOptions, + ...observability?.transportOptions, onDiagnostic: (message) => { - process.stderr.write(`[eval-session runnerd] ${message}\n`); + process.stderr.write(`[eval-session runnerd] ${redact(message)}\n`); }, }; const service = options.serviceFactory?.(runnerdPath, transportOptions) @@ -468,13 +477,14 @@ export async function runEvalSessionCli( await closeSession(session, "eval session complete"); snapshot = session.snapshot(); - await writeFile(cli.outputPath, `${JSON.stringify({ + await writeFile(cli.outputPath, redact(`${JSON.stringify({ schema: "paperclip-runner/eval-session-artifact/v1", attemptId: request.attemptId, build: PAPERCLIP_RUNNER_BUILD_METADATA, runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` }, requestedModel: request.model, ...(gateway ? { requestRouting: gateway.evidence } : {}), + ...(observability ? { nativeObservability: observability.evidence } : {}), ...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }), provider: requestedProvider, driver: requestedDriver, @@ -520,7 +530,7 @@ export async function runEvalSessionCli( finishedAt: new Date().toISOString(), durationMs: Date.now() - startedAtMs, }, - }, null, 2)}\n`); + }, null, 2)}\n`)); return accountingError === null ? 0 : 2; } catch (error) { if (session !== null) { @@ -546,7 +556,7 @@ export async function runEvalSessionCli( snapshot = session.snapshot(); } const usage = usageIfAvailable(request, snapshot); - await writeFile(cli.outputPath, `${JSON.stringify({ + await writeFile(cli.outputPath, redact(`${JSON.stringify({ schema: "paperclip-runner/eval-session-artifact/v1", attemptId: request.attemptId, infrastructureError: error instanceof Error ? error.message : String(error), @@ -555,6 +565,7 @@ export async function runEvalSessionCli( runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` }, requestedModel: request.model, ...(gateway ? { requestRouting: gateway.evidence } : {}), + ...(observability ? { nativeObservability: observability.evidence } : {}), ...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }), provider: requestedProvider, driver: requestedDriver, @@ -579,7 +590,7 @@ export async function runEvalSessionCli( finishedAt: new Date().toISOString(), durationMs: Date.now() - startedAtMs, }, - }, null, 2)}\n`); + }, null, 2)}\n`)); return 2; } finally { await gateway?.close();