Add explicit native observability route to Runner eval CLI

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -159,6 +159,45 @@ change the hosted matrix. The controller must enforce request admission,
 deadlines, and limits before forwarding traffic. A local fake-API probe only
 proves routing; it does not qualify live provider or Braintrust behavior.
 
+## Controller-assigned native observability
+
+The same CLI accepts `--observability-config /absolute/path/observability.json`
+only with `--gateway-config`. This opt-in route supports native Codex and ACPX
+Claude. Without that argument, the gateway route exports no native traces.
+
+The descriptor uses schema `paperclip.runner-eval.observability/v1`. It contains
+`campaignId`, `executionId`, and `attemptId` matching the gateway and request;
+`projectId` (UUID), `traceId` (32 lowercase hex characters), and `spanId` (16
+lowercase hex characters); `baseUrl` in the form
+`http://127.0.0.1:PORT/otel`; and `launch`, the secret-free result from the existing
+`buildProviderObservabilityLaunch` in `packages/adapter-utils`. The CLI accepts
+only its known Codex/Claude exporter settings and the mock session's exact
+company and agent attributes. Bind `revision` to `compatibilityKey`. Use a
+compatibility key of at most 128 letters, digits, underscores, or hyphens.
+
+Build the launch with `/v1/logs` and `/v1/traces` under that base URL, a timeout
+of at most 3 seconds, and only the `x-bt-parent` header from
+`exportBraintrustAttemptParent` for those project/trace/span IDs. Do not include
+upstream authorization in the file. Supply the attempt's separate 32-byte
+hexadecimal relay credential through `PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN`. The CLI
+adds authorization to the provider environment and redacts it from diagnostics
+and artifacts. Codex configuration is written only into the disposable gateway
+home; the existing model route is preserved. The descriptor must be a regular
+file, not a symlink, and fit within 128 KiB.
+
+Success and failure artifacts record `nativeObservability` with the declared
+project/trace/span IDs, descriptor SHA-256, `configured: true`, and
+`verification: "unverified"`. Configuration does not prove export, ingestion,
+indexing, complete capture, or billing coverage. Relay receipts and destination
+readback must establish those separate facts. Provider errors keep their
+original outcome even when traces arrive.
+
+The controller owns the relay, credentials, ingestion quotas, deadline, drain,
+and private receipt retention. This CLI option does not start a relay, set an
+operating allowance, or enable a paid campaign. The existing Product E2E relay
+and launch generator remain the integration owners; the hosted Runner workflow
+still needs its own qualified controller integration.
+
 ## Hosted campaign
 
 Use the `Runner Direct Live Protocol Evals` workflow. Dispatch the workflow

packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts

@@ -1,6 +1,6 @@
 import { createHash } from "node:crypto";
 import { existsSync } from "node:fs";
-import { chmod, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
+import { chmod, mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, expect, it, vi } from "vitest";
@@ -10,7 +10,11 @@ import { createCapabilityFixtureState } from "../mock-core/capability-control-pl
 import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js";
 import { parseEvalSessionRequest } from "./eval-session-contract.js";
 import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js";
-import { runEvalSessionCli } from "./eval-session.js";
+import { prepareEvalObservabilityRuntime } from "./eval-observability-runtime.js";
+import { buildProviderObservabilityLaunch } from "../../../adapter-utils/src/agent-observability.js";
+import { exportBraintrustAttemptParent } from "../../../../tests/runner-e2e/braintrust.js";
+import { parse } from "smol-toml";
+import { parseEvalSessionCliArgs, runEvalSessionCli } from "./eval-session.js";
 
 const roots: string[] = [];
 afterEach(async () => {
@@ -42,6 +46,82 @@ async function fixture(provider = "codex", acpxAgent?: string) {
   return { root, request, descriptor, descriptorPath };
 }
 
+const otlpToken = "b".repeat(64);
+async function observabilityFixture(f: Awaited<ReturnType<typeof fixture>>) {
+  const identity = { projectId: "11111111-1111-4111-8111-111111111111", traceId: "1".repeat(32), spanId: "2".repeat(16) };
+  const baseUrl = "http://127.0.0.1:4568/otel";
+  const descriptor = {
+    schema: "paperclip.runner-eval.observability/v1", campaignId: f.descriptor.campaignId,
+    executionId: f.descriptor.executionId, attemptId: f.request.attemptId, ...identity, baseUrl,
+    launch: buildProviderObservabilityLaunch({ kind: f.request.provider === "codex" ? "codex" : "claude",
+      companyId: "company-1", agentId: "actor-1", revision: "revision-1", compatibilityKey: "revision-1", enabled: true,
+      capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true },
+      logsEndpoint: `${baseUrl}/v1/logs`, tracesEndpoint: `${baseUrl}/v1/traces`,
+      headers: { "x-bt-parent": exportBraintrustAttemptParent(identity) },
+    }),
+  };
+  const file = join(f.root, "observability.json");
+  await writeFile(file, JSON.stringify(descriptor));
+  return { descriptor, file };
+}
+
+it.each([["codex", undefined], ["acpx", "claude"]])("applies the existing %s/%s launch contract only to its owned gateway home", async (provider, agent) => {
+  const f = await fixture(provider, agent), observation = await observabilityFixture(f);
+  const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token });
+  try {
+    const result = await prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken, BRAINTRUST_API_KEY: "upstream-secret" });
+    const launch = result.transportOptions.observability;
+    expect(result.evidence).toMatchObject({ mode: "controller_otlp", configured: true, verification: "unverified", projectId: observation.descriptor.projectId });
+    expect(JSON.stringify(result.evidence)).not.toContain(otlpToken);
+    expect(JSON.stringify(result)).not.toContain("upstream-secret");
+    expect(launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toContain(`authorization=Bearer%20${otlpToken}`);
+    expect(result.transportOptions.environment.BRAINTRUST_API_KEY).toBeUndefined();
+    if (provider === "codex") {
+      const raw = await readFile(join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"), "utf8"), config = parse(raw);
+      expect(config.model_provider).toBe("paperclip_eval");
+      expect(config.otel).toEqual(observation.descriptor.launch.otelConfig);
+      expect(raw).not.toContain(otlpToken);
+      expect((await stat(join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"))).mode & 0o777).toBe(0o600);
+    } else {
+      const sanitized = createSanitizedAcpxSpawnInput(result.transportOptions.environment, "claude").env;
+      expect(sanitized.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT).toBe(observation.descriptor.baseUrl + "/v1/traces");
+      expect(sanitized.OTEL_EXPORTER_OTLP_TRACES_HEADERS).toContain(otlpToken);
+      expect(sanitized.ANTHROPIC_API_KEY).toBe(token);
+    }
+  } finally { await gateway.close(); }
+});
+
+it.each(["identity", "endpoint", "parent", "credential", "company", "rawBodies", "modelEnvironment", "extraField", "unknownExporter"])("rejects observability %s drift before changing the gateway home", async fault => {
+  const f = await fixture(), observation = await observabilityFixture(f);
+  const descriptor = observation.descriptor as unknown as Record<string, any>;
+  if (fault === "identity") descriptor.attemptId = "other";
+  if (fault === "endpoint") descriptor.launch.otelConfig.exporter["otlp-http"].endpoint = "https://api.braintrust.dev/otel/v1/logs";
+  if (fault === "parent") descriptor.traceId = "3".repeat(32);
+  if (fault === "credential") descriptor.launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS += ",authorization=Bearer%20upstream-secret";
+  if (fault === "company") descriptor.launch.otelConfig.span_attributes["paperclip.company.id"] = "other";
+  if (fault === "rawBodies") descriptor.launch.otelConfig.log_guardian_assessments = true;
+  if (fault === "modelEnvironment") descriptor.launch.environment.OPENAI_API_KEY = "upstream-secret";
+  if (fault === "extraField") descriptor.extra = "unknown";
+  if (fault === "unknownExporter") descriptor.launch.otelConfig.extra_exporter = "console";
+  await writeFile(observation.file, JSON.stringify(descriptor));
+  const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token });
+  const config = join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"), before = await readFile(config);
+  try {
+    await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow("Invalid controlled eval observability");
+    expect(await readFile(config)).toEqual(before);
+  } finally { await gateway.close(); }
+});
+
+it("refuses missing relay credentials and oversized observability files before launch", async () => {
+  const f = await fixture(), observation = await observabilityFixture(f);
+  const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token });
+  try {
+    await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, {})).rejects.toThrow("OTLP token");
+    await writeFile(observation.file, " ".repeat(131073));
+    await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow("Invalid controlled eval observability");
+  } finally { await gateway.close(); }
+});
+
 it.each([["codex", undefined], ["acpx", "claude"]])("routes %s/%s with only its proxy credential and an owned home", async (provider, agent) => {
   const f = await fixture(provider, agent);
   const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, {
@@ -102,8 +182,10 @@ it("rejects missing tokens and unsupported provider routes before runtime creati
   await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })).rejects.toThrow("does not support");
 });
 
-it.each([false, true])("the real CLI retains route identity and removes the owned home after failure=%s", async (fail) => {
+it.each([[false, false], [true, false], [false, true], [true, true]])("the real CLI retains route identity and removes its home after failure=%s, observability=%s", async (fail, withObservability) => {
   const f = await fixture();
+  const observation = withObservability ? await observabilityFixture(f) : undefined;
+  vi.stubEnv("PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN", otlpToken);
   vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token);
   vi.stubEnv("OPENAI_API_KEY", "upstream-must-not-reach-provider");
   const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json");
@@ -122,24 +204,27 @@ it.each([false, true])("the real CLI retains route identity and removes the owne
   let home: string | undefined;
   const factory = vi.fn((_binary, options) => {
     expect(options.homeMode).toBe("preconfigured");
+    expect(Boolean(options.observability)).toBe(Boolean(withObservability));
     expect(options.environment.OPENAI_API_KEY).toBeUndefined();
     expect(options.environment.PAPERCLIP_AI_PROVIDER_KEY).toBe(token);
     home = options.environment.HOME;
     expect(existsSync(home!)).toBe(true);
     return { create: async () => ({
-      sendMessage: async () => { if (fail) throw new Error("simulated provider failure"); return { turnId: "turn-1", status: "completed", snapshot }; },
+      sendMessage: async () => { if (fail) throw new Error(`simulated provider failure${withObservability ? ` Bearer ${otlpToken}` : ""}`); return { turnId: "turn-1", status: "completed", snapshot }; },
       completeAttempt: async () => undefined, shutdown: async () => { snapshot.status = "closed"; }, snapshot: () => snapshot,
     }) } as never;
   });
-  const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath], { serviceFactory: factory });
+  const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath, ...(observation ? ["--observability-config", observation.file] : [])], { serviceFactory: factory });
   expect(result).toBe(fail ? 2 : 0);
   expect(factory).toHaveBeenCalledTimes(1);
   expect(existsSync(home!)).toBe(false);
   const raw = await readFile(outputPath, "utf8"), artifact = JSON.parse(raw);
   expect(artifact.requestRouting).toMatchObject({ mode: "controller_gateway", provider: "openai", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1" });
   expect(raw).not.toContain(token);
+  expect(raw).not.toContain(otlpToken);
+  expect(artifact.nativeObservability).toEqual(withObservability ? expect.objectContaining({ configured: true, verification: "unverified" }) : undefined);
   expect(raw).not.toContain("upstream-must-not-reach-provider");
-  if (fail) expect(artifact.infrastructureError).toBe("simulated provider failure");
+  if (fail) expect(artifact.infrastructureError).toBe(`simulated provider failure${withObservability ? " [REDACTED]" : ""}`);
 });
 
 
@@ -154,3 +239,18 @@ it.each(["codex", undefined])("rejects ACPX Codex (%s) before creating any provi
   expect(factory).not.toHaveBeenCalled();
   expect(existsSync(outputPath)).toBe(false);
 });
+
+it("requires gateway ownership before accepting observability", () => {
+  expect(() => parseEvalSessionCliArgs(["--request", "request.json", "--output", "artifact.json", "--observability-config", "otel.json"]))
+    .toThrow("requires an explicit controller gateway");
+});
+
+it("rejects a symlinked observability descriptor", async () => {
+  const f = await fixture(), observation = await observabilityFixture(f);
+  const linked = join(f.root, "linked.json");
+  await symlink(observation.file, linked);
+  const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token });
+  try {
+    await expect(prepareEvalObservabilityRuntime(linked, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow();
+  } finally { await gateway.close(); }
+});

packages/paperclip-runner/src/cli/eval-observability-runtime.ts

@@ -0,0 +1,135 @@
+import { createHash } from "node:crypto";
+import { constants } from "node:fs";
+import { open } from "node:fs/promises";
+import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js";
+import {
+  applyProviderObservabilityEnvironment,
+  writeManagedCodexObservability,
+  type ProviderObservabilityLaunch,
+} from "../drivers/agent-observability.js";
+import type { EvalSessionRequest } from "./eval-session-contract.js";
+import type { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js";
+
+type Gateway = Awaited<ReturnType<typeof prepareEvalGatewayRuntime>>;
+const invalid = () => new Error("Invalid controlled eval observability configuration");
+function object(value: unknown, names: string[]): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)
+      || Object.keys(value).length !== names.length || Object.keys(value).some(key => !names.includes(key))) throw invalid();
+  return value as Record<string, unknown>;
+}
+function text(value: unknown): string {
+  if (typeof value !== "string" || !value || value.length > 4096 || /[\r\n\0]/.test(value)) throw invalid();
+  return value;
+}
+function decodedMap(value: unknown): Record<string, string> {
+  const result: Record<string, string> = {};
+  for (const item of text(value).split(",")) {
+    const index = item.indexOf("=");
+    if (index < 1) throw invalid();
+    const key = item.slice(0, index);
+    if (Object.hasOwn(result, key)) throw invalid();
+    result[key] = decodeURIComponent(item.slice(index + 1));
+  }
+  return result;
+}
+function same(value: unknown, expected: Record<string, unknown>) {
+  const actual = object(value, Object.keys(expected));
+  if (Object.entries(expected).some(([key, item]) => actual[key] !== item)) throw invalid();
+}
+function parentHeader(value: unknown, projectId: string, traceId: string, spanId: string): string {
+  const headers = decodedMap(value), parent = text(headers["x-bt-parent"]);
+  object(headers, ["x-bt-parent"]);
+  const bytes = Buffer.from(parent, "base64");
+  if (bytes.toString("base64") !== parent || !bytes.subarray(0, 3).equals(Buffer.from([4, 2, 0]))) throw invalid();
+  same(JSON.parse(bytes.subarray(3).toString("utf8")), { object_id: projectId, row_id: spanId, span_id: spanId, root_span_id: traceId });
+  return parent;
+}
+
+export async function prepareEvalObservabilityRuntime(
+  file: string,
+  request: EvalSessionRequest,
+  gateway: Gateway,
+  environment: NodeJS.ProcessEnv = process.env,
+) {
+  let raw: Buffer;
+  const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+  try {
+    const stat = await handle.stat();
+    if (!stat.isFile() || stat.size > 131_072) throw invalid();
+    raw = Buffer.alloc(stat.size + 1);
+    const { bytesRead } = await handle.read(raw, 0, raw.length, 0);
+    if (bytesRead !== stat.size) throw invalid();
+    raw = raw.subarray(0, bytesRead);
+  } finally { await handle.close(); }
+  let launch: ProviderObservabilityLaunch;
+  let projectId: string, traceId: string, spanId: string;
+  try {
+    const config = object(JSON.parse(raw.toString("utf8")), ["schema", "campaignId", "executionId", "attemptId", "projectId", "traceId", "spanId", "baseUrl", "launch"]);
+    projectId = text(config.projectId); traceId = text(config.traceId); spanId = text(config.spanId);
+    if (config.schema !== "paperclip.runner-eval.observability/v1" || config.attemptId !== request.attemptId
+        || config.campaignId !== gateway.evidence.campaignId || config.executionId !== gateway.evidence.executionId
+        || !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(projectId)
+        || !/^[a-f0-9]{32}$/.test(traceId) || /^0+$/.test(traceId) || !/^[a-f0-9]{16}$/.test(spanId) || /^0+$/.test(spanId)
+        || !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}\/otel$/.test(text(config.baseUrl))
+        || Number(new URL(config.baseUrl as string).port) > 65535) throw invalid();
+    const kind = request.provider === "codex" ? "codex" : request.provider === "acpx" && request.acpxAgent === "claude" ? "claude" : null;
+    if (!kind) throw invalid();
+    const source = object(config.launch, ["kind", "compatibilityKey", "environment", ...(kind === "codex" ? ["otelConfig"] : [])]);
+    if (source.kind !== kind || !/^[A-Za-z0-9_-]{1,128}$/.test(text(source.compatibilityKey))) throw invalid();
+    const state = new CapabilityMockControlPlaneAdapter(request.session.seed).snapshot();
+    const attributes = {
+      "paperclip.company.id": request.session.companyId ?? state.company.id,
+      "paperclip.agent.id": request.session.actorId ?? state.actors[0]!.id,
+      "paperclip.observability.revision": source.compatibilityKey,
+    };
+    const common = ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"];
+    const env = object(source.environment, kind === "codex" ? common : [...common,
+      "CLAUDE_CODE_ENABLE_TELEMETRY", "CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", "OTEL_LOGS_EXPORTER", "OTEL_TRACES_EXPORTER", "OTEL_METRICS_EXPORTER",
+      "OTEL_LOG_USER_PROMPTS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_TOOL_CONTENT", "OTEL_LOG_RAW_API_BODIES",
+      "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_RESOURCE_ATTRIBUTES",
+    ]);
+    for (const value of Object.values(env)) text(value);
+    for (const signal of ["LOGS", "TRACES"]) {
+      if (!/^[1-9][0-9]{0,3}$/.test(env[`OTEL_EXPORTER_OTLP_${signal}_TIMEOUT`] as string) || Number(env[`OTEL_EXPORTER_OTLP_${signal}_TIMEOUT`]) > 3000) throw invalid();
+      parentHeader(env[`OTEL_EXPORTER_OTLP_${signal}_HEADERS`], projectId, traceId, spanId);
+    }
+    if (kind === "codex") {
+      const otel = object(source.otelConfig, ["exporter", "trace_exporter", "metrics_exporter", "log_user_prompt", "log_agent_responses", "log_guardian_assessments", "tool_result", "span_attributes"]);
+      for (const [key, signal] of [["exporter", "logs"], ["trace_exporter", "traces"]]) {
+        const exporter = object(otel[key!], ["otlp-http"]);
+        const http = object(exporter["otlp-http"], ["endpoint", "protocol", "headers"]);
+        if (http.endpoint !== `${config.baseUrl}/v1/${signal}` || http.protocol !== "binary") throw invalid();
+        object(http.headers, []);
+      }
+      const tool = object(otel.tool_result, ["max_bytes"]);
+      if (otel.metrics_exporter !== "none" || otel.log_guardian_assessments !== false
+          || typeof otel.log_user_prompt !== "boolean" || typeof otel.log_agent_responses !== "boolean"
+          || ![0, 65536].includes(tool.max_bytes as number)) throw invalid();
+      same(otel.span_attributes, attributes);
+    } else {
+      const fixed: Record<string, string> = { CLAUDE_CODE_ENABLE_TELEMETRY: "1", CLAUDE_CODE_ENHANCED_TELEMETRY_BETA: "1",
+        OTEL_LOGS_EXPORTER: "otlp", OTEL_TRACES_EXPORTER: "otlp", OTEL_METRICS_EXPORTER: "none", OTEL_LOG_RAW_API_BODIES: "0",
+        OTEL_EXPORTER_OTLP_LOGS_PROTOCOL: "http/protobuf", OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: "http/protobuf",
+        OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: `${config.baseUrl}/v1/logs`, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: `${config.baseUrl}/v1/traces` };
+      if (Object.entries(fixed).some(([key, value]) => env[key] !== value)
+          || ["OTEL_LOG_USER_PROMPTS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_TOOL_CONTENT"].some(key => !["0", "1"].includes(env[key] as string))) throw invalid();
+      same(decodedMap(env.OTEL_RESOURCE_ATTRIBUTES), attributes);
+    }
+    launch = structuredClone(source) as unknown as ProviderObservabilityLaunch;
+  } catch { throw invalid(); }
+  const token = environment.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN;
+  if (typeof token !== "string" || !/^[a-f0-9]{64}$/.test(token)) throw new Error("Missing or invalid controlled eval OTLP token");
+  const launchEnvironment = { ...launch.environment };
+  for (const signal of ["LOGS", "TRACES"]) {
+    const key = `OTEL_EXPORTER_OTLP_${signal}_HEADERS`;
+    launchEnvironment[key] += `,authorization=${encodeURIComponent(`Bearer ${token}`)}`;
+  }
+  launch = { ...launch, environment: launchEnvironment };
+  if (launch.kind === "codex") writeManagedCodexObservability(gateway.transportOptions.environment.CODEX_HOME!, launch.otelConfig!);
+  return {
+    transportOptions: { ...gateway.transportOptions, observability: launch,
+      environment: applyProviderObservabilityEnvironment(gateway.transportOptions.environment, launch) },
+    evidence: { mode: "controller_otlp" as const, configured: true, verification: "unverified" as const,
+      projectId, traceId, spanId, descriptorSha256: createHash("sha256").update(raw).digest("hex") },
+  };
+}

packages/paperclip-runner/src/cli/eval-session.ts

@@ -37,6 +37,8 @@ import {
 } from "./eval-session-contract.js";
 import { evalProviderTransportOptions } from "./eval-provider-runtime.js";
 import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js";
+import { prepareEvalObservabilityRuntime } from "./eval-observability-runtime.js";
+import { redactObservabilityDiagnostic } from "../drivers/agent-observability.js";
 import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js";
 
 interface EvalSessionCliOptions {
@@ -45,6 +47,7 @@ interface EvalSessionCliOptions {
   candidateProfile?: EvalCandidateProfile;
   expectedAcpxProfile?: Record<string, unknown>;
   gatewayConfigPath?: string;
+  observabilityConfigPath?: string;
 }
 
 function argument(args: string[], name: string): string {
@@ -55,7 +58,7 @@ function argument(args: string[], name: string): string {
 }
 
 export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions {
-  const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile", "--gateway-config"]);
+  const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile", "--gateway-config", "--observability-config"]);
   const seen = new Set<string>();
   for (let index = 0; index < args.length; index += 2) {
     if (!allowed.has(args[index] ?? "")) {
@@ -65,6 +68,7 @@ export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions {
     seen.add(args[index]!);
     if (args[index + 1] === undefined || args[index + 1]!.startsWith("--")) throw new Error(`missing ${args[index]}`);
   }
+  if (seen.has("--observability-config") && !seen.has("--gateway-config")) throw new Error("Eval observability requires an explicit controller gateway");
   const candidateIndex = args.indexOf("--candidate-profile");
   const candidateProfile = candidateIndex < 0 ? undefined : args[candidateIndex + 1];
   if (candidateProfile !== undefined && candidateProfile !== "pi" && candidateProfile !== "cursor" && candidateProfile !== "copilot") {
@@ -93,6 +97,7 @@ export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions {
     ...(candidateProfile === undefined ? {} : { candidateProfile }),
     ...(expectedAcpxProfile === undefined ? {} : { expectedAcpxProfile }),
     ...(seen.has("--gateway-config") ? { gatewayConfigPath: argument(args, "--gateway-config") } : {}),
+    ...(seen.has("--observability-config") ? { observabilityConfigPath: argument(args, "--observability-config") } : {}),
     requestPath: argument(args, "--request"),
     outputPath: argument(args, "--output"),
   };
@@ -406,11 +411,14 @@ export async function runEvalSessionCli(
   const gateway = cli.gatewayConfigPath
     ? await prepareEvalGatewayRuntime(cli.gatewayConfigPath, request)
     : undefined;
+  let observability: Awaited<ReturnType<typeof prepareEvalObservabilityRuntime>> | undefined;
+  const redact = (message: string) => redactObservabilityDiagnostic(message, observability?.transportOptions.environment, observability?.transportOptions.observability.otelConfig);
   let session: CapabilityLiveSession | null = null;
   let turn: CapabilityLiveTurnResult | null = null;
   let snapshot: CapabilityLiveSessionSnapshot | null = null;
 
   try {
+    if (cli.observabilityConfigPath) observability = await prepareEvalObservabilityRuntime(cli.observabilityConfigPath, request, gateway!);
     const runtimeContext = await prepareEvalRuntimeContext(
       resolve(request.session.workingDirectory ?? process.cwd()),
     );
@@ -422,8 +430,9 @@ export async function runEvalSessionCli(
       baseInstructions: evalRuntimeSystemInstructions(runtimeContext),
       environment: process.env,
       ...gateway?.transportOptions,
+      ...observability?.transportOptions,
       onDiagnostic: (message) => {
-        process.stderr.write(`[eval-session runnerd] ${message}\n`);
+        process.stderr.write(`[eval-session runnerd] ${redact(message)}\n`);
       },
     };
     const service = options.serviceFactory?.(runnerdPath, transportOptions)
@@ -468,13 +477,14 @@ export async function runEvalSessionCli(
     await closeSession(session, "eval session complete");
     snapshot = session.snapshot();
 
-    await writeFile(cli.outputPath, `${JSON.stringify({
+    await writeFile(cli.outputPath, redact(`${JSON.stringify({
       schema: "paperclip-runner/eval-session-artifact/v1",
       attemptId: request.attemptId,
       build: PAPERCLIP_RUNNER_BUILD_METADATA,
       runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` },
       requestedModel: request.model,
       ...(gateway ? { requestRouting: gateway.evidence } : {}),
+      ...(observability ? { nativeObservability: observability.evidence } : {}),
       ...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }),
       provider: requestedProvider,
       driver: requestedDriver,
@@ -520,7 +530,7 @@ export async function runEvalSessionCli(
         finishedAt: new Date().toISOString(),
         durationMs: Date.now() - startedAtMs,
       },
-    }, null, 2)}\n`);
+    }, null, 2)}\n`));
     return accountingError === null ? 0 : 2;
   } catch (error) {
     if (session !== null) {
@@ -546,7 +556,7 @@ export async function runEvalSessionCli(
       snapshot = session.snapshot();
     }
     const usage = usageIfAvailable(request, snapshot);
-    await writeFile(cli.outputPath, `${JSON.stringify({
+    await writeFile(cli.outputPath, redact(`${JSON.stringify({
       schema: "paperclip-runner/eval-session-artifact/v1",
       attemptId: request.attemptId,
       infrastructureError: error instanceof Error ? error.message : String(error),
@@ -555,6 +565,7 @@ export async function runEvalSessionCli(
       runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` },
       requestedModel: request.model,
       ...(gateway ? { requestRouting: gateway.evidence } : {}),
+      ...(observability ? { nativeObservability: observability.evidence } : {}),
       ...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }),
       provider: requestedProvider,
       driver: requestedDriver,
@@ -579,7 +590,7 @@ export async function runEvalSessionCli(
         finishedAt: new Date().toISOString(),
         durationMs: Date.now() - startedAtMs,
       },
-    }, null, 2)}\n`);
+    }, null, 2)}\n`));
     return 2;
   } finally {
     await gateway?.close();