diff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts index d4e889ecf32..cd1cb1e03e1 100644 --- a/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts +++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; -import { chmod, mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { chmod, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; @@ -10,11 +10,7 @@ import { createCapabilityFixtureState } from "../mock-core/capability-control-pl import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js"; import { parseEvalSessionRequest } from "./eval-session-contract.js"; import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js"; -import { prepareEvalObservabilityRuntime } from "./eval-observability-runtime.js"; -import { buildProviderObservabilityLaunch } from "../../../adapter-utils/src/agent-observability.js"; -import { exportBraintrustAttemptParent } from "../../../../tests/runner-e2e/braintrust.js"; -import { parse } from "smol-toml"; -import { parseEvalSessionCliArgs, runEvalSessionCli } from "./eval-session.js"; +import { runEvalSessionCli } from "./eval-session.js"; const roots: string[] = []; afterEach(async () => { @@ -46,83 +42,6 @@ async function fixture(provider = "codex", acpxAgent?: string) { return { root, request, descriptor, descriptorPath }; } -const otlpToken = "b".repeat(64); -async function observabilityFixture(f: Awaited>) { - const identity = { projectId: "11111111-1111-4111-8111-111111111111", traceId: "1".repeat(32), spanId: "2".repeat(16) }; - const baseUrl = "http://127.0.0.1:4568/otel"; - const descriptor = { - schema: "paperclip.runner-eval.observability/v1", campaignId: f.descriptor.campaignId, - executionId: f.descriptor.executionId, attemptId: f.request.attemptId, ...identity, baseUrl, - launch: buildProviderObservabilityLaunch({ kind: f.request.provider === "codex" ? "codex" : "claude", - companyId: "company-1", agentId: "actor-1", revision: "revision-1", compatibilityKey: "revision-1", enabled: true, - capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true }, - logsEndpoint: `${baseUrl}/v1/logs`, tracesEndpoint: `${baseUrl}/v1/traces`, - headers: { "x-bt-parent": exportBraintrustAttemptParent(identity) }, - }), - }; - const file = join(f.root, "observability.json"); - await writeFile(file, JSON.stringify(descriptor)); - return { descriptor, file }; -} - -it.each([["codex", undefined], ["acpx", "claude"]])("applies the existing %s/%s launch contract only to its owned gateway home", async (provider, agent) => { - const f = await fixture(provider, agent), observation = await observabilityFixture(f); - const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); - try { - const result = await prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken, BRAINTRUST_API_KEY: "upstream-secret" }); - const launch = result.transportOptions.observability; - expect(result.evidence).toMatchObject({ mode: "controller_otlp", configured: true, verification: "unverified", projectId: observation.descriptor.projectId }); - expect(JSON.stringify(result.evidence)).not.toContain(otlpToken); - expect(JSON.stringify(result)).not.toContain("upstream-secret"); - expect(launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toContain(`authorization=Bearer%20${otlpToken}`); - expect(result.transportOptions.environment.BRAINTRUST_API_KEY).toBeUndefined(); - if (provider === "codex") { - const raw = await readFile(join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"), "utf8"), config = parse(raw); - expect(config.model_provider).toBe("paperclip_eval"); - expect(config.otel).toEqual(observation.descriptor.launch.otelConfig); - expect(raw).not.toContain(otlpToken); - expect((await stat(join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"))).mode & 0o777).toBe(0o600); - } else { - const sanitized = createSanitizedAcpxSpawnInput(result.transportOptions.environment, "claude").env; - expect(sanitized.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT).toBe(observation.descriptor.baseUrl + "/v1/traces"); - expect(sanitized.OTEL_EXPORTER_OTLP_TRACES_HEADERS).toContain(otlpToken); - expect(sanitized.ANTHROPIC_API_KEY).toBe(token); - } - } finally { await gateway.close(); } -}); - -it.each(["identity", "endpoint", "parent", "credential", "company", "rawBodies", "modelEnvironment", "extraField", "unknownExporter", "prototypeHeader"])("rejects observability %s drift before changing the gateway home", async fault => { - const f = await fixture(), observation = await observabilityFixture(f); - const descriptor = observation.descriptor as unknown as Record; - if (fault === "identity") descriptor.attemptId = "other"; - if (fault === "endpoint") descriptor.launch.otelConfig.exporter["otlp-http"].endpoint = "https://api.braintrust.dev/otel/v1/logs"; - if (fault === "parent") descriptor.traceId = "3".repeat(32); - if (fault === "prototypeHeader") descriptor.launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS += ",__proto__=ignored"; - if (fault === "credential") descriptor.launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS += ",authorization=Bearer%20upstream-secret"; - if (fault === "company") descriptor.launch.otelConfig.span_attributes["paperclip.company.id"] = "other"; - if (fault === "rawBodies") descriptor.launch.otelConfig.log_guardian_assessments = true; - if (fault === "modelEnvironment") descriptor.launch.environment.OPENAI_API_KEY = "upstream-secret"; - if (fault === "extraField") descriptor.extra = "unknown"; - if (fault === "unknownExporter") descriptor.launch.otelConfig.extra_exporter = "console"; - await writeFile(observation.file, JSON.stringify(descriptor)); - const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); - const config = join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"), before = await readFile(config); - try { - await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow("Invalid controlled eval observability"); - expect(await readFile(config)).toEqual(before); - } finally { await gateway.close(); } -}); - -it("refuses missing relay credentials and oversized observability files before launch", async () => { - const f = await fixture(), observation = await observabilityFixture(f); - const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); - try { - await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, {})).rejects.toThrow("OTLP token"); - await writeFile(observation.file, " ".repeat(131073)); - await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow("Invalid controlled eval observability"); - } finally { await gateway.close(); } -}); - it.each([["codex", undefined], ["acpx", "claude"]])("routes %s/%s with only its proxy credential and an owned home", async (provider, agent) => { const f = await fixture(provider, agent); const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { @@ -183,10 +102,8 @@ it("rejects missing tokens and unsupported provider routes before runtime creati await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })).rejects.toThrow("does not support"); }); -it.each([[false, false], [true, false], [false, true], [true, true]])("the real CLI retains route identity and removes its home after failure=%s, observability=%s", async (fail, withObservability) => { +it.each([false, true])("the real CLI retains route identity and removes the owned home after failure=%s", async (fail) => { const f = await fixture(); - const observation = withObservability ? await observabilityFixture(f) : undefined; - vi.stubEnv("PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN", otlpToken); vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token); vi.stubEnv("OPENAI_API_KEY", "upstream-must-not-reach-provider"); const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json"); @@ -205,27 +122,24 @@ it.each([[false, false], [true, false], [false, true], [true, true]])("the real let home: string | undefined; const factory = vi.fn((_binary, options) => { expect(options.homeMode).toBe("preconfigured"); - expect(Boolean(options.observability)).toBe(Boolean(withObservability)); expect(options.environment.OPENAI_API_KEY).toBeUndefined(); expect(options.environment.PAPERCLIP_AI_PROVIDER_KEY).toBe(token); home = options.environment.HOME; expect(existsSync(home!)).toBe(true); return { create: async () => ({ - sendMessage: async () => { if (fail) throw new Error(`simulated provider failure${withObservability ? ` Bearer ${otlpToken}` : ""}`); return { turnId: "turn-1", status: "completed", snapshot }; }, + sendMessage: async () => { if (fail) throw new Error("simulated provider failure"); return { turnId: "turn-1", status: "completed", snapshot }; }, completeAttempt: async () => undefined, shutdown: async () => { snapshot.status = "closed"; }, snapshot: () => snapshot, }) } as never; }); - const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath, ...(observation ? ["--observability-config", observation.file] : [])], { serviceFactory: factory }); + const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath], { serviceFactory: factory }); expect(result).toBe(fail ? 2 : 0); expect(factory).toHaveBeenCalledTimes(1); expect(existsSync(home!)).toBe(false); const raw = await readFile(outputPath, "utf8"), artifact = JSON.parse(raw); expect(artifact.requestRouting).toMatchObject({ mode: "controller_gateway", provider: "openai", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1" }); expect(raw).not.toContain(token); - expect(raw).not.toContain(otlpToken); - expect(artifact.nativeObservability).toEqual(withObservability ? expect.objectContaining({ configured: true, verification: "unverified" }) : undefined); expect(raw).not.toContain("upstream-must-not-reach-provider"); - if (fail) expect(artifact.infrastructureError).toBe(`simulated provider failure${withObservability ? " [REDACTED]" : ""}`); + if (fail) expect(artifact.infrastructureError).toBe("simulated provider failure"); }); @@ -240,18 +154,3 @@ it.each(["codex", undefined])("rejects ACPX Codex (%s) before creating any provi expect(factory).not.toHaveBeenCalled(); expect(existsSync(outputPath)).toBe(false); }); - -it("requires gateway ownership before accepting observability", () => { - expect(() => parseEvalSessionCliArgs(["--request", "request.json", "--output", "artifact.json", "--observability-config", "otel.json"])) - .toThrow("requires an explicit controller gateway"); -}); - -it("rejects a symlinked observability descriptor", async () => { - const f = await fixture(), observation = await observabilityFixture(f); - const linked = join(f.root, "linked.json"); - await symlink(observation.file, linked); - const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); - try { - await expect(prepareEvalObservabilityRuntime(linked, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow(); - } finally { await gateway.close(); } -}); diff --git a/tests/runner-e2e/runner-eval-observability.test.ts b/tests/runner-e2e/runner-eval-observability.test.ts new file mode 100644 index 00000000000..1ce5fca2f12 --- /dev/null +++ b/tests/runner-e2e/runner-eval-observability.test.ts @@ -0,0 +1,185 @@ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { chmod, mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { createSanitizedAcpxSpawnInput } from "../../packages/paperclip-runner/src/drivers/acpx/environment.js"; +import { createCapabilityFixtureState } from "../../packages/paperclip-runner/src/mock-core/capability-control-plane-types.js"; +import type { CapabilityLiveSessionSnapshot } from "../../packages/paperclip-runner/src/live/live-session.js"; +import { parseEvalSessionRequest } from "../../packages/paperclip-runner/src/cli/eval-session-contract.js"; +import { prepareEvalGatewayRuntime } from "../../packages/paperclip-runner/src/cli/eval-gateway-runtime.js"; +import { prepareEvalObservabilityRuntime } from "../../packages/paperclip-runner/src/cli/eval-observability-runtime.js"; +import { buildProviderObservabilityLaunch } from "../../packages/adapter-utils/src/agent-observability.js"; +import { exportBraintrustAttemptParent } from "./braintrust.js"; +import { createRequire } from "node:module"; +import { parseEvalSessionCliArgs, runEvalSessionCli } from "../../packages/paperclip-runner/src/cli/eval-session.js"; + +const { parse } = createRequire(new URL("../../packages/paperclip-runner/package.json", import.meta.url))("smol-toml") as { parse(text: string): Record }; +const roots: string[] = []; +afterEach(async () => { + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) { + for (const entry of await readdir(root).catch(() => [])) { + if (entry.startsWith(".paperclip-eval-runtime-context-")) await chmod(join(root, entry, "instructions"), 0o700).catch(() => undefined); + } + await rm(root, { recursive: true, force: true }); + } +}); +const token = "a".repeat(64); +async function fixture(provider = "codex", acpxAgent?: string) { + const root = await mkdtemp(join(tmpdir(), "runner-gateway-test-")); roots.push(root); + const model = acpxAgent === "claude" ? "claude-sonnet-5" : "gpt-5.6-sol"; + const binary = join(root, "runnerd"); + await writeFile(binary, "fake runnerd"); + const request = parseEvalSessionRequest({ + schema: "paperclip-runner/eval-session-request/v1", attemptId: "attempt-1", prompt: "Read the governed task.", + provider, ...(acpxAgent ? { acpxAgent } : {}), model, + runnerd: { path: binary, sha256: createHash("sha256").update("fake runnerd").digest("hex") }, + limits: { turnTimeoutMs: 120000, maxAgentTurns: 1, maxEstimatedCostNanodollars: 100000000 }, + session: { workingDirectory: root }, + }); + const descriptor = { schema: "paperclip.runner-eval.gateway/v1", provider: acpxAgent === "claude" ? "anthropic" : "openai", + model, baseUrl: "http://127.0.0.1:4567", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1", upstreamRoute: "braintrust" }; + const descriptorPath = join(root, "gateway.json"); + await writeFile(descriptorPath, JSON.stringify(descriptor)); + return { root, request, descriptor, descriptorPath }; +} + +const otlpToken = "b".repeat(64); +async function observabilityFixture(f: Awaited>) { + const identity = { projectId: "11111111-1111-4111-8111-111111111111", traceId: "1".repeat(32), spanId: "2".repeat(16) }; + const baseUrl = "http://127.0.0.1:4568/otel"; + const descriptor = { + schema: "paperclip.runner-eval.observability/v1", campaignId: f.descriptor.campaignId, + executionId: f.descriptor.executionId, attemptId: f.request.attemptId, ...identity, baseUrl, + launch: buildProviderObservabilityLaunch({ kind: f.request.provider === "codex" ? "codex" : "claude", + companyId: "company-1", agentId: "actor-1", revision: "revision-1", compatibilityKey: "revision-1", enabled: true, + capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true }, + logsEndpoint: `${baseUrl}/v1/logs`, tracesEndpoint: `${baseUrl}/v1/traces`, + headers: { "x-bt-parent": exportBraintrustAttemptParent(identity) }, + }), + }; + const file = join(f.root, "observability.json"); + await writeFile(file, JSON.stringify(descriptor)); + return { descriptor, file }; +} + +it.each([["codex", undefined], ["acpx", "claude"]])("applies the existing %s/%s launch contract only to its owned gateway home", async (provider, agent) => { + const f = await fixture(provider, agent), observation = await observabilityFixture(f); + const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); + try { + const result = await prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken, BRAINTRUST_API_KEY: "upstream-secret" }); + const launch = result.transportOptions.observability; + expect(result.evidence).toMatchObject({ mode: "controller_otlp", configured: true, verification: "unverified", projectId: observation.descriptor.projectId }); + expect(JSON.stringify(result.evidence)).not.toContain(otlpToken); + expect(JSON.stringify(result)).not.toContain("upstream-secret"); + expect(launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toContain(`authorization=Bearer%20${otlpToken}`); + expect(result.transportOptions.environment.BRAINTRUST_API_KEY).toBeUndefined(); + if (provider === "codex") { + const raw = await readFile(join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"), "utf8"), config = parse(raw); + expect(config.model_provider).toBe("paperclip_eval"); + expect(config.otel).toEqual(observation.descriptor.launch.otelConfig); + expect(raw).not.toContain(otlpToken); + expect((await stat(join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"))).mode & 0o777).toBe(0o600); + } else { + const sanitized = createSanitizedAcpxSpawnInput(result.transportOptions.environment, "claude").env; + expect(sanitized.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT).toBe(observation.descriptor.baseUrl + "/v1/traces"); + expect(sanitized.OTEL_EXPORTER_OTLP_TRACES_HEADERS).toContain(otlpToken); + expect(sanitized.ANTHROPIC_API_KEY).toBe(token); + } + } finally { await gateway.close(); } +}); + +it.each(["identity", "endpoint", "parent", "credential", "company", "rawBodies", "modelEnvironment", "extraField", "unknownExporter", "prototypeHeader"])("rejects observability %s drift before changing the gateway home", async fault => { + const f = await fixture(), observation = await observabilityFixture(f); + const descriptor = observation.descriptor as unknown as Record; + if (fault === "identity") descriptor.attemptId = "other"; + if (fault === "endpoint") descriptor.launch.otelConfig.exporter["otlp-http"].endpoint = "https://api.braintrust.dev/otel/v1/logs"; + if (fault === "parent") descriptor.traceId = "3".repeat(32); + if (fault === "prototypeHeader") descriptor.launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS += ",__proto__=ignored"; + if (fault === "credential") descriptor.launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS += ",authorization=Bearer%20upstream-secret"; + if (fault === "company") descriptor.launch.otelConfig.span_attributes["paperclip.company.id"] = "other"; + if (fault === "rawBodies") descriptor.launch.otelConfig.log_guardian_assessments = true; + if (fault === "modelEnvironment") descriptor.launch.environment.OPENAI_API_KEY = "upstream-secret"; + if (fault === "extraField") descriptor.extra = "unknown"; + if (fault === "unknownExporter") descriptor.launch.otelConfig.extra_exporter = "console"; + await writeFile(observation.file, JSON.stringify(descriptor)); + const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); + const config = join(gateway.transportOptions.environment.CODEX_HOME!, "config.toml"), before = await readFile(config); + try { + await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow("Invalid controlled eval observability"); + expect(await readFile(config)).toEqual(before); + } finally { await gateway.close(); } +}); + +it("refuses missing relay credentials and oversized observability files before launch", async () => { + const f = await fixture(), observation = await observabilityFixture(f); + const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); + try { + await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, {})).rejects.toThrow("OTLP token"); + await writeFile(observation.file, " ".repeat(131073)); + await expect(prepareEvalObservabilityRuntime(observation.file, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow("Invalid controlled eval observability"); + } finally { await gateway.close(); } +}); + +it.each([false, true])("the real CLI redacts native relay credentials and removes its home after failure=%s", async (fail) => { + const f = await fixture(); + const observation = await observabilityFixture(f); + vi.stubEnv("PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN", otlpToken); + vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token); + vi.stubEnv("OPENAI_API_KEY", "upstream-must-not-reach-provider"); + const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json"); + await writeFile(requestPath, JSON.stringify(f.request)); + const state = JSON.stringify(createCapabilityFixtureState({})); + const snapshot = { + sessionId: "session-1", revision: 1, providerThreadId: "provider-1", providerSessionId: "provider-1", + providerModel: { id: f.request.model, provider: "openai" }, status: "idle", activeTurnId: null, + createdAt: "2026-10-24T00:00:00Z", updatedAt: "2026-10-24T00:00:01Z", + authority: { companyId: "company-1", actorId: "actor-1", taskId: "task-1", runId: "run-1", sessionId: "session-1", scenarioId: "gateway-test" }, + config: { provider: "codex", driver: "codex_app_server", seedState: state, workingDirectory: f.root, scenario: { id: "gateway-test" }, capabilities: [], explicitClaims: [], turnTimeoutMs: 1000 }, + mockState: state, process: null, networkEvidence: { realPaperclipRequests: 0, childPaperclipEnvironmentKeys: [] }, + transcript: [], evidence: [], authorizationRecords: [], attempts: [], terminalTurns: [{ turnId: "turn-1", status: "completed" }], + usageLedger: [{ receiptId: "receipt-1", turnId: "turn-1", providerCalls: 1, providerRequests: 1, inputTokens: 100, outputTokens: 10, cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0 }], usageUnavailable: [], + } as unknown as CapabilityLiveSessionSnapshot; + let home: string | undefined; + const factory = vi.fn((_binary, options) => { + expect(options.homeMode).toBe("preconfigured"); + expect(options.observability).toBeDefined(); + expect(options.environment.OPENAI_API_KEY).toBeUndefined(); + expect(options.environment.PAPERCLIP_AI_PROVIDER_KEY).toBe(token); + home = options.environment.HOME; + expect(existsSync(home!)).toBe(true); + return { create: async () => ({ + sendMessage: async () => { if (fail) throw new Error(`simulated provider failure Bearer ${otlpToken}`); return { turnId: "turn-1", status: "completed", snapshot }; }, + completeAttempt: async () => undefined, shutdown: async () => { snapshot.status = "closed"; }, snapshot: () => snapshot, + }) } as never; + }); + const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath, "--observability-config", observation.file], { serviceFactory: factory }); + expect(result).toBe(fail ? 2 : 0); + expect(factory).toHaveBeenCalledTimes(1); + expect(existsSync(home!)).toBe(false); + const raw = await readFile(outputPath, "utf8"), artifact = JSON.parse(raw); + expect(artifact.requestRouting).toMatchObject({ mode: "controller_gateway", provider: "openai", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1" }); + expect(raw).not.toContain(token); + expect(raw).not.toContain(otlpToken); + expect(artifact.nativeObservability).toEqual(expect.objectContaining({ configured: true, verification: "unverified" })); + expect(raw).not.toContain("upstream-must-not-reach-provider"); + if (fail) expect(artifact.infrastructureError).toBe("simulated provider failure [REDACTED]"); +}); + + +it("requires gateway ownership before accepting observability", () => { + expect(() => parseEvalSessionCliArgs(["--request", "request.json", "--output", "artifact.json", "--observability-config", "otel.json"])) + .toThrow("requires an explicit controller gateway"); +}); + +it("rejects a symlinked observability descriptor", async () => { + const f = await fixture(), observation = await observabilityFixture(f); + const linked = join(f.root, "linked.json"); + await symlink(observation.file, linked); + const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }); + try { + await expect(prepareEvalObservabilityRuntime(linked, f.request, gateway, { PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN: otlpToken })).rejects.toThrow(); + } finally { await gateway.close(); } +});