diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index a51bd022762..6f4a3a643e1 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -156,7 +156,17 @@ keys. The server environment strips the gateway token and upstream credentials. Every upstream attempt reserves its maximum estimated charge in an fsynced journal before dispatch. Requests, retries, and cancelled calls retain distinct -IDs. Unknown usage keeps the reservation held. One shared writer serializes +IDs. Braintrust gateway requests propagate the reserved request ID, campaign, +execution, attempt and phase as allowlisted `paperclip_eval_*` metadata in the +SDK-compatible parent context. The exact context is retained in the private +receipt before dispatch, so a request canceled before response headers can +still be looked up by `metadata.paperclip_eval_request_id` in project logs. +This context contains no provider credential, prompt or response, and does not +create an extra model call or a synthetic parent span. Direct provider routes +receive no Braintrust context. Older project-only receipts cannot acquire this +identity retroactively; time or session matches remain candidate evidence. +A matching logged span does not itself prove complete usage or final billing. +Unknown usage keeps the reservation held. One shared writer serializes admission across cells, so concurrent requests cannot each spend the same remaining allowance. Campaign deadline, request count, and in-flight count are also enforced. Private request receipts retain transport IDs, response IDs, usage and cache TTLs, diff --git a/tests/runner-e2e/braintrust-gateway-context.ts b/tests/runner-e2e/braintrust-gateway-context.ts new file mode 100644 index 00000000000..8364f371a20 --- /dev/null +++ b/tests/runner-e2e/braintrust-gateway-context.ts @@ -0,0 +1,17 @@ +import { validEvalLedgerId, type EvalChargeIdentity } from "./cost-ledger.js"; + +export function braintrustGatewayRequestParent(projectId: string, requestId: string, campaignId: string, identity: EvalChargeIdentity): string { + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(projectId) + || ![requestId, campaignId, identity.executionId, identity.attemptId].every(validEvalLedgerId) + || !["setup", "agent", "judge", "feedback_review", "tool", "infrastructure"].includes(identity.phase)) { + throw new Error("Invalid eval gateway logging identity"); + } + return Buffer.concat([Buffer.from([4, 2, 0]), Buffer.from(JSON.stringify({ + object_id: projectId, + propagated_event: { metadata: { + paperclip_eval_request_id: requestId, paperclip_eval_campaign_id: campaignId, + paperclip_eval_execution_id: identity.executionId, paperclip_eval_attempt_id: identity.attemptId, + paperclip_eval_phase: identity.phase, + } }, + }))]).toString("base64"); +} diff --git a/tests/runner-e2e/eval-gateway.test.ts b/tests/runner-e2e/eval-gateway.test.ts index 4fa02239ca9..ba5266e72f3 100644 --- a/tests/runner-e2e/eval-gateway.test.ts +++ b/tests/runner-e2e/eval-gateway.test.ts @@ -31,12 +31,54 @@ it("fsyncs reservation before dispatch, pins gateway routing, and replaces reser const result = await forwardEvalRequest({ ...f, transport }); expect(reservedBeforeDispatch).toBe(true); expect(transport.mock.calls[0]![0]).toBe("https://gateway.braintrust.dev/responses"); - expect(captured).toMatchObject({ redirect: "error", headers: { authorization: `Bearer ${route.apiKey}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", "x-bt-project-id": route.projectId, "x-bt-endpoint-name": route.endpointName, "x-bt-parent": `project_id:${route.projectId}` } }); + expect(captured).toMatchObject({ redirect: "error", headers: { authorization: `Bearer ${route.apiKey}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", "x-bt-project-id": route.projectId, "x-bt-endpoint-name": route.endpointName, "x-bt-parent": result.braintrustParent } }); expect(JSON.parse(captured!.body as string)).toMatchObject({ max_output_tokens: 100, service_tier: "default" }); expect(result).toMatchObject({ outcome: "succeeded", costUsd: "0.000068000", final: true, usageComplete: true, gatewaySpanId: "gateway-span" }); expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000068000", heldUsd: "0.000000000", inflight: 0 }); expect(await readFile(f.file, "utf8")).not.toContain(route.apiKey); }); +it.each(["openai", "anthropic"] as const)("correlates %s gateway logs before response headers even when the client cancels", async provider => { + const f = await fixture(); + const selectedRoute: EvalGatewayRoute = provider === "openai" ? route : { ...route, provider, endpointName: "ANTHROPIC_API_KEY", + price: { ...route.price, provider, cacheWrite5mUsdPerMillion: "2", cacheWrite1hUsdPerMillion: "4" } }; + const parents: string[] = []; + const initialReceipts: EvalGatewayReceipt[] = []; + for (let attempt = 0; attempt < 2; attempt++) { + const controller = new AbortController(); + const receipt = await forwardEvalRequest({ ...f, route: selectedRoute, signal: controller.signal, + transport: async (_url, init) => { + parents.push(new Headers(init?.headers).get("x-bt-parent")!); + initialReceipts.push(structuredClone(f.receipts.at(-1)!)); + controller.abort(); + throw new DOMException("client disconnected", "AbortError"); + }, + }); + expect(receipt).toMatchObject({ outcome: "cancelled", errorCode: "request_cancelled", costUsd: null, final: false }); + expect(receipt.gatewaySpanId).toBeUndefined(); + } + expect(parents[0]).not.toBe(parents[1]); + for (const [index, encoded] of parents.entries()) { + const bytes = Buffer.from(encoded, "base64"); + expect([...bytes.subarray(0, 3)]).toEqual([4, 2, 0]); + expect(JSON.parse(bytes.subarray(3).toString())).toEqual({ object_id: route.projectId, propagated_event: { metadata: { + paperclip_eval_request_id: initialReceipts[index]!.requestId, paperclip_eval_campaign_id: "campaign", + paperclip_eval_execution_id: identity.executionId, paperclip_eval_attempt_id: identity.attemptId, paperclip_eval_phase: "agent", + } } }); + expect(initialReceipts[index]).toMatchObject({ braintrustParent: encoded, final: false, costUsd: null }); + } + expect(f.ledger.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 2, knownUsd: "0.000000000" }); + expect(f.ledger.snapshot().heldUsd).not.toBe("0.000000000"); + expect(JSON.stringify(initialReceipts)).not.toContain(route.apiKey); +}); +it("keeps Braintrust request metadata out of direct provider requests", async () => { + const f = await fixture(); + let headers: Headers | undefined; + const receipt = await forwardEvalRequest({ ...f, route: { ...route, mode: "direct" }, transport: async (_url, init) => { + headers = new Headers(init?.headers); return response(); + } }); + expect(headers!.has("x-bt-parent")).toBe(false); + expect(receipt).not.toHaveProperty("braintrustParent"); +}); it("stops at the campaign cap before any upstream request", async () => { const f = await fixture("0.002"), transport = vi.fn(); expect(evalRequestReservation(route)).toBe("0.003000000"); diff --git a/tests/runner-e2e/eval-gateway.ts b/tests/runner-e2e/eval-gateway.ts index 72528908662..26af76ff106 100644 --- a/tests/runner-e2e/eval-gateway.ts +++ b/tests/runner-e2e/eval-gateway.ts @@ -1,6 +1,7 @@ import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { once } from "node:events"; +import { braintrustGatewayRequestParent } from "./braintrust-gateway-context.js"; import { usdToUnits } from "../../packages/shared/src/money.js"; import { estimateProviderCost, ProviderUsageAccumulator, type EvalModelPrice, type ProviderUsageReceipt } from "./provider-usage.js"; import { EvalAdmissionError, validEvalLedgerId, type EvalChargeIdentity, type openEvalCostLedger } from "./cost-ledger.js"; @@ -19,7 +20,7 @@ export interface EvalGatewayRoute { } export interface EvalGatewayReceipt { requestId: string; status?: number; route: "direct" | "braintrust"; - gatewayRequestId?: string; gatewaySpanId?: string; providerRequestId?: string; + gatewayRequestId?: string; gatewaySpanId?: string; providerRequestId?: string; braintrustParent?: string; endpoint?: string; cache?: string; failoverFrom?: string; failoverTo?: string; errorOrigin?: string; outcome: "succeeded" | "failed" | "cancelled"; errorCode?: string; usageComplete: boolean; final: boolean; costUsd: string | null; @@ -111,7 +112,8 @@ export async function forwardEvalRequest(input: { if (input.signal?.aborted) throw new EvalRequestError("request_cancelled"); if (Date.now() >= input.deadlineAt) throw new EvalAdmissionError("request_deadline_reached"); const requestId = randomUUID(); - const receipt: EvalGatewayReceipt = { requestId, route: route.mode, outcome: "failed", usageComplete: false, final: false, costUsd: null }; + const receipt: EvalGatewayReceipt = { requestId, route: route.mode, outcome: "failed", usageComplete: false, final: false, costUsd: null, + ...(route.mode === "braintrust" ? { braintrustParent: braintrustGatewayRequestParent(route.projectId!, requestId, ledger.snapshot().campaignId, input.identity) } : {}) }; const usage = new ProviderUsageAccumulator(route.provider, body); await ledger.reserve(requestId, input.identity, reservation); const abort = new AbortController(); @@ -122,7 +124,7 @@ export async function forwardEvalRequest(input: { const headers: Record = { "content-type": "application/json", "accept-encoding": "identity" }; if (route.mode === "braintrust") Object.assign(headers, { authorization: `Bearer ${route.apiKey}`, "x-bt-project-id": route.projectId!, "x-bt-endpoint-name": route.endpointName!, - "x-bt-parent": `project_id:${route.projectId}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", + "x-bt-parent": receipt.braintrustParent!, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", }); else if (route.provider === "openai") headers.authorization = `Bearer ${route.apiKey}`; else headers["x-api-key"] = route.apiKey;