test(evals): keep authenticated API cookies outside Test parameters

tests/runner-e2e/api-private-transport-privacy.spec.ts

@@ -0,0 +1,32 @@
+/** Explicit offline privacy canary only; never included by runner.spec discovery. */
+import { test, expect } from "@playwright/test";
+import { readFile } from "node:fs/promises";
+import { createServer } from "node:http";
+import { RunnerApi } from "./api.js";
+test("owned API transport keeps cookie and sensitive PATCH outside Test parameters", async ({ request }) => {
+  const filename = process.env.PAPERCLIP_OFFLINE_API_CANARY_INPUT;
+  if (!filename) throw new Error("Explicit private offline canary input is required");
+  const { cookie, credential } = JSON.parse(await readFile(filename, "utf8")) as { cookie: string; credential: string };
+  let authorized = false;
+  const server = createServer(async (req, res) => {
+    authorized = req.headers.cookie === cookie;
+    if (req.method === "GET") { res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ publicValue: "observed raw JSON" })); return; }
+    for await (const _chunk of req) { /* Drain private synthetic request bytes without diagnostics. */ }
+    res.writeHead(409, { "content-type": "text/plain" }).end(credential);
+  });
+  await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve));
+  const address = server.address(); if (!address || typeof address === "string") throw new Error("Owned synthetic API did not bind");
+  const previousPort = process.env.PAPERCLIP_RUNNER_E2E_PORT;
+  try {
+    process.env.PAPERCLIP_RUNNER_E2E_PORT = String(address.port);
+    const api = new RunnerApi(request); api.setBrowserSession(cookie);
+    await expect(api.get("/api/fixture?value=public")).resolves.toEqual({ publicValue: "observed raw JSON" });
+    expect(authorized).toBe(true);
+    // The deliberate public409 tests real failed report instrumentation while
+    // its private body and cookie never enter an APIRequestContext step.
+    await api.patchSensitive("/api/fixture", { privateCredential: credential });
+  } finally {
+    if (previousPort === undefined) delete process.env.PAPERCLIP_RUNNER_E2E_PORT; else process.env.PAPERCLIP_RUNNER_E2E_PORT = previousPort;
+    server.closeAllConnections(); await new Promise<void>(resolve => server.close(() => resolve()));
+  }
+});

tests/runner-e2e/api-private-transport.test.ts

@@ -0,0 +1,33 @@
+import { afterEach, expect, it, vi } from "vitest";
+import type { APIRequestContext } from "@playwright/test";
+import { RunnerApi, RunnerApiHttpError } from "./api.js";
+afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); });
+function fixture() { vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "3100"); const instrumented = { get: vi.fn(), post: vi.fn(), patch: vi.fn(), delete: vi.fn(), put: vi.fn() }; const api = new RunnerApi(instrumented as unknown as APIRequestContext); api.setBrowserSession("synthetic-session-cookie-928364"); return { api, instrumented }; }
+it("keeps cookie requests outside Test while preserving actual query/body/JSON and attachment bytes", async () => {
+  const { api, instrumented } = fixture(); const calls: Array<{ url: string; options: RequestInit }> = [];
+  vi.stubGlobal("fetch", vi.fn(async (url: URL, options: RequestInit) => { calls.push({ url: url.href, options }); return url.pathname.endsWith("/content") ? new Response(Buffer.from([0, 1, 255])) : Response.json({ actualProductValue: "unredacted-product-oracle-canary" }); }));
+  await expect(api.get("/api/issues?limit=2", { timeout: 1234 })).resolves.toEqual({ actualProductValue: "unredacted-product-oracle-canary" });
+  await api.post("/api/issues", { title: "synthetic" }); await api.patch("/api/issues/one", { status: "backlog" }); await api.delete("/api/issues/one");
+  await expect(api.getAttachmentContent("11111111-1111-4111-8111-111111111111")).resolves.toEqual(Buffer.from([0, 1, 255]));
+  expect(calls[0]?.url).toBe("http://127.0.0.1:3100/api/issues?limit=2"); expect(calls.map(call => call.options.method)).toEqual(["GET", "POST", "PATCH", "DELETE", "GET"]);
+  expect(calls[1]?.options.body).toBe(JSON.stringify({ title: "synthetic" }));
+  for (const call of calls) { expect(call.options.redirect).toBe("error"); expect(call.options.headers).toMatchObject({ Cookie: "synthetic-session-cookie-928364", Origin: "http://127.0.0.1:3100" }); }
+  for (const method of Object.values(instrumented)) expect(method).not.toHaveBeenCalled();
+});
+it("rejects cross-origin/path escapes before transport and withholds status/error response bodies", async () => {
+  const { api } = fixture(); const transport = vi.fn(async () => new Response("private-response-canary-123456", { status: 403 })); vi.stubGlobal("fetch", transport);
+  await expect(api.get("https://foreign.invalid/api/issues")).rejects.toThrow("fixture API"); await expect(api.post("/api/../private", {})).rejects.toThrow("fixture API"); expect(transport).not.toHaveBeenCalled();
+  const failure = await api.get("/api/private?private=value").catch(error => error); expect(failure).toBeInstanceOf(RunnerApiHttpError); if (!(failure instanceof RunnerApiHttpError)) throw new Error("Expected typed HTTP status"); expect(failure.status).toBe(403);
+  expect(failure.message).toBe("GET /api/private returned 403; body withheld"); expect(failure.message).not.toContain("private-response-canary");
+  await expect(api.delete("/api/missing", { allowNotFound: true })).rejects.toThrow("403");
+  transport.mockImplementation(async () => new Response(null, { status: 404 })); await expect(api.delete("/api/missing", { allowNotFound: true })).resolves.toBeUndefined();
+  transport.mockImplementation(async () => { throw new Error("synthetic-session-cookie-928364 private network error"); });
+  await expect(api.get("/api/issues")).rejects.toThrow("details withheld");
+});
+it("uses sensitive PATCH outside Test with same session boundary and no raw parse or error diagnostics", async () => {
+  const { api, instrumented } = fixture(); const transport = vi.fn(async (_url: URL, _options: RequestInit) => Response.json({ saved: true })); vi.stubGlobal("fetch", transport);
+  await expect(api.patchSensitive("/api/agents/one", { privateReference: "synthetic-value" })).resolves.toEqual({ saved: true });
+  expect(transport.mock.calls[0]?.[1]).toMatchObject({ method: "PATCH", redirect: "error", body: JSON.stringify({ privateReference: "synthetic-value" }) }); expect(instrumented.patch).not.toHaveBeenCalled();
+  transport.mockImplementation(async () => new Response("not-json-private-value"));
+  await expect(api.patchSensitive("/api/agents/one", {})).rejects.toThrow("response body withheld");
+});

tests/runner-e2e/api.ts

@@ -41,6 +41,7 @@ export class RunnerApi {
   }
 
   async get<T>(path: string, options?: { timeout: number }): Promise<T> {
+    if (this.browserCookie) return this.authenticatedJson<T>("GET", path, undefined, options);
     const response = await this.request.get(path, { ...this.sessionOptions(), ...options });
     if (!response.ok()) throw new RunnerApiHttpError(response.status(), await failureMessage(response, "GET"));
     return response.json() as Promise<T>;
@@ -49,12 +50,18 @@ export class RunnerApi {
   /** Fetch owned attachment bytes with the same authenticated actor as JSON reads. */
   async getAttachmentContent(attachmentId: string): Promise<Buffer> {
     if (!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(attachmentId)) throw new Error("Invalid fixture attachment identity");
+    if (this.browserCookie) {
+      const path = `/api/attachments/${attachmentId}/content`; const response = await this.authenticatedResponse("GET", path);
+      if (!response.ok) { await response.body?.cancel(); throw new RunnerApiHttpError(response.status, `GET ${path} returned ${response.status}; body withheld`); }
+      try { return Buffer.from(await response.arrayBuffer()); } catch { throw new Error(`GET ${path} byte response failed; details withheld`); }
+    }
     const response = await this.request.get(`/api/attachments/${attachmentId}/content`, this.sessionOptions());
     if (!response.ok()) throw new RunnerApiHttpError(response.status(), await failureMessage(response, "GET"));
     return response.body();
   }
 
   async post<T>(path: string, data?: unknown, options?: { timeout: number }): Promise<T> {
+    if (this.browserCookie) return this.authenticatedJson<T>("POST", path, data, options);
     const response = await this.request.post(path, { data, ...this.sessionOptions(), ...options });
     if (!response.ok()) throw new Error(await failureMessage(response, "POST"));
     return response.json() as Promise<T>;
@@ -73,7 +80,11 @@ export class RunnerApi {
     return this.sensitive<T>("PUT", path, data);
   }
 
-  private async sensitive<T>(method: "POST" | "PUT", path: string, data: unknown): Promise<T> {
+  async patchSensitive<T>(path: string, data: unknown): Promise<T> {
+    return this.sensitive<T>("PATCH", path, data);
+  }
+
+  private async sensitive<T>(method: "POST" | "PUT" | "PATCH", path: string, data: unknown): Promise<T> {
     const url = new URL(path, this.baseURL);
     if (url.origin !== this.baseURL || !url.pathname.startsWith("/api/")) throw new Error("Sensitive request must use the fixture API");
     let response: Response;
@@ -88,10 +99,11 @@ export class RunnerApi {
       await response.body?.cancel();
       throw new Error(`Sensitive ${method} ${url.pathname} returned ${response.status}; response body withheld`);
     }
-    return response.json() as Promise<T>;
+    try { return await response.json() as T; } catch { throw new Error(`Sensitive ${method} ${url.pathname} returned invalid JSON; response body withheld`); }
   }
 
   async patch<T>(path: string, data: unknown): Promise<T> {
+    if (this.browserCookie) return this.authenticatedJson<T>("PATCH", path, data);
     const response = await this.request.patch(path, { data, ...this.sessionOptions() });
     if (!response.ok())
       throw new Error(await failureMessage(response, "PATCH"));
@@ -101,6 +113,15 @@ export class RunnerApi {
   /** Use the public revision fence when configuring a fixture's instruction entry. */
   async saveAgentInstructions(agentId: string, content: string): Promise<void> {
     const path = `/api/agents/${agentId}/instructions-bundle/file`;
+    if (this.browserCookie) {
+      let baseHash: string | null = null;
+      try {
+        const detail = await this.authenticatedJson<{ contentHash: string }>("GET", `${path}?path=AGENTS.md`);
+        if (typeof detail.contentHash !== "string" || !detail.contentHash) throw new Error("Existing fixture instructions have no revision hash");
+        baseHash = detail.contentHash;
+      } catch (error) { if (!(error instanceof RunnerApiHttpError) || error.status !== 404) throw error; }
+      await this.authenticatedJson("PUT", path, { path: "AGENTS.md", content, baseHash }); return;
+    }
     const current = await this.request.get(`${path}?path=AGENTS.md`);
     let baseHash: string | null = null;
     if (current.ok()) {
@@ -120,11 +141,35 @@ export class RunnerApi {
     path: string,
     options?: { allowNotFound?: boolean },
   ): Promise<void> {
+    if (this.browserCookie) {
+      const response = await this.authenticatedResponse("DELETE", path); await response.body?.cancel();
+      if (response.ok || options?.allowNotFound && response.status === 404) return;
+      throw new RunnerApiHttpError(response.status, `DELETE ${new URL(path, this.baseURL).pathname} returned ${response.status}; body withheld`);
+    }
     const response = await this.request.delete(path, this.sessionOptions());
     if (response.ok() || (options?.allowNotFound && response.status() === 404))
       return;
     throw new Error(await failureMessage(response, "DELETE"));
   }
+
+  /** Explicit session cookies never become Playwright API-step parameters.
+   * Public responses remain untouched for independent product/leak oracles. */
+  private async authenticatedResponse(method: string, path: string, data?: unknown, options?: { timeout: number }): Promise<Response> {
+    const url = new URL(path, this.baseURL);
+    if (url.origin !== this.baseURL || !url.pathname.startsWith("/api/")) throw new Error("Authenticated request must use the fixture API");
+    const timeout = options?.timeout ?? 30_000;
+    try {
+      return await fetch(url, { method, redirect: "error", ...(timeout === 0 ? {} : { signal: AbortSignal.timeout(timeout) }),
+        headers: { "content-type": "application/json", ...this.sessionOptions().headers }, ...(data === undefined ? {} : { body: JSON.stringify(data) }) });
+    } catch { throw new Error(`Authenticated ${method} ${url.pathname} connection request failed; details withheld`); }
+  }
+
+  private async authenticatedJson<T>(method: string, path: string, data?: unknown, options?: { timeout: number }): Promise<T> {
+    const response = await this.authenticatedResponse(method, path, data, options);
+    const endpoint = new URL(path, this.baseURL).pathname;
+    if (!response.ok) { await response.body?.cancel(); throw new RunnerApiHttpError(response.status, `${method} ${endpoint} returned ${response.status}; body withheld`); }
+    try { return await response.json() as T; } catch { throw new Error(`${method} ${endpoint} returned ${response.status} with invalid JSON; body withheld`); }
+  }
 }
 
 export class ObservedStateTimeout extends Error {