fix(evals): keep repeated run observations out of browser trace

This commit is contained in:
Dotta committed 2026-10-27 03:41:37 -05:00
1 parent aae4f384e0
commit 56eb41b819
4 files changed
+63 -13

No files matched your search

+37 -1
View File
@@ -5,7 +5,7 @@ const response = (status: number, data: unknown = {}): APIResponse => ({
ok: () => status >= 200 && status < 300, status: () => status,
json: async () => data, text: async () => JSON.stringify(data), url: () => "http://fixture.invalid/instructions",
}) as APIResponse;
afterEach(() => vi.unstubAllEnvs());
afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
describe("fixture instruction revision fence", () => {
function api(current: APIResponse, saved = response(200)) {
vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "3100");
@@ -67,3 +67,39 @@ it("keeps existing polling retries when the strict error policy is absent", asyn
load, accept: Boolean, intervalMs: 1 })).resolves.toBe(true);
expect(load).toHaveBeenCalledTimes(2);
});
it("observes identical native poll results with the actual actor and no traced request", async () => {
vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "3100");
const get = vi.fn(() => { throw new Error("Poll must not enter the Playwright API recorder"); });
const api = new RunnerApi({ get } as unknown as APIRequestContext);
api.setBrowserSession("synthetic-cookie");
const rows = [{ id: "11111111-1111-4111-8111-111111111111", status: "running" }];
const fetcher = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(JSON.stringify(rows), { status: 200 }));
vi.stubGlobal("fetch", fetcher);
const endpoint = "/api/companies/22222222-2222-4222-8222-222222222222/heartbeat-runs?limit=100";
await expect(api.getRunObservation(endpoint, { timeout: 1234 })).resolves.toEqual(rows);
expect(get).not.toHaveBeenCalled(); expect(fetcher).toHaveBeenCalledTimes(1);
expect(String(fetcher.mock.calls[0]![0])).toBe(`http://127.0.0.1:3100${endpoint}`);
expect(fetcher.mock.calls[0]![1]).toMatchObject({ method: "GET", redirect: "error", headers: { Cookie: "synthetic-cookie", Origin: "http://127.0.0.1:3100" } });
expect((fetcher.mock.calls[0]![1] as RequestInit).signal).toBeInstanceOf(AbortSignal);
});
it("keeps local-trusted run observations cookie-free and preserves typed HTTP failure without retry", async () => {
vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "3100");
const api = new RunnerApi({} as APIRequestContext);
const fetcher = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) => new Response("private failure body", { status: 403 })); vi.stubGlobal("fetch", fetcher);
await expect(api.getRunObservation("/api/heartbeat-runs/11111111-1111-4111-8111-111111111111/events?afterSeq=0&limit=1000")).rejects.toMatchObject({ name: "RunnerApiHttpError", status: 403 });
expect(fetcher).toHaveBeenCalledTimes(1);
expect(fetcher.mock.calls[0]![1]).toMatchObject({ headers: { "content-type": "application/json" } });
expect((fetcher.mock.calls[0]![1] as RequestInit).headers).not.toHaveProperty("Cookie");
});
it.each([
"https://foreign.invalid/api/heartbeat-runs/11111111-1111-4111-8111-111111111111",
"/api/companies/22222222-2222-4222-8222-222222222222/agents",
"/api/heartbeat-runs/11111111-1111-4111-8111-111111111111?token=anything",
"/api/heartbeat-runs/11111111-1111-4111-8111-111111111111/events?afterSeq=0&limit=10&extra=1",
])("rejects non-observation transport target %s before dispatch", async endpoint => {
vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "3100");
const fetcher = vi.fn(); vi.stubGlobal("fetch", fetcher);
await expect(new RunnerApi({} as APIRequestContext).getRunObservation(endpoint)).rejects.toThrow("allowlisted same-origin GET");
expect(fetcher).not.toHaveBeenCalled();
});