From ba04d3761b088f28d8dfd31d0f524650e029fad7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 18:37:03 -0500 Subject: [PATCH] test(evals): retain identity probe credentials for one attempt --- .../identity-probes.test.ts | 46 +++++++++++++++++++ .../chat-catalog-domains/identity-support.ts | 39 ++++++++++++---- 2 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-probes.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-probes.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-probes.test.ts new file mode 100644 index 00000000000..8bc16e88550 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-probes.test.ts @@ -0,0 +1,46 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { cleanupIdentityProbes, identityDefinition, probeAgentPermission, readAgentResource } from "./identity-support.js"; +afterEach(() => vi.restoreAllMocks()); +function context() { + const create = vi.fn(async () => ({ id: "key", token: "attempt-private-token" })); + const revoke = vi.fn(async () => {}); const privateSecrets: string[] = []; + const ctx = { input: { nonce: "nonce", privateSecrets, fixtures: { agent: { id: "ceo" } }, api: { baseURL: "http://127.0.0.1:3100", postSensitive: create, delete: revoke } } } as unknown as ChatCatalogContext; + return { ctx, create, revoke, privateSecrets }; +} +describe("identity attempt-private credential lifetime", () => { + it("shares one credential for concurrent probes but performs every read freshly", async () => { + const f = context(); let reads = 0; + const fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async () => new Response(JSON.stringify({ revision: ++reads }), { status: 200 })); + const results = await Promise.all([readAgentResource(f.ctx, "/api/issues/task"), readAgentResource(f.ctx, "/api/issues/task")]); + expect(results.map(row => row.data)).toEqual([{ revision: 1 }, { revision: 2 }]); + await probeAgentPermission(f.ctx, "/api/companies/company/budgets/policies", "POST", {}); + expect(f.create).toHaveBeenCalledTimes(1); expect(fetch).toHaveBeenCalledTimes(3); expect(f.revoke).not.toHaveBeenCalled(); + expect(f.privateSecrets).toEqual(["attempt-private-token"]); + await cleanupIdentityProbes(f.ctx); await cleanupIdentityProbes(f.ctx); + expect(f.revoke).toHaveBeenCalledTimes(1); + }); + it("retains cleanup responsibility after a transport failure and isolates attempts", async () => { + const a = context(); const b = context(); + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("transport failed")); + await expect(readAgentResource(a.ctx, "/api/issues/task")).rejects.toThrow(); + await expect(readAgentResource(b.ctx, "/api/issues/task")).rejects.toThrow(); + await cleanupIdentityProbes(a.ctx); await cleanupIdentityProbes(b.ctx); + expect(a.create).toHaveBeenCalledTimes(1); expect(b.create).toHaveBeenCalledTimes(1); + expect(a.revoke).toHaveBeenCalledTimes(1); expect(b.revoke).toHaveBeenCalledTimes(1); + }); + it("cleans a partial setup and composes the domain cleanup", async () => { + const f = context(); vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response("{}")); + const ownCleanup = vi.fn(async () => {}); + const definition = identityDefinition("access.private-resource", async ctx => { await readAgentResource(ctx, "/api/issues/task"); throw new Error("setup failed"); }, + () => ({ passed: false, checks: [] }), { cleanup: ownCleanup }); + await expect(definition.setup(f.ctx)).rejects.toThrow("setup failed"); expect(f.revoke).toHaveBeenCalledTimes(1); + await definition.cleanup!(f.ctx, {}); expect(ownCleanup).toHaveBeenCalledTimes(1); + }); + it("rejects absent redaction ownership and foreign URLs before issuing credentials", async () => { + const f = context(); + await expect(readAgentResource(f.ctx, "https://elsewhere.test/api/issues/task")).rejects.toThrow("escaped"); + delete f.ctx.input.privateSecrets; + await expect(readAgentResource(f.ctx, "/api/issues/task")).rejects.toThrow("redaction"); expect(f.create).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-support.ts b/tests/runner-e2e/chat-catalog-domains/identity-support.ts index 0e14e447403..0503abdb2f3 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-support.ts @@ -32,10 +32,20 @@ export function identityObservation(value: unknown): IdentityObservation | undef && [o.tasks, o.projects, o.goals, o.activity, o.documents, o.links, o.runs, o.companies, o.agents, o.interactions].every(Array.isArray) ? o : undefined; } export function identityDefinition(scenarioId: string, setup: ChatCatalogCase["setup"], grade: ChatCatalogCase["grade"], extra: Partial = {}): ChatCatalogCase { - return { scenarioId, setup, grade, observe: observeIdentity, oracleVersion: "paperclip.chat-catalog.identity.v1", + const definition: ChatCatalogCase = { scenarioId, setup, grade, observe: observeIdentity, oracleVersion: "paperclip.chat-catalog.identity.v1", sourceFiles: ["chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", `chat-catalog-domains/identity-${scenarioId.split(".")[0]}.ts`, `chat-scenarios/${scenarioId.split(".")[0]}.yaml`], expectedRunCount: 1, turnTimeoutMs: 120_000, attemptTimeoutMs: 300_000, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, ...extra }; + return { ...definition, + async setup(ctx) { + try { return await definition.setup(ctx); } + catch (error) { await cleanupIdentityProbes(ctx); throw error; } + }, + async cleanup(ctx, state) { + try { await cleanupIdentityProbes(ctx); } + finally { await definition.cleanup?.(ctx, state); } + }, + }; } export async function waitForReply(ctx: ChatCatalogContext) { const { pollUntil } = await import("../api.js"); @@ -56,27 +66,38 @@ export async function identityWorker(ctx: ChatCatalogContext, name = "Morgan") { } /** Real agent bearer, public route and HTTP status. Never pass bearer into * Playwright requests, case state, screenshots, model context or evidence. */ +const identityProbeKeys = new WeakMap>(); +async function identityProbeKey(ctx: ChatCatalogContext) { + if (!ctx.input.privateSecrets) throw new Error("Identity probes require attempt-private credential redaction"); + let pending = identityProbeKeys.get(ctx); + if (!pending) { + pending = ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `chat-eval-probe-${ctx.input.nonce}` }) + .then(key => { ctx.input.privateSecrets!.push(key.token); return key; }); + identityProbeKeys.set(ctx, pending); + } + return pending; +} +export async function cleanupIdentityProbes(ctx: ChatCatalogContext) { + const pending = identityProbeKeys.get(ctx); if (!pending) return; + const key = await pending; + await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`, { allowNotFound: true }); + identityProbeKeys.delete(ctx); +} export async function probeAgentPermission(ctx: ChatCatalogContext, path: string, method: string, body?: unknown) { - const id = ctx.input.fixtures.agent.id; - const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${id}/keys`, { name: `chat-eval-probe-${ctx.input.nonce}` }); - try { const target = new URL(path, ctx.input.api.baseURL); if (target.origin !== ctx.input.api.baseURL || !target.pathname.startsWith("/api/")) throw new Error("Agent permission probe escaped fixture API"); + const key = await identityProbeKey(ctx); const response = await fetch(target, { method, redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}`, "content-type": "application/json" }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); await response.body?.cancel(); return response.status; - } finally { await ctx.input.api.delete(`/api/agents/${id}/keys/${key.id}`); } } export async function readAgentResource(ctx: ChatCatalogContext, path: string): Promise<{ status: number; data?: unknown }> { - const id = ctx.input.fixtures.agent.id; - const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${id}/keys`, { name: `chat-eval-read-${ctx.input.nonce}` }); - try { const target = new URL(path, ctx.input.api.baseURL); if (target.origin !== ctx.input.api.baseURL || !target.pathname.startsWith("/api/")) throw new Error("Agent read escaped fixture API"); + const key = await identityProbeKey(ctx); const response = await fetch(target, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}` } }); if (!response.ok) { await response.body?.cancel(); return { status: response.status }; } return { status: response.status, data: await response.json() }; - } finally { await ctx.input.api.delete(`/api/agents/${id}/keys/${key.id}`); } } export async function followRenderedLink(ctx: ChatCatalogContext, href: string) { const link = ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first();