test(evals): retain identity probe credentials for one attempt

tests/runner-e2e/chat-catalog-domains/identity-probes.test.ts

@@ -0,0 +1,46 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { cleanupIdentityProbes, identityDefinition, probeAgentPermission, readAgentResource } from "./identity-support.js";
+afterEach(() => vi.restoreAllMocks());
+function context() {
+  const create = vi.fn(async () => ({ id: "key", token: "attempt-private-token" }));
+  const revoke = vi.fn(async () => {}); const privateSecrets: string[] = [];
+  const ctx = { input: { nonce: "nonce", privateSecrets, fixtures: { agent: { id: "ceo" } }, api: { baseURL: "http://127.0.0.1:3100", postSensitive: create, delete: revoke } } } as unknown as ChatCatalogContext;
+  return { ctx, create, revoke, privateSecrets };
+}
+describe("identity attempt-private credential lifetime", () => {
+  it("shares one credential for concurrent probes but performs every read freshly", async () => {
+    const f = context(); let reads = 0;
+    const fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async () => new Response(JSON.stringify({ revision: ++reads }), { status: 200 }));
+    const results = await Promise.all([readAgentResource(f.ctx, "/api/issues/task"), readAgentResource(f.ctx, "/api/issues/task")]);
+    expect(results.map(row => row.data)).toEqual([{ revision: 1 }, { revision: 2 }]);
+    await probeAgentPermission(f.ctx, "/api/companies/company/budgets/policies", "POST", {});
+    expect(f.create).toHaveBeenCalledTimes(1); expect(fetch).toHaveBeenCalledTimes(3); expect(f.revoke).not.toHaveBeenCalled();
+    expect(f.privateSecrets).toEqual(["attempt-private-token"]);
+    await cleanupIdentityProbes(f.ctx); await cleanupIdentityProbes(f.ctx);
+    expect(f.revoke).toHaveBeenCalledTimes(1);
+  });
+  it("retains cleanup responsibility after a transport failure and isolates attempts", async () => {
+    const a = context(); const b = context();
+    vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("transport failed"));
+    await expect(readAgentResource(a.ctx, "/api/issues/task")).rejects.toThrow();
+    await expect(readAgentResource(b.ctx, "/api/issues/task")).rejects.toThrow();
+    await cleanupIdentityProbes(a.ctx); await cleanupIdentityProbes(b.ctx);
+    expect(a.create).toHaveBeenCalledTimes(1); expect(b.create).toHaveBeenCalledTimes(1);
+    expect(a.revoke).toHaveBeenCalledTimes(1); expect(b.revoke).toHaveBeenCalledTimes(1);
+  });
+  it("cleans a partial setup and composes the domain cleanup", async () => {
+    const f = context(); vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response("{}"));
+    const ownCleanup = vi.fn(async () => {});
+    const definition = identityDefinition("access.private-resource", async ctx => { await readAgentResource(ctx, "/api/issues/task"); throw new Error("setup failed"); },
+      () => ({ passed: false, checks: [] }), { cleanup: ownCleanup });
+    await expect(definition.setup(f.ctx)).rejects.toThrow("setup failed"); expect(f.revoke).toHaveBeenCalledTimes(1);
+    await definition.cleanup!(f.ctx, {}); expect(ownCleanup).toHaveBeenCalledTimes(1);
+  });
+  it("rejects absent redaction ownership and foreign URLs before issuing credentials", async () => {
+    const f = context();
+    await expect(readAgentResource(f.ctx, "https://elsewhere.test/api/issues/task")).rejects.toThrow("escaped");
+    delete f.ctx.input.privateSecrets;
+    await expect(readAgentResource(f.ctx, "/api/issues/task")).rejects.toThrow("redaction"); expect(f.create).not.toHaveBeenCalled();
+  });
+});

tests/runner-e2e/chat-catalog-domains/identity-support.ts

@@ -32,10 +32,20 @@ export function identityObservation(value: unknown): IdentityObservation | undef
     && [o.tasks, o.projects, o.goals, o.activity, o.documents, o.links, o.runs, o.companies, o.agents, o.interactions].every(Array.isArray) ? o : undefined;
 }
 export function identityDefinition(scenarioId: string, setup: ChatCatalogCase["setup"], grade: ChatCatalogCase["grade"], extra: Partial<ChatCatalogCase> = {}): ChatCatalogCase {
-  return { scenarioId, setup, grade, observe: observeIdentity, oracleVersion: "paperclip.chat-catalog.identity.v1",
+  const definition: ChatCatalogCase = { scenarioId, setup, grade, observe: observeIdentity, oracleVersion: "paperclip.chat-catalog.identity.v1",
     sourceFiles: ["chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", `chat-catalog-domains/identity-${scenarioId.split(".")[0]}.ts`, `chat-scenarios/${scenarioId.split(".")[0]}.yaml`],
     expectedRunCount: 1, turnTimeoutMs: 120_000, attemptTimeoutMs: 300_000,
     async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, ...extra };
+  return { ...definition,
+    async setup(ctx) {
+      try { return await definition.setup(ctx); }
+      catch (error) { await cleanupIdentityProbes(ctx); throw error; }
+    },
+    async cleanup(ctx, state) {
+      try { await cleanupIdentityProbes(ctx); }
+      finally { await definition.cleanup?.(ctx, state); }
+    },
+  };
 }
 export async function waitForReply(ctx: ChatCatalogContext) {
   const { pollUntil } = await import("../api.js");
@@ -56,27 +66,38 @@ export async function identityWorker(ctx: ChatCatalogContext, name = "Morgan") {
 }
 /** Real agent bearer, public route and HTTP status. Never pass bearer into
  * Playwright requests, case state, screenshots, model context or evidence. */
+const identityProbeKeys = new WeakMap<ChatCatalogContext, Promise<{ id: string; token: string }>>();
+async function identityProbeKey(ctx: ChatCatalogContext) {
+  if (!ctx.input.privateSecrets) throw new Error("Identity probes require attempt-private credential redaction");
+  let pending = identityProbeKeys.get(ctx);
+  if (!pending) {
+    pending = ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `chat-eval-probe-${ctx.input.nonce}` })
+      .then(key => { ctx.input.privateSecrets!.push(key.token); return key; });
+    identityProbeKeys.set(ctx, pending);
+  }
+  return pending;
+}
+export async function cleanupIdentityProbes(ctx: ChatCatalogContext) {
+  const pending = identityProbeKeys.get(ctx); if (!pending) return;
+  const key = await pending;
+  await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`, { allowNotFound: true });
+  identityProbeKeys.delete(ctx);
+}
 export async function probeAgentPermission(ctx: ChatCatalogContext, path: string, method: string, body?: unknown) {
-  const id = ctx.input.fixtures.agent.id;
-  const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${id}/keys`, { name: `chat-eval-probe-${ctx.input.nonce}` });
-  try {
     const target = new URL(path, ctx.input.api.baseURL);
     if (target.origin !== ctx.input.api.baseURL || !target.pathname.startsWith("/api/")) throw new Error("Agent permission probe escaped fixture API");
+    const key = await identityProbeKey(ctx);
     const response = await fetch(target, { method, redirect: "error", signal: AbortSignal.timeout(15_000),
       headers: { Authorization: `Bearer ${key.token}`, "content-type": "application/json" }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) });
     await response.body?.cancel(); return response.status;
-  } finally { await ctx.input.api.delete(`/api/agents/${id}/keys/${key.id}`); }
 }
 export async function readAgentResource(ctx: ChatCatalogContext, path: string): Promise<{ status: number; data?: unknown }> {
-  const id = ctx.input.fixtures.agent.id;
-  const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${id}/keys`, { name: `chat-eval-read-${ctx.input.nonce}` });
-  try {
     const target = new URL(path, ctx.input.api.baseURL);
     if (target.origin !== ctx.input.api.baseURL || !target.pathname.startsWith("/api/")) throw new Error("Agent read escaped fixture API");
+    const key = await identityProbeKey(ctx);
     const response = await fetch(target, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}` } });
     if (!response.ok) { await response.body?.cancel(); return { status: response.status }; }
     return { status: response.status, data: await response.json() };
-  } finally { await ctx.input.api.delete(`/api/agents/${id}/keys/${key.id}`); }
 }
 export async function followRenderedLink(ctx: ChatCatalogContext, href: string) {
   const link = ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first();