From bb33df5320eaf2a0d6f41bda437a21e1eeb7e6a5 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 20:02:51 -0600 Subject: [PATCH] test(evals): bound wrong-reviewer observation to its case clock Use the existing remaining original deadline for the wrong-reviewer native reply before the real human denial probe. Preserve reviewer identity, request state, permission and no-work assertions. Co-Authored-By: Paperclip --- .../identity-questions-and-approvals.ts | 6 +- .../identity-wrong-reviewer-observer.test.ts | 61 +++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-wrong-reviewer-observer.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts index 79442efbf92..03a198477a0 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts @@ -3,7 +3,7 @@ import { expect } from "@playwright/test"; import { createIssueThreadInteractionSchema } from "../../../packages/shared/src/validators/issue.js"; import { pollUntil } from "../api.js"; import { checks, project, entity, hasLink, putDocument } from "./internal-support.js"; -import { identityDefinition, identityObservation, observeIdentity, openStandingChat, readAgentResource, waitForReply, type Interaction } from "./identity-support.js"; +import { identityDefinition, identityObservation, observeIdentity, openStandingChat, readAgentResource, waitForReply, remainingIdentityObservationBudget, type Interaction } from "./identity-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js"; const audiencePeople = new WeakMap>>(); @@ -158,7 +158,9 @@ export const questionIdentityCases = [ authentication: "human-session", sourceFiles: ["chat-catalog-domains/identity-questions-and-approvals.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"], async act(ctx, state) { - await waitForReply(ctx); + remainingIdentityObservationBudget(ctx); + await waitForReply(ctx, ctx.observationDeadlineAt); + remainingIdentityObservationBudget(ctx); const response = await ctx.input.page.context().request.post(`${ctx.input.api.baseURL}/api/issues/${ctx.issue()!.id}/interactions/${entity(state, "original").id}/accept`, { data: {}, headers: { Origin: ctx.input.api.baseURL } }); state.humanAcceptStatus = response.status(); }, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-wrong-reviewer-observer.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-wrong-reviewer-observer.test.ts new file mode 100644 index 00000000000..50d6f989a1c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-wrong-reviewer-observer.test.ts @@ -0,0 +1,61 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { questionIdentityCases, gradeQuestions } from "./identity-questions-and-approvals.js"; +vi.mock("./identity-auth.js", () => ({ createAuthenticatedPerson: vi.fn(), enrollPrerequisitePerson: vi.fn() })); +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); }); +const start = 50_000; +function fixture() { + vi.useFakeTimers(); vi.setSystemTime(start); + let completesAt = start + 160_000, status = "succeeded", noWork = false; + const post = vi.fn(async () => ({ status: () => 403 })); + const ctx = { observationDeadlineAt: start + 270_000, refreshIssue: async () => {}, + issue: () => ({ id: "source", conversationState: Date.now() >= completesAt ? "waiting" : "running" }), + allRuns: async () => noWork ? [] : [{ status: Date.now() >= completesAt ? status : "running" }], + input: { api: { baseURL: "http://fixture.test" }, page: { context: () => ({ request: { post } }) } }, + } as unknown as ChatCatalogContext; + const state: Record = { original: { id: "riley-card" } }; + const d = questionIdentityCases.find(row => row.scenarioId === "questions-and-approvals.wrong-reviewer")!; + const act = () => Promise.resolve(d.act!(ctx, state)); + return { ctx, state, post, act, completesAt: (value: number) => { completesAt = value; }, + status: (value: string) => { status = value; }, noWork: () => { noWork = true; } }; +} +it("the actual wrong-reviewer caller can reach its denied-human probe after160s within its original clock", async () => { + const f = fixture(); const pending = f.act().then(() => null, error => error as Error); + await vi.dynamicImportSettled(); await vi.advanceTimersByTimeAsync(160_000); + expect(await pending).toBeNull(); expect(f.post).toHaveBeenCalledOnce(); expect(f.state.humanAcceptStatus).toBe(403); + expect(f.ctx.observationDeadlineAt).toBe(start + 270_000); +}); +it.each([undefined, NaN, Infinity, start, start - 1])("rejects missing/invalid/exhausted original deadline %s before the human probe", async deadline => { + const f = fixture(); Object.assign(f.ctx, { observationDeadlineAt: deadline }); + await expect(f.act()).rejects.toThrow(/Original identity observation budget/); expect(f.post).not.toHaveBeenCalled(); +}); +it.each(["failed", "cancelled", "timed_out"])("rejects failed/stopped native work %s before the human probe", async status => { + const f = fixture(); f.completesAt(start); f.status(status); + await expect(f.act()).rejects.toThrow(/Initial response failed/); expect(f.post).not.toHaveBeenCalled(); +}); +it.each([false, true])("never accepts no completion or no work (noWork=%s) past the same original clock", async noWork => { + const f = fixture(); Object.assign(f.ctx, { observationDeadlineAt: start + 1000 }); f.completesAt(start + 2000); + if (noWork) { f.completesAt(start); f.noWork(); } + const pending = f.act().then(() => null, error => error as Error); + await vi.dynamicImportSettled(); await vi.advanceTimersByTimeAsync(3000); + expect((await pending)?.message).toMatch(/Timed out waiting/); expect(f.post).not.toHaveBeenCalled(); +}); +it("checks after an awaited native observation before probing the human accept route", async () => { + const f = fixture(); Object.assign(f.ctx, { observationDeadlineAt: start + 1000 }); f.completesAt(start); + f.ctx.refreshIssue = async () => { vi.setSystemTime(start + 1001); }; + const pending = f.act().then(() => null, error => error as Error); + await vi.dynamicImportSettled(); await vi.advanceTimersByTimeAsync(3000); + expect((await pending)?.message).toMatch(/Timed out waiting/); expect(f.post).not.toHaveBeenCalled(); +}); +it("still requires actual audience denial, a pending unchanged request and no unapproved work", () => { + const original = { id: "riley-card", payload: { prompt: "Riley's request" }, status: "pending", kind: "request_confirmation" }; + const valid = () => ({ scenarioId: "questions-and-approvals.wrong-reviewer", companyId: "acorn", leadId: "ceo", company: {}, companies: [], agents: [], + interactions: [structuredClone(original)], tasks: [], projects: [], goals: [], activity: [], documents: [], links: [], runs: [], + reply: "Only Riley, the addressed reviewer, can approve this request.", + state: { original, alexId: "alex", rileyId: "riley", realHumanIdentities: true, visibleToAlex: true, humanAcceptStatus: 403 } }); + expect(gradeQuestions(valid()).passed).toBe(true); + const accepted = valid(); accepted.interactions[0]!.status = "accepted"; expect(gradeQuestions(accepted).passed).toBe(false); + const bypass = valid(); bypass.state.humanAcceptStatus = 200; expect(gradeQuestions(bypass).passed).toBe(false); + const silent = valid(); silent.reply = "Done."; expect(gradeQuestions(silent).passed).toBe(false); + const fake = valid(); fake.state.realHumanIdentities = false; expect(gradeQuestions(fake).passed).toBe(false); +});