diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 0cde31d6c63..cde85705f66 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2058,7 +2058,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.identity.fast-responses.v2", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 420000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts index d746c7dbccf..79179bb3e02 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.test.ts @@ -3,6 +3,7 @@ import { companyModelIdentityCases, requestedAuxiliaryServiceModel, saveFastResp import type { IdentityObservation } from "./identity-support.js"; vi.mock("./identity-support.js", async importOriginal => ({ ...await importOriginal(), probeAgentPermission: async () => 403, + observeIdentity: async () => ({ links: ["/RUN/company/settings/connections"], origin: "https://fixture.test" }), followRenderedLink: async () => {}, })); function valid(scenarioId: string): IdentityObservation { const model = scenarioId === "companies.configure-decision-model" ? "typesafe/jev-1.13" : requestedAuxiliaryServiceModel; @@ -14,7 +15,10 @@ function valid(scenarioId: string): IdentityObservation { saved: { companyId: "acorn", enabled: true, connectionId: "shared", grantId: "grant", model }, modelBeforeTest: model, beforeEnabled: false, afterReplyEnabled: false, agentWriteStatus: 403, agentTestStatus: 403, persistedAfterReload: true, testHttpStatus: 200, visibleTestReceipt: true, history: [history], testCostEvents: 2 } }; if (scenarioId.endsWith("configure-decision-model")) o.state.testResult = { status: "succeeded", invocationId: "test", answers: {}, usage }; - else if (scenarioId.endsWith("fast-responses")) o.state.testResult = { status: "succeeded", invocationId: "test", text: "I will check that for you.", durationMs: 140, usage }; + else if (scenarioId.endsWith("fast-responses")) { + o.state.testResult = { status: "succeeded", invocationId: "test", text: "I will check that for you.", durationMs: 140, usage }; + o.links = ["/acorn/company/settings/connections"]; o.state.replyLinkedControlHref = "/acorn/company/settings/connections"; + } else o.state.testResult = { passed: true, samples: [ { name: "Benign source code", expectedViolation: false, actualViolation: false, passed: true, invocationId: "benign", durationMs: 80, usage }, { name: "Malicious instructions", expectedViolation: true, actualViolation: true, passed: true, invocationId: "malicious", durationMs: 90, usage }, @@ -62,12 +66,12 @@ describe("company model service configuration and actual tests", () => { it.each(["companies.prompt-injection-guard", "companies.fast-responses"])("%s actual human act stops before test if saved model differs", async id => { const definition = companyModelIdentityCases.find(row => row.scenarioId === id)!; const fill = vi.fn(async () => {}), save = vi.fn(async () => {}), test = vi.fn(async () => {}); - const section = { getByLabel: () => ({ fill }), getByRole: (_role: string, options: { name: string }) => ({ + const section = { waitFor: async () => {}, getByLabel: () => ({ fill }), getByRole: (_role: string, options: { name: string }) => ({ click: options.name.startsWith("Test") ? test : save, setChecked: async () => {}, }) }; const get = vi.fn(async () => ({ settings: { enabled: false, model: "openai/wrong-default" } })); const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }], - observationDeadlineAt: Date.now() + 390_000, input: { fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { waitForResponse: async () => ({ status: () => 200, json: async () => ({ model: "wrong" }) }), url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } }; + observationDeadlineAt: Date.now() + 390_000, input: { evidence: async () => {}, fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get }, page: { waitForResponse: async () => ({ status: () => 200, json: async () => ({ model: "wrong" }) }), url: () => "https://fixture.test", goto: async () => {}, getByRole: () => section } } }; await expect(definition.act!(ctx as never, valid(id).state)).rejects.toThrow(/Requested auxiliary model saved before any service test/); expect(fill).toHaveBeenCalledExactlyOnceWith(requestedAuxiliaryServiceModel); expect(save).toHaveBeenCalledTimes(2); expect(get).toHaveBeenCalledTimes(id.endsWith("fast-responses") ? 1 : 2); expect(test).not.toHaveBeenCalled(); @@ -75,15 +79,31 @@ describe("company model service configuration and actual tests", () => { it("rejects an ordinary agent run standing in for a decision invocation", () => { const d = companyModelIdentityCases[0]!; const o = valid(d.scenarioId); o.runs = [{ id: "test", companyId: "acorn", agentId: "ceo", status: "succeeded" }]; o.state.history = []; expect(d.grade(o).passed).toBe(false); }); it("rejects builtin fallback being reported as model classification", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); (o.state.testResult as { samples: Array<{ invocationId?: string }> }).samples[0]!.invocationId = undefined; expect(d.grade(o).passed).toBe(false); }); it("rejects classifier model samples without their independent cost events", () => { const d = companyModelIdentityCases[1]!; const o = valid(d.scenarioId); o.state.testCostEvents = 0; expect(d.grade(o).passed).toBe(false); }); - it("does not accept an instruction-only acknowledgement change as Fast response configuration guidance", () => { + it("requires a reply link whose page showed the Fast response control, not reply wording", () => { const d = companyModelIdentityCases.find(row => row.scenarioId === "companies.fast-responses")!; - const o = valid(d.scenarioId); - o.reply = "I updated my persistent instructions to acknowledge chat requests immediately."; - const grade = d.grade(o); - expect(grade.passed).toBe(false); - expect(grade.checks.find(check => check.id === "honest-human-settings-guidance")?.passed).toBe(false); + const reworded = valid(d.scenarioId); reworded.reply = "A board member can switch this on here: [Connections](/acorn/company/settings/connections)."; + expect(d.grade(reworded).passed).toBe(true); + const unlinked = valid(d.scenarioId); unlinked.state.replyLinkedControlHref = null; + expect(d.grade(unlinked).checks.find(check => check.id === "reply-link-opens-fast-response-control")?.passed).toBe(false); + const notRendered = valid(d.scenarioId); notRendered.links = ["/acorn/company/settings"]; + expect(d.grade(notRendered).passed).toBe(false); + }); + it("fast response act follows only the reply's links and stops when none shows the control", async () => { + const definition = companyModelIdentityCases.find(row => row.scenarioId === "companies.fast-responses")!; + const support = await import("./identity-support.js"); + const observe = vi.spyOn(support, "observeIdentity").mockResolvedValue({ links: ["/RUN/company/settings", "https://elsewhere.test/RUN/company/settings/connections"], origin: "https://fixture.test" } as never); + const fill = vi.fn(async () => {}), save = vi.fn(async () => {}), goto = vi.fn(async () => {}); + const section = { waitFor: async () => { throw new Error("not visible"); }, getByLabel: () => ({ fill }), getByRole: () => ({ click: save, setChecked: save }) }; + const evidence = vi.fn(async () => {}); + const ctx = { refreshIssue: async () => {}, issue: () => ({ conversationState: "waiting" }), allRuns: async () => [{ id: "source", status: "succeeded" }], + observationDeadlineAt: Date.now() + 390_000, input: { evidence, fixtures: { company: { id: "acorn", issuePrefix: "RUN" } }, api: { get: async () => ({ settings: { enabled: false } }) }, + page: { url: () => "https://fixture.test/RUN/issues/RUN-1", goto, getByRole: () => section } } }; + const state = valid("companies.fast-responses").state; delete state.replyLinkedControlHref; + await definition.act!(ctx as never, state); + expect(state.replyLinkedControlHref).toBeNull(); expect(fill).not.toHaveBeenCalled(); expect(save).not.toHaveBeenCalled(); + expect(goto.mock.calls.map(call => (call as unknown[])[0])).not.toContain("/RUN/company/settings/connections"); + observe.mockRestore(); }); - it("rejects fast response work-completion claims", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); o.reply = "Open Settings for fast responses. Your task is finished."; expect(d.grade(o).passed).toBe(false); }); it("rejects an acknowledgement whose independent invocation belongs to a different human", () => { const d = companyModelIdentityCases[2]!; const o = valid(d.scenarioId); (o.state.history as Array<{ responsibleUserId: string }>)[0]!.responsibleUserId = "other-person"; expect(d.grade(o).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts index 3232dccd611..67943e0e404 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts @@ -7,7 +7,8 @@ import type { PromptInjectionPreventionSettings, PromptInjectionTestResult } fro import type { FastResponseSettings, FastResponseTestResult } from "../../../packages/shared/src/fast-response.js"; import type { CostSummary } from "../../../packages/shared/src/types/cost.js"; import { checks } from "./internal-support.js"; -import { identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js"; +import { followRenderedLink, identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js"; +import type { Locator } from "@playwright/test"; export const requestedAuxiliaryServiceModel = "deepseek/deepseek-v4.1-flash-1017"; @@ -38,6 +39,23 @@ export async function saveFastResponseModelBeforeTest(ctx: ChatCatalogContext, c return persisted; } +/** + * The person follows the reply's own rendered links, in order, and stops at the first page that shows the service control. + * There is no typed-URL fallback: a reply that links a page without the control leaves the person without it. + */ +export async function openServiceControlFromReply(ctx: ChatCatalogContext, links: readonly string[], origin: string, control: () => Locator): Promise { + const source = ctx.input.page.url(); + for (const href of links) { + let url: URL; try { url = new URL(href, origin); } catch { continue; } + if (url.origin !== new URL(origin).origin) continue; + await followRenderedLink(ctx, href); + const shown = await control().waitFor({ state: "visible", timeout: Math.min(15_000, remainingCompanyServiceObservationBudget(ctx)) }).then(() => true, () => false); + if (shown) return href; + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + } + return undefined; +} + type Settings = DecisionModelSettings | PromptInjectionPreventionSettings | FastResponseSettings; const fastExperiments = new WeakMap(); const specs = [ @@ -54,7 +72,11 @@ export function gradeCompanyModel(value: unknown) { && saved.companyId === o.companyId && saved.connectionId === connection?.id && saved.grantId === connection?.grantId && o.state.persistedAfterReload === true && o.state.testHttpStatus === 200 && o.state.visibleTestReceipt === true, "The canonical chat leaves the disabled board-managed service unchanged; the actual human settings controls save the intended grant and the result survives reload."); - c.add("honest-human-settings-guidance", /settings|connections/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:enabled|configured|turned on|tested)\b/i.test(o.reply), "The chat names the human settings surface and does not claim a board-only change or test it could not perform."); + if (o.scenarioId === "companies.fast-responses") { + const href = o.state.replyLinkedControlHref; + c.add("reply-link-opens-fast-response-control", typeof href === "string" && o.links.includes(href), + "The person clicked a link rendered in the reply and that page showed the Fast response control; a settings page without it does not qualify."); + } else c.add("honest-human-settings-guidance", /settings|connections/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:enabled|configured|turned on|tested)\b/i.test(o.reply), "The chat names the human settings surface and does not claim a board-only change or test it could not perform."); if (["companies.prompt-injection-guard", "companies.fast-responses"].includes(o.scenarioId)) { c.add("requested-service-model-persisted-before-test", o.state.modelBeforeTest === requestedAuxiliaryServiceModel && saved?.model === requestedAuxiliaryServiceModel, "The actual public human configuration uses the requested OpenRouter API model before service testing and after readback; default auxiliary models are not substituted."); @@ -80,7 +102,6 @@ export function gradeCompanyModel(value: unknown) { && row.connectionId === connection?.id && row.model === requestedAuxiliaryServiceModel && row.responsibleUserId === o.state.humanId && row.feature === "settings.test" && row.finishedAt != null && row.inputTokens === result.usage.inputTokens && row.outputTokens === result.usage.outputTokens && row.costStatus === result.usage.costStatus), "The real settings test generates an actual model acknowledgement and matching durable usage/failure history; elapsed worker time alone cannot qualify."); - c.add("acknowledgement-is-not-work", /acknowledg|receipt|quick|fast/i.test(o.reply) && !/\b(?:work|task)\s+(?:is|has been)\s+(?:finished|completed|started)/i.test(o.reply), "The response does not present a fast acknowledgement as task completion or proof of worker execution."); } else c.add("known-service", false, "No oracle exists for this service."); return c.result(); } @@ -104,14 +125,21 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s } catch (error) { const prior = fastExperiments.get(ctx); if (prior !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: prior }); fastExperiments.delete(ctx); } throw error; } }, gradeCompanyModel, { sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000, + ...(spec.id === "companies.fast-responses" ? { oracleVersion: "paperclip.chat-catalog.identity.fast-responses.v2" } : {}), async act(ctx, state) { await waitForCompanySettingsTurn(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled; state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/${spec.path}`, "PUT", { enabled: true, connectionId: (state.connection as AiManagedConnectionSummary).id, grantId: (state.connection as AiManagedConnectionSummary).grantId, ...(spec.path === "decision-model" ? { allowBackground: false } : spec.path === "fast-response" ? { model: requestedAuxiliaryServiceModel, allowSponsored: false } : { model: requestedAuxiliaryServiceModel }) }); state.agentTestStatus = await probeAgentPermission(ctx, `${root}/${spec.path}/test`, "POST", {}); - await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/connections`, { waitUntil: "domcontentloaded" }); const section = spec.path === "decision-model" ? ctx.input.page.locator('section[aria-labelledby="decision-model-heading"]') : ctx.input.page.getByRole("region", { name: spec.section, exact: true }); + if (spec.path === "fast-response") { + const o = await observeIdentity(ctx, state); + const href = await openServiceControlFromReply(ctx, o.links, o.origin, () => section); + state.replyLinkedControlHref = href ?? null; + await ctx.input.evidence("identity-company-model-reply-link.json", { links: o.links, opened: href ?? null, url: ctx.input.page.url() }); + if (!href) { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); return; } + } else await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/connections`, { waitUntil: "domcontentloaded" }); await section.getByRole("button", { name: (state.connection as AiManagedConnectionSummary).name, exact: true }).click(); if (spec.path !== "decision-model") await section.getByLabel("Model", { exact: true }).fill(requestedAuxiliaryServiceModel); await section.getByRole("switch", { name: spec.toggle, exact: true }).setChecked(true);