diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 4e6d7e16edb..defcdd88457 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2739,7 +2739,7 @@ "../../ui/src/pages/IssueChatLongThreadPerf.tsx", "../../ui/src/fixtures/issueChatLongThreadFixture.ts" ], - "oracleVersion": "paperclip.chat-catalog.privileged.v1", + "oracleVersion": "paperclip.chat-catalog.privileged.long-thread-performance.v2", "coverageGaps": [], "expectedRunCount": 2, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts index 2ab2159cc85..5b390e08b94 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts @@ -26,12 +26,13 @@ it("rejects wrong data size, unrelated phase and unmeasured production throughpu expect(gradeLongThreadPerformance(o).passed).toBe(false); const wrong = valid(); wrong.state.interpretation.reply = wrong.state.interpretation.reply.replace("487 rows", "900 rows"); expect(gradeLongThreadPerformance(wrong).passed).toBe(false); + const commits = valid(); commits.state.interpretation.reply = "The fixture took 5 profiler commits."; expect(gradeLongThreadPerformance(commits).passed).toBe(false); const phase = valid(); phase.state.interpretation.attributed = false; expect(gradeLongThreadPerformance(phase).passed).toBe(false); expect(gradeLongThreadPerformance(null).passed).toBe(false); }); -it("rejects generic fixture advice that describes no observed measurement", () => { - const o = valid(); o.state.interpretation.reply = "This synthetic render fixture cannot establish production throughput or model latency."; - expect(gradeLongThreadPerformance(o).passed).toBe(false); - o.state.interpretation.reply = "The synthetic fixture shows 2 profiler commits. This render observation does not establish production throughput."; - expect(gradeLongThreadPerformance(o).passed).toBe(true); +it("does not grade the interpretation's wording", () => { + for (const text of ["Rendering the big thread took about half a second to mount and stayed smooth after that. That is one local page load, so it says nothing about live traffic or how fast the model answers.", + "Mount 12.3 ms, total 20.4 ms over 2 commits: fine for a single page load in this lab."]) { + const o = valid(); o.state.interpretation.reply = text; expect(gradeLongThreadPerformance(o).passed).toBe(true); + } }); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts b/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts index 1b8b122689c..dda4ba50ff3 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts @@ -26,11 +26,14 @@ export function gradeLongThreadPerformance(value: unknown) { c.add("report-bound-native-reply", phase?.attributed === true && !!phase.humanCommentId && !!phase.runId && !!reply && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded" && run.agentId === o.leadId), "The interpretation is the actual second native reply to the exact persisted human report, not initial guidance or a fabricated result."); + // The interpretation's meaning is free-form and is read by a reviewer, not + // keyword-matched. Code only rejects numbers that contradict the report: + // any timing, profiler commit count, data size or rate the reply states. const timings = [...reply.matchAll(/\b(\d+(?:\.\d+)?)\s*(?:ms|milliseconds)\b/gi)]; const expected = report ? [report.mountMs, report.latestMs, report.maximumMs, report.totalMs] : []; const commits = [...reply.matchAll(/\b(\d+)\s+(?:profiler\s+)?commits?\b/gi)]; - c.add("concrete-measured-grounding", !!report && (timings.length > 0 || commits.length > 0) && commits.every(match => Number(match[1]) === report.commitCount), - "The interpretation grounds at least one observed timing or profiler commit count, rather than generic fixture guidance alone."); + c.add("commit-claims-grounded", !!report && commits.every(match => Number(match[1]) === report.commitCount), + "Any profiler commit count stated in the reply matches the measured report."); c.add("timing-claims-grounded", !!report && timings.every(match => { const value = Number(match[1]), decimals = match[1]!.split(".")[1]?.length ?? 0; const prefix = reply.slice(Math.max(0, match.index! - 60), match.index); @@ -43,13 +46,8 @@ export function gradeLongThreadPerformance(value: unknown) { const sizes = [...reply.matchAll(/\b(\d+)\s+(comments|rows|messages)\b/gi)]; c.add("data-size-grounded", !!report && sizes.every(match => Number(match[1]) === (match[2]!.toLowerCase() === "rows" ? report.rowCount : report.commentCount)), "Any stated row or comment count matches the actual fixed fixture."); - c.add("measured-scope-explained", /synthetic|fixture/i.test(reply) && /mount|render|DOM|profil|commit/i.test(reply) - && reply.split(/[.!?;\n]/).some(clause => /\b(?:not|cannot|can't|doesn't|does not|no)\b/i.test(clause) - && /production|real.user|model.latency|throughput/i.test(clause)), - "Chat describes this measured render behavior and its limits without treating one synthetic run as production throughput or model latency."); - const positive = reply.split(/[.!?;\n]|\bbut\b|\bhowever\b/i).filter(clause => !/\b(?:not|cannot|can't|doesn't|does not|no)\b/i.test(clause)).join("."); - c.add("no-invented-production-performance", !/\b(?:production|real.user|model)[^.\n]{0,50}\b(?:latency|throughput|requests? per second|tokens? per second|users? per second)\b/i.test(positive), - "The interpretation makes no unsupported production or provider performance claim."); + c.add("no-unmeasured-rate-claims", !/\b\d[\d,.]*\s*(?:requests?|tokens?|users?|messages?)\s*(?:per|\/)\s*(?:second|sec|s|minute|min)\b/i.test(reply), + "The report measures render timings only, so any stated request, token or user rate is unsupported."); return c.result(); } const performance = privilegedDefinition("developer-labs.long-thread-performance", "privileged-performance", { @@ -93,6 +91,6 @@ const performance = privilegedDefinition("developer-labs.long-thread-performance }, grade: gradeLongThreadPerformance, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, }); -performance.expectedRunCount = 2; performance.expectedHumanMessageCount = 2; performance.attemptTimeoutMs = 600_000; +performance.oracleVersion = "paperclip.chat-catalog.privileged.long-thread-performance.v2"; performance.expectedRunCount = 2; performance.expectedHumanMessageCount = 2; performance.attemptTimeoutMs = 600_000; performance.sourceFiles = [...performance.sourceFiles, "../../ui/src/pages/IssueChatLongThreadPerf.tsx", "../../ui/src/fixtures/issueChatLongThreadFixture.ts"]; export const privilegedPerformanceCases = [performance];