Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-10 12:37:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): stop parsing numbers out of the long-thread interpretation

Browse Source 
A correct interpretation rounded (~470 comments, ~0.27s), proposed other
sizes (50 comments) and failed the number checks. The grade now covers the
measured report and the attributed reply; the reviewer reads its meaning.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-09 03:43:11 -06:00
1 parent ae20a63031
commit c5a0d8eae3
3 files changed
+18 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-catalog-descriptors.json
+1 -1
View File
@@ -2739,7 +2739,7 @@
"../../ui/src/pages/IssueChatLongThreadPerf.tsx",
"../../ui/src/fixtures/issueChatLongThreadFixture.ts"
],
"oracleVersion": "paperclip.chat-catalog.privileged.long-thread-performance.v2",
"oracleVersion": "paperclip.chat-catalog.privileged.long-thread-performance.v3",
"coverageGaps": [],
"expectedRunCount": 2,
"expectedHumanMessageCount": 2,
tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts
+11 -22
View File
@@ -11,28 +11,17 @@ function valid() {
reply: "The synthetic fixture rendered 487 rows with 469 comments. Mount was 12.3 ms; total was 20.4 ms. This measures render cost, not production throughput or model latency." } } };
}
it("accepts actual fixed-fixture measurements and attributed limited interpretation", () => expect(gradeLongThreadPerformance(valid()).passed).toBe(true));
it("accepts ordinary display rounding and the reversed phrasing of a production limitation", () => {
const o = valid(); o.state.interpretation.reply = "The synthetic fixture mount was 12 ms. Production throughput cannot be inferred from these render timings.";
expect(gradeLongThreadPerformance(o).passed).toBe(true);
});
it("rejects invented timings and swaps between named metrics", () => {
for (const text of ["The synthetic fixture mount was 500 ms. It does not prove production throughput.",
"The synthetic fixture mount was 20.4 ms. It does not prove production throughput."]) {
const o = valid(); o.state.interpretation.reply = text; expect(gradeLongThreadPerformance(o).passed).toBe(false);
}
});
it("rejects wrong data size, unrelated phase and unmeasured production throughput", () => {
const o = valid(); o.state.interpretation.reply += " Production throughput is 100 requests per second.";
expect(gradeLongThreadPerformance(o).passed).toBe(false);
const wrong = valid(); wrong.state.interpretation.reply = wrong.state.interpretation.reply.replace("487 rows", "900 rows");
expect(gradeLongThreadPerformance(wrong).passed).toBe(false);
const commits = valid(); commits.state.interpretation.reply = "The fixture took 5 profiler commits."; expect(gradeLongThreadPerformance(commits).passed).toBe(false);
const phase = valid(); phase.state.interpretation.attributed = false; expect(gradeLongThreadPerformance(phase).passed).toBe(false);
expect(gradeLongThreadPerformance(null).passed).toBe(false);
});
it("does not grade the interpretation's wording", () => {
for (const text of ["Rendering the big thread took about half a second to mount and stayed smooth after that. That is one local page load, so it says nothing about live traffic or how fast the model answers.",
"Mount 12.3 ms, total 20.4 ms over 2 commits: fine for a single page load in this lab."]) {
it("does not parse the interpretation's numbers or wording", () => {
for (const text of ["Encouraging at ~470 comments (mount ~0.27s, append ~22ms); add points such as 50 comments to see the growth curve.",
"Rendering the big thread took about half a second to mount. That is one local page load, so it says nothing about live traffic."]) {
const o = valid(); o.state.interpretation.reply = text; expect(gradeLongThreadPerformance(o).passed).toBe(true);
}
});
it("rejects a reply not bound to the persisted report, or an unmeasured report", () => {
const phase = valid(); phase.state.interpretation.attributed = false; expect(gradeLongThreadPerformance(phase).passed).toBe(false);
const empty = valid(); empty.state.interpretation.reply = ""; expect(gradeLongThreadPerformance(empty).passed).toBe(false);
const extra = valid(); extra.runs.push({ id: "third", agentId: "lead", status: "succeeded" }); expect(gradeLongThreadPerformance(extra).passed).toBe(false);
const pending = valid(); pending.state.performanceReport.mountMs = Number.NaN; expect(gradeLongThreadPerformance(pending).passed).toBe(false);
const size = valid(); size.state.performanceReport.rowCount = 12; expect(gradeLongThreadPerformance(size).passed).toBe(false);
expect(gradeLongThreadPerformance(null).passed).toBe(false);
});
tests/runner-e2e/chat-catalog-domains/privileged-performance.ts
+6 -23
View File
@@ -26,28 +26,11 @@ export function gradeLongThreadPerformance(value: unknown) {
c.add("report-bound-native-reply", phase?.attributed === true && !!phase.humanCommentId && !!phase.runId && !!reply
&& o.runs.length === 2 && o.runs.every(run => run.status === "succeeded" && run.agentId === o.leadId),
"The interpretation is the actual second native reply to the exact persisted human report, not initial guidance or a fabricated result.");
// The interpretation's meaning is free-form and is read by a reviewer, not
// keyword-matched. Code only rejects numbers that contradict the report:
// any timing, profiler commit count, data size or rate the reply states.
const timings = [...reply.matchAll(/\b(\d+(?:\.\d+)?)\s*(?:ms|milliseconds)\b/gi)];
const expected = report ? [report.mountMs, report.latestMs, report.maximumMs, report.totalMs] : [];
const commits = [...reply.matchAll(/\b(\d+)\s+(?:profiler\s+)?commits?\b/gi)];
c.add("commit-claims-grounded", !!report && commits.every(match => Number(match[1]) === report.commitCount),
"Any profiler commit count stated in the reply matches the measured report.");
c.add("timing-claims-grounded", !!report && timings.every(match => {
const value = Number(match[1]), decimals = match[1]!.split(".")[1]?.length ?? 0;
const prefix = reply.slice(Math.max(0, match.index! - 60), match.index);
const label = [...prefix.matchAll(/\b(mount|latest|maximum|max|total)\b/gi)].at(-1)?.[1]?.toLowerCase();
const candidates = label === "mount" ? [report!.mountMs] : label === "latest" ? [report!.latestMs]
: label === "maximum" || label === "max" ? [report!.maximumMs] : label === "total" ? [report!.totalMs] : expected;
return candidates.some(actual => Math.abs(actual - value) <= Math.max(0.5 * 10 ** -decimals + 0.001, Math.abs(actual) * 0.01));
}),
"Any timing repeated in the reply matches an actual displayed measurement, allowing ordinary display rounding.");
const sizes = [...reply.matchAll(/\b(\d+)\s+(comments|rows|messages)\b/gi)];
c.add("data-size-grounded", !!report && sizes.every(match => Number(match[1]) === (match[2]!.toLowerCase() === "rows" ? report.rowCount : report.commentCount)),
"Any stated row or comment count matches the actual fixed fixture.");
c.add("no-unmeasured-rate-claims", !/\b\d[\d,.]*\s*(?:requests?|tokens?|users?|messages?)\s*(?:per|\/)\s*(?:second|sec|s|minute|min)\b/i.test(reply),
"The report measures render timings only, so any stated request, token or user rate is unsupported.");
// The interpretation is free-form analysis: correct replies round
// ("~470 comments", "~0.27s"), propose other sizes ("50 comments") and
// combine phases, so parsing its numbers or words rejects valid answers.
// Code grades the measured report and that the reply is the attributed
// answer to it; a reviewer reads the interpretation itself.
return c.result();
}
const performance = privilegedDefinition("developer-labs.long-thread-performance", "privileged-performance", {
@@ -91,6 +74,6 @@ const performance = privilegedDefinition("developer-labs.long-thread-performance
}, grade: gradeLongThreadPerformance,
async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
});
performance.oracleVersion = "paperclip.chat-catalog.privileged.long-thread-performance.v2"; performance.expectedRunCount = 2; performance.expectedHumanMessageCount = 2; performance.attemptTimeoutMs = 600_000;
performance.oracleVersion = "paperclip.chat-catalog.privileged.long-thread-performance.v3"; performance.expectedRunCount = 2; performance.expectedHumanMessageCount = 2; performance.attemptTimeoutMs = 600_000;
performance.sourceFiles = [...performance.sourceFiles, "../../ui/src/pages/IssueChatLongThreadPerf.tsx", "../../ui/src/fixtures/issueChatLongThreadFixture.ts"];
export const privilegedPerformanceCases = [performance];