Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-06 23:20:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): accept concise truthful routine history answers

Browse Source
This commit is contained in:
Dotta committed 2026-10-26 20:28:08 -05:00
1 parent d162a92c3f
commit 94cf8870d6
2 files changed
+4 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-catalog-domains/integrator-routine-history.test.ts
+1
View File
@@ -14,6 +14,7 @@ function valid() {
describe("routine invocation and native failure distinction", () => {
it("accepts distinct genuinely admitted failure and completed invocation, never absent evidence", () => {
expect(gradeIntegratorRoutineHistory(valid()).passed).toBe(true);
expect(gradeIntegratorRoutineHistory({ ...valid(), reply: "Two manual runs this week: ACORN-1 failed; ACORN-2 completed successfully." }).passed).toBe(true);
expect(gradeIntegratorRoutineHistory(undefined).passed).toBe(false);
const missing = valid(); missing.state.failedEvents = [];
expect(gradeIntegratorRoutineHistory(missing).passed).toBe(false);
tests/runner-e2e/chat-catalog-domains/integrator-routine-history.ts
+3 -1
View File
@@ -36,7 +36,9 @@ export function gradeIntegratorRoutineHistory(value: unknown) {
c.add("failed-native-not-inferred-routine-completion", !!failedTask?.identifier && !!completedTask?.identifier
&& groundedStatement(o.reply, new RegExp(escaped(failedTask.identifier), "i"), /\b(?:failed|failure|unsuccessful)\b/i, new RegExp(`${escaped(failedTask.identifier)}[^\n.!?]{0,60}\\b(?:succeeded|successfully completed)\\b`, "i"))
&& groundedStatement(o.reply, new RegExp(escaped(completedTask.identifier), "i"), /\b(?:completed|succeeded|successful)\b/i, /\ball (?:runs|executions) (?:succeeded|completed successfully)\b/i)
&& /\bmanual\b/i.test(o.reply) && /\b(?:scheduled|schedule)\b/i.test(o.reply) && /\b(?:no (?:recorded |confirmed )?scheduled|not (?:proof|evidence)|cannot confirm|can't confirm|missing)\b/i.test(o.reply), "The response preserves which actual native execution failed and which invocation completed, identifies the manual sources, and does not infer scheduled morning success from a saved schedule.");
&& !o.reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:scheduled|scheduler|automatically|automatic|on schedule)\b/i.test(sentence)
&& /\b(?:ran|executed|succeeded|completed|delivered|successful)\b/i.test(sentence)
&& !/\b(?:no|not|cannot|can't|unconfirmed|unverified|unknown|missing)\b/i.test(sentence)), "The response preserves which actual native execution failed and which invocation completed without inferring unsupported scheduled success. A truthful concise account of the actual manual runs needs no unprompted schedule-denial sentence.");
return c.result();
}
export const integratorRoutineHistoryCases = [integratorDefinition("routines", "routines.read-history", async ctx => {