mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(evals): accept bound reduced-passive timeline handoffs
Version the measured timeline oracle to v8 without changing query, evidence joins, or execution bounds. Co-Authored-By: Paperclip <paperclip@paperclip.ing>
This commit is contained in:
1 parent
5991e10109
commit
d52f38db7a
4 files changed
+19
-4
No files matched your search
@@ -258,7 +258,7 @@
|
||||
"../../server/src/services/native-runtime/status-decision-committer.ts",
|
||||
"../../server/src/services/work-timeline.ts"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.work-timeline.v7",
|
||||
"oracleVersion": "paperclip.chat-catalog.work-timeline.v8",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 3,
|
||||
"minimumRunCount": 3,
|
||||
|
||||
@@ -29,7 +29,7 @@ export function truthfulTimelineAccount(input: {
|
||||
const assignment = "\\b(?:reassign(?:ed|ment)?|assign(?:ed|ment)?|hand(?:ed)?(?:\\s+\\w+){0,4}\\s+(?:off|over)|handed|handoff)\\b";
|
||||
const target = `\\bto\\s+${escape(repairWorker)}\\b`;
|
||||
const actorAction = `${humanActor}(?:\\s+(?:then|later|now|personally|explicitly|actually|had|has|have|also)){0,3}\\s+${assignment}`;
|
||||
const passiveAction = `\\b(?:was|got|had been|has been)\\s+${assignment}`;
|
||||
const passiveAction = `(?:\\b(?:was|got|had been|has been)\\s+${assignment}|\\b(?:the|same|this)\\s+(?:task|work)\\s+(?:reassigned|assigned|handed\\s+(?:off|over))\\b)`;
|
||||
const handoffToRepairWorker = new RegExp(`${actorAction}${withinClause}{0,60}${target}|${passiveAction}${withinClause}{0,60}${target}${withinClause}{0,40}\\bby\\s+(?:(?:the|a|an)\\s+)?${humanActor}`, "i");
|
||||
const passiveHandoffToRepairWorker = new RegExp(`${passiveAction}${withinClause}{0,60}${target}`, "gi");
|
||||
const successorOwnership = new RegExp(`\\b${escape(repairWorker)}\\s+took\\s+(?:over\\s+(?:the\\s+)?(?:same\\s+)?(?:task|work)|over\\s+from\\s+${escape(firstWorker)}|over(?=\\s*(?:[,;.!?\\n]|$))|the\\s+same\\s+(?:task|work))\\b`, "gi");
|
||||
|
||||
@@ -42,6 +42,21 @@ it("accepts an ordered recorded reassignment without an unrequested human identi
|
||||
o.reply = o.reply.replace("Then a human reassigned the task to Riley.", "Then the task was reassigned to Riley.");
|
||||
expect(gradeWorkTimeline(o).passed).toBe(true);
|
||||
});
|
||||
it.each([
|
||||
{ handoff: "Same task reassigned to Riley.", expected: true },
|
||||
{ handoff: "Same task reassigned to Morgan.", expected: false },
|
||||
{ handoff: "Same task reassigned to Riley by Morgan.", expected: false },
|
||||
{ handoff: "Same task continued.", expected: false },
|
||||
])("binds reduced-passive task handoffs: $handoff", ({ handoff, expected }) => {
|
||||
const o = fixture();
|
||||
o.reply = `Morgan ran verification which failed and work paused.\n\n## Repair — ${handoff}\nRiley repaired the label and the check passed.`;
|
||||
expect(gradeWorkTimeline(o).passed).toBe(expected);
|
||||
});
|
||||
it("does not move the repair before a reduced-passive handoff", () => {
|
||||
const o = fixture();
|
||||
o.reply = "Morgan ran verification which failed and work paused. Riley repaired the label and the check passed. Same task reassigned to Riley.";
|
||||
expect(gradeWorkTimeline(o).passed).toBe(false);
|
||||
});
|
||||
it("accepts the successor's natural ownership transition in the ordered account", () => {
|
||||
const o = fixture();
|
||||
o.reply = o.reply.replace("Then a human reassigned the task to Riley.", "Then Riley took over the task.");
|
||||
@@ -236,7 +251,7 @@ it("admits the selected live timeline metadata without invoking fixture callback
|
||||
const spies = callbacks.map(key => vi.spyOn(implementation, key));
|
||||
try {
|
||||
expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 });
|
||||
expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v7");
|
||||
expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v8");
|
||||
expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles);
|
||||
expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 });
|
||||
for (const spy of spies) expect(spy).not.toHaveBeenCalled();
|
||||
|
||||
@@ -92,7 +92,7 @@ function timeline(): ChatCatalogCase {
|
||||
const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name };
|
||||
await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state;
|
||||
}, gradeWorkTimeline);
|
||||
return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v7", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000,
|
||||
return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v8", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000,
|
||||
sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts"],
|
||||
async observe(ctx, state): Promise<Timeline> { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; },
|
||||
async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },
|
||||
|
||||
Reference in new issue
Block a user