diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 653b5ee13bd..d14ae9b3a2f 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -258,7 +258,7 @@ "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts" ], - "oracleVersion": "paperclip.chat-catalog.work-timeline.v5", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v6", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts index f306b4c9595..6764835d8a1 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts @@ -109,3 +109,23 @@ it.each([ it("still rejects an explicitly different worker under a heading", () => { expect(grade(account.replace("The verification command failed", "The verification command another worker ran failed"))).toBe(false); }); + +const bulletedAccount = `**First attempt (Morgan)** +- The check genuinely failed: exit code 1. +- Morgan reported the task as blocked. + +**Repair (Riley)** +The task was reassigned to Riley. +- Riley added the label while keeping the heading. +- Verification on the saved bytes passed: exit code 0, matching the file.`; +it("joins a worker's explicit repair and adjacent verified result within the same bulleted section", () => { + expect(grade(bulletedAccount)).toBe(true); +}); +it.each([ + bulletedAccount.replace("Verification on the saved bytes passed", "Morgan's verification on the saved bytes passed"), + bulletedAccount.replace("- Verification", "- Morgan inspected another file.\n- Verification"), + bulletedAccount.replace("- Verification", "\n- Verification"), + bulletedAccount.replace("Verification on the saved bytes passed", "An unrelated verification on the saved bytes passed"), +])("does not borrow another actor, paragraph or unrelated result for a repair: %s", reply => { + expect(grade(reply)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index f4f7420a48e..15481798489 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -22,14 +22,18 @@ export function truthfulTimelineAccount(input: { const actorAction = `${humanActor}(?:\\s+(?:then|later|now|personally|explicitly|actually|had|has|have|also)){0,3}\\s+${assignment}`; const passiveAction = `\\b(?:was|got|had been|has been)\\s+${assignment}`; const handoffToRepairWorker = new RegExp(`${actorAction}${withinClause}{0,60}${target}|${passiveAction}${withinClause}{0,60}${target}${withinClause}{0,40}\\bby\\s+(?:(?:the|a|an)\\s+)?${humanActor}`, "i"); + const passiveHandoffToRepairWorker = new RegExp(`${passiveAction}${withinClause}{0,60}${target}`, "gi"); const differentNamedPerson = (text: string) => { const namedRole = /^\s*(?:Then\s+)?([A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+)*)\s*,?\s*(?:the\s+)?(?:human|person|user|operator)\b/u.exec(text); return !!namedRole && !["The", "A", "An", input.personName].some(name => name?.toLowerCase() === namedRole[1]!.toLowerCase()) || /\b(?:another|different|other)\s+(?:human|person|user|operator)\b/i.test(text); }; - const humanHandoffMatches = (text: string) => { + const handoffMatches = (text: string) => { if (differentNamedPerson(text) || /\b(?:not|never|no)\b[^.!?]{0,50}\b(?:reassign|assign|hand|handoff)/i.test(text)) return []; - return [...text.matchAll(new RegExp(handoffToRepairWorker.source, "gi"))].filter(match => { + const actorUnspecified = [...text.matchAll(passiveHandoffToRepairWorker)].filter(match => + !/\bby\b/i.test(text.slice(match.index + match[0].length)) + && !/\b(?:automatically|autonomously|self-assigned)\b/i.test(text)); + return [...text.matchAll(new RegExp(handoffToRepairWorker.source, "gi")), ...actorUnspecified].filter(match => { // An explicit other person's role must not become the current human just // because earlier milestones share this sentence. if (!/^(?:human|person|user|operator|board)\b/i.test(match[0])) return true; @@ -48,14 +52,14 @@ export function truthfulTimelineAccount(input: { const resolvedPronouns = new Set(); const headingPrefix = heading?.[0] ?? ""; const unwrapped = headingPrefix + raw.slice(headingPrefix.length).replace(/\n(?![ \t]*(?:[-+*]\s+|\d+[.)]\s+|#{1,6} +|\*\*))/g, " "); - const text = unwrapped.replace(/[*`]/g, "") + const text = unwrapped.replace(/^[ \t]*(?:[-+*]\s+|\d+[.)]\s+)/gm, "").replace(/[*`]/g, "") .replace(/^[ \t]*(?:(?:[-+] |\d+[.)] )\s*)?\d{1,2}:\d{2}(?::\d{2})?(?:[ \t]+(?:AM|PM))?(?:[ \t]*[—–:-][ \t]*|[ \t]+)/gim, "") .replace(/<([a-z][a-z0-9]*)(?:\s+[^<>]*|\s*\/?)>/gi, "$1 ").replace(/<\/[a-z][a-z0-9]*\s*>/gi, "") .replace(/(^|\n|[.!?]\s+)(he|she|they)\b/gi, (match, lead, _pronoun, at, paragraph) => { if (!owner || !heading) return match; const before = paragraph.slice(heading[0].replace(/[*`]/g, "").length, at).trim(); const previous = before.match(/[^.!?\n]+[.!?]?$/)?.[0]?.trim() ?? ""; - const directHandoff = owner === repairWorker && humanHandoffMatches(previous) + const directHandoff = owner === repairWorker && handoffMatches(previous) .some(match => !namesOtherActor(previous.replace(match[0], ""), owner)); const sameSubject = (new RegExp(`^${escape(owner)}\\b`, "i").test(previous) || resolvedPronouns.has(paragraph.lastIndexOf(previous, at))) && !namesOtherActor(previous, owner); @@ -71,7 +75,7 @@ export function truthfulTimelineAccount(input: { // Search milestone clauses, not the first occurrence of a worker or outcome. // A summary or heading may mention the repair before the ordered account. const clauses = [...body.matchAll(new RegExp(`${withinClause}+(?:[!?]|\\.(?=\\s|$)|(?=\\n|$))`, "g"))] - .map(match => ({ text: match[0], at: match.index, owner: paragraphs.find(row => row.start <= match.index && row.end > match.index)?.owner })); + .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, paragraphStart: paragraph?.start }; }); // A command noun can have ordinary modifiers, but never absorb a new event, // actor, negation or clause boundary while connecting this pause to failure. const commandBoundaries = [ @@ -89,7 +93,7 @@ export function truthfulTimelineAccount(input: { const failures = clauses.flatMap(({ text, at, owner }) => (mentions(text, firstWorker) || owner === firstWorker && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker)) && new RegExp(`\\b${commandSubject}\\b`, "i").test(text) ? [...text.matchAll(failureOutcome)].map(match => ({ at: at + match.index, end: at + match.index + match[0].length, clauseAt: at })) : []); - const handoffs = clauses.flatMap(({ text, at }) => humanHandoffMatches(text).map(match => ({ at: at + match.index }))); + const handoffs = clauses.flatMap(({ text, at }) => handoffMatches(text).map(match => ({ at: at + match.index }))); const repairGap = "(?:\\s+|\\s*[,:—–-]\\s*)"; const repairModifier = "(?:then|later|now|personally|explicitly|actually|had|has|have|also|who|successfully|finally|subsequently|after\\s+the\\s+handoff)"; const repairVerb = [ @@ -114,10 +118,11 @@ export function truthfulTimelineAccount(input: { // Only an immediately adjacent, complete command result can finish this // repair account. No intervening sentence or earlier/unrelated result joins. const adjacentVerification = new RegExp(`^\\s*(?:(?:the|a|an)\\s+|${escape(repairWorker)}['’]s\\s+)?${commandSubject}\\s+${successModifiers}(?:${repairSuccess.source})(?:\\s+with\\s+exit(?:\\s+(?:code|status))?\\s*[:=]?\\s*0)?\\s*[.!]?\\s*$`, "i"); + const contextualCommandLead = new RegExp(`^\\s*(?:(?:the|a|an)\\s+)?${commandSubject}(?:[ \\t]+${commandModifier}){0,5}\\s+${successModifiers}$`, "i"); const repairObjectTail = new RegExp(`^(?:[ \\t]+${commandModifier}){0,6}\\s*[.!]?\\s*$`, "i"); const passiveRepairSubject = new RegExp(`^\\s*(?:${commandModifier}[ \\t]+){1,6}$`, "i"); const deniedRepairObject = /^\s*(?:[a-z]+ly\s+){0,2}(?:(?:no|none|neither|nothing)\b|zero(?=\s|[.!?]|$))/i; - const repairs = clauses.flatMap(({ text, at }, clauseIndex) => [...text.matchAll(repairAction)].filter(match => { + const repairs = clauses.flatMap(({ text, at, owner, paragraphStart }, clauseIndex) => [...text.matchAll(repairAction)].filter(match => { const end = match.index + match[0].length; const laterSuccess = [...text.slice(end).matchAll(new RegExp(repairSuccess.source, "gi"))].some(success => { const outcomeAt = end + success.index; @@ -131,9 +136,17 @@ export function truthfulTimelineAccount(input: { const passiveRepair = /^\b(?:was|got|had been|has been)\b/i.test(match[0]); const assertedRepair = /^\s*(?:(?:then|later|finally)\s+)?$/i.test(lead) || passiveRepair && passiveRepairSubject.test(lead); + const nextClause = clauses[clauseIndex + 1]; + const contextualOutcome = next && repairSuccess.exec(next); + const contextualResult = owner === repairWorker && nextClause?.owner === repairWorker + && paragraphStart === nextClause.paragraphStart && !!contextualOutcome + && contextualCommandLead.test(next.slice(0, contextualOutcome.index)) + && successIsOwned(next, contextualOutcome.index) + && !new RegExp(failureOutcome.source, "i").test(next) + && !namesOtherActor(text + " " + next, repairWorker) && !mentions(text + " " + next, firstWorker); if (next && assertedRepair && !deniedRepairObject.test(passiveRepair ? lead : text.slice(end)) - && repairObjectTail.test(text.slice(end)) - && !text.trimEnd().endsWith("?") && adjacentVerification.test(next) + && (repairObjectTail.test(text.slice(end)) || contextualResult) + && !text.trimEnd().endsWith("?") && !next.trimEnd().endsWith("?") && (adjacentVerification.test(next) || contextualResult) && !/\b(?:if|unless|whether|before|earlier|previous(?:ly)?|already|unrelated|different|other)\b/i.test(text + next)) return true; // An earlier success word counts only with the bounded causal relationship // "the check passed after [this worker's repair]", never by itself. diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index 55ca85e62c1..93779007197 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -37,6 +37,28 @@ function fixture() { verification: [{ taskId: "task", runId: "first", filename: "launch.html", command: "node accessible-heading-and-email-label-check", sha256: "a".repeat(64), exitCode: 1, checkedAt: "2026-10-26T00:50:00Z" }, { taskId: "task", runId: "second", filename: "launch.html", command: "node accessible-heading-and-email-label-check", sha256: "b".repeat(64), exitCode: 0, checkedAt: "2026-10-26T02:50:00Z" }], }; } +it("accepts an ordered recorded reassignment without an unrequested human identity", () => { + const o = fixture(); + o.reply = o.reply.replace("Then a human reassigned the task to Riley.", "Then the task was reassigned to Riley."); + expect(gradeWorkTimeline(o).passed).toBe(true); +}); +it.each([ + "Then the task was reassigned to Riley by Morgan.", + "Then the task was reassigned to Riley by Jordan.", + "Then the task was reassigned to Morgan.", + "Then Riley took the same task.", + "Then the task was reassigned to Riley automatically.", +])("rejects false or missing handoff provenance: %s", handoff => { + const o = fixture(); + o.reply = o.reply.replace("Then a human reassigned the task to Riley.", handoff); + expect(gradeWorkTimeline(o).passed).toBe(false); +}); +it("keeps failure and pause before an actor-unspecified reassignment", () => { + const o = fixture(); + o.reply = "The task was reassigned to Riley. Morgan ran verification which failed and work paused. Riley repaired the label and the check passed."; + expect(gradeWorkTimeline(o).passed).toBe(false); +}); + it("accepts an account without unrequested date or timezone labels but rejects prior-week ISO dates", () => { const o = fixture(); o.reply = o.reply.replace("Today (UTC): ", "") + " [Source](/RUN/issues/task?at=2026-10-25)"; @@ -205,7 +227,7 @@ it("admits the selected live timeline metadata without invoking fixture callback const spies = callbacks.map(key => vi.spyOn(implementation, key)); try { expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 }); - expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v5"); + expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v6"); expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles); expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 }); for (const spy of spies) expect(spy).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts index 7117855eebe..3114b73a79a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts @@ -57,7 +57,7 @@ export function gradeWorkTimeline(value: unknown) { }); c.add("ordered-and-truthful-account", worker.id === first.agentId && repairWorker.id === second.agentId && dateClaimsInWindow && truthfulTimelineAccount({ reply: o.reply, firstWorker: typeof worker.name === "string" ? worker.name : "", repairWorker: typeof repairWorker.name === "string" ? repairWorker.name : "", - personName: typeof o.state.personName === "string" ? o.state.personName : undefined }), "The reply attributes and orders the actual command failure/pause, human reassignment to the different repair worker and successful repair, keeps explicit ISO dates within the current-week observation window, and does not invent a failed native run."); + personName: typeof o.state.personName === "string" ? o.state.personName : undefined }), "The reply attributes and orders the actual command failure/pause, saved reassignment to the different repair worker and successful repair, keeps explicit ISO dates within the current-week observation window, and does not invent a failed native run."); c.add("inspectable-major-milestones", hasLink(o, "issues", t) && hasLink(o, "issues", prep), "Both actual launch work items are linked for inspection."); const hidden = entity(o.state, "hiddenTask"); c.add("hidden-work-never-disclosed", [403, 404].includes(o.hiddenStatus) && ![hidden.id, hidden.title, hidden.description].filter(Boolean).some(text => o.reply.includes(text!)) @@ -92,7 +92,7 @@ function timeline(): ChatCatalogCase { const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name }; await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state; }, gradeWorkTimeline); - return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v5", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, + return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v6", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts"], async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; }, async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },