diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json
index b50df3afaf2..798187b7f08 100644
--- a/tests/runner-e2e/chat-catalog-descriptors.json
+++ b/tests/runner-e2e/chat-catalog-descriptors.json
@@ -249,7 +249,6 @@
"chat-catalog-domains/internal-activity.ts",
"chat-scenarios/activity.yaml",
"chat-catalog-domains/internal-work-timeline.ts",
- "chat-catalog-domains/internal-timeline-narrative.ts",
"chat-catalog-domains/internal-timeline-verifier.ts",
"chat-catalog-domains/internal-privacy.ts",
"chat-catalog-domains/identity-auth.ts",
@@ -259,7 +258,7 @@
"../../server/src/services/work-timeline.ts",
"../../server/src/services/native-runtime/runner-api-catalog.ts"
],
- "oracleVersion": "paperclip.chat-catalog.work-timeline.v13",
+ "oracleVersion": "paperclip.chat-catalog.work-timeline.v14",
"coverageGaps": [],
"expectedRunCount": 3,
"minimumRunCount": 3,
diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts
deleted file mode 100644
index 8f5208968f1..00000000000
--- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts
+++ /dev/null
@@ -1,311 +0,0 @@
-import { expect, it } from "vitest";
-import { truthfulTimelineAccount } from "./internal-timeline-narrative.js";
-
-const account = `**First attempt (Morgan)**
-Morgan built the page. The verification command failed with exit 1. Morgan reported the task as blocked.
-
-**Repair (Riley)**
-You reassigned the task to Riley. He added \`\`, preserved the heading, and verification exited 0.`;
-const grade = (reply: string) => truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley" });
-it("resolves a named section's command and pronoun without requiring repeated names", () => {
- expect(grade(account)).toBe(true);
-});
-it.each([
- account.replace("You reassigned the task to Riley. ", ""),
- account.replace("You reassigned", "Morgan reassigned"),
- account.replace("He added", "Morgan added"),
- account.replace("First attempt (Morgan)", "First attempt (Riley)"),
- account + " A native run failed.",
- account.replace("verification exited 0", "verification exited 1"),
-])("retains consequential actor and outcome requirements: %s", reply => {
- expect(grade(reply)).toBe(false);
-});
-it("does not infer the initial worker over a contrary possessive within the command clause", () => {
- const reply = account.replace("**First attempt (Morgan)**\nMorgan built the page. ", "")
- .replace("The verification command failed", "The verification command failed on Riley's attempt");
- expect(grade(reply)).toBe(false);
-});
-it("does not credit a different worker's execution as the first failed attempt", () => {
- const reply = account.replace("**First attempt (Morgan)**\nMorgan built the page. ", "")
- .replace("command failed", "command failed during Riley's execution");
- expect(grade(reply)).toBe(false);
-});
-it("keeps a contrary worker heading as invalidation across blank lines", () => {
- const reply = account.replace("First attempt (Morgan)", "Initial attempt (Riley)")
- .replace("Morgan built the page. ", "Riley built the page.\n\n");
- expect(grade(reply)).toBe(false);
-});
-it("keeps a legitimate anonymous command description across a blank line", () => {
- expect(grade(account.replace("The verification", "\n\nThe verification"))).toBe(true);
-});
-
-it("does not turn a project prototype possessive into a worker", () => {
- expect(grade(account.replace("**First attempt (Morgan)**\nMorgan built the page. ", "")
- .replace("command failed", "command failed on Launch's prototype"))).toBe(true);
-});
-it("starts fresh worker scope at a consecutive Markdown heading", () => {
- expect(grade(account.replace("**First attempt (Morgan)**\nMorgan built the page. ", "## Earlier work (Riley)\nSummary.\n## Initial build (Morgan)\nThe engineer built the page.\n\n"))).toBe(true);
-});
-it.each(["run", "initial attempt", "saved check"])("retains unknown-person %s ownership as a contrary actor", execution => {
- expect(grade(account.replace("**First attempt (Morgan)**\nMorgan built the page. ", "")
- .replace("command failed", `command failed on Avery's ${execution}`))).toBe(false);
-});
-
-it("does not credit a desired reassignment as a completed handoff", () => {
- expect(grade("Morgan ran verification which failed and work paused. You wanted the task reassigned to Riley. Riley repaired the label and the check passed.")).toBe(false);
-});
-it("binds a known human's possessive ownership independently of the object noun", () => {
- const reply = account.replace("**First attempt (Morgan)**\nMorgan built the page. ", "")
- .replace("command failed", "command failed on Avery's build");
- expect(truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley", personName: "Avery" })).toBe(false);
-});
-
-it("separates intent about the label from a subsequent completed handoff", () => {
- expect(grade("Morgan ran verification which failed and work paused. You wanted the label fixed, so you reassigned the task to Riley. Riley repaired the label and the check passed.")).toBe(true);
-});
-it("does not turn a known person's requirements into execution ownership", () => {
- const reply = "The verification command failed with exit 1 against Avery’s requirements and task paused. You reassigned the task to Riley. Riley repaired the label and the check passed.";
- expect(truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley", personName: "Avery" })).toBe(true);
-});
-it("does not credit a planned reassignment as completed", () => {
- expect(grade("Morgan ran verification which failed and work paused. You planned to have the task reassigned to Riley. Riley repaired the label and the check passed.")).toBe(false);
-});
-
-it("does not give a heading credit for another actor's command", () => {
- expect(grade(account.replace("The verification command failed", "The human operator ran verification and it failed"))).toBe(false);
-});
-
-it("accepts a transition before a command subject", () => {
- expect(grade(account.replace("The verification", "Then the verification"))).toBe(true);
-});
-it("resolves the first subject after the worker heading", () => {
- expect(grade(account.replace("**Repair (Riley)**\nYou reassigned the task to Riley. He", "You reassigned the task to Riley.\n\n**Repair (Riley)**\nHe"))).toBe(true);
-});
-it.each([
- account.replace("You reassigned the task to Riley. He", "You reassigned the task to Riley. Avery inspected the page. He"),
- account.replace("You reassigned the task to Riley. He", "You reassigned the task to Riley. The human operator inspected the page. He"),
- account.replace("The verification command failed", "The verification command that Avery ran failed"),
-])("does not resolve another actor through a heading: %s", reply => {
- expect(grade(reply)).toBe(false);
-});
-
-it("does not treat a weekday in a command result as an actor", () => {
- expect(grade(account.replace("failed with exit 1", "failed on Tuesday with exit 1"))).toBe(true);
-});
-it("retains the safely resolved subject across consecutive pronouns", () => {
- expect(grade(account.replace("He added", "He preserved the heading. He added"))).toBe(true);
-});
-
-const timestampedAccount = `**Initial build (Morgan)**
-- 14:51:50 — Verification ran on saved bytes and returned exit 1.
-- 14:52:04 — Morgan reported the failure and paused the task blocked.
-
-**Repair (Riley)**
-- 14:52:05 — You reassigned the task to Riley with the brief "complete only on real success."
-- 14:52:27 — Riley added the label and verification returned exit 0.`;
-it("reads timestamped milestone bullets and a handoff ending in a quoted sentence", () => {
- expect(grade(timestampedAccount)).toBe(true);
-});
-it.each([
- timestampedAccount.replace(/- 14:52:05[^\n]+\n/, ""),
- timestampedAccount.replace("You reassigned", "Morgan reassigned"),
- timestampedAccount.replace("Riley added", "Morgan added"),
-])("keeps attribution required in timestamped milestones: %s", reply => {
- expect(grade(reply)).toBe(false);
-});
-
-it("keeps wrapped conditional context without hiding an asserted native failure", () => {
- expect(grade(account + "\n\nIf a\nnative run failed\nit would be recorded; both native runs succeeded.")).toBe(true);
- expect(grade(account + "\n\nA\nnative run failed\nbefore repair.")).toBe(false);
-});
-it.each([
- "The human reassigned the task to Riley.",
- "The human operator reassigned the task to Riley.",
- "You reassigned the task to Riley after the failure.",
-])("uses the same validated human handoff for the following pronoun: %s", handoff => {
- expect(grade(account.replace("You reassigned the task to Riley.", handoff))).toBe(true);
-});
-it.each([
- "The verification command run by Avery failed",
- "The verification command Avery ran failed",
-])("does not give a heading another runner's command: %s", command => {
- expect(grade(account.replace("The verification command failed", command))).toBe(false);
-});
-it.each(["2:51:50 PM —", "14:51:50:", "14:51:50"])("reads an equivalent display clock %s", clock => {
- expect(grade(timestampedAccount.replace("14:51:50 —", clock))).toBe(true);
-});
-it.each(["## Repair (Riley)\n\n", "**Repair (Riley)** "])("reads an adjacent heading layout %s", heading => {
- expect(grade(account.replace("**Repair (Riley)**\n", heading))).toBe(true);
-});
-
-it.each(["on Monday's run", "during Today's run"])("does not turn a possessive day into the command actor: %s", day => {
- expect(grade(account.replace("failed with exit 1", `failed ${day} with exit 1`))).toBe(true);
-});
-it.each(["engineer", "reviewer"])("permits an unbound %s role to refer to the named heading worker", role => {
- expect(grade(account.replace("The verification command failed", `The verification command that the ${role} ran failed`))).toBe(true);
-});
-
-it.each([
- "The verification command failed with exit 1 after the test was executed.",
- "The verification command run by the native runner failed with exit 1.",
- "The verification command run by the test harness failed with exit 1.",
- "The verification command the runner ran failed with exit 1.",
-])("does not turn command execution into another worker: %s", result => {
- expect(grade(account.replace("The verification command failed with exit 1.", result))).toBe(true);
-});
-it("still rejects an explicitly different worker under a heading", () => {
- expect(grade(account.replace("The verification command failed", "The verification command another worker ran failed"))).toBe(false);
-});
-
-const bulletedAccount = `**First attempt (Morgan)**
-- The check genuinely failed: exit code 1.
-- Morgan reported the task as blocked.
-
-**Repair (Riley)**
-The task was reassigned to Riley.
-- Riley added the label while keeping the heading.
-- Verification on the saved bytes passed: exit code 0, matching the file.`;
-it("joins a worker's explicit repair and adjacent verified result within the same bulleted section", () => {
- expect(grade(bulletedAccount)).toBe(true);
-});
-it.each([
- bulletedAccount.replace("Verification on the saved bytes passed", "Morgan's verification on the saved bytes passed"),
- bulletedAccount.replace("- Verification", "- Morgan inspected another file.\n- Verification"),
- bulletedAccount.replace("- Verification", "\n- Verification"),
- bulletedAccount.replace("Verification on the saved bytes passed", "An unrelated verification on the saved bytes passed"),
-])("does not borrow another actor, paragraph or unrelated result for a repair: %s", reply => {
- expect(grade(reply)).toBe(false);
-});
-
-const coordinatedAccount = bulletedAccount
- .replace("The task was reassigned to Riley.\n", "")
- .replace("- Riley added the label while keeping the heading.\n- Verification on the saved bytes passed", "- Riley took the same task, added the label while keeping the heading, and re-ran the check on the saved bytes. It passed");
-it("retains the successor subject through coordinated repair and its nearest command result", () => {
- expect(grade(coordinatedAccount)).toBe(true);
-});
-it.each([
- coordinatedAccount.replace("and re-ran", "and Morgan re-ran"),
- coordinatedAccount.replace("It passed", "Morgan inspected another task. It passed"),
- coordinatedAccount.replace("It passed", "\n\nIt passed"),
- coordinatedAccount.replace("added the label", "did not add the label"),
-])("keeps coordinated command ownership and adjacency bounded: %s", reply => {
- expect(grade(reply)).toBe(false);
-});
-
-it.each([
- { handoff: "The task was reassigned by Morgan to Riley.", expected: false },
- { handoff: "By the runner, the task was reassigned to Riley.", expected: false },
- { handoff: "By the human operator, the task was reassigned to Riley.", expected: true },
-])("checks explicit handoff provenance throughout the clause: $handoff", ({ handoff, expected }) => {
- expect(grade(account.replace("You reassigned the task to Riley.", handoff).replace("He added", "Riley added"))).toBe(expected);
-});
-it("rejects an asserted pass whose command exit is nonzero", () => {
- expect(grade(bulletedAccount.replace("exit code 0, matching the file", "exit code 2"))).toBe(false);
-});
-
-it.each(["Friday", "5 PM", "the end of the day"])("does not invent an actor from a by-deadline adjunct: %s", deadline => {
- expect(grade(account.replace("You reassigned the task to Riley.", `You reassigned the task to Riley by ${deadline}.`))).toBe(true);
-});
-
-const explicitRepairContinuity = "Morgan ran verification which failed and work paused.\n\nYou reassigned the task to Riley.\n\n## Repair and verification\nRiley added the label, preserved the heading, then re-ran verify_launch_html. It passed with exit code 0 on the saved bytes.";
-it("binds the explicit repair subject to its adjacent command result without a named heading", () => {
- expect(grade(explicitRepairContinuity)).toBe(true);
-});
-it.each([
- explicitRepairContinuity.replace("Riley added", "Morgan added"),
- explicitRepairContinuity.replace("then re-ran", "then Morgan re-ran"),
- explicitRepairContinuity.replace("It passed", "Avery inspected the page. It passed"),
- explicitRepairContinuity.replace("It passed", "The unrelated test passed"),
- explicitRepairContinuity.replace("Riley added the label", "Riley repaired no label"),
- explicitRepairContinuity.replace("Riley added the label", "Riley planned to add the label"),
- explicitRepairContinuity.replace("It passed", "It did not pass"),
- explicitRepairContinuity.replace("You reassigned the task to Riley.", "") + " You reassigned the task to Riley.",
-])("keeps actor, event, negation and chronology boundaries for adjacent repair results: %s", reply => {
- expect(grade(reply)).toBe(false);
-});
-
-it("stops repair subject continuity when a coordinated clause introduces another worker", () => {
- expect(grade("Morgan ran verification which failed and work paused. You reassigned the task to Riley. Riley added the label, and Avery inspected the saved bytes and then re-ran verify_launch_html. It passed with exit code 0 on the saved bytes.")).toBe(false);
-});
-
-const editedArtifactAccount = `Morgan ran verification which failed and work paused.
-
-You reassigned RUN-3 from Morgan to Riley.
-
-**Repair and passing verification**
-Riley edited \`launch.html\` to add \`\` while preserving the \`
\`, then re-ran \`verify_launch_html\` on the saved bytes:
-- Result: exit code 0 (passed), checked 00:09:08.`;
-const gradeEditedArtifact = (reply: string) => truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley", taskIdentifier: "RUN-3" });
-it("binds a completed artifact edit to its immediately attached command-result bullet", () => {
- expect(gradeEditedArtifact(editedArtifactAccount)).toBe(true);
-});
-it.each([
- editedArtifactAccount.replace("Riley edited", "Morgan edited"),
- editedArtifactAccount.replace("Riley edited", "Riley planned to edit"),
- editedArtifactAccount.replace("Riley edited", "Riley never edited"),
- editedArtifactAccount.replace("then re-ran", "then Avery re-ran"),
- editedArtifactAccount.replace("Result: exit code 0 (passed)", "Result: Avery ran the check which passed"),
- editedArtifactAccount.replace("- Result:", "- Avery inspected the page.\n- Result:"),
- editedArtifactAccount.replace("- Result:", "\n- Result:"),
- editedArtifactAccount.replace("- Result:", "## Another milestone\n- Result:"),
- editedArtifactAccount.replace("exit code 0 (passed)", "an unrelated check passed"),
- editedArtifactAccount.replace("exit code 0 (passed)", "did not pass"),
- editedArtifactAccount.replace("exit code 0 (passed)", "exit code 1 (failed)"),
- editedArtifactAccount.replace("You reassigned RUN-3 from Morgan to Riley.", "") + " You reassigned RUN-3 from Morgan to Riley.",
- editedArtifactAccount + " A native run failed.",
-])("keeps completed-edit command results bounded by actor, assertion, paragraph and order: %s", reply => {
- expect(gradeEditedArtifact(reply)).toBe(false);
-});
-
-
-const bulletCommandAccount = `**Timeline (Nov 6, UTC)**
-- **02:31:30–02:32:37 — Morgan:** built launch.html without the label per the brief. Ran authorized verify_launch_html on the saved bytes: exitCode 1. Morgan set the task to blocked.
-- **02:32:38 — you reassigned RUN-3 to Riley**, cleared the blocker.
-- **02:32:38–02:33:04 — Riley:** added , preserved the heading, reran verify_launch_html on the saved bytes: exitCode 0. Run succeeded; committed to done.`;
-const gradeBulletCommand = (reply: string) => truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley", taskIdentifier: "RUN-3" });
-it("keeps a completed command bound to the preceding worker-labeled bullet subject", () => {
- expect(gradeBulletCommand(bulletCommandAccount)).toBe(true);
-});
-it.each([
- bulletCommandAccount.replace("Ran authorized", "Avery ran authorized"),
- bulletCommandAccount.replace("brief. Ran", "brief. Avery inspected the page. Ran"),
- bulletCommandAccount.replace("brief. Ran", "brief.\n- Ran"),
- bulletCommandAccount.replace("brief. Ran", "brief.\n## Another milestone\nRan"),
- bulletCommandAccount.replace("Ran authorized", "Planned to run authorized"),
- bulletCommandAccount.replace("exitCode 1", "did not fail"),
- bulletCommandAccount.replace("exitCode 0", "exitCode 2"),
- bulletCommandAccount.replace("Riley:** added", "Riley:** planned to add"),
- bulletCommandAccount.replace("- **02:32:38 — you reassigned RUN-3 to Riley**, cleared the blocker.\n", "") + " You reassigned RUN-3 to Riley.",
-])("does not borrow omitted command subjects across a material boundary: %s", reply => {
- expect(gradeBulletCommand(reply)).toBe(false);
-});
-
-const implicitTransitionAccount = `## Signup implementation — RUN-3
-- Morgan built the page. The verification command returned exit 1. The task paused as blocked.
-- Riley added the missing label and re-ran verification: exit 0.`;
-const gradeImplicitTransition = (reply: string) => truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley", taskIdentifier: "RUN-3" });
-it("accepts a same-task named-worker transition without requiring an assignment phrase", () => {
- expect(gradeImplicitTransition(implicitTransitionAccount)).toBe(true);
- expect(gradeImplicitTransition(implicitTransitionAccount.replace("## Signup implementation — RUN-3", "2. **Signup implementation — RUN-3**"))).toBe(true);
-});
-it("keeps automatic verification and another task's requirements as context", () => {
- expect(gradeImplicitTransition(implicitTransitionAccount.replace("The verification command returned exit 1", "The verification command failed. The validator automatically reported exit 1"))).toBe(true);
- expect(gradeImplicitTransition(implicitTransitionAccount.replace("RUN-3", "RUN-3 implements requirements from [RUN-2](/RUN/issues/RUN-2)"))).toBe(true);
- expect(gradeImplicitTransition(implicitTransitionAccount.replace("Riley added", "Riley automatically added"))).toBe(true);
- expect(gradeImplicitTransition(implicitTransitionAccount.replace("the missing label and", "the missing label automatically and"))).toBe(true);
-});
-it.each([
- implicitTransitionAccount.replace("Riley added", "Riley repaired RUN-4 and added"),
- implicitTransitionAccount.replace("The task paused as blocked.", "The task paused as blocked. The task recovered automatically."),
- implicitTransitionAccount + " Then Riley self-assigned the task.",
- implicitTransitionAccount + " Then Riley recorded a self-takeover of the task.",
- implicitTransitionAccount.replace("- Riley", "\n## Separate work — RUN-4\n- Riley"),
- implicitTransitionAccount.replace("The task paused as blocked.\n- Riley added the missing label and re-ran verification: exit 0.", "Riley added the missing label and re-ran verification: exit 0. The task paused as blocked."),
- implicitTransitionAccount.replace("The task paused as blocked.", ""),
- implicitTransitionAccount.replace("returned exit 1", "returned exit 0"),
- implicitTransitionAccount.replace("Riley added", "Morgan added"),
- implicitTransitionAccount.replace("Riley added", "Riley planned to add"),
- implicitTransitionAccount.replace("exit 0", "exit 2"),
-])("retains task, pause, failure, actor and chronology boundaries for an implicit transition: %s", reply => {
- expect(gradeImplicitTransition(reply)).toBe(false);
-});
diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts
deleted file mode 100644
index 9498c39c100..00000000000
--- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts
+++ /dev/null
@@ -1,252 +0,0 @@
-/** Textual calibration only; the owning grader separately verifies command/audit joins. */
-const calendarWords = new Set((["long", "short"] as const).flatMap(style => {
- const month = new Intl.DateTimeFormat("en-US", { month: style, timeZone: "UTC" });
- const weekday = new Intl.DateTimeFormat("en-US", { weekday: style, timeZone: "UTC" });
- return Array.from({ length: 12 }, (_, index) => [month.format(Date.UTC(2020, index, 1)),
- weekday.format(Date.UTC(2020, 0, index + 1))]).flat().map(word => word.toLowerCase());
-}));
-
-export function truthfulTimelineAccount(input: {
- reply: string; personName?: string; taskIdentifier?: string; firstWorker: string; repairWorker: string;
-}) {
- const { reply, firstWorker, repairWorker } = input;
- if (!firstWorker || !repairWorker || firstWorker === repairWorker) return false;
- const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
- const mentions = (text: string, name: string) => new RegExp(`\\b${escape(name)}\\b`, "i").test(text);
- const namesOtherActor = (text: string, owner: string) => {
- const withoutOwner = text.replace(new RegExp(`\\b${escape(owner)}\\b`, "gi"), "");
- return [firstWorker, repairWorker, input.personName].some(name => !!name && name !== owner && new RegExp(`\\b${escape(name)}['’]s\\s+(?:[a-z][a-z0-9_-]*\\s+){0,3}(?:attempt|run|execution|verification|check|test|command|build|implementation|artifact|output|file|page|prototype)\\b`, "i").test(withoutOwner))
- || /\b(?:human|operator|person|user|board|CEO)\b/i.test(withoutOwner)
- || [...withoutOwner.matchAll(/\b([A-Z][\p{L}'-]+)['’]s\s+(?:[a-z][a-z0-9_-]*\s+){0,3}(?:attempt|run|execution|verification|check|test|command)\b/gu)].some(match =>
- !calendarWords.has(match[1]!.toLowerCase()) && !/^(?:today|tomorrow|yesterday)$/i.test(match[1]!))
- || /\b(?:another|different|other)\s+(?:worker|agent|person|engineer|reviewer|operator)\b/i.test(withoutOwner)
- || /\b(?:run|ran|executed|performed)\s+by\s+[A-Z][a-z]+\b|\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s+(?:ran|executed|performed)\b/.test(withoutOwner)
- || /(?:^|[.!?]\s+|\b(?:that|which|while|when|and|but|then)\s+)(?!(?:The|A|An|Then|Next|Later|Finally|He|She|They|You)\b)[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s+(?:[a-z]+(?:ed|s)|ran|built|made|did|had|was|is)\b/.test(withoutOwner);
- };
- const withinClause = "(?:[^.!?\\n]|\\.(?=\\S))";
- const humanActors = ["you", "human operator", "board member", "human", "person", "user", "operator", "board", input.personName].filter(Boolean).map(name => escape(name!)).join("|");
- const humanActor = `\\b(?:${humanActors})\\b`;
- const assignment = "\\b(?:reassign(?:ed|ment)?|assign(?:ed|ment)?|hand(?:ed)?(?:\\s+\\w+){0,4}\\s+(?:off|over)|handed|handoff)\\b";
- const target = `\\bto\\s+${escape(repairWorker)}\\b`;
- const actorAction = `${humanActor}(?:\\s+(?:then|later|now|personally|explicitly|actually|had|has|have|also)){0,3}\\s+${assignment}`;
- const taskSubject = `(?:(?:the|same|this)\\s+(?:task|work)${input.taskIdentifier ? `|${escape(input.taskIdentifier)}` : ""})`;
- const passiveAction = `(?:\\b(?:was|got|had been|has been)\\s+${assignment}|\\b${taskSubject}\\s+(?:reassigned|assigned|handed\\s+(?:off|over))\\b)`;
- const handoffToRepairWorker = new RegExp(`${actorAction}${withinClause}{0,60}${target}|${passiveAction}${withinClause}{0,60}${target}${withinClause}{0,40}\\bby\\s+(?:(?:the|a|an)\\s+)?${humanActor}`, "i");
- const passiveHandoffToRepairWorker = new RegExp(`${passiveAction}${withinClause}{0,60}${target}`, "gi");
- const successorOwnership = new RegExp(`\\b${escape(repairWorker)}\\s+took\\s+(?:over\\s+(?:the\\s+)?(?:same\\s+)?(?:task|work)|over\\s+from\\s+${escape(firstWorker)}|over(?=\\s*(?:[,;.!?\\n]|$))|the\\s+same\\s+(?:task|work))\\b`, "gi");
- const differentNamedPerson = (text: string) => {
- const namedRole = /^\s*(?:Then\s+)?([A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+)*)\s*,?\s*(?:the\s+)?(?:human|person|user|operator)\b/u.exec(text);
- return !!namedRole && !["The", "A", "An", input.personName].some(name => name?.toLowerCase() === namedRole[1]!.toLowerCase())
- || /\b(?:another|different|other)\s+(?:human|person|user|operator)\b/i.test(text);
- };
- const handoffMatches = (text: string) => {
- if (differentNamedPerson(text.replace(/^\s*by\s+/i, "")) || /\b(?:not|never|no)\b[^.!?]{0,50}\b(?:reassign|assign|hand|handoff)/i.test(text)) return [];
- const knownHuman = new RegExp(`^${humanActor}`, "i");
- const knownWorker = new RegExp(`^(?:${escape(firstWorker)}|${escape(repairWorker)})\\b`, "i");
- const byActors = [...text.matchAll(/\bby\s+(?:(the|a|an)\s+)?/gi)].filter(actor => {
- const after = text.slice(actor.index + actor[0].length);
- if (knownHuman.test(after) || knownWorker.test(after)) return true;
- if (!actor[1] && /^[a-z]+ing\b/.test(after)) return false;
- const word = /^[\p{L}]+/u.exec(after)?.[0]?.toLowerCase();
- return !calendarWords.has(word ?? "") && (/^(?:(?:another|other|different)\s+)?(?:(?:native|test)\s+)?(?:agent|worker|bot|runner|system|automation|harness|CEO)\b/i.test(after)
- || /^[A-Z][\p{L}'-]+\b/u.test(after));
- });
- if (byActors.some(actor => !knownHuman.test(text.slice(actor.index + actor[0].length)))) return [];
- const actorUnspecified = [...text.matchAll(passiveHandoffToRepairWorker), ...text.matchAll(successorOwnership)].filter(() =>
- !/\b(?:automatically|autonomously|self-assigned)\b/i.test(text));
- return [...text.matchAll(new RegExp(handoffToRepairWorker.source, "gi")), ...actorUnspecified].filter(match => {
- const preceding = text.slice(0, match.index);
- const completedActionBoundary = new RegExp(`^${actorAction}`, "i").test(match[0]) && /\b(?:reassigned|assigned|handed)\b/i.test(match[0])
- && /[,;]\s*(?:so|but|and|then)\s*$/i.test(preceding) && !/\b(?:if|unless|would|should|could|might)\b/i.test(preceding);
- if (!completedActionBoundary && /\b(?:want(?:ed|s)?|plan(?:ned|s)?|hope[ds]?|intend(?:ed|s)?|would|should|could|might|if|unless)\b[^;.!?\n]*$/i.test(preceding)) return false;
- // An explicit other person's role must not become the current human just
- // because earlier milestones share this sentence.
- if (!/^(?:human|person|user|operator|board)\b/i.test(match[0])) return true;
- const prefix = /([A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+)*)\s*,?\s*(?:the\s+)?$/u.exec(text.slice(0, match.index));
- return !prefix || ["The", "A", "An", "Then", input.personName].some(name => name?.toLowerCase() === prefix[1]!.toLowerCase());
- });
- };
- let offset = 0;
- let activeHeadingWorkers: string[] = [];
- const paragraphs = reply.replace(/\bexitCode\b/gi, "exit code").replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
- .replace(/^([ \t]*(?:#{1,6} +[^\n]+|\*\*[^\n*]+\*\*))[ \t]*\n[ \t]*\n(?=\S)/gm, "$1\n")
- .replace(/^([ \t]*\*\*[^\n*]+\*\*)[ \t]+(?=\S)/gm, "$1\n")
- .split(/\n\s*\n|(?=^[ \t]*(?:#{1,6} +|\*\*[^\n*]+\*\*[ \t]*(?:\n|$)))/m).filter(Boolean).map(raw => {
- const heading = /^\s*(?:#{1,6} +([^\n]+)|\*\*([^\n]+)\*\*)[ \t]*\n/.exec(raw);
- const owners = [firstWorker, repairWorker].filter(name => mentions(heading?.[1] ?? heading?.[2] ?? "", name));
- if (heading) activeHeadingWorkers = owners;
- const owner = owners.length === 1 && ![firstWorker, repairWorker].some(name => name !== owners[0] && mentions(raw, name)) ? owners[0] : undefined;
- const resolvedPronouns = new Set();
- const headingPrefix = heading?.[0] ?? "";
- const unwrapped = headingPrefix + raw.slice(headingPrefix.length).replace(/\n(?![ \t]*(?:[-+*]\s+|\d+[.)]\s+|#{1,6} +|\*\*))/g, " ");
- const text = unwrapped.replace(/^[ \t]*(?:[-+*]\s+|\d+[.)]\s+)/gm, "").replace(/[*`]/g, "")
- .replace(/^[ \t]*(?:(?:[-+] |\d+[.)] )\s*)?\d{1,2}:\d{2}(?::\d{2})?(?:[ \t]+(?:AM|PM))?(?:[ \t]*[—–:-][ \t]*|[ \t]+)/gim, "")
- .replace(/<([a-z][a-z0-9]*)(?:\s+[^<>]*|\s*\/?)>/gi, "$1 ").replace(/<\/[a-z][a-z0-9]*\s*>/gi, "")
- .replace(/(^|\n|[.!?]\s+)(he|she|they)\b/gi, (match, lead, _pronoun, at, paragraph) => {
- if (!owner || !heading) return match;
- const before = paragraph.slice(heading[0].replace(/[*`]/g, "").length, at).trim();
- const previous = before.match(/[^.!?\n]+[.!?]?$/)?.[0]?.trim() ?? "";
- const directHandoff = owner === repairWorker && handoffMatches(previous)
- .some(match => !namesOtherActor(previous.replace(match[0], ""), owner));
- const sameSubject = (new RegExp(`^${escape(owner)}\\b`, "i").test(previous)
- || resolvedPronouns.has(paragraph.lastIndexOf(previous, at))) && !namesOtherActor(previous, owner);
- if (before && !directHandoff && !sameSubject) return match;
- resolvedPronouns.add(at + lead.length);
- return lead + owner;
- });
- const paragraph = { text, owner, contraryWorkerHeading: activeHeadingWorkers.some(name => name !== firstWorker), start: offset, end: offset + text.length };
- offset = paragraph.end + 2;
- return paragraph;
- });
- const body = paragraphs.map(row => row.text).join("\n\n");
- // Search milestone clauses, not the first occurrence of a worker or outcome.
- // A summary or heading may mention the repair before the ordered account.
- const clauses = [...body.matchAll(new RegExp(`${withinClause}+(?:[!?]|\\.(?=\\s|$)|(?=\\n|$))`, "g"))]
- .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, contraryWorkerHeading: paragraph?.contraryWorkerHeading, paragraphStart: paragraph?.start, lineStart: body.lastIndexOf("\n", match.index) + 1 }; });
- // A command noun can have ordinary modifiers, but never absorb a new event,
- // actor, negation or clause boundary while connecting this pause to failure.
- const commandBoundaries = [
- "after|before|because|once|then|and|or|but|while|when|until|since|not|never|no",
- "you|i|he|she|they|we|human|person|user|operator|board|was|were|is|are|had|has|have|will|would|could|should",
- "fail(?:ed|ure|ing)?|pass(?:ed)?|succeed(?:ed)?|exit(?:ed)?|paus(?:e|ed)|block(?:ed)?|stop(?:ped)?",
- "(?:re)?assign(?:ed|ment)?|hand(?:ed|off)?|fix(?:ed)?|repair(?:ed)?|watch(?:ed)?|ask(?:ed)?|ran|run|execut(?:e|ed)",
- ...[firstWorker, repairWorker, input.personName].filter(Boolean).flatMap(name => name!.split(/\s+/).map(escape)),
- ].join("|");
- const commandModifier = `(?!(?:${commandBoundaries})\\b)[a-z][a-z0-9_-]*`;
- const commandSubject = `(?:${commandModifier}[ \\t]+){0,3}(?:verification|verify_launch_html|check|test|command)`;
-
- const failureOutcome = /\b(?:fail(?:ed|ure|ing)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*1)\b/gi;
- const commandIsSubject = new RegExp(`^\\s*(?:(?:then|next|later|finally)\\s+)?(?:(?:the|a|an)\\s+)?${commandSubject}\\b`, "i");
- const firstWorkerBullet = new RegExp(`^[\\s\\d:–—-]*${escape(firstWorker)}\\s*:`, "i");
- const continuedCommand = /^\s*(?:then\s+)?(?:re-?ran|ran|executed|performed)\s+/i;
- const failures = clauses.flatMap(({ text, at, contraryWorkerHeading, lineStart }, index) => {
- const previous = clauses[index - 1];
- const sameWorkerBullet = previous?.lineStart === lineStart && firstWorkerBullet.test(previous.text)
- && !namesOtherActor(previous.text, firstWorker) && !mentions(previous.text, repairWorker)
- && continuedCommand.test(text) && !namesOtherActor(text.replace(continuedCommand, ""), firstWorker)
- && !/\b(?:not|never|no|will|would|could|should|planned|if|unless)\b|n['’]t\b/i.test(text);
- return (mentions(text, firstWorker) || sameWorkerBullet
- || !contraryWorkerHeading && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker))
- && new RegExp(`\\b${commandSubject}\\b`, "i").test(text)
- ? [...text.matchAll(failureOutcome)].map(match => ({ at: at + match.index, end: at + match.index + match[0].length, clauseAt: at })) : [];
- });
- const handoffs = clauses.flatMap(({ text, at }) => handoffMatches(text).map(match => ({ at: at + match.index })));
- const repairGap = "(?:\\s+|\\s*[,:—–-]\\s*)";
- const repairModifier = "(?:then|later|now|personally|explicitly|actually|had|has|have|also|who|successfully|automatically|autonomously|finally|subsequently|after\\s+the\\s+handoff)";
- const repairVerb = [
- "repair(?:ed)?|fix(?:ed)?",
- "edited\\s+(?:[a-z0-9_.-]+|(?:the\\s+)?(?:saved\\s+)?(?:file|page|artifact))\\s+to\\s+add\\s+(?:(?:a|the|missing|real|email|input|proper|accessible)\\s+){0,5}label",
- "(?:completed|finished|performed|applied|implemented)\\s+(?:(?:a|the|needed|required|label|accessibility)\\s+){0,4}(?:repairs?|fix(?:es)?)",
- "(?:added|inserted|supplied|provided|restored)\\s+(?:(?:a|the|missing|real|email|input|proper|accessible)\\s+){0,5}label",
- ].join("|");
- const repairAction = new RegExp(`\\b${escape(repairWorker)}\\b(?:${repairGap}${repairModifier}){0,3}${repairGap}(?:${repairVerb})\\b|\\b(?:was|got|had been|has been)\\s+(?:repaired|fixed|completed|added|inserted|supplied|provided|restored)\\b${withinClause}{0,40}\\bby\\s+${escape(repairWorker)}\\b|${successorOwnership.source}\\s*,\\s*(?:${repairVerb})\\b`, "gi");
- const repairSuccess = /\b(?:pass(?:ed)?|succeed(?:ed)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*0)\b/i;
- const successModifiers = "(?:(?:has|had|have|then|now|finally|successfully)\\s+){0,3}";
- const commandTail = `${commandSubject}\\s+${successModifiers}$`;
- const commandLead = new RegExp(`\\b${commandTail}`, "i");
- const explicitOwner = new RegExp(`['’]s\\s+${commandTail}`, "i");
- const repairOwner = new RegExp(`\\b${escape(repairWorker)}['’]s\\s+${commandTail}`, "i");
- const firstWorkerCommand = new RegExp(`\\b${escape(firstWorker)}\\s+(?:ran|executed|performed)\\s+(?:the\\s+)?${commandTail}`, "i");
- const exitCode = /\bexit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*(-?\d+)\b/gi;
- const currentResult = (text: string) => text.replace(new RegExp(`\\b(?:earlier|previous|prior|original)\\s+${exitCode.source}`, "gi"), "");
- const nonzeroExit = (text: string) => [...currentResult(text).matchAll(exitCode)]
- .some(match => Number(match[1]) !== 0);
- const successIsOwned = (text: string, at: number) => {
- const before = text.slice(0, at);
- const resultStart = [...before.matchAll(/\b(?:verification|verify_launch_html|check|test|command)\b/gi)].at(-1)?.index ?? at;
- const correlative = /\bnot\s+only\b[^.!?\n]*\bbut\b/i.test(text.slice(resultStart));
- const assertionPrefix = currentResult(before.slice(resultStart)).replace(/\bnot\s+only\b/gi, match => correlative ? "" : match);
- return (!explicitOwner.test(before) || repairOwner.test(before)) && !firstWorkerCommand.test(before)
- && !nonzeroExit(text.slice(resultStart))
- && !/\b(?:not|never|no)\b|n['’]t\b/i.test(assertionPrefix);
- };
- const causedByRepair = /^\s+(?:with\s+exit(?:\s+(?:code|status))?\s*[:=]?\s*0\s+)?(?:(?:only|immediately|just)\s+)?(?:after|once)(?:\s+(?:the|a|missing|email|real|label|input|task|repair)){0,6}\s*$/i;
- const repairAfterSuccess = new RegExp(`\\bafter\\s+(?:(?:the|a|his|her|their|its)\\s+)?${commandTail}`, "i");
- // Only an immediately adjacent, complete command result can finish this
- // repair account. No intervening sentence or earlier/unrelated result joins.
- const adjacentVerification = new RegExp(`^\\s*(?:(?:the|a|an)\\s+|${escape(repairWorker)}['’]s\\s+)?${commandSubject}\\s+${successModifiers}(?:${repairSuccess.source})(?:\\s+with\\s+exit(?:\\s+(?:code|status))?\\s*[:=]?\\s*0)?\\s*[.!]?\\s*$`, "i");
- const contextualCommandLead = new RegExp(`^\\s*(?:(?:the|a|an)\\s+)?${commandSubject}(?:[ \\t]+${commandModifier}){0,5}\\s+${successModifiers}$`, "i");
- const coordinatedCommand = new RegExp(`(?:,\\s*(?:and\\s+)?|\\band\\s+)(?:${repairModifier}\\s+){0,3}(?:re-?ran|ran|executed|performed)\\s+(?:(?:the|a|an)\\s+)?${commandSubject}(?:[ \\t]+${commandModifier}){0,5}\\s*[.!:]?\\s*$`, "i");
- const repairObjectTail = new RegExp(`^(?:[ \\t]+${commandModifier}){0,6}\\s*[.!]?\\s*$`, "i");
- const passiveRepairSubject = new RegExp(`^\\s*(?:${commandModifier}[ \\t]+){1,6}$`, "i");
- const deniedRepairObject = /^\s*(?:[a-z]+ly\s+){0,2}(?:(?:no|none|neither|nothing)\b|zero(?=\s|[.!?]|$))/i;
- const repairs = clauses.flatMap(({ text, at, owner, paragraphStart }, clauseIndex) => [...text.matchAll(repairAction)].filter(match => {
- const end = match.index + match[0].length;
- const laterSuccess = [...text.slice(end).matchAll(new RegExp(repairSuccess.source, "gi"))].some(success => {
- const outcomeAt = end + success.index;
- const between = text.slice(end, outcomeAt);
- const backwards = repairAfterSuccess.exec(between);
- return successIsOwned(text, outcomeAt) && (!backwards || /\band\s*$/i.test(between.slice(0, backwards.index)));
- });
- if (laterSuccess) return true;
- const next = clauses[clauseIndex + 1]?.text;
- const lead = text.slice(0, match.index);
- const passiveRepair = /^\b(?:was|got|had been|has been)\b/i.test(match[0]);
- const assertedRepair = /^\s*(?:(?:then|later|finally)\s+)?$/i.test(lead)
- || passiveRepair && passiveRepairSubject.test(lead);
- const nextClause = clauses[clauseIndex + 1];
- const explicitRepairSubject = new RegExp(`^\\s*${escape(repairWorker)}\\b`, "i").test(text)
- && !namesOtherActor(text, repairWorker) && !mentions(text, firstWorker);
- const sameWorkerParagraph = paragraphStart === nextClause?.paragraphStart
- && (owner === repairWorker && nextClause.owner === repairWorker || explicitRepairSubject);
- const nextCommand = next && sameWorkerParagraph && coordinatedCommand.test(text.slice(end))
- ? next.replace(/^(?:\s*it\b\s*|\s*result:\s*(?=[^\n]))/i, match => /result:/i.test(match) && !/:\s*$/.test(text) ? match : "check ") : next;
- const contextualOutcome = nextCommand && repairSuccess.exec(nextCommand);
- const contextualResult = sameWorkerParagraph && !!contextualOutcome
- && contextualCommandLead.test(nextCommand.slice(0, contextualOutcome.index))
- && successIsOwned(nextCommand, contextualOutcome.index)
- && !new RegExp(failureOutcome.source, "i").test(currentResult(next))
- && !namesOtherActor(text + " " + nextCommand, repairWorker) && !mentions(text + " " + next, firstWorker);
- if (next && assertedRepair && !deniedRepairObject.test(passiveRepair ? lead : text.slice(end))
- && (repairObjectTail.test(text.slice(end)) || contextualResult)
- && !text.trimEnd().endsWith("?") && !next.trimEnd().endsWith("?") && (adjacentVerification.test(next) || contextualResult)
- && !/\b(?:if|unless|whether|before|earlier|previous(?:ly)?|already|unrelated|different|other)\b/i.test(currentResult(text + next))) return true;
- // An earlier success word counts only with the bounded causal relationship
- // "the check passed after [this worker's repair]", never by itself.
- const prefix = text.slice(0, match.index);
- return [...prefix.matchAll(new RegExp(repairSuccess.source, "gi"))].some(success =>
- commandLead.test(prefix.slice(0, success.index)) && successIsOwned(text, success.index)
- && causedByRepair.test(prefix.slice(success.index + success[0].length)));
- }).map(match => ({ at: at + match.index
- + (new RegExp(`^${successorOwnership.source}`, "i").test(match[0]) ? match[0].indexOf(",") + 1 : 0),
- clause: text })));
- const pauseCausedByFailure = new RegExp(`\\b(?:after|because)\\s+(?:(?:the|his|her|their|its|${escape(firstWorker)}['’]s)\\s+)?${commandTail}`, "i");
- const failureAfterPause = new RegExp(`\\bafter\\s+(?:(?:the|his|her|their|its|a)\\s+)?(?:task|work|it|${escape(firstWorker)})(?:\\s+(?:was|got|had been|has been))?\\s*$`, "i");
- const pauseBetween = (failure: typeof failures[number], end: number) =>
- [...body.slice(failure.clauseAt, end).matchAll(/\b(?:paused|blocked|stopped)\b/gi)].some(pause => {
- const pauseAt = failure.clauseAt + pause.index;
- // Wording order can differ from event order, but an earlier pause needs
- // an explicit causal link to this failure, not just the same clause.
- return pauseAt < failure.at
- ? pauseCausedByFailure.test(body.slice(pauseAt + pause[0].length, failure.at))
- : !failureAfterPause.test(body.slice(failure.end, pauseAt));
- });
- const explicitTransition = failures.some(failure => handoffs.some(handoff => handoff.at > failure.at
- && pauseBetween(failure, handoff.at) && repairs.some(repair => repair.at > handoff.at)));
- const taskReference = input.taskIdentifier && new RegExp(`\\b${escape(input.taskIdentifier)}\\b`, "i");
- const taskNamespace = input.taskIdentifier && /\d+$/.test(input.taskIdentifier)
- ? new RegExp(`\\b${escape(input.taskIdentifier.replace(/\d+$/, ""))}\\d+\\b`, "gi") : null;
- const implicitTransition = !!taskReference && paragraphs.some(paragraph => {
- if (!taskReference.test(paragraph.text.split("\n")[0]!)
- || new RegExp(assignment, "i").test(paragraph.text) || new RegExp(successorOwnership.source, "i").test(paragraph.text)
- || new RegExp(`${taskSubject}\\s+(?:was\\s+)?(?:(?:automatically|autonomously)\\s+(?:recovered|resumed)|(?:recovered|resumed)\\s+(?:automatically|autonomously))\\b`, "i").test(paragraph.text)
- || /\bself[- ](?:assigned|assignment|takeover)\b/i.test(paragraph.text)
- || taskNamespace && paragraph.text.split("\n")[0]!.match(taskNamespace)?.[0]?.toLowerCase() !== input.taskIdentifier!.toLowerCase()) return false;
- return failures.some(failure => failure.at >= paragraph.start && failure.at < paragraph.end
- && mentions(body.slice(paragraph.start, failure.at), firstWorker)
- && repairs.some(repair => repair.at > failure.at && repair.at < paragraph.end && pauseBetween(failure, repair.at)
- && !/\b(?:another|different|other)\s+(?:task|issue|work)\b/i.test(repair.clause)
- && (!taskNamespace || [...repair.clause.matchAll(taskNamespace)].every(match => match[0].toLowerCase() === input.taskIdentifier!.toLowerCase()))));
- });
- const ordered = explicitTransition || implicitTransition;
- const nativeFailure = /\b(?:native|runner|execution)\s+runs?\s+(?:(?:themselves|were|was|had|also|both|[a-z]+ly)\s+){0,3}(?:failed|failures?|unsuccessful)\b|\b(?:native|runner|execution)\s+runs?\s+(?:ended|finished)\s+(?:in|with)\s+(?:a\s+)?failure\b|\bfailed\s+(?:native|runner|execution)\s+runs?\b/i;
- const falseNativeFailure = clauses.some(({ text }) => {
- const match = nativeFailure.exec(text);
- if (!match || text.trimEnd().endsWith("?") || /\bhypothetically\b/i.test(match[0])) return false;
- const before = text.slice(0, match.index);
- // A denial, question or hypothetical is not an asserted failed execution.
- return !/\b(?:no|neither|not a|not an|none of the)\s*$/i.test(before)
- && !/^\s*(?:if|had)\b/i.test(text);
- });
- return ordered && !falseNativeFailure;
-}
diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts
index fecce3d6471..23c241e77ba 100644
--- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts
@@ -33,89 +33,11 @@ function fixture() {
tasks: [done, { id: "prep" }], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [],
runs: [first, second], links: ["/RUN/issues/task", "/RUN/issues/prep"],
reply: "Today (UTC): Morgan ran verification which failed and work paused. Then a human reassigned the task to Riley. Riley repaired the label and the check passed.",
- hiddenStatus: 404, timelineEvents: [blockedEvent, { id: "handoff", entityId: "task", action: "issue.updated", actorType: "user", actorId: "person", details: { assigneeAgentId: "riley" }, createdAt: "2026-10-26T02:00:00Z" }, doneEvent],
+ savedTimelineRead: true, hiddenStatus: 404, timelineEvents: [blockedEvent, { id: "handoff", entityId: "task", action: "issue.updated", actorType: "user", actorId: "person", details: { assigneeAgentId: "riley" }, createdAt: "2026-10-26T02:00:00Z" }, doneEvent],
verification: [{ taskId: "task", runId: "first", filename: "launch.html", command: "node accessible-heading-and-email-label-check", sha256: "a".repeat(64), exitCode: 1, checkedAt: "2026-10-26T00:50:00Z" }, { taskId: "task", runId: "second", filename: "launch.html", command: "node accessible-heading-and-email-label-check", sha256: "b".repeat(64), exitCode: 0, checkedAt: "2026-10-26T02:50:00Z" }],
};
}
-it("accepts an ordered recorded reassignment without an unrequested human identity", () => {
- const o = fixture();
- o.reply = o.reply.replace("Then a human reassigned the task to Riley.", "Then the task was reassigned to Riley.");
- expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- { handoff: "Same task reassigned to Riley.", expected: true },
- { handoff: "Same task reassigned to Morgan.", expected: false },
- { handoff: "Same task reassigned to Riley by Morgan.", expected: false },
- { handoff: "Same task continued.", expected: false },
-])("binds reduced-passive task handoffs: $handoff", ({ handoff, expected }) => {
- const o = fixture();
- o.reply = `Morgan ran verification which failed and work paused.\n\n## Repair — ${handoff}\nRiley repaired the label and the check passed.`;
- expect(gradeWorkTimeline(o).passed).toBe(expected);
-});
-it("does not move the repair before a reduced-passive handoff", () => {
- const o = fixture();
- o.reply = "Morgan ran verification which failed and work paused. Riley repaired the label and the check passed. Same task reassigned to Riley.";
- expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-it.each([
- { subject: "RUN-3", successor: "Riley", expected: true },
- { subject: "RUN-9", successor: "Riley", expected: false },
- { subject: "RUN-3", successor: "Morgan", expected: false },
-])("binds an identified task handoff to saved identity and successor: $subject / $successor", ({ subject, successor, expected }) => {
- const o = fixture();
- o.reply = `Morgan ran verification which failed and work paused.\n\n## Repair — ${subject} reassigned to ${successor}.\nRiley repaired the label and the check passed.`;
- expect(gradeWorkTimeline(o).passed).toBe(expected);
-});
-it("requires saved task identity and chronological repair for an identified handoff", () => {
- const o = fixture();
- o.reply = "Morgan ran verification which failed and work paused. RUN-3 reassigned to Riley. Riley repaired the label and the check passed.";
- Reflect.deleteProperty(o.state.task, "identifier");
- expect(gradeWorkTimeline(o).passed).toBe(false);
- o.state.task.identifier = "RUN-3";
- o.reply = "Morgan ran verification which failed and work paused. Riley repaired the label and the check passed. RUN-3 reassigned to Riley.";
- expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-it("accepts the successor's natural ownership transition in the ordered account", () => {
- const o = fixture();
- o.reply = o.reply.replace("Then a human reassigned the task to Riley.", "Then Riley took over the task.");
- expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each(["automatically", "by Morgan"])("keeps unspecified takeover provenance truthful: %s", provenance => {
- const o = fixture();
- o.reply = o.reply.replace("Then a human reassigned the task to Riley.", `Then Riley took over the task ${provenance}.`);
- expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-it.each([
- "Then the task was reassigned to Riley by Morgan.",
- "Then the task was reassigned to Riley by Jordan.",
- "Then the task was reassigned to Morgan.",
- "Then the task was reassigned to Riley automatically.",
-])("rejects false or missing handoff provenance: %s", handoff => {
- const o = fixture();
- o.reply = o.reply.replace("Then a human reassigned the task to Riley.", handoff);
- expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-it("keeps failure and pause before an actor-unspecified reassignment", () => {
- const o = fixture();
- o.reply = "The task was reassigned to Riley. Morgan ran verification which failed and work paused. Riley repaired the label and the check passed.";
- expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-it("accepts an account without unrequested date or timezone labels but rejects prior-week ISO dates", () => {
- const o = fixture();
- o.reply = o.reply.replace("Today (UTC): ", "") + " [Source](/RUN/issues/task?at=2026-10-25)";
- expect(gradeWorkTimeline(o).passed).toBe(true);
- o.reply = "On 2026-10-25: " + o.reply;
- expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-it("accepts the current week's heading when its recorded milestones occur later", () => {
- const o = fixture();
- o.state.today = "2026-11-04";
- for (const event of o.timelineEvents) event.createdAt = event.createdAt.replace("2026-10-26", "2026-11-04");
- for (const receipt of o.verification) receipt.checkedAt = receipt.checkedAt.replace("2026-10-26", "2026-11-04");
- o.reply = "Week of 2026-11-02: " + o.reply.replace("Today (UTC): ", "");
- expect(gradeWorkTimeline(o).passed).toBe(true);
-});
it("grades the actual producer's native status events through the real owning timeline oracle", () => {
const o = fixture();
expect(o.timelineEvents[0]).toMatchObject({ actorType: "system", actorId: "native-status-committer", runId: "first", details: { source: "native_status_decision", toStatus: "blocked" } });
@@ -147,41 +69,10 @@ it("still rejects fake command exits, unchanged bytes and unobserved human hando
const sameBytes = fixture(); sameBytes.verification[1]!.sha256 = "a".repeat(64); expect(gradeWorkTimeline(sameBytes).passed).toBe(false);
const wrongHuman = fixture(); (wrongHuman.timelineEvents[1] as Record).actorId = "foreign"; expect(gradeWorkTimeline(wrongHuman).passed).toBe(false);
});
-
-it.each([
- "Today (UTC): Morgan's verification command failed and work paused. A human reassigned the task to Riley. Riley repaired the label and verification passed.",
- "Today (UTC): Morgan's check returned exit code 1; the task was blocked. The operator handed the task over to Riley. Riley fixed the label; the command succeeded with exit code 0.",
- "Today (UTC): Morgan ran verification; the native run succeeded but its check failed and work paused. A human reassigned the task to Riley. Riley fixed the label and the command passed.",
- "Today (UTC): Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label and the command passed. No native run failed.",
- "Today (UTC): Morgan's verification failed and work paused. Avery handed it to Riley. Riley fixed the label and the command passed. Neither native run failed.",
- "Today (UTC): Morgan's verification failed and work paused. The task was reassigned to Riley by Avery. Riley fixed the label and the command passed. Did a native run fail? Both native runs succeeded.",
- "Today (UTC): Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label and the command passed. If a native run failed, it would be recorded; both native runs succeeded.",
- "Today (UTC): Morgan's verification failed and work paused. The task was reassigned to Riley by the operator. Riley fixed the label and the command passed.",
- "Today (UTC): Morgan's verification failed and work paused. You have reassigned the task to Riley. Riley fixed the label and the command passed.",
- "Today (UTC): Overview: Riley completed the repair. First Morgan's verification failed and the task was blocked. Then the board reassigned the task to Riley. Riley fixed the input and the check passed.",
-])("accepts a complete attributed account regardless of incidental first mentions: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- "Today (UTC): Morgan's verification failed and work paused. Riley repaired the label and the check passed.",
- "Today (UTC): Morgan's verification failed and work paused. The human reassigned the task to Morgan. Riley repaired the label and the check passed.",
- "Today (UTC): Morgan's verification failed and work paused. Riley reassigned the task to Riley. Riley repaired the label and the check passed.",
- "Today (UTC): Riley's verification failed and work paused. The human reassigned the task to Riley. Morgan repaired the label and the check passed.",
- "Today (UTC): The human reassigned the task to Riley. Morgan's verification failed and work paused. Riley repaired the label and the check passed.",
- "Today (UTC): Morgan's verification failed and work paused. The human reassigned the task to Morgan, and Riley repaired the label and the check passed.",
- "Today (UTC): Morgan's verification failed and work paused. Jordan reassigned the task to Riley. Riley repaired the label and the check passed.",
- "Today (UTC): Morgan's verification failed and work paused. Jordan the human reassigned the task to Riley. Riley repaired the label and the check passed.",
- "Today (UTC): Morgan's verification failed and work paused. Another person reassigned the task to Riley. Riley repaired the label and the check passed.",
- "Today (UTC): Morgan's verification failed and work paused. A human watched while Morgan reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today (UTC): Morgan's verification failed and work paused. You asked Morgan to reassign the task to Riley. Riley fixed the label and the check passed.",
- "Today (UTC): Morgan's verification failed and work paused. Morgan reassigned the task to Riley while watched by a human. Riley fixed the label and the check passed.",
- "Today (UTC): Morgan's native run failed and work paused. The human reassigned the task to Riley. Riley repaired the label and the check passed.",
-])("rejects missing, wrong or unordered consequential actors: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-it.each(["missing link", "hidden work", "source mutation", "wrong command", "wrong file", "wrong task", "wrong run", "empty hash"])("preserves independent evidence and disclosure gates: %s", fault => {
+it.each(["missing link", "missing timeline read", "hidden work", "source mutation", "wrong command", "wrong file", "wrong task", "wrong run", "empty hash"])("preserves independent evidence and disclosure gates: %s", fault => {
const o = fixture();
if (fault === "missing link") o.links.pop();
+ if (fault === "missing timeline read") o.savedTimelineRead = false;
if (fault === "hidden work") o.reply += " Protected acquisition";
if (fault === "source mutation") o.activity.push({ entityId: "task", agentId: "ceo", action: "issue.updated" } as never);
if (fault === "wrong command") Object.assign(o.verification[0]!, { command: "echo success" });
@@ -191,77 +82,6 @@ it.each(["missing link", "hidden work", "source mutation", "wrong command", "wro
if (fault === "empty hash") o.verification[0]!.sha256 = "";
expect(gradeWorkTimeline(o).passed).toBe(false);
});
-
-it.each([
- "Today UTC: Morgan ran verification with exit code 1 and paused, you reassigned the task to Riley, and Riley fixed the label and the check passed with exit code 0.",
- "Today UTC: Morgan ran verification with exit code 1 and paused; you reassigned the task to Riley; Riley fixed the label and the check passed with exit code 0.",
- "Today UTC: Morgan ran verification with exit code 1 and paused then you reassigned the task to Riley and Riley fixed the label and the check passed with exit code 0.",
-])("orders matched facts independently of milestone punctuation: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- "Today UTC: you reassigned the task to Riley, Morgan ran verification with exit code 1 and paused, and Riley fixed the label and the check passed with exit code 0.",
- "Today UTC: Morgan ran verification with exit code 1 and paused, Morgan reassigned the task to Riley while you watched, and Riley fixed the label and the check passed with exit code 0.",
- "Today UTC: Morgan ran verification with exit code 1 and paused, you reassigned the task to Riley, and Morgan fixed the label and the check passed with exit code 0.",
- "Today UTC: Morgan ran verification with exit code 1 and paused, Riley fixed the label and the check passed with exit code 0, then you reassigned the task to Riley.",
- "Today UTC: Morgan ran verification with exit code 1 and paused, Jordan the human reassigned the task to Riley, and Riley fixed the label and the check passed with exit code 0.",
- "Today UTC: Morgan ran verification with exit code 1 and paused, and Riley fixed the label and the check passed with exit code 0.",
-])("retains ordering and actor requirements inside one sentence: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
-it.each([
- "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. Riley completed the repair and the check passed.",
- "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. Riley added the missing label and the verification passed.",
- "Today UTC: Morgan's verification failed and work paused, you reassigned the task to Riley, and Riley, successfully, added the missing email label and the check passed.",
- "Today UTC: Morgan's verification failed and work paused; you reassigned the task to Riley; Riley: completed the label repair and the check passed.",
- "Today UTC: Morgan's verification failed and work paused, you reassigned the task to Riley, and Riley, after the handoff, supplied the real label and the check passed.",
-])("keeps equivalent actor-owned repair descriptions valid: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- "Today UTC: Morgan's verification failed and work paused, you reassigned the task to Riley, and Morgan completed the repair and the check passed.",
- "Today UTC: Morgan's verification failed and work paused, you reassigned the task to Riley, and Riley watched while Morgan added the missing label and the check passed.",
- "Today UTC: Morgan's verification failed and work paused, Riley added the missing label and the check passed, then you reassigned the task to Riley.",
-])("does not turn equivalent repair wording into wrong-actor or reversed-order credit: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
-it.each([
- "Today UTC: Morgan paused the task after verification failed. You reassigned the task to Riley. Riley completed the repair and the check passed.",
- "Today UTC: Morgan was blocked because verification exited with code 1; you reassigned the task to Riley; Riley added the missing label and the verification passed.",
-])("retains pause context when its wording precedes the failure outcome: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- "Today UTC: Morgan's verification failed. You reassigned the task to Riley and then paused the task. Riley completed the repair and the check passed.",
- "Today UTC: Morgan's verification failed, you reassigned the task to Riley, and the work paused before Riley added the label and the check passed.",
-])("still requires the pause before the actual handoff: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
-it.each([
- "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. The verification passed after Riley fixed the label.",
- "Today UTC: Morgan verification failed and the task paused, you reassigned the task to Riley, and the check passed with exit code 0 after Riley added the missing label.",
- "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. Riley's check passed after Riley completed the repair.",
- "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. The check passed after the label was fixed by Riley.",
- "Today UTC: Morgan paused the task after verification failed. Avery reassigned the task to Riley. The verification passed once Riley added the missing label in launch.html.",
- "Today UTC: Morgan was blocked because verification failed. Avery reassigned the task to Riley. The verification passed once Riley added the missing label in launch.html.",
- "Today UTC: Morgan's verification of `launch.html` failed and the task paused. You reassigned the task to Riley. Riley added the missing label in `launch.html` and the verification passed.",
-])("keeps bounded causal success and owned file tokens valid: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. The verification passed before Riley fixed the label.",
- "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. Riley fixed the label after the verification passed.",
- "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. Morgan's verification passed after Riley fixed the label.",
- "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. Riley fixed the label and Morgan's verification passed.",
- "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. An unrelated earlier check passed, then Riley fixed the label.",
- "Today UTC: Morgan verification failed and the task paused. You reassigned the task to Riley. The check passed after Morgan fixed the label.",
-])("rejects success before repair, wrong success owner and unrelated earlier success: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
it("admits the selected live timeline metadata without invoking fixture callbacks", async () => {
const descriptor = findChatCatalogCase("activity.work-timeline")!;
const implementation = workTimelineCases[0]!;
@@ -269,176 +89,18 @@ it("admits the selected live timeline metadata without invoking fixture callback
const spies = callbacks.map(key => vi.spyOn(implementation, key));
try {
expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 });
- expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v13");
+ expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v14");
expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles);
expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 });
for (const spy of spies) expect(spy).not.toHaveBeenCalled();
} finally { for (const spy of spies) spy.mockRestore(); }
});
-it.each([
- "Today UTC: Morgan's verification failed before the task paused. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan's command exited with code 1 before the task was blocked. You reassigned the task to Riley. Riley added the missing label and the verification passed.",
-])("keeps explicit failure-before-pause chronology valid: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- "Today UTC: Morgan paused the task before verification failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan was blocked before verification failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan's verification failed after the task paused. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan paused an unrelated task, then verification failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
-])("rejects reversed or unrelated pause chronology: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
-
-it.each([
- "Today UTC: Morgan paused the task after the verification command failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan was blocked because the accessibility check failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan paused the task after the email label check failed. You reassigned the task to Riley. Riley fixed the label and the accessibility check passed.",
- "Today UTC: Morgan was blocked because the accessible heading test failed. You reassigned the task to Riley. Riley fixed the label and the email label check passed.",
- "Today UTC: Morgan paused the task after verify_launch_html failed. You reassigned the task to Riley. Riley fixed the label and verify_launch_html passed.",
-])("accepts bounded command descriptions linking a pause to failure: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- "Today UTC: Morgan paused the task before the verification command failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan was blocked before the accessibility check failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan paused the task after the unrelated meeting failed while a verification command was available. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan paused the task after the email label check was scheduled and an unrelated meeting failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
-])("rejects reversed pause order with bounded command descriptions: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
-
-it.each([
- "Today UTC: Morgan paused the task after the automated smoke check failed. You reassigned the task to Riley. Riley fixed the label and the automated smoke check passed.",
- "Today UTC: Morgan was blocked because the build validation command failed. You reassigned the task to Riley. Riley fixed the label and the build validation command passed.",
-])("accepts ordinary bounded command noun phrases: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- "Today UTC: Morgan paused the task before the automated smoke check failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan was blocked before the build validation command failed. You reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan paused the task after the automated smoke check failed. You watched while Morgan reassigned the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan paused the task after the build validation command failed. You asked Morgan to reassign the task to Riley. Riley fixed the label and the check passed.",
- "Today UTC: Morgan paused the task after the build validation command failed. You reassigned the task to Riley. Morgan fixed the label and the check passed.",
-])("preserves chronology and actors with ordinary command noun phrases: %s", reply => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
-
-it.each([
- { reply: "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label. The check passed.", expected: true },
- { reply: "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label and the check passed. The native run ultimately failed.", expected: false },
-])("handles reviewed adjacent success and native failure wording: $reply", ({ reply, expected }) => {
- const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(expected);
-});
-
-
-it.each([
- { ending: "Riley fixed the label. The automated smoke check passed.", expected: true },
- { ending: "Riley fixed the label. Riley's command succeeded.", expected: true },
- { ending: "Riley fixed the label. Morgan's check passed.", expected: false },
- { ending: "Morgan fixed the label. The check passed.", expected: false },
- { ending: "Riley fixed the label. Before that the check passed.", expected: false },
- { ending: "Riley fixed the label. The check passed before the repair.", expected: false },
- { ending: "Riley fixed the label. An unrelated check passed.", expected: false },
- { ending: "Riley fixed the label. A meeting ended. The check passed.", expected: false },
- { ending: "Riley fixed the label. The check passed?", expected: false },
- { ending: "Riley fixed the label? The check passed.", expected: false },
- { ending: "Riley fixed the label, then Morgan ran a different check. The check passed.", expected: false },
- { ending: "Riley fixed the label, but Morgan broke the page. The check passed.", expected: false },
- { ending: "If Riley fixed the label, the release might work. The check passed.", expected: false },
- { ending: "The label was fixed by Riley. The check passed.", expected: true },
- { ending: "Riley fixed the missing accessible email label. The check passed.", expected: true },
- { ending: "The check passed. Riley fixed the label.", expected: false },
-])("keeps adjacent success bound to the repair and event order: $ending", ({ ending, expected }) => {
- const o = fixture(); o.reply = "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. " + ending;
- expect(gradeWorkTimeline(o).passed).toBe(expected);
-});
-it.each([
- { ending: "The native run unexpectedly failed.", expected: false },
- { ending: "The runner runs eventually failed.", expected: false },
- { ending: "No native run ultimately failed.", expected: true },
- { ending: "The native run never ultimately failed.", expected: true },
- { ending: "Did the native run ultimately fail?", expected: true },
- { ending: "If a native run ultimately failed, it would be recorded.", expected: true },
- { ending: "The native run hypothetically failed.", expected: true },
- { ending: "The native run's verification unexpectedly failed, while the native run succeeded.", expected: true },
-])("distinguishes bounded native failure assertions from denials and checks: $ending", ({ ending, expected }) => {
- const o = fixture(); o.reply += " " + ending; expect(gradeWorkTimeline(o).passed).toBe(expected);
-});
-
-
-it("rejects the reviewed denied adjacent repair object", () => {
- const o = fixture();
- o.reply = "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed nothing. The check passed.";
- expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
-
-it.each([
- { repair: "Riley fixed absolutely nothing.", expected: false },
- { repair: "Riley fixed none of the labels.", expected: false },
- { repair: "Riley fixed neither label.", expected: false },
- { repair: "Riley fixed no label.", expected: false },
- { repair: "Riley fixed zero issues.", expected: false },
- { repair: "Nothing was fixed by Riley.", expected: false },
- { repair: "Riley fixed only the missing label.", expected: true },
- { repair: "Riley fixed the label with zero extra changes.", expected: true },
-])("distinguishes denied object heads from substantive repairs: $repair", ({ repair, expected }) => {
- const o = fixture(); o.reply = "Today UTC: Morgan's verification failed and work paused. You reassigned the task to Riley. " + repair + " The check passed.";
- expect(gradeWorkTimeline(o).passed).toBe(expected);
-});
-
-it.each([
- { handoff: "Riley took over a different task.", repair: "Riley repaired the label and the check passed.", expected: false },
- { handoff: "Riley took the same task,", repair: "added the label, but the check did not pass.", expected: false },
- { handoff: "You reassigned the task to Riley by updating the assignee.", repair: "Riley repaired the label and the check passed.", expected: true },
- { handoff: "You reassigned the task to Riley.", repair: "Riley fixed the label and the check passed with exit 0, unlike the earlier exit 1.", expected: true },
-])("respects reviewed object, assertion, method and result boundaries: $handoff $repair", ({ handoff, repair, expected }) => {
- const o = fixture();
- o.reply = `Morgan's verification failed and work paused. ${handoff} ${repair}`;
- expect(gradeWorkTimeline(o).passed).toBe(expected);
-});
-
-it.each([
- "Morgan's verification failed and work paused. Riley took the same task. Riley fixed the label and the check not only passed but exited 0.",
- "Morgan's verification failed and work paused. Riley took over from Morgan. Riley fixed the label and the check passed.",
-])("accepts the reviewed correlative assertion or known-source takeover: %s", reply => {
- const o = fixture(); o.reply = reply;
- expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-
-it.each(["another agent", "a bot"])("retains explicit nonhuman handoff attribution for %s", actor => {
- const o = fixture();
- o.reply = o.reply.replace("Then a human reassigned the task to Riley.", `The task was reassigned to Riley by ${actor}.`);
- expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
-const unnamedAttempt = "The verification command failed with exit code 1 and the task paused. You reassigned the task to Riley. Riley repaired the label and the check passed.";
-it("accepts a recorded failed attempt without requiring every worker's name", () => {
- const o = fixture(); o.reply = unnamedAttempt;
- expect(gradeWorkTimeline(o).passed).toBe(true);
-});
-it.each([
- unnamedAttempt.replace("You reassigned the task to Riley. ", ""),
- unnamedAttempt.replace("the check passed", "the check failed"),
- unnamedAttempt.replace("The verification command failed", "The verification command failed on Riley's attempt"),
-])("keeps missing handoff, failed repair and explicit wrong initial actor rejected: %s", reply => {
- const o = fixture(); o.reply = reply;
- expect(gradeWorkTimeline(o).passed).toBe(false);
-});
-
-it("corroborates a same-task implicit worker transition with the saved handoff and inspection links", () => {
- const o = fixture();
- o.reply = "## Signup implementation — RUN-3\nMorgan built the page. The verification command failed with exit 1 and the task paused as blocked. Riley added the missing label and verification exited 0.";
- expect(gradeWorkTimeline(o).passed).toBe(true);
- const withoutHandoff = { ...o, timelineEvents: o.timelineEvents.filter(event => event.id !== "handoff") };
- expect(gradeWorkTimeline(withoutHandoff).passed).toBe(false);
- const wrongAssignee = { ...o, timelineEvents: o.timelineEvents.map(event => event.id === "handoff"
- ? { ...event, details: { assigneeAgentId: "morgan" } } : event) };
- expect(gradeWorkTimeline(wrongAssignee).passed).toBe(false);
- expect(gradeWorkTimeline({ ...o, links: [] }).passed).toBe(false);
+it("grades saved milestones and the timeline read, never the wording of the model's reply", () => {
+ // Retained run: correct account that refers to Morgan as "He" and the check as "It".
+ const retained = "**2. Build and first verification — RUN-3**\n- Morgan built `launch.html`.\n- He ran `verify_launch_html` on the saved file. It failed with real exit code 1. The task was paused as blocked.\n\n**3. Repair and sign-off**\n- The task was reassigned from Morgan to Riley.\n- Riley added the label and reran `verify_launch_html`. It passed with exit code 0.";
+ for (const reply of [retained, "Morgan's check failed, the work paused, you handed it to Riley, and Riley's repair passed."]) {
+ expect(gradeWorkTimeline({ ...fixture(), reply }).passed).toBe(true);
+ expect(gradeWorkTimeline({ ...fixture(), reply, savedTimelineRead: false }).passed).toBe(false);
+ }
});
diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline-grounding.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline-grounding.test.ts
new file mode 100644
index 00000000000..0293ba81fdc
--- /dev/null
+++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline-grounding.test.ts
@@ -0,0 +1,25 @@
+import { expect, it } from "vitest";
+import { readSavedTaskTimeline } from "./internal-work-timeline.js";
+
+const taskId = "77ca2c1f-1aab-4acc-aca2-80d8057a1900";
+const toolRow = (name: string, output: string, status = "completed", eventType = "tool.execution.completed", sourceKind = "runner") => ({
+ eventType, payload: { prpEvent: { sourceKind, eventType, payload: { name, status, output } } },
+});
+const activity = (id: string) => JSON.stringify({ result: { apiOperationId: "GET /api/issues/{id}/activity", data: [{ entityId: id, action: "issue.updated" }] } });
+
+it("accepts a completed product read of the launch task's saved activity, comments or history", () => {
+ expect(readSavedTaskTimeline([toolRow("paperclip_call_api", activity(taskId))], taskId)).toBe(true);
+ expect(readSavedTaskTimeline([toolRow("paperclip_call_api", JSON.stringify({ result: { apiOperationId: "GET /api/issues/{id}/comments", data: [{ issueId: taskId }] } }))], taskId)).toBe(true);
+ expect(readSavedTaskTimeline([toolRow("paperclip_get_task_history", JSON.stringify({ taskId, entries: [] }))], taskId)).toBe(true);
+});
+
+it("rejects an answer that never read this task's saved events", () => {
+ expect(readSavedTaskTimeline([], taskId)).toBe(false);
+ // Another task's activity, a failed read, an unfinished read, or a non-runner frame are not grounding.
+ expect(readSavedTaskTimeline([toolRow("paperclip_call_api", activity("other-task"))], taskId)).toBe(false);
+ expect(readSavedTaskTimeline([toolRow("paperclip_call_api", activity(taskId), "failed")], taskId)).toBe(false);
+ expect(readSavedTaskTimeline([toolRow("paperclip_call_api", activity(taskId), "completed", "tool.execution.started")], taskId)).toBe(false);
+ expect(readSavedTaskTimeline([toolRow("paperclip_call_api", activity(taskId), "completed", "tool.execution.completed", "provider")], taskId)).toBe(false);
+ // Reading only the task record is not reading its saved timeline.
+ expect(readSavedTaskTimeline([toolRow("paperclip_get_task", JSON.stringify({ result: { apiOperationId: "GET /api/issues/{id}", data: { id: taskId } } }))], taskId)).toBe(false);
+});
diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts
index 187a6e34808..e7ca32b92ec 100644
--- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts
+++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts
@@ -5,10 +5,10 @@ import { fixtureWorker, makeDefinition } from "./internal-definition.js";
import { agentStatus, cleanupPrivacyProbes } from "./internal-privacy.js";
import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js";
import { startTimelineVerifier, type VerificationReceipt } from "./internal-timeline-verifier.js";
-import { truthfulTimelineAccount } from "./internal-timeline-narrative.js";
+import { collectRunEvents } from "../run-observations.js";
import { localDay } from "./internal-routine-audit.js";
import { checks, entity, hasLink, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js";
-interface Timeline extends InternalObservation { verification: VerificationReceipt[]; hiddenStatus: number; timelineEvents: Entity[] }
+interface Timeline extends InternalObservation { verification: VerificationReceipt[]; hiddenStatus: number; timelineEvents: Entity[]; savedTimelineRead?: boolean }
const objectRecord = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {};
/** Native outcomes are projected by the canonical committer, not an agent PATCH. */
export function timelineStatusMilestone(row: Entity, task: Entity, run: Entity, workerId: string, status: "blocked" | "done") {
@@ -27,6 +27,20 @@ export function timelineStatusMilestone(row: Entity, task: Entity, run: Entity,
&& result.issueStatusAfter === status && result.issueStatusBefore === row.details.fromStatus
&& typeof result.assessmentId === "string" && !!result.assessmentId && row.details.assessmentId === result.assessmentId;
}
+/**
+ * The chat run read the launch task's saved activity, comments or history
+ * through a product tool before answering. Tool outputs are product data, so
+ * this is checked exactly; the agent's own wording is never matched.
+ */
+export function readSavedTaskTimeline(events: readonly Record[], taskId: string) {
+ return !!taskId && events.some(row => {
+ const envelope = objectRecord(objectRecord(row.payload).prpEvent), payload = objectRecord(envelope.payload);
+ const output = typeof payload.output === "string" ? payload.output : "";
+ return row.eventType === "tool.execution.completed" && envelope.sourceKind === "runner" && payload.status === "completed"
+ && output.includes(taskId)
+ && (payload.name === "paperclip_get_task_history" || /GET \/api\/issues\/\{id\}\/(?:activity|comments)\b/.test(output));
+ });
+}
export function gradeWorkTimeline(value: unknown) {
const c = checks(); const o = observation(value) as Timeline | undefined;
c.add("real-timeline-and-command-receipts", !!o && Array.isArray(o.verification) && Array.isArray(o.timelineEvents) && typeof o.hiddenStatus === "number", "Real command exits, current-week state events and independent private access are required.");
@@ -47,17 +61,11 @@ export function gradeWorkTimeline(value: unknown) {
&& events.every(row => !!row.createdAt && localDay(row.createdAt, zone) >= monday.toISOString().slice(0, 10) && localDay(row.createdAt, zone) <= today)
&& Date.parse(events.find(row => row.id === o.state.blockedEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!)
&& Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.doneEventId)!.createdAt!), "The saved actual blocked → human handoff → agent completion events fall in the person's real current week and retain chronological attribution.");
- const worker = objectRecord(o.state.worker), repairWorker = objectRecord(o.state.repairWorker);
- const accountText = o.reply.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1");
- const claimedDates = accountText.match(/\b\d{4}-\d{2}-\d{2}\b/g) ?? [];
- const dateClaimsInWindow = claimedDates.every(day => {
- const parsed = new Date(`${day}T12:00:00Z`);
- return Number.isFinite(parsed.getTime()) && parsed.toISOString().slice(0, 10) === day
- && day >= monday.toISOString().slice(0, 10) && day <= today;
- });
- c.add("ordered-and-truthful-account", worker.id === first.agentId && repairWorker.id === second.agentId && dateClaimsInWindow && truthfulTimelineAccount({ reply: o.reply,
- taskIdentifier: typeof t.identifier === "string" ? t.identifier : undefined, firstWorker: typeof worker.name === "string" ? worker.name : "", repairWorker: typeof repairWorker.name === "string" ? repairWorker.name : "",
- personName: typeof o.state.personName === "string" ? o.state.personName : undefined }), "The reply attributes and orders the actual command failure/pause, saved reassignment to the different repair worker and successful repair, keeps explicit ISO dates within the current-week observation window, and does not invent a failed native run.");
+ // The model's prose is nondeterministic, so it is not pattern-matched. The
+ // saved milestones above are checked exactly; here the account must come
+ // from reading those saved events, and the links below must point at them.
+ c.add("account-grounded-in-saved-timeline", o.savedTimelineRead === true,
+ "Before answering, the chat run read the launch task's saved activity, comments or history through a product tool.");
c.add("inspectable-major-milestones", hasLink(o, "issues", t) && hasLink(o, "issues", prep), "Both actual launch work items are linked for inspection.");
const hidden = entity(o.state, "hiddenTask");
c.add("hidden-work-never-disclosed", [403, 404].includes(o.hiddenStatus) && ![hidden.id, hidden.title, hidden.description].filter(Boolean).some(text => o.reply.includes(text!))
@@ -92,9 +100,17 @@ function timeline(): ChatCatalogCase {
const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name };
await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state;
}, gradeWorkTimeline);
- return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v13", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000,
- sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts","../../server/src/services/native-runtime/runner-api-catalog.ts"],
- async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; },
+ return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v14", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000,
+ sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts","../../server/src/services/native-runtime/runner-api-catalog.ts"],
+ async observe(ctx, state): Promise {
+ const o = await observeInternal(ctx, state), taskId = entity(state, "task").id ?? "";
+ let savedTimelineRead = false;
+ for (const run of o.runs.filter(row => row.agentId === o.leadId && row.contextSnapshot?.issueId === ctx.issue()?.id)) {
+ const events = await collectRunEvents>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`));
+ savedTimelineRead = savedTimelineRead || readSavedTaskTimeline(events, taskId);
+ }
+ return { ...o, savedTimelineRead, verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${taskId}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) };
+ },
async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },
async cleanup(ctx) { try { await verifier?.close(); } finally { verifier = undefined; await cleanupPrivacyProbes(ctx); await owner?.dispose(); owner = undefined; } },
};