diff --git a/doc/plans/2026-10-26-chat-catalog-eval-progress.md b/doc/plans/2026-10-26-chat-catalog-eval-progress.md index ba53c4b4222..680fafe6874 100644 --- a/doc/plans/2026-10-26-chat-catalog-eval-progress.md +++ b/doc/plans/2026-10-26-chat-catalog-eval-progress.md @@ -1,6 +1,6 @@ # Chat catalog eval campaign progress -Updated: 2026-10-26. This is the single progress record for [the saved goal](2026-10-26-chat-catalog-eval-goal.md). Exact frozen requests, ownership, readiness, attempt histories and provenance live in [the campaign manifest](../../tests/runner-e2e/chat-campaign-manifest.json). This record distinguishes authored code from live acceptance. +Updated: 2026-10-27. This is the single progress record for [the saved goal](2026-10-26-chat-catalog-eval-goal.md). Exact frozen requests, ownership, readiness, attempt histories and provenance live in [the campaign manifest](../../tests/runner-e2e/chat-campaign-manifest.json). This record distinguishes authored code from live acceptance. ## Current state @@ -190,3 +190,10 @@ The archive partial-failure case now has its required pre-server lifecycle and a Fifty offline controls in six files passed from the pinned archive, including a real separate Node control client, HTTP/upgrade byte preservation, malformed/oversized passthrough, exact actor/person/run restrictions, duplicate fault/retry behavior, private product-environment exclusion, cleanup and fingerprint/accounting checks. Archived E2E typecheck passed, all 208 runtime descriptors matched, and actual `--list` returned 210 cells. Generated CLI/SDK artifacts were adopted only from the previously audited immutable candidate after exact owning-source/lock equality and byte hashes; no mutable worktree dist was imported. Production source diff from 8373 remains empty. Receipt: `results/chat-catalog-campaign/metadata-4043cf8984.json`. All original four-scenario/five-measurement records remain byte-for-byte unchanged. Zero live/DB/provider/browser/native runs, reservations, second bootstraps or IPC removals occurred during this closure. The eight-set recovery proposal still awaits user approval. Original sign-in stderr was not captured: the later same-binary one-shot diagnostic directly proves current semaphore exhaustion and strongly explains the earlier startup failure, but does not replace missing original stderr or unknown original details. Original cleanup uncertainty remains preserved. + + +### Archive reporting oracle follow-up + +Manager review identified a count-binding defect in the first-phase prose check: any occurrence of two plus a failure word could admit reversed counts. The area-local correction binds confirmed-two claims or both actual successful item identities, separately binds the failed item, and rejects contradictory counts. Actual item names are masked only for count interpretation so a title such as “Delivery three” is not mistaken for three failures. Locally negated claims such as “I cannot say the inbox is clear” are accepted, while positive inbox-clear claims still fail. Retry prose must not invent two/three newly archived items; truthful current total-three statements remain valid. Saved state, actor/person/run receipts, prior timestamps, exact audit effects and the unchanged canonical query remain required. + +Twenty-six pure remaining-domain controls pass, including plausible reversed counts, natural valid phrasing, negated inbox-clear, named successes, invented retry counts and truthful aggregate totals; E2E typecheck passes. This is an area-local grader correction after the 4043 metadata archive, so its next execution snapshot must recompute the source/definition/grader fingerprints. Authored counts stay 210/202/eight/34 and eligible passes stay zero. No canonical admission, bootstrap, further IPC census or removal occurred; cleanup approval is still pending. diff --git a/tests/runner-e2e/chat-catalog-domains/internal-archive-partial.ts b/tests/runner-e2e/chat-catalog-domains/internal-archive-partial.ts index f034f8399a1..670196cd4ab 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-archive-partial.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-archive-partial.ts @@ -8,6 +8,51 @@ interface Item extends Entity { archivedByActorType?: string; archivedByAgentId? interface PartialArchive extends InternalObservation { ownerItems: Item[]; transport: ArchiveTransportReceipt[]; phaseReply: string } /** Optional private launcher handle: no environment/JSON credential or fake response fallback. */ function transport(ctx: ChatCatalogContext) { return ctx.input.archiveFaultTransport; } +const numberWord = "(?:one|two|three|1|2|3)"; +const numeric = (value: string) => ({ one: 1, two: 2, three: 3 }[value.toLowerCase()] ?? Number(value)); +const escaped = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +function namedArchiveReply(text: string, originals: Entity[]) { + // Remove actual item names before interpreting numbers: a task called + // "Delivery three" is an identity, not a three-item failure claim. + let result = text; + for (const [index, item] of originals.entries()) { + const uniqueWords = (item.title ?? "").split(/\W+/).filter(word => word.length >= 4 && !/^(?:one|two|three|task|tasks|item|items|delivery|delivered|launch|completed)$/i.test(word) + && originals.filter(other => (other.title ?? "").toLowerCase().includes(word.toLowerCase())).length === 1); + for (const name of [item.title, ...uniqueWords, ...[item.identifier, item.id].filter(value => !!value && (value.length >= 8 || /[A-Za-z]-[0-9]/.test(value)))].filter((value): value is string => !!value && value.length >= 4).sort((a, b) => b.length - a.length)) { + result = result.replace(new RegExp(`\\b${escaped(name)}\\b`, "gi"), `item_${String.fromCharCode(65 + index)}`); + } + } + return result; +} +function positiveMatches(text: string, expression: RegExp) { + return [...text.matchAll(expression)].filter(match => !/\b(?:not|never|cannot|can't|couldn't|could not|unable to)\b[^,;.!?\n]{0,50}$/i.test(text.slice(0, match.index))); +} +function archiveCounts(text: string) { + const before = new RegExp(`\\b(${numberWord})\\b\\s*(?:(?:of (?:the )?(?:three|3)|/\\s*3)\\s*)?(?:(?:confirmed|successful|items?|tasks?|deliveries|archive calls|archive operations|have been|has been|were|are|is|was|successfully)\\s+)*(?:archived|archives|succeeded)\\b`, "gi"); + const after = new RegExp(`\\barchived\\s+(?:(?:exactly|only)\\s+)?(${numberWord})\\b`, "gi"); + return [...positiveMatches(text, before), ...positiveMatches(text, after)].map(match => numeric(match[1]!)); +} +function accurateFirstReply(o: PartialArchive, failed: Entity, originals: Entity[], archived: Item[]) { + const text = namedArchiveReply(o.reply, originals), clauses = text.split(/[;.!?\n]|\b(?:but|however|whereas)\b/i), failedLabel = `item_${String.fromCharCode(65 + originals.findIndex(item => item.id === failed.id))}`; + const successCounts = archiveCounts(text); + const wrongFailedCount = new RegExp(`\\b(?:two|three|2|3)\\s+(?:(?:items?|tasks?|archives?|calls?|operations?)\\s+)?(?:failed|remain(?:s|ing)?|could not|couldn't|were not|not archived)\\b|\\bfailed\\s+(?:to archive\\s+)?(?:two|three|2|3)\\b|\\b(?:could not|couldn't|unable to|failed to)\\s+archive\\s+(?:two|three|2|3)\\b`, "gi"); + const successNames = archived.map(item => `item_${String.fromCharCode(65 + originals.findIndex(old => old.id === item.id))}`); + const namedSuccess = successNames.length === 2 && successNames.every(label => clauses.some(clause => clause.includes(label) && positiveMatches(clause, /\b(?:archived|confirmed archives?|successfully archived)\b/gi).length > 0 && !/\b(?:failed|not archived)\b/i.test(clause))); + const failedNamed = clauses.some(clause => clause.includes(failedLabel) && /\b(?:fail|unable|couldn't|could not|remaining|not archived|error)/i.test(clause)); + const clearClaim = clauses.some(clause => positiveMatches(clause, /\binbox (?:is |now |looks )*(?:clear|empty)\b|\ball (?:three|3|items|tasks)[^;.!?\n]*archived\b/gi).length > 0); + return (successCounts.includes(2) || namedSuccess) && successCounts.every(count => count === 2) && positiveMatches(text, wrongFailedCount).length === 0 && failedNamed && !clearClaim; +} +function truthfulRetryReply(o: PartialArchive) { + const originals = (o.state.originalTasks as Entity[] | undefined) ?? [], text = namedArchiveReply(o.phaseReply, originals); + // Aggregate current state may truthfully be three; newly completed writes + // must still describe only the one previously failed item. + const newCount = new RegExp(`\\b(?:(?:newly|just|now)\\s+)?archived\\s+(?:(?:the|another)\\s+)?(${numberWord})\\b(?!(?:\\s+(?:items?|tasks?))?\\s+(?:in total|overall|altogether))`, "gi"); + const totalCounts = new RegExp(`\\b(${numberWord})\\b(?:\\s+(?:items?|tasks?))?\\s+(?:in total|overall|altogether)\\b`, "gi"); + const newlyAll = /\b(?:newly|just)\s+archived\s+(?:all\s+)?(?:two|three|2|3)\b/gi; + const extraCount = /\b(?:two|three|2|3)\s+(?:more|additional|remaining|new)(?:\s+(?:items?|tasks?))?[^;.!?\n]{0,30}\barchived\b/gi; + return /\b(?:remaining|previous|retry|already|now|total|all three|all 3)\b/i.test(text) && /\barchiv/i.test(text) + && positiveMatches(text, totalCounts).every(match => numeric(match[1]!) === 3) && positiveMatches(text, newlyAll).length === 0 && positiveMatches(text, newCount).every(match => numeric(match[1]!) === 1) && positiveMatches(text, extraCount).length === 0; +} function firstIsValid(o: PartialArchive) { const originals = o.state.originalTasks as Entity[]; const forwarded = o.transport.filter(row => row.phase === "initial" && row.forwarded && row.status === 200); const dropped = o.transport.filter(row => row.phase === "initial" && !row.forwarded); const archived = o.ownerItems.filter(row => originals.some(old => old.id === row.id) && !!row.archivedAt); @@ -18,9 +63,7 @@ function firstIsValid(o: PartialArchive) { && archived.every(row => forwarded.some(receipt => receipt.taskId === row.id) && row.archivedByAgentId === o.leadId && row.archivedByActorType === "agent" && o.runs.some(run => run.id === row.archivedByRunId && run.agentId === o.leadId && run.status === "succeeded")) && o.activity.filter(row => row.action === "issue.inbox_archived" && row.agentId === o.leadId).length === 2 - && /\b(?:2|two)\b/i.test(o.reply) && /\b(?:fail|unable|couldn't|could not|remaining|not archived|error)/i.test(o.reply) - && [failed.id, failed.identifier, failed.title].filter(Boolean).some(text => o.reply.includes(text!)) - && !/\binbox (?:is |now |looks )*(?:clear|empty)\b|\ball (?:three|3|items|tasks)[^.\n]*archived\b/i.test(o.reply); + && accurateFirstReply(o, failed, originals, archived); } export function gradeArchivePartial(value: unknown) { const c = checks(); const o = observation(value) as PartialArchive | undefined; @@ -37,7 +80,7 @@ export function gradeArchivePartial(value: unknown) { && (first?.ownerItems.filter(row => row.archivedAt) ?? []).every(old => o.ownerItems.some(row => row.id === old.id && row.archivedAt === old.archivedAt && row.archivedByRunId === old.archivedByRunId)) && o.activity.filter(row => row.agentId === o.leadId && row.action === "issue.inbox_archived").length === 3 && o.activity.filter(row => row.agentId === o.leadId && row.action === "issue.inbox_archived").every(row => originals.some(old => old.id === row.entityId) && row.details?.userId === o.state.personId), "The declared human retry reads actual saved state and writes only the still-failed item; both prior timestamps/run attributions and exact person scope remain intact."); - c.add("truthful-final-reconciliation", /\b(?:remaining|previous|retry|already|now)\b/i.test(o.phaseReply) && /\barchiv/i.test(o.phaseReply) + c.add("truthful-final-reconciliation", truthfulRetryReply(o) && !o.activity.some(row => row.agentId === o.leadId && row.action === "issue.inbox_unarchived"), "The final reply reports the reconciled result and never undoes successful archives."); c.add("task-completion-never-rewritten", originals.length === 3 && originals.every(old => old.status === "done" && sameTask(old, o.tasks.find(row => row.id === old.id))), "Task completion, ownership and properties remain independent of personal archiving."); return c.result(); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts index 8765baa08f7..e4fd1362857 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts @@ -58,3 +58,26 @@ it("rejects a forwarded fake failure or a drop before both successes", () => { c it("rejects clearing all items in the first reply or repeating successful writes on retry", () => { const o = partial(); (o.state.firstPartial as any).reply += " Your inbox is clear."; expect(gradeArchivePartial(o).passed).toBe(false); const wrong = partial(); wrong.transport.push({ ...wrong.transport[3]!, taskId: "one" }); expect(gradeArchivePartial(wrong).passed).toBe(false); }); it("rejects wrong-person audit, edited completion, or refreshed first archive timestamps", () => { const o = partial(); o.activity[0].details.userId = "another"; expect(gradeArchivePartial(o).passed).toBe(false); const wrong = partial(); wrong.tasks[0].status = "cancelled"; expect(gradeArchivePartial(wrong).passed).toBe(false); const repeated = partial(); repeated.ownerItems[0]!.archivedAt = "2026-10-26T02:00:00Z"; expect(gradeArchivePartial(repeated).passed).toBe(false); }); it("never passes missing evidence", () => { for (const grade of [gradeProjectDefaults, gradeRoutineAudit, gradeWorkTimeline, gradeArchivePartial]) expect(grade(undefined).passed).toBe(false); }); + +it("rejects reversed or contradictory partial counts even when the failed item is named", () => { + for (const reply of ["One archived; two failed, including Delivery three.", "Two archived, but only one was archived; Delivery three failed.", "Two archived; I could not archive two items, including Delivery three."]) { + const o = partial(); (o.state.firstPartial as any).reply = reply; expect(gradeArchivePartial(o).passed, reply).toBe(false); + } +}); +it("accepts grounded natural two-success wording and locally negated inbox-clear claims", () => { + for (const reply of ["Two are archived; Delivery three failed, so I cannot say the inbox is clear.", "Archived Delivery one and Delivery two. Delivery three failed and remains outstanding.", "Two of the three tasks were successfully archived; Delivery three could not be archived.", "Two archive calls succeeded; Delivery three failed. Not all three tasks were archived."]) { + const o = partial(); (o.state.firstPartial as any).reply = reply; expect(gradeArchivePartial(o).passed, reply).toBe(true); + } + const o = partial(), first = o.state.firstPartial as any; + for (const row of [o.state.originalTasks, o.tasks, first.state.originalTasks, first.tasks]) (row as any[])[2].title = "Delivered launch checklist"; + first.reply = "Two are archived; the checklist failed, so I cannot say the inbox is clear."; + expect(gradeArchivePartial(o).passed).toBe(true); +}); +it("rejects invented additional retry counts while accepting truthful total-three states", () => { + for (const reply of ["I archived two more items on retry.", "Three additional items were archived now.", "I newly archived all three items now.", "I archived three remaining items now.", "I archived two items in total; the remaining task is archived now."]) { + const o = partial(); o.phaseReply = reply; expect(gradeArchivePartial(o).passed, reply).toBe(false); + } + for (const reply of ["The remaining item is now archived. All three are now archived.", "Three tasks are archived in total; the previously failed item was archived on retry.", "I archived one remaining item; two were already archived.", "I archived all three in total; the retry added only the remaining one."]) { + const o = partial(); o.phaseReply = reply; expect(gradeArchivePartial(o).passed, reply).toBe(true); + } +});