From fefc475470bcef6a6b82485067a110e6631f3743 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 23:19:31 -0500 Subject: [PATCH] fix(evals): accept exact natural historical delivery dates --- .../internal-artifact-search.test.ts | 23 +++++++++++++ .../internal-artifact-search.ts | 32 ++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts index 338e0887518..195dd101527 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.test.ts @@ -13,6 +13,29 @@ function valid() { } describe("migrated historical website artifact calibration", () => { it("accepts preserved original last-week delivery and actual later import", () => expect(gradeArtifactSearch(valid()).passed).toBe(true)); + it.each([ + "This is the final approved website delivered on October 21, 2026; today was only its import.", + "The final website was delivered 21 October 2026.", + "The approved final was delivered Oct 21.", + "The final was delivered 2026-10-21 and imported October 26, 2026.", + ])("accepts correct source dates without a literal last-week phrase: %s", reply => { + const o = valid(); o.reply = reply; expect(gradeArtifactSearch(o).passed).toBe(true); + }); + it.each([ + "The final approved website was delivered October 22, 2026.", + "The final approved website was delivered October 21, 2025.", + "The final approved website was delivered last week, on October 26, 2026.", + "The final approved website was not delivered last week.", + "The final approved website was not delivered on October 21, 2026.", + ])("rejects wrong or contradicted delivery dates: %s", reply => { + const o = valid(); o.reply = reply; expect(gradeArtifactSearch(o).passed).toBe(false); + }); + it("does not let natural-date wording rescue wrong final bytes or source", () => { + const bytes = valid(); bytes.reply = "Final website delivered October 21, 2026."; bytes.downloadedHash = "b".repeat(64); + expect(gradeArtifactSearch(bytes).passed).toBe(false); + const source = valid(); source.reply = bytes.reply; source.finalAttachment.issueId = "draft-task"; + expect(gradeArtifactSearch(source).passed).toBe(false); + }); it("builds real inline migration files with SHA-bound bytes and comment linkage", () => { const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID"); const yaml = parse(bundle.files[".paperclip.yaml"] as string); expect(yaml.schemaVersion).toBe(7); for (const row of bundle.receipts) { expect(bundle.files[`blobs/${row.sha256}`]).toMatchObject({ encoding: "base64" }); expect(yaml.tasks[row.slug].attachments[0]).toMatchObject({ sha256: row.sha256, commentIndex: 0 }); } expect(yaml.tasks["final-website"].completedAt).toBe("2026-10-21T12:00:00.000Z"); }); it("passes the actual production portability parser with all original dates/comments/attachments and no database access", async () => { const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID"); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts index 7bef494cf69..bd1f0f4ae9f 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts @@ -55,6 +55,36 @@ export function historicalWebsitePreviewMatches(value: unknown, bundle: ReturnTy function fileLink(o: InternalObservation, file: IssueAttachment) { return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === file.contentPath; } catch { return false; } }); } +/** Compare stated delivery dates with the preserved source date, not today's import date. */ +export function describesHistoricalDelivery(reply: string, deliveredAt: string) { + const expected = new Date(deliveredAt); + if (!Number.isFinite(expected.getTime())) return false; + const day = expected.toISOString().slice(0, 10); + const months = "(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)"; + const dates = new RegExp(`\\b(?:\\d{4}-\\d{2}-\\d{2}|${months}\\s+\\d{1,2}(?:st|nd|rd|th)?(?:,?\\s+\\d{4})?|\\d{1,2}(?:st|nd|rd|th)?\\s+(?:of\\s+)?${months}(?:,?\\s+\\d{4})?)\\b`, "gi"); + let matched = false; + for (const part of reply.split(/\n+|;|(?<=[.!?])\s+/)) { + // Qualified older history may have a different date; it is not the selected final. + if (/\b(?:older|earlier|previous|superseded|outside)\b/i.test(part)) continue; + if (/\blast\s+week\b/i.test(part)) { + if (/\b(?:not|never)\s+(?:(?:actually|delivered|from|during|in|the)\s+){0,3}last\s+week\b/i.test(part)) return false; + matched = true; + } + for (const date of part.matchAll(dates)) { + const prefix = part.slice(0, date.index); + const anchors = [...prefix.matchAll(/\b(deliver(?:ed|y)?|completed|import(?:ed)?|upload(?:ed)?)\b/gi)]; + // A separately identified import/upload time must not rewrite delivery time. + if (/^(?:import|upload)/i.test(anchors.at(-1)?.[1] ?? "")) continue; + if (/\b(?:not|never)\s+(?:(?:actually|deliver(?:ed)?|completed|dated|on|from|during)\s+){0,3}$/i.test(prefix)) return false; + let text = date[0].replace(/(\d)(?:st|nd|rd|th)\b/gi, "$1").replace(/\bof\s+/gi, ""); + if (!/\d{4}/.test(text)) text += ` ${expected.getUTCFullYear()}`; + const parsed = new Date(`${text} UTC`); + if (!Number.isFinite(parsed.getTime()) || parsed.toISOString().slice(0, 10) !== day) return false; + matched = true; + } + } + return matched; +} export function gradeArtifactSearch(value: unknown) { const c = checks(); const o = observation(value) as SearchObservation | undefined; c.add("real-imported-artifact-history", !!o && !!o.finalAttachment && Array.isArray(o.deliveryComments) && Array.isArray(o.searchResults) && !!o.currentSource, "Public migration receipts, original source dates and actual imported attachment/download evidence are required."); @@ -69,7 +99,7 @@ export function gradeArtifactSearch(value: unknown) { "The final belongs to the requested prior week through preserved completion and attachment-bound source-comment dates; the actual attachment import happened later."); c.add("exact-accessible-final-and-download", o.finalAttachment.id === file.id && o.finalAttachment.issueId === target.id && o.finalAttachment.sha256 === o.state.finalHash && o.downloadedHash === o.state.finalHash && hasLink(o, "issues", target) && !!fileLink(o, o.finalAttachment) - && /\b(?:final|approved|delivered)\b/i.test(o.reply) && (/\blast week\b/i.test(o.reply) || o.reply.includes(String(o.state.deliveryAt).slice(0, 10))), + && /\b(?:final|approved|delivered)\b/i.test(o.reply) && describesHistoricalDelivery(o.reply, String(o.state.deliveryAt)), "The answer selects the actual final from the requested source time, with both exact source and usable verified download links."); const distractors = o.state.distractors as Entity[] | undefined; c.add("draft-older-final-and-private-snippets-excluded", !!distractors && distractors.every(row => !hasLink(o, "issues", row) || row.id !== hidden.id && o.reply.split(/\n/).some(part => [row.id, row.identifier, row.title].filter(Boolean).some(name => part.includes(name!))