fix(evals): accept exact natural historical delivery dates

This commit is contained in:
Dotta committed 2026-10-28 00:25:04 -05:00
1 parent 8b29ca5489
commit fefc475470
2 files changed
+54 -1

No files matched your search

@@ -13,6 +13,29 @@ function valid() {
}
describe("migrated historical website artifact calibration", () => {
it("accepts preserved original last-week delivery and actual later import", () => expect(gradeArtifactSearch(valid()).passed).toBe(true));
it.each([
"This is the final approved website delivered on October 21, 2026; today was only its import.",
"The final website was delivered 21 October 2026.",
"The approved final was delivered Oct 21.",
"The final was delivered 2026-10-21 and imported October 26, 2026.",
])("accepts correct source dates without a literal last-week phrase: %s", reply => {
const o = valid(); o.reply = reply; expect(gradeArtifactSearch(o).passed).toBe(true);
});
it.each([
"The final approved website was delivered October 22, 2026.",
"The final approved website was delivered October 21, 2025.",
"The final approved website was delivered last week, on October 26, 2026.",
"The final approved website was not delivered last week.",
"The final approved website was not delivered on October 21, 2026.",
])("rejects wrong or contradicted delivery dates: %s", reply => {
const o = valid(); o.reply = reply; expect(gradeArtifactSearch(o).passed).toBe(false);
});
it("does not let natural-date wording rescue wrong final bytes or source", () => {
const bytes = valid(); bytes.reply = "Final website delivered October 21, 2026."; bytes.downloadedHash = "b".repeat(64);
expect(gradeArtifactSearch(bytes).passed).toBe(false);
const source = valid(); source.reply = bytes.reply; source.finalAttachment.issueId = "draft-task";
expect(gradeArtifactSearch(source).passed).toBe(false);
});
it("builds real inline migration files with SHA-bound bytes and comment linkage", () => { const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID"); const yaml = parse(bundle.files[".paperclip.yaml"] as string); expect(yaml.schemaVersion).toBe(7); for (const row of bundle.receipts) { expect(bundle.files[`blobs/${row.sha256}`]).toMatchObject({ encoding: "base64" }); expect(yaml.tasks[row.slug].attachments[0]).toMatchObject({ sha256: row.sha256, commentIndex: 0 }); } expect(yaml.tasks["final-website"].completedAt).toBe("2026-10-21T12:00:00.000Z"); });
it("passes the actual production portability parser with all original dates/comments/attachments and no database access", async () => {
const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID");
@@ -55,6 +55,36 @@ export function historicalWebsitePreviewMatches(value: unknown, bundle: ReturnTy
function fileLink(o: InternalObservation, file: IssueAttachment) {
return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === file.contentPath; } catch { return false; } });
}
/** Compare stated delivery dates with the preserved source date, not today's import date. */
export function describesHistoricalDelivery(reply: string, deliveredAt: string) {
const expected = new Date(deliveredAt);
if (!Number.isFinite(expected.getTime())) return false;
const day = expected.toISOString().slice(0, 10);
const months = "(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)";
const dates = new RegExp(`\\b(?:\\d{4}-\\d{2}-\\d{2}|${months}\\s+\\d{1,2}(?:st|nd|rd|th)?(?:,?\\s+\\d{4})?|\\d{1,2}(?:st|nd|rd|th)?\\s+(?:of\\s+)?${months}(?:,?\\s+\\d{4})?)\\b`, "gi");
let matched = false;
for (const part of reply.split(/\n+|;|(?<=[.!?])\s+/)) {
// Qualified older history may have a different date; it is not the selected final.
if (/\b(?:older|earlier|previous|superseded|outside)\b/i.test(part)) continue;
if (/\blast\s+week\b/i.test(part)) {
if (/\b(?:not|never)\s+(?:(?:actually|delivered|from|during|in|the)\s+){0,3}last\s+week\b/i.test(part)) return false;
matched = true;
}
for (const date of part.matchAll(dates)) {
const prefix = part.slice(0, date.index);
const anchors = [...prefix.matchAll(/\b(deliver(?:ed|y)?|completed|import(?:ed)?|upload(?:ed)?)\b/gi)];
// A separately identified import/upload time must not rewrite delivery time.
if (/^(?:import|upload)/i.test(anchors.at(-1)?.[1] ?? "")) continue;
if (/\b(?:not|never)\s+(?:(?:actually|deliver(?:ed)?|completed|dated|on|from|during)\s+){0,3}$/i.test(prefix)) return false;
let text = date[0].replace(/(\d)(?:st|nd|rd|th)\b/gi, "$1").replace(/\bof\s+/gi, "");
if (!/\d{4}/.test(text)) text += ` ${expected.getUTCFullYear()}`;
const parsed = new Date(`${text} UTC`);
if (!Number.isFinite(parsed.getTime()) || parsed.toISOString().slice(0, 10) !== day) return false;
matched = true;
}
}
return matched;
}
export function gradeArtifactSearch(value: unknown) {
const c = checks(); const o = observation(value) as SearchObservation | undefined;
c.add("real-imported-artifact-history", !!o && !!o.finalAttachment && Array.isArray(o.deliveryComments) && Array.isArray(o.searchResults) && !!o.currentSource, "Public migration receipts, original source dates and actual imported attachment/download evidence are required.");
@@ -69,7 +99,7 @@ export function gradeArtifactSearch(value: unknown) {
"The final belongs to the requested prior week through preserved completion and attachment-bound source-comment dates; the actual attachment import happened later.");
c.add("exact-accessible-final-and-download", o.finalAttachment.id === file.id && o.finalAttachment.issueId === target.id && o.finalAttachment.sha256 === o.state.finalHash
&& o.downloadedHash === o.state.finalHash && hasLink(o, "issues", target) && !!fileLink(o, o.finalAttachment)
&& /\b(?:final|approved|delivered)\b/i.test(o.reply) && (/\blast week\b/i.test(o.reply) || o.reply.includes(String(o.state.deliveryAt).slice(0, 10))),
&& /\b(?:final|approved|delivered)\b/i.test(o.reply) && describesHistoricalDelivery(o.reply, String(o.state.deliveryAt)),
"The answer selects the actual final from the requested source time, with both exact source and usable verified download links.");
const distractors = o.state.distractors as Entity[] | undefined;
c.add("draft-older-final-and-private-snippets-excluded", !!distractors && distractors.every(row => !hasLink(o, "issues", row) || row.id !== hidden.id && o.reply.split(/\n/).some(part => [row.id, row.identifier, row.title].filter(Boolean).some(name => part.includes(name!))