mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-06 14:02:36 +01:00
fix(evals): accept exact natural historical delivery dates
This commit is contained in:
1 parent
8b29ca5489
commit
fefc475470
2 files changed
+54
-1
No files matched your search
@@ -13,6 +13,29 @@ function valid() {
|
||||
}
|
||||
describe("migrated historical website artifact calibration", () => {
|
||||
it("accepts preserved original last-week delivery and actual later import", () => expect(gradeArtifactSearch(valid()).passed).toBe(true));
|
||||
it.each([
|
||||
"This is the final approved website delivered on October 21, 2026; today was only its import.",
|
||||
"The final website was delivered 21 October 2026.",
|
||||
"The approved final was delivered Oct 21.",
|
||||
"The final was delivered 2026-10-21 and imported October 26, 2026.",
|
||||
])("accepts correct source dates without a literal last-week phrase: %s", reply => {
|
||||
const o = valid(); o.reply = reply; expect(gradeArtifactSearch(o).passed).toBe(true);
|
||||
});
|
||||
it.each([
|
||||
"The final approved website was delivered October 22, 2026.",
|
||||
"The final approved website was delivered October 21, 2025.",
|
||||
"The final approved website was delivered last week, on October 26, 2026.",
|
||||
"The final approved website was not delivered last week.",
|
||||
"The final approved website was not delivered on October 21, 2026.",
|
||||
])("rejects wrong or contradicted delivery dates: %s", reply => {
|
||||
const o = valid(); o.reply = reply; expect(gradeArtifactSearch(o).passed).toBe(false);
|
||||
});
|
||||
it("does not let natural-date wording rescue wrong final bytes or source", () => {
|
||||
const bytes = valid(); bytes.reply = "Final website delivered October 21, 2026."; bytes.downloadedHash = "b".repeat(64);
|
||||
expect(gradeArtifactSearch(bytes).passed).toBe(false);
|
||||
const source = valid(); source.reply = bytes.reply; source.finalAttachment.issueId = "draft-task";
|
||||
expect(gradeArtifactSearch(source).passed).toBe(false);
|
||||
});
|
||||
it("builds real inline migration files with SHA-bound bytes and comment linkage", () => { const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID"); const yaml = parse(bundle.files[".paperclip.yaml"] as string); expect(yaml.schemaVersion).toBe(7); for (const row of bundle.receipts) { expect(bundle.files[`blobs/${row.sha256}`]).toMatchObject({ encoding: "base64" }); expect(yaml.tasks[row.slug].attachments[0]).toMatchObject({ sha256: row.sha256, commentIndex: 0 }); } expect(yaml.tasks["final-website"].completedAt).toBe("2026-10-21T12:00:00.000Z"); });
|
||||
it("passes the actual production portability parser with all original dates/comments/attachments and no database access", async () => {
|
||||
const bundle = buildHistoricalWebsiteImport("2026-10-21T12:00:00.000Z", "ORCHID");
|
||||
|
||||
@@ -55,6 +55,36 @@ export function historicalWebsitePreviewMatches(value: unknown, bundle: ReturnTy
|
||||
function fileLink(o: InternalObservation, file: IssueAttachment) {
|
||||
return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === file.contentPath; } catch { return false; } });
|
||||
}
|
||||
/** Compare stated delivery dates with the preserved source date, not today's import date. */
|
||||
export function describesHistoricalDelivery(reply: string, deliveredAt: string) {
|
||||
const expected = new Date(deliveredAt);
|
||||
if (!Number.isFinite(expected.getTime())) return false;
|
||||
const day = expected.toISOString().slice(0, 10);
|
||||
const months = "(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)";
|
||||
const dates = new RegExp(`\\b(?:\\d{4}-\\d{2}-\\d{2}|${months}\\s+\\d{1,2}(?:st|nd|rd|th)?(?:,?\\s+\\d{4})?|\\d{1,2}(?:st|nd|rd|th)?\\s+(?:of\\s+)?${months}(?:,?\\s+\\d{4})?)\\b`, "gi");
|
||||
let matched = false;
|
||||
for (const part of reply.split(/\n+|;|(?<=[.!?])\s+/)) {
|
||||
// Qualified older history may have a different date; it is not the selected final.
|
||||
if (/\b(?:older|earlier|previous|superseded|outside)\b/i.test(part)) continue;
|
||||
if (/\blast\s+week\b/i.test(part)) {
|
||||
if (/\b(?:not|never)\s+(?:(?:actually|delivered|from|during|in|the)\s+){0,3}last\s+week\b/i.test(part)) return false;
|
||||
matched = true;
|
||||
}
|
||||
for (const date of part.matchAll(dates)) {
|
||||
const prefix = part.slice(0, date.index);
|
||||
const anchors = [...prefix.matchAll(/\b(deliver(?:ed|y)?|completed|import(?:ed)?|upload(?:ed)?)\b/gi)];
|
||||
// A separately identified import/upload time must not rewrite delivery time.
|
||||
if (/^(?:import|upload)/i.test(anchors.at(-1)?.[1] ?? "")) continue;
|
||||
if (/\b(?:not|never)\s+(?:(?:actually|deliver(?:ed)?|completed|dated|on|from|during)\s+){0,3}$/i.test(prefix)) return false;
|
||||
let text = date[0].replace(/(\d)(?:st|nd|rd|th)\b/gi, "$1").replace(/\bof\s+/gi, "");
|
||||
if (!/\d{4}/.test(text)) text += ` ${expected.getUTCFullYear()}`;
|
||||
const parsed = new Date(`${text} UTC`);
|
||||
if (!Number.isFinite(parsed.getTime()) || parsed.toISOString().slice(0, 10) !== day) return false;
|
||||
matched = true;
|
||||
}
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
export function gradeArtifactSearch(value: unknown) {
|
||||
const c = checks(); const o = observation(value) as SearchObservation | undefined;
|
||||
c.add("real-imported-artifact-history", !!o && !!o.finalAttachment && Array.isArray(o.deliveryComments) && Array.isArray(o.searchResults) && !!o.currentSource, "Public migration receipts, original source dates and actual imported attachment/download evidence are required.");
|
||||
@@ -69,7 +99,7 @@ export function gradeArtifactSearch(value: unknown) {
|
||||
"The final belongs to the requested prior week through preserved completion and attachment-bound source-comment dates; the actual attachment import happened later.");
|
||||
c.add("exact-accessible-final-and-download", o.finalAttachment.id === file.id && o.finalAttachment.issueId === target.id && o.finalAttachment.sha256 === o.state.finalHash
|
||||
&& o.downloadedHash === o.state.finalHash && hasLink(o, "issues", target) && !!fileLink(o, o.finalAttachment)
|
||||
&& /\b(?:final|approved|delivered)\b/i.test(o.reply) && (/\blast week\b/i.test(o.reply) || o.reply.includes(String(o.state.deliveryAt).slice(0, 10))),
|
||||
&& /\b(?:final|approved|delivered)\b/i.test(o.reply) && describesHistoricalDelivery(o.reply, String(o.state.deliveryAt)),
|
||||
"The answer selects the actual final from the requested source time, with both exact source and usable verified download links.");
|
||||
const distractors = o.state.distractors as Entity[] | undefined;
|
||||
c.add("draft-older-final-and-private-snippets-excluded", !!distractors && distractors.every(row => !hasLink(o, "issues", row) || row.id !== hidden.id && o.reply.split(/\n/).some(part => [row.id, row.identifier, row.title].filter(Boolean).some(name => part.includes(name!))
|
||||
|
||||
Reference in new issue
Block a user