From 470db33754fad2d47685e0aa52d8db5102a0c21f Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 20:56:55 -0600 Subject: [PATCH] fix(evals): follow a rendered task link in revise-delivery only when one exists The revision request attaches its source task, so the shared result-navigation check's requirement for a rendered task link rejected a correct reply that returned both version downloads. The case still opens the returned download and both versions' library stacks. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../chat-catalog-domains/internal-website-delivery.ts | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 46094a31977..4d5353ba0f7 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2872,7 +2872,7 @@ "chat-catalog-domains/internal-website-delivery.ts", "registered-artifact.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.website-revision.v2", + "oracleVersion": "paperclip.chat-catalog.internal.website-revision.v3", "coverageGaps": [], "expectedRunCount": 2, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts index 18edbd65af0..16292d7d921 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts @@ -102,7 +102,7 @@ function websiteDelivery(revising: boolean): ChatCatalogCase { const state = { project: p, task: handedOff, worker, priorRunId, filename, producedBody: actual, producedHash: hash(actual), marker, bodyFact, signupHref, originalAttachment, originalProductId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment }; await ctx.input.evidence("chat-catalog-actual-prior-website.json", state); return state; }, gradeWebsiteDelivery); - return { ...base, ...(revising ? { oracleVersion: "paperclip.chat-catalog.internal.website-revision.v2" } : {}), authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000, + return { ...base, ...(revising ? { oracleVersion: "paperclip.chat-catalog.internal.website-revision.v3" } : {}), authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md"), expectedRunCount: 2, minimumRunCount: 2, attemptTimeoutMs: 600_000, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-website-delivery.ts", "registered-artifact.ts"], async observe(ctx, state): Promise { const o = await observeInternal(ctx, state); const ids = [entity(state, "task").id, ...(ctx.issue() ? [ctx.issue()!.id] : [])]; @@ -112,7 +112,10 @@ function websiteDelivery(revising: boolean): ChatCatalogCase { return { ...o, attachments, products, downloaded, ...(ctx.issue() ? { chatIssueId: ctx.issue()!.id } : {}) }; }, async verifyResult(ctx, state, value) { - const o = value as DeliveryObservation; await base.verifyResult(ctx, state, value); + const o = value as DeliveryObservation; + // The revision request already attaches the source task; follow a rendered task link when one exists, + // while the returned download and both library stacks below are always opened. + if (!revising || o.tasks.some(row => hasLink(o, "issues", row))) await base.verifyResult(ctx, state, value); const file = deliveredFiles(o).find(row => downloadHref(o, row)); if (!file) throw new Error("Missing exact registered download result"); const href = downloadHref(o, file)!; const page = await ctx.input.page.context().newPage(); try {