From 1c7cbdc25bae7404f909084359b44754c59cb0b7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 28 Oct 2026 01:57:54 -0500 Subject: [PATCH] test(evals): preserve artifact outcome without extra task status gate Co-Authored-By: Paperclip --- .../identity-confirmation-workflows.test.ts | 2 +- .../chat-catalog-domains/identity-confirmation-workflows.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts index ec6bf564db6..0686a689512 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts @@ -77,7 +77,7 @@ describe("answer continuation and ordinary deliverable custody", () => { const o = valid("questions-and-approvals.duplicate-answer"); if (kind === "foreign-task") o.documents[0]!.issueId = "foreign"; if (kind === "wrong-key") o.activity.find(row => row.id === "document")!.details!.key = "other"; - if (kind === "unfinished") o.tasks[0]!.status = "in_progress"; + if (kind === "unfinished") o.runs[2]!.status = "running"; if (kind === "wrong-company") o.runs[2]!.companyId = "foreign"; if (kind === "unstarted") o.runs[2]!.startedAt = null; if (kind === "unattributed-plan") o.documents.push({ id: "plan", key: "plan", issueId: "draft", latestRevisionId: "missing" }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts index a3d7b15dd9d..bbccc96f3ba 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts @@ -64,13 +64,13 @@ export function gradeConfirmationWorkflow(value: unknown) { && o.runs.some(run => run.id === revision.createdByRunId && run.status === "succeeded" && !!run.startedAt && run.companyId === o.companyId && run.agentId === revision.createdByAgentId && [o.leadId, (o.state.worker as Entity | undefined)?.id].includes(run.agentId))); - c.add("one-actual-answer-artifact", o.tasks.length === 1 && o.tasks[0]?.companyId === o.companyId && o.tasks[0]?.status === "done" && hasLink(o, "issues", o.tasks[0]) + c.add("one-actual-answer-artifact", o.tasks.length === 1 && o.tasks[0]?.companyId === o.companyId && hasLink(o, "issues", o.tasks[0]) && introductions.length === 1 && !!introduction && (introduction.body ?? "").includes("Acorn") && authored(introduction) && o.documents.filter(document => document.key === "plan").length <= 1 && o.documents.every(authored) && o.activity.some(row => row.action === "issue.document_created" && row.entityId === o.tasks[0]?.id && row.details?.documentId === introduction?.id && row.details.key === introduction?.key && [o.leadId, (o.state.worker as Entity | undefined)?.id].includes(row.agentId ?? "")), - "One linked completed task has one attributed saved introduction using Acorn; its separately keyed platform plan is not another introduction."); + "One linked ordinary task has one attributed saved introduction using Acorn; its separately keyed platform plan is not another introduction."); } else c.add("known-workflow", false, "This workflow has no independent oracle."); return c.result(); }