From 6565111dfcc7711cee643ffadca299485d03abbf Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 04:55:34 -0500 Subject: [PATCH] test: bind review handoff and race actual human answers Co-Authored-By: Paperclip --- .../identity-confirmation-workflows.test.ts | 4 ++++ .../identity-confirmation-workflows.ts | 7 ++++++- .../identity-plan-review.test.ts | 16 ++++++++++++++++ .../chat-catalog-domains/identity-plan-review.ts | 10 +++++++++- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts index 69f98c5228c..76af69184e7 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts @@ -37,6 +37,10 @@ describe("conversational proposal and simultaneous human answer oracles", () => it("accepting the proposal twice cannot produce two durable tasks", () => { const o = valid("questions-and-approvals.accept-one-proposal"); o.tasks.push({ ...o.tasks[0]!, id: "duplicate" }); expect(gradeConfirmationWorkflow(o).passed).toBe(false); }); + it("a task created before the protected answer cannot be credited despite a correct final artifact", () => { + const o = valid("questions-and-approvals.duplicate-answer"); o.state.noTaskBeforeAnswers = false; + expect(gradeConfirmationWorkflow(o).passed).toBe(false); + }); it("a duplicated answer continuation is rejected even with identical final output", () => { const o = valid("questions-and-approvals.duplicate-answer"); o.runs.push({ ...o.runs[1]!, id: "extra-resume" }); expect(gradeConfirmationWorkflow(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts index 79331be9517..bbf133edaec 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts @@ -102,8 +102,13 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [ authentication: "human-session", sourceFiles: sources, expectedRunCount: 4, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, attemptTimeoutMs: 600_000, observe: observeWorkflow, async act(ctx, state) { - await waitForReply(ctx); const other = tabs.get(ctx); if (!other) throw new Error("The second real browser tab is missing"); + const other = tabs.get(ctx); if (!other) throw new Error("The second real browser tab is missing"); + await pollUntil({ label: "Canonical name reply has an admitted native run", deadlineAt: Date.now() + 120_000, + load: () => ctx.allRuns(), accept: runs => runs.some(run => run.agentId === ctx.input.fixtures.agent.id && ["running", "succeeded"].includes(run.status)) }); + const pending = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`); + if (pending.find(row => row.id === entity(state, "original").id)?.status !== "pending") throw new Error("The original human-only question is not pending before the two-tab answer race"); const before = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true`); state.noTaskBeforeAnswers = before.filter(row => row.id !== ctx.issue()!.id).length === 0; + if (!state.noTaskBeforeAnswers) throw new Error("Work started before the protected human question was actually answered"); for (const page of [ctx.input.page, other]) { const form = page.getByTestId("task-chat-composer-takeover"); if (!(await form.isVisible())) await page.locator(`#interaction-${entity(state, "original").id}`).click(); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts index 00d3b103388..8c8199cf34a 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts @@ -22,6 +22,22 @@ describe("versioned approval and revised publication scope", () => { it(`${d.scenarioId}: changed target revision rejected`, () => { const o = valid(d.scenarioId); (o.state.plan as { latestRevisionId: string }).latestRevisionId = "unreviewed"; expect(d.grade(o).passed).toBe(false); }); it(`${d.scenarioId}: response promise without executed result rejected`, () => { const o = valid(d.scenarioId); o.runs = []; expect(d.grade(o).passed).toBe(false); }); } + it("accepts the approved scope in the actual execution task plan and rejects wrong document, revision or writer bindings", () => { + const d = planReviewIdentityCases[0]!; const o = valid(d.scenarioId); + o.tasks[0]!.title = "Prepare research checklist"; o.tasks[0]!.description = "See the saved execution plan."; + o.documents = [{ id: "execution-plan", issueId: "task", key: "plan", body: "Acorn research launch. Approved source revision reviewed.", latestRevisionId: "handoff" }]; + o.revisions = [{ id: "handoff", documentId: "execution-plan", createdByRunId: "lead-run", createdByAgentId: "ceo" }]; + expect(d.grade(o).passed).toBe(true); + o.documents[0]!.issueId = "unrelated-task"; expect(d.grade(o).passed).toBe(false); o.documents[0]!.issueId = "task"; + o.documents[0]!.body = "Acorn research launch. A different source revision."; expect(d.grade(o).passed).toBe(false); + o.documents[0]!.body = "Acorn research launch. Approved source revision reviewed."; + o.revisions[0]!.createdByRunId = "unrelated-run"; expect(d.grade(o).passed).toBe(false); + o.revisions[0]!.createdByRunId = "lead-run"; o.links = []; expect(d.grade(o).passed).toBe(false); + }); + it("rejects a still-pending publication proposal even when a correct local review draft exists", () => { + const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.interactions[0]!.status = "pending"; + expect(d.grade(o).passed).toBe(false); + }); it("rejects acceptance of a stale revision despite successful current work", () => { const d = planReviewIdentityCases[0]!; const o = valid(d.scenarioId); o.state.staleAttemptStatus = 200; expect(d.grade(o).passed).toBe(false); }); it("rejects reusing approval on the publication after the draft request", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.interactions[0]!.status = "accepted"; expect(d.grade(o).passed).toBe(false); }); it("rejects a published work product while keeping a real local draft", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.state.noPublishedWorkProduct = false; expect(d.grade(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts index a51db4a5f8c..0a28411dc87 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts @@ -24,7 +24,15 @@ export function gradePlanReview(value: unknown) { && o.state.sourcePlanRevisionAfter === plan?.latestRevisionId && typeof o.state.canonicalCommentId === "string" && current.result?.commentId === o.state.canonicalCommentId && o.activity.some(row => row.action === "issue.thread_interaction_accepted" && row.details?.interactionId === original?.id && row.details?.responseCommentId === o.state.canonicalCommentId), "The exact canonical conversational approval is attached to the real reviewed revision and decision audit."); - c.add("reviewed-scope-handed-off", !!saved && `${saved.title} ${saved.description}`.includes(String(o.state.scopeMarker)), "The actual ordinary task retains the substantive scope of the approved plan."); + const scopeInBrief = !!saved && `${saved.title} ${saved.description}`.includes(String(o.state.scopeMarker)); + const scopeInSavedPlan = !!saved && o.documents.some(doc => doc.issueId === saved.id && doc.key === "plan" + && (doc.body ?? "").includes(String(o.state.scopeMarker)) && typeof plan?.latestRevisionId === "string" + && (doc.body ?? "").includes(plan.latestRevisionId) && typeof doc.latestRevisionId === "string" + && o.revisions.some(revision => revision.id === doc.latestRevisionId && revision.documentId === doc.id + && o.runs.some(run => run.id === revision.createdByRunId && run.agentId === revision.createdByAgentId + && (run.agentId === o.leadId || run.agentId === worker?.id + && (run.contextSnapshot?.issueId === saved.id || (run as unknown as { nativeIssueId?: string }).nativeIssueId === saved.id))))); + c.add("reviewed-scope-handed-off", scopeInBrief || scopeInSavedPlan, "The actual ordinary task retains the approved scope in its brief or an attributed saved plan bound to the reviewed source revision."); c.add("stale-review-shows-new-version", o.state.staleAttemptStatus === 409 && o.state.staleErrorCode === "interaction_stale_target" && o.state.newerRevisionVisible === true && o.state.newerRevisionId !== o.state.staleRevisionId && o.state.staleExpired === true, "A separately opened real old revision becomes unapprovable after a public edit; the browser shows the actual newer version instead of approving unseen work.");