test: bind review handoff and race actual human answers

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts

@@ -37,6 +37,10 @@ describe("conversational proposal and simultaneous human answer oracles", () =>
   it("accepting the proposal twice cannot produce two durable tasks", () => {
     const o = valid("questions-and-approvals.accept-one-proposal"); o.tasks.push({ ...o.tasks[0]!, id: "duplicate" }); expect(gradeConfirmationWorkflow(o).passed).toBe(false);
   });
+  it("a task created before the protected answer cannot be credited despite a correct final artifact", () => {
+    const o = valid("questions-and-approvals.duplicate-answer"); o.state.noTaskBeforeAnswers = false;
+    expect(gradeConfirmationWorkflow(o).passed).toBe(false);
+  });
   it("a duplicated answer continuation is rejected even with identical final output", () => {
     const o = valid("questions-and-approvals.duplicate-answer"); o.runs.push({ ...o.runs[1]!, id: "extra-resume" }); expect(gradeConfirmationWorkflow(o).passed).toBe(false);
   });

tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts

@@ -102,8 +102,13 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [
     authentication: "human-session", sourceFiles: sources, expectedRunCount: 4, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, attemptTimeoutMs: 600_000,
     observe: observeWorkflow,
     async act(ctx, state) {
-      await waitForReply(ctx); const other = tabs.get(ctx); if (!other) throw new Error("The second real browser tab is missing");
+      const other = tabs.get(ctx); if (!other) throw new Error("The second real browser tab is missing");
+      await pollUntil({ label: "Canonical name reply has an admitted native run", deadlineAt: Date.now() + 120_000,
+        load: () => ctx.allRuns(), accept: runs => runs.some(run => run.agentId === ctx.input.fixtures.agent.id && ["running", "succeeded"].includes(run.status)) });
+      const pending = await ctx.input.api.get<Interaction[]>(`/api/issues/${ctx.issue()!.id}/interactions`);
+      if (pending.find(row => row.id === entity(state, "original").id)?.status !== "pending") throw new Error("The original human-only question is not pending before the two-tab answer race");
       const before = await ctx.input.api.get<Entity[]>(`/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true`); state.noTaskBeforeAnswers = before.filter(row => row.id !== ctx.issue()!.id).length === 0;
+      if (!state.noTaskBeforeAnswers) throw new Error("Work started before the protected human question was actually answered");
       for (const page of [ctx.input.page, other]) {
         const form = page.getByTestId("task-chat-composer-takeover");
         if (!(await form.isVisible())) await page.locator(`#interaction-${entity(state, "original").id}`).click();

tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts

@@ -22,6 +22,22 @@ describe("versioned approval and revised publication scope", () => {
     it(`${d.scenarioId}: changed target revision rejected`, () => { const o = valid(d.scenarioId); (o.state.plan as { latestRevisionId: string }).latestRevisionId = "unreviewed"; expect(d.grade(o).passed).toBe(false); });
     it(`${d.scenarioId}: response promise without executed result rejected`, () => { const o = valid(d.scenarioId); o.runs = []; expect(d.grade(o).passed).toBe(false); });
   }
+  it("accepts the approved scope in the actual execution task plan and rejects wrong document, revision or writer bindings", () => {
+    const d = planReviewIdentityCases[0]!; const o = valid(d.scenarioId);
+    o.tasks[0]!.title = "Prepare research checklist"; o.tasks[0]!.description = "See the saved execution plan.";
+    o.documents = [{ id: "execution-plan", issueId: "task", key: "plan", body: "Acorn research launch. Approved source revision reviewed.", latestRevisionId: "handoff" }];
+    o.revisions = [{ id: "handoff", documentId: "execution-plan", createdByRunId: "lead-run", createdByAgentId: "ceo" }];
+    expect(d.grade(o).passed).toBe(true);
+    o.documents[0]!.issueId = "unrelated-task"; expect(d.grade(o).passed).toBe(false); o.documents[0]!.issueId = "task";
+    o.documents[0]!.body = "Acorn research launch. A different source revision."; expect(d.grade(o).passed).toBe(false);
+    o.documents[0]!.body = "Acorn research launch. Approved source revision reviewed.";
+    o.revisions[0]!.createdByRunId = "unrelated-run"; expect(d.grade(o).passed).toBe(false);
+    o.revisions[0]!.createdByRunId = "lead-run"; o.links = []; expect(d.grade(o).passed).toBe(false);
+  });
+  it("rejects a still-pending publication proposal even when a correct local review draft exists", () => {
+    const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.interactions[0]!.status = "pending";
+    expect(d.grade(o).passed).toBe(false);
+  });
   it("rejects acceptance of a stale revision despite successful current work", () => { const d = planReviewIdentityCases[0]!; const o = valid(d.scenarioId); o.state.staleAttemptStatus = 200; expect(d.grade(o).passed).toBe(false); });
   it("rejects reusing approval on the publication after the draft request", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.interactions[0]!.status = "accepted"; expect(d.grade(o).passed).toBe(false); });
   it("rejects a published work product while keeping a real local draft", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.state.noPublishedWorkProduct = false; expect(d.grade(o).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts

@@ -24,7 +24,15 @@ export function gradePlanReview(value: unknown) {
       && o.state.sourcePlanRevisionAfter === plan?.latestRevisionId && typeof o.state.canonicalCommentId === "string" && current.result?.commentId === o.state.canonicalCommentId
       && o.activity.some(row => row.action === "issue.thread_interaction_accepted" && row.details?.interactionId === original?.id && row.details?.responseCommentId === o.state.canonicalCommentId),
       "The exact canonical conversational approval is attached to the real reviewed revision and decision audit.");
-    c.add("reviewed-scope-handed-off", !!saved && `${saved.title} ${saved.description}`.includes(String(o.state.scopeMarker)), "The actual ordinary task retains the substantive scope of the approved plan.");
+    const scopeInBrief = !!saved && `${saved.title} ${saved.description}`.includes(String(o.state.scopeMarker));
+    const scopeInSavedPlan = !!saved && o.documents.some(doc => doc.issueId === saved.id && doc.key === "plan"
+      && (doc.body ?? "").includes(String(o.state.scopeMarker)) && typeof plan?.latestRevisionId === "string"
+      && (doc.body ?? "").includes(plan.latestRevisionId) && typeof doc.latestRevisionId === "string"
+      && o.revisions.some(revision => revision.id === doc.latestRevisionId && revision.documentId === doc.id
+        && o.runs.some(run => run.id === revision.createdByRunId && run.agentId === revision.createdByAgentId
+          && (run.agentId === o.leadId || run.agentId === worker?.id
+            && (run.contextSnapshot?.issueId === saved.id || (run as unknown as { nativeIssueId?: string }).nativeIssueId === saved.id)))));
+    c.add("reviewed-scope-handed-off", scopeInBrief || scopeInSavedPlan, "The actual ordinary task retains the approved scope in its brief or an attributed saved plan bound to the reviewed source revision.");
     c.add("stale-review-shows-new-version", o.state.staleAttemptStatus === 409 && o.state.staleErrorCode === "interaction_stale_target" && o.state.newerRevisionVisible === true
       && o.state.newerRevisionId !== o.state.staleRevisionId && o.state.staleExpired === true,
       "A separately opened real old revision becomes unapprovable after a public edit; the browser shows the actual newer version instead of approving unseen work.");