From febb47fb9268f617aff4aaff636f2f075a34b0f3 Mon Sep 17 00:00:00 2001
From: Dotta <bippadotta@protonmail.com>
Date: Fri, 6 Nov 2026 05:37:36 -0600
Subject: [PATCH] fix(evals): await current skill review hydration within
 original budget

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 .../integrator-skill-review-visible.test.ts   | 23 +++++++++++++++++--
 .../integrator-skill-update.ts                |  9 +++++++-
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-review-visible.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-review-visible.test.ts
index daa0f6f1484..be87eade060 100644
--- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-review-visible.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-review-visible.test.ts
@@ -1,6 +1,6 @@
 // @vitest-environment jsdom
 import { afterEach, beforeEach, expect, it, vi } from "vitest";
-import { readVisibleSkillReview } from "./integrator-skill-update.js";
+import { readVisibleSkillReview, waitForVisibleSkillReview } from "./integrator-skill-update.js";
 const input = { runId: "run", humanCommentId: "human" };
 const thread = (body: string) => `<section data-testid="task-chat-thread"><div id="comment-human">Review before updating.</div>${body}</section>`;
 const response = '<div data-thread-anchor="run:turn:0"><div data-testid="task-chat-final-response"><strong>Installed 0.4</strong><p>Regression tests; split forms into smaller steps. No update applied yet.</p></div></div>';
@@ -10,7 +10,7 @@ beforeEach(() => {
   Object.defineProperty(HTMLElement.prototype, "innerText", { configurable: true, get() { return this.textContent ?? ""; } });
   document.body.innerHTML = thread(response);
 });
-afterEach(() => vi.restoreAllMocks());
+afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers(); });
 it("reads rendered current completion only, excluding reasoning and tool output", () => {
   document.querySelector('[data-thread-anchor]')!.insertAdjacentHTML("afterbegin", '<div data-testid="task-chat-reasoning">PRIVATE REASONING</div><pre>TOOL STDOUT</pre>');
   expect(readVisibleSkillReview(input)).toContain("Installed 0.4");
@@ -23,3 +23,22 @@ it("rejects hidden, foreign, old and ambiguously anchored conversation surfaces"
   document.body.innerHTML = thread(response).replace('comment-human', 'comment-foreign'); expect(readVisibleSkillReview(input)).toBe("");
   document.body.innerHTML = thread(response) + thread(response); expect(readVisibleSkillReview(input)).toBe("");
 });
+
+it("waits for eventual current response hydration within the original deadline only", async () => {
+  vi.useFakeTimers(); vi.setSystemTime(1_000);
+  document.body.innerHTML = thread("");
+  const evaluate = vi.fn(async () => readVisibleSkillReview(input));
+  const ctx = { observationDeadlineAt: 2_000, input: { page: { evaluate } } } as unknown as Parameters<typeof waitForVisibleSkillReview>[0];
+  const pending = waitForVisibleSkillReview(ctx, input);
+  await vi.advanceTimersByTimeAsync(250);
+  document.body.innerHTML = thread(response);
+  await vi.advanceTimersByTimeAsync(250);
+  expect(await pending).toContain("Installed 0.4");
+  expect(evaluate).toHaveBeenCalledTimes(3);
+  expect(ctx.observationDeadlineAt).toBe(2_000);
+  document.body.innerHTML = thread(response.replace('run:turn:0', 'foreign:turn:0'));
+  const timeout = expect(waitForVisibleSkillReview(ctx, input)).rejects.toThrow("Timed out waiting");
+  await vi.advanceTimersByTimeAsync(500); await timeout;
+  expect(ctx.observationDeadlineAt).toBe(2_000);
+  await expect(waitForVisibleSkillReview(ctx, input)).rejects.toThrow("budget exhausted");
+});
diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts
index f53ef111d31..0a0399a3ef6 100644
--- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts
+++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-update.ts
@@ -49,6 +49,13 @@ export function readVisibleSkillReview(input: { runId: string; humanCommentId: s
   }).flatMap(turn => [...turn.querySelectorAll('[data-testid="task-chat-final-response"]')]
     .filter(node => visible(node)).map(node => (node as HTMLElement).innerText)).join("\n");
 }
+export async function waitForVisibleSkillReview(ctx: ChatCatalogContext, input: { runId: string; humanCommentId: string }) {
+  remainingIdentityObservationBudget(ctx);
+  return pollUntil({ label: "current visible skill-review completion", deadlineAt: ctx.observationDeadlineAt!, intervalMs: 250,
+    load: () => ctx.input.page.evaluate(readVisibleSkillReview, input),
+    accept: text => Date.now() < ctx.observationDeadlineAt! && !!text.trim(),
+  });
+}
 export function skillReviewPreviewText(o: UpdateObservation) {
   const evidence = o.visibleReview, issueId = o.state.sourceIssueId;
   if (!evidence || typeof issueId !== "string") return o.reply;
@@ -80,7 +87,7 @@ async function observeUpdate(ctx: ChatCatalogContext, state: Record<string, unkn
     const run = acceptedSkillReviewRun(base, issueId);
     if (run) {
       const humanCommentId = run.contextSnapshot!.commentId as string;
-      const visibleText = await ctx.input.page.evaluate(readVisibleSkillReview, { runId: run.id, humanCommentId });
+      const visibleText = await waitForVisibleSkillReview(ctx, { runId: run.id, humanCommentId });
       if (visibleText.trim()) visibleReview = { runId: run.id, issueId, humanCommentId, summary: run.resultJson!.summary as string, visibleText };
     }
   }