From bcd7ca5e77c18e031ba289585ba406db1fe0625e Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 21:52:31 -0500 Subject: [PATCH] test(evals): distinguish substantive blue steering and observed branch --- .../runtime-active-steering.test.ts | 15 ++++++++---- .../runtime-active-steering.ts | 24 +++++++++++++------ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts index 0c1efa73281..2deb1d60fa2 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeActiveSteering, isBlueColor } from "./runtime-active-steering.js"; +import { gradeActiveSteering, isBlueColor, hasVisibleBlueDesign } from "./runtime-active-steering.js"; function valid() { return { scenarioId: "steering.steer-active", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [], runs: [{ id: "worker", status: "succeeded" }, { id: "source", status: "succeeded" }], @@ -7,14 +7,14 @@ function valid() { queueBefore: { issueId: "website", targetRunId: "worker", protocol: "paperclip_runner_v1", steeringDisposition: "available", entries: [{ comment: { id: "blue", body: "Switch to a blue design while you are working.", authorUserId: "human", authorAgentId: null } }] }, queueAfterAction: { entries: [] as Array<{ comment: { id: string; body: string } }> }, humanSteerSelected: true, agentSteerStatus: 403, queryReply: "Use Steer beside your saved message to send this direction to Morgan's active turn; waiting in the queue is separate from delivery.", - steerStatus: 200, acknowledgedSameRun: true, steerAuditBound: true, idempotentRetryStatus: 200, ordinaryConsumerCount: 0, + steerObservedBranch: "same-run-acknowledged", steerStatus: 200, acknowledgedSameRun: true, steerAuditBound: true, idempotentRetryStatus: 200, ordinaryConsumerCount: 0, actualBlueArtifact: true, blueRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true, - providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } }; + steerFailureCode: "", providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } }; } it("accepts real explicit same-turn delivery and actual blue result", () => expect(gradeActiveSteering(valid()).passed).toBe(true)); it("accepts natural human-control guidance", () => { const o = valid(); o.state.queryReply = "Your direction is saved. Select the Steer action in Morgan's task to apply it during this execution."; expect(gradeActiveSteering(o).passed).toBe(true); }); it("accepts an actual visible provider rejection with retained one-time normal delivery", () => { - const o = valid(); Object.assign(o.state, { steerStatus: 409, providerFailureVisible: true, failureRetainedMessage: true, supportedContinuationVisible: true, ordinaryConsumerCount: 1, continuationInputBound: true, acknowledgedSameRun: false, steerAuditBound: false }); + const o = valid(); Object.assign(o.state, { steerObservedBranch: "provider-rejected-normal-continuation", steerStatus: 409, steerFailureCode: "steering_rejected", providerFailureVisible: true, failureRetainedMessage: true, supportedContinuationVisible: true, ordinaryConsumerCount: 1, continuationInputBound: true, acknowledgedSameRun: false, steerAuditBound: false }); o.state.queueAfterAction.entries = [{ comment: { id: "blue", body: "Switch to a blue design while you are working." } }]; o.runs.push({ id: "ordinary", status: "succeeded" }); expect(gradeActiveSteering(o).passed).toBe(true); o.state.failureRetainedMessage = false; expect(gradeActiveSteering(o).passed).toBe(false); @@ -30,3 +30,10 @@ it("rejects wrong human ownership", () => { const o = valid(); o.state.queueBefo it("rejects absent actual evidence", () => expect(gradeActiveSteering(undefined).passed).toBe(false)); it.each(["rgb(0, 0, 255)", "rgb(42, 90, 180)", "rgba(15, 30, 98, 1)"])("accepts actual blue computed colors %s", color => expect(isBlueColor(color)).toBe(true)); it.each(["rgb(255, 0, 0)", "rgb(240, 240, 240)", "rgb(100, 150, 100)", "transparent"])("rejects other computed colors %s", color => expect(isBlueColor(color)).toBe(false)); + +it.each(["rgba(0, 0, 255, 0)", "rgba(0, 0, 255, 0.01)"])("rejects transparent blue computed colors %s", color => expect(isBlueColor(color)).toBe(false)); +it("rejects claiming the unobserved branch as qualified", () => { const o = valid(); o.state.steerObservedBranch = "provider-rejected-normal-continuation"; expect(gradeActiveSteering(o).passed).toBe(false); }); + +it("rejects a red page with only default blue links", () => expect(hasVisibleBlueDesign([{ tag: "BODY", color: "rgb(0, 0, 0)", background: "rgb(255, 0, 0)" }, { tag: "A", color: "rgb(0, 0, 238)", background: "rgba(0, 0, 0, 0)" }])).toBe(false)); +it("accepts a natural blue heading accent on a white page", () => expect(hasVisibleBlueDesign([{ tag: "BODY", color: "rgb(0, 0, 0)", background: "rgb(255, 255, 255)" }, { tag: "H1", color: "rgb(25, 70, 180)", background: "rgba(0, 0, 0, 0)" }])).toBe(true)); +it("rejects transparent blue design elements", () => expect(hasVisibleBlueDesign([{ tag: "SECTION", color: "rgb(0, 0, 0)", background: "rgba(0, 0, 255, 0)" }])).toBe(false)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts index e0304c18ecd..f9a4e9e218a 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts @@ -13,8 +13,14 @@ export function isBlueColor(color: string) { const parts = color.match(/^rgba?\(\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)/i); if (!parts) return false; const [, red, green, blue] = parts.map(Number); + const alpha = color.match(/^rgba?\([^)]*[,/]\s*(\d*(?:\.\d+)?)\s*\)$/i); + if (/^rgba\(/i.test(color) && (!alpha || Number(alpha[1]) < 0.25)) return false; return blue! >= 64 && blue! > red! + 24 && blue! >= green!; } +export function hasVisibleBlueDesign(elements: Array<{ tag: string; role?: string | null; color: string; background: string }>) { + return elements.some(element => ["BODY", "MAIN", "SECTION", "HEADER", "H1", "H2", "H3", "BUTTON"].includes(element.tag) || element.role === "button" + ? isBlueColor(element.background) || element.tag !== "BODY" && isBlueColor(element.color) : false); +} const direction = "Switch to a blue design while you are working."; export function gradeActiveSteering(value: unknown) { const c = checks(), o = runtimeObservation(value); @@ -29,10 +35,10 @@ export function gradeActiveSteering(value: unknown) { && /steer|steering/i.test(String(s.queryReply)) && !/I(?:'ve| have)? (?:already )?(?:steered|delivered|implemented)|blue (?:design|page) (?:is|has been) (?:ready|complete)/i.test(String(s.queryReply)), "Chat identifies the supported human Steer control; it cannot forge the action or claim queue admission already changed the active result."); const accepted = s.steerStatus === 200; - c.add("delivery-distinct-from-queue-with-no-loss", accepted + c.add("delivery-distinct-from-queue-with-no-loss", s.steerObservedBranch === (accepted ? "same-run-acknowledged" : "provider-rejected-normal-continuation") && (accepted ? s.acknowledgedSameRun === true && after?.entries.length === 0 && s.steerAuditBound === true && s.idempotentRetryStatus === 200 && s.ordinaryConsumerCount === 0 - : s.steerStatus === 409 && s.providerFailureVisible === true && s.failureRetainedMessage === true && after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true - && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true, + : ["steering_rejected", "steering_unsupported"].includes(String(s.steerFailureCode)) && s.steerStatus === 409 && s.providerFailureVisible === true && s.failureRetainedMessage === true && after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true + && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true), "A durable same-run ACK is required for delivery; an actual provider rejection instead retains the exact message and visible failure until one normal continuation consumes it."); c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.blueRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true && o.runs.length === (accepted ? 2 : 3) && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1, @@ -72,19 +78,23 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act const result = await response; state.steerStatus = result.status(); const after = await ctx.input.api.get(endpoint); state.queueAfterAction = after; if (result.status() === 200) { + state.steerObservedBranch = "same-run-acknowledged"; const run = await ctx.input.api.get<{ resultJson?: { queuedSteeringAcknowledgements?: Record } }>(`/api/heartbeat-runs/${state.workerRunId}`); const ack = run.resultJson?.queuedSteeringAcknowledgements?.[targetId]; state.acknowledgedSameRun = ack?.status === "acknowledged" && ack.queueId === before.queueId && typeof ack.turnId === "string" && ack.turnId.length > 0; const retry = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.idempotentRetryStatus = retry.status(); await retry.dispose(); } else { if (result.status() !== 409) throw new Error("Steering failed outside its supported visible rejection contract"); + state.steerObservedBranch = "provider-rejected-normal-continuation"; const body = await result.json() as { code?: string; details?: { code?: string } }; - if (!/steering/.test(body.code ?? body.details?.code ?? "")) throw new Error("A non-provider queue conflict does not qualify as a provider steering rejection"); + state.steerFailureCode = body.code ?? body.details?.code; + if (!["steering_rejected", "steering_unsupported"].includes(String(state.steerFailureCode))) throw new Error("A non-provider queue conflict does not qualify as a provider steering rejection"); await expect(ctx.input.page.getByText(/steer|temporarily unavailable|timed out/i).filter({ hasNot: ctx.input.page.locator("button") }).first()).toBeVisible(); state.providerFailureVisible = true; state.failureRetainedMessage = after.entries.some(row => row.comment.id === targetId && row.comment.body === direction); await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible(); await expect(ctx.input.page.getByTestId("task-chat-composer-input").last()).toBeVisible(); state.supportedContinuationVisible = true; } + await ctx.input.evidence("runtime-steering-observed-branch.json", { branch: state.steerObservedBranch, failureCode: state.steerFailureCode ?? null, status: state.steerStatus, issueId: task.id, runId: state.workerRunId, commentId: targetId, otherBranchLiveQualified: false }); await ctx.input.capture("actual-steering-delivery", "Selected human steering action and actual native delivery or retained rejection", "runtime-steer-active.png"); await writeFile((state.gate as { gate: string }).gate, "The human brief is ready. Implement a small semantic website.html using the actual design direction you received, with a visible h1 and a labelled signup action. Register the real website.html file as a downloadable artifact work product. Save the actual chosen color and design in a non-plan website-delivery task document, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.", "utf8"); const finished = await pollUntil({ label: "Actual selected steering or preserved normal continuation produces its website", deadlineAt: Date.now() + 240_000, @@ -112,11 +122,11 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act try { const page = await preview.newPage(); await page.route("**/*", route => route.abort()); await page.setContent(artifact.content, { waitUntil: "domcontentloaded" }); - const colors = await page.locator("body, body *").evaluateAll(nodes => nodes.filter(node => { + const colors = await page.locator("body, main, section, header, h1, h2, h3, button, [role=button]").evaluateAll(nodes => nodes.filter(node => { const box = node.getBoundingClientRect(), style = getComputedStyle(node); return box.width > 0 && box.height > 0 && style.visibility !== "hidden" && style.display !== "none"; - }).flatMap(node => { const style = getComputedStyle(node); return [style.color, style.backgroundColor, style.borderColor]; })); - state.actualBlueArtifact = colors.some(isBlueColor); + }).map(node => { const style = getComputedStyle(node); return { tag: node.tagName, role: node.getAttribute("role"), color: style.color, background: style.backgroundColor }; })); + state.actualBlueArtifact = hasVisibleBlueDesign(colors); } finally { await preview.close(); } } state.blueRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id && /blue/i.test(row.body ?? "")