From ac8ac79d68b9e331dccab68110ab87572787282e Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 21:48:43 -0500 Subject: [PATCH] test(evals): author actual native same-turn steering journey --- .../runtime-active-steering.test.ts | 32 +++++ .../runtime-active-steering.ts | 132 ++++++++++++++++++ .../runtime-capability-audit.json | 4 +- 3 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts new file mode 100644 index 00000000000..0c1efa73281 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts @@ -0,0 +1,32 @@ +import { expect, it } from "vitest"; +import { gradeActiveSteering, isBlueColor } from "./runtime-active-steering.js"; +function valid() { + return { scenarioId: "steering.steer-active", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [], + runs: [{ id: "worker", status: "succeeded" }, { id: "source", status: "succeeded" }], + state: { task: { id: "website" }, targetId: "blue", humanId: "human", workerRunId: "worker", workerWasReallyWaiting: true, + queueBefore: { issueId: "website", targetRunId: "worker", protocol: "paperclip_runner_v1", steeringDisposition: "available", entries: [{ comment: { id: "blue", body: "Switch to a blue design while you are working.", authorUserId: "human", authorAgentId: null } }] }, + queueAfterAction: { entries: [] as Array<{ comment: { id: string; body: string } }> }, humanSteerSelected: true, agentSteerStatus: 403, + queryReply: "Use Steer beside your saved message to send this direction to Morgan's active turn; waiting in the queue is separate from delivery.", + steerStatus: 200, acknowledgedSameRun: true, steerAuditBound: true, idempotentRetryStatus: 200, ordinaryConsumerCount: 0, + actualBlueArtifact: true, blueRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true, + providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } }; +} +it("accepts real explicit same-turn delivery and actual blue result", () => expect(gradeActiveSteering(valid()).passed).toBe(true)); +it("accepts natural human-control guidance", () => { const o = valid(); o.state.queryReply = "Your direction is saved. Select the Steer action in Morgan's task to apply it during this execution."; expect(gradeActiveSteering(o).passed).toBe(true); }); +it("accepts an actual visible provider rejection with retained one-time normal delivery", () => { + const o = valid(); Object.assign(o.state, { steerStatus: 409, providerFailureVisible: true, failureRetainedMessage: true, supportedContinuationVisible: true, ordinaryConsumerCount: 1, continuationInputBound: true, acknowledgedSameRun: false, steerAuditBound: false }); + o.state.queueAfterAction.entries = [{ comment: { id: "blue", body: "Switch to a blue design while you are working." } }]; o.runs.push({ id: "ordinary", status: "succeeded" }); + expect(gradeActiveSteering(o).passed).toBe(true); + o.state.failureRetainedMessage = false; expect(gradeActiveSteering(o).passed).toBe(false); +}); +it("rejects queue admission without actual same-run acknowledgement", () => { const o = valid(); o.state.acknowledgedSameRun = false; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects hidden normal replay after accepted same-turn delivery", () => { const o = valid(); o.state.ordinaryConsumerCount = 1; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects steering a foreign run", () => { const o = valid(); o.state.queueBefore.targetRunId = "other"; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects falsely claiming the queued direction was already implemented", () => { const o = valid(); o.state.queryReply = "I have implemented the blue design with Steer."; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects a forged human steering action", () => { const o = valid(); o.state.agentSteerStatus = 200; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects saved blue prose without a rendered blue deliverable", () => { const o = valid(); o.state.actualBlueArtifact = false; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects unavailable advertised capability", () => { const o = valid(); o.state.queueBefore.steeringDisposition = "unsupported"; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects wrong human ownership", () => { const o = valid(); o.state.queueBefore.entries[0]!.comment.authorUserId = "other"; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects absent actual evidence", () => expect(gradeActiveSteering(undefined).passed).toBe(false)); +it.each(["rgb(0, 0, 255)", "rgb(42, 90, 180)", "rgba(15, 30, 98, 1)"])("accepts actual blue computed colors %s", color => expect(isBlueColor(color)).toBe(true)); +it.each(["rgb(255, 0, 0)", "rgb(240, 240, 240)", "rgb(100, 150, 100)", "transparent"])("rejects other computed colors %s", color => expect(isBlueColor(color)).toBe(false)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts new file mode 100644 index 00000000000..e0304c18ecd --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts @@ -0,0 +1,132 @@ +import { expect } from "@playwright/test"; +import { writeFile } from "node:fs/promises"; +import type { IssueQueuedCommentQueue } from "../../../packages/shared/src/types/issue.js"; +import { pollUntil } from "../api.js"; +import { submitTaskReply } from "../user-actions.js"; +import { readRegisteredArtifacts } from "../registered-artifact.js"; +import { checks, type Entity } from "./internal-support.js"; +import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; +import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { runtimeQueueCases } from "./runtime-queue.js"; +import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js"; +export function isBlueColor(color: string) { + const parts = color.match(/^rgba?\(\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)/i); + if (!parts) return false; + const [, red, green, blue] = parts.map(Number); + return blue! >= 64 && blue! > red! + 24 && blue! >= green!; +} +const direction = "Switch to a blue design while you are working."; +export function gradeActiveSteering(value: unknown) { + const c = checks(), o = runtimeObservation(value); + c.add("actual-native-steering-context", !!o, "A genuine compatible active native run and saved human message are required."); if (!o) return c.result(); + const s = o.state, before = s.queueBefore as IssueQueuedCommentQueue, after = s.queueAfterAction as IssueQueuedCommentQueue; + const entry = before?.entries.find(row => row.comment.id === s.targetId); + c.add("saved-owned-message-compatible-active-target", !!entry && entry.comment.body === direction && entry.comment.authorUserId === s.humanId && !entry.comment.authorAgentId + && before.entries.length === 1 && before.issueId === (s.task as Entity).id && before.targetRunId === s.workerRunId && before.protocol === "paperclip_runner_v1" + && before.steeringDisposition === "available" && s.workerWasReallyWaiting === true, + "The actual harness advertises steering for the exact saved human-owned message and running task, before the canonical request."); + c.add("explicit-human-control-and-truthful-guidance", s.humanSteerSelected === true && s.agentSteerStatus === 403 + && /steer|steering/i.test(String(s.queryReply)) && !/I(?:'ve| have)? (?:already )?(?:steered|delivered|implemented)|blue (?:design|page) (?:is|has been) (?:ready|complete)/i.test(String(s.queryReply)), + "Chat identifies the supported human Steer control; it cannot forge the action or claim queue admission already changed the active result."); + const accepted = s.steerStatus === 200; + c.add("delivery-distinct-from-queue-with-no-loss", accepted + ? s.acknowledgedSameRun === true && after?.entries.length === 0 && s.steerAuditBound === true && s.idempotentRetryStatus === 200 && s.ordinaryConsumerCount === 0 + : s.steerStatus === 409 && s.providerFailureVisible === true && s.failureRetainedMessage === true && after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true + && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true, + "A durable same-run ACK is required for delivery; an actual provider rejection instead retains the exact message and visible failure until one normal continuation consumes it."); + c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.blueRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true + && o.runs.length === (accepted ? 2 : 3) && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1, + "Actual delivered HTML and its task revision reflect blue under the correct consuming native run, with no duplicate work or hidden run."); + return c.result(); +} +export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-active", async ctx => { + const state: Record = await busyRuntimeWorker(ctx), task = state.task as Entity, source = ctx.input.page.url(); + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); + await submitTaskReply(ctx.input.page, direction); + const before = await pollUntil({ label: "Actual native same-turn steering advertised for saved human blue message", deadlineAt: Date.now() + 30_000, + load: () => ctx.input.api.get(`/api/issues/${task.id}/queued-comments`), + accept: q => q.entries.length === 1 && q.entries[0]?.comment.body === direction && q.targetRunId === state.workerRunId && q.protocol === "paperclip_runner_v1" && q.steeringDisposition === "available" }); + const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); + if (before.entries[0]?.comment.authorUserId !== session.user.id || before.entries[0].comment.authorAgentId) throw new Error("Steering fixture lacks actual human message ownership"); + await expect(ctx.input.page.getByTestId(`task-chat-queued-steer-${before.entries[0].comment.id}`)).toBeEnabled(); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + return { ...state, humanId: session.user.id, queueBefore: before, targetId: before.entries[0].comment.id }; +}, gradeActiveSteering, { + authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, + setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, + sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], + async act(ctx, state) { + await pollUntil({ label: "Canonical steering guidance while the original native owner remains active", deadlineAt: Date.now() + 150_000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running") + && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") }); + state.queryReply = (await observeRuntime(ctx, state)).reply; + const source = ctx.input.page.url(), task = state.task as Entity, endpoint = `/api/issues/${task.id}/queued-comments`, targetId = String(state.targetId); + const before = await ctx.input.api.get(endpoint); + if (before.entries.length !== 1 || before.entries[0]?.comment.id !== targetId || before.targetRunId !== state.workerRunId) throw new Error("Canonical guidance lost or duplicated the saved human direction"); + const mutation = { queueId: before.queueId, revision: before.revision, targetRunId: before.targetRunId }; + state.agentSteerStatus = await probeAgentPermission(ctx, `${endpoint}/${targetId}/steer`, "POST", mutation); + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); + const response = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `${endpoint}/${targetId}/steer`); + await ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`).click(); state.humanSteerSelected = true; + const result = await response; state.steerStatus = result.status(); + const after = await ctx.input.api.get(endpoint); state.queueAfterAction = after; + if (result.status() === 200) { + const run = await ctx.input.api.get<{ resultJson?: { queuedSteeringAcknowledgements?: Record } }>(`/api/heartbeat-runs/${state.workerRunId}`); + const ack = run.resultJson?.queuedSteeringAcknowledgements?.[targetId]; + state.acknowledgedSameRun = ack?.status === "acknowledged" && ack.queueId === before.queueId && typeof ack.turnId === "string" && ack.turnId.length > 0; + const retry = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.idempotentRetryStatus = retry.status(); await retry.dispose(); + } else { + if (result.status() !== 409) throw new Error("Steering failed outside its supported visible rejection contract"); + const body = await result.json() as { code?: string; details?: { code?: string } }; + if (!/steering/.test(body.code ?? body.details?.code ?? "")) throw new Error("A non-provider queue conflict does not qualify as a provider steering rejection"); + await expect(ctx.input.page.getByText(/steer|temporarily unavailable|timed out/i).filter({ hasNot: ctx.input.page.locator("button") }).first()).toBeVisible(); + state.providerFailureVisible = true; state.failureRetainedMessage = after.entries.some(row => row.comment.id === targetId && row.comment.body === direction); + await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible(); + await expect(ctx.input.page.getByTestId("task-chat-composer-input").last()).toBeVisible(); state.supportedContinuationVisible = true; + } + await ctx.input.capture("actual-steering-delivery", "Selected human steering action and actual native delivery or retained rejection", "runtime-steer-active.png"); + await writeFile((state.gate as { gate: string }).gate, "The human brief is ready. Implement a small semantic website.html using the actual design direction you received, with a visible h1 and a labelled signup action. Register the real website.html file as a downloadable artifact work product. Save the actual chosen color and design in a non-plan website-delivery task document, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.", "utf8"); + const finished = await pollUntil({ label: "Actual selected steering or preserved normal continuation produces its website", deadlineAt: Date.now() + 240_000, + load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), + accept: v => v.task.status === "done" && v.queue.entries.length === 0 && v.runs.length === (state.steerStatus === 200 ? 2 : 3) && v.runs.every(row => row.status === "succeeded"), + reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Steered work failed or admitted an undeclared execution" : undefined }); + const successors = finished.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); + state.ordinaryConsumerCount = successors.length; state.outputRunId = state.steerStatus === 200 ? state.workerRunId : successors[0]?.id; + if (successors.length === 1) { + const full = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${successors[0]!.id}`); + state.continuationInputBound = full.contextSnapshot?.wakeCommentIds?.filter(id => id === targetId).length === 1 && full.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(direction) === true; + } + state.workerTaskDone = finished.task.status === "done"; state.queueEmptyAtEnd = finished.queue.entries.length === 0; + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async observe(ctx, state) { + const o = await observeRuntime(ctx, state), task = state.task as Entity; + state.steerAuditBound = o.activity.some(row => row.action === "issue.queued_comment_steered" && row.entityId === task.id && row.actorId === state.humanId + && row.details?.commentId === state.targetId && row.details?.targetRunId === state.workerRunId && row.details?.duplicate === false); + if (state.outputRunId) { + const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, String(state.outputRunId), ["website.html"]); + if (artifact?.contentVerified) { + const browser = ctx.input.page.context().browser(); if (!browser) throw new Error("Actual delivered color inspection requires the existing browser"); + const preview = await browser.newContext({ javaScriptEnabled: false }); + try { + const page = await preview.newPage(); await page.route("**/*", route => route.abort()); + await page.setContent(artifact.content, { waitUntil: "domcontentloaded" }); + const colors = await page.locator("body, body *").evaluateAll(nodes => nodes.filter(node => { + const box = node.getBoundingClientRect(), style = getComputedStyle(node); + return box.width > 0 && box.height > 0 && style.visibility !== "hidden" && style.display !== "none"; + }).flatMap(node => { const style = getComputedStyle(node); return [style.color, style.backgroundColor, style.borderColor]; })); + state.actualBlueArtifact = colors.some(isBlueColor); + } finally { await preview.close(); } + } + state.blueRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id && /blue/i.test(row.body ?? "") + && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); + } + return o; + }, + async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Entity; + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("heading", { name: task.title!, exact: true }).first()).toBeVisible(); + await expect(ctx.input.page.getByTestId("task-chat-queued-messages")).toHaveCount(0); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, + async cleanup(ctx, state) { try { await cleanupIdentityProbes(ctx); } finally { await releaseRuntimeWorker(state); } }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json index c0b6a0ff188..cd9a0f5f1de 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json @@ -221,8 +221,8 @@ }, { "scenarioId": "steering.steer-active", - "readinessClass": "local", - "needs": "Real compatible active harness and explicit Steer; delivered acknowledgement and actual output; provider rejection continuation branch." + "readinessClass": "implemented", + "needs": "Actual running native owner and saved human message with real advertised steering; canonical human-control guidance and ordinary-agent403, actual Steer click, independently retained same-run ACK/audit and idempotent replay, or actual provider rejection retaining visible exact message for one normal continuation. Actual registered HTML is rendered offline with network and JavaScript disabled and computed visible blue colors plus latest native-attributed design revision. Both outcome branches authored; neither branch live qualified." }, { "scenarioId": "steering.interrupt-with-direction",