From 26e77362c586a363e4bb0423d2ec961ec2b59269 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 11 Nov 2026 12:23:18 -0600 Subject: [PATCH] fix(evals): grade steering.steer-active honestly on runtimes that cannot steer The OpenCode harness reports steering: false, so the queue for its running task says steeringDisposition "unsupported" and the page offers Interrupt instead of Steer. The fixture waited only for "available" and timed out. Accept either definitive capability. When steering is available, the same-turn ACK branch is graded as before. When it is unsupported, grade the honest fallback the scenario names: no usable Steer control, a visible continuation control, a refused steer attempt (409 steering_unsupported) that records no steering and keeps the exact message queued, and one normal continuation that consumes it and renders the blue page. Drop the reply keyword checks and the /blue/ match on the delivery document; the rendered page proves the colour. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../runtime-active-steering.test.ts | 45 ++++++- .../runtime-active-steering.ts | 123 ++++++++++++------ 3 files changed, 121 insertions(+), 49 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index b49ec36fcc9..925e40d2779 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5612,7 +5612,7 @@ "registered-artifact.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime.steer-active.v3", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts index 2deb1d60fa2..370a2a28894 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts @@ -1,18 +1,18 @@ import { expect, it } from "vitest"; -import { gradeActiveSteering, isBlueColor, hasVisibleBlueDesign } from "./runtime-active-steering.js"; +import { gradeActiveSteering, isBlueColor, hasVisibleBlueDesign, steeringBranch } from "./runtime-active-steering.js"; function valid() { return { scenarioId: "steering.steer-active", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [], runs: [{ id: "worker", status: "succeeded" }, { id: "source", status: "succeeded" }], state: { task: { id: "website" }, targetId: "blue", humanId: "human", workerRunId: "worker", workerWasReallyWaiting: true, queueBefore: { issueId: "website", targetRunId: "worker", protocol: "paperclip_runner_v1", steeringDisposition: "available", entries: [{ comment: { id: "blue", body: "Switch to a blue design while you are working.", authorUserId: "human", authorAgentId: null } }] }, - queueAfterAction: { entries: [] as Array<{ comment: { id: string; body: string } }> }, humanSteerSelected: true, agentSteerStatus: 403, + queueAfterAction: { entries: [] as Array<{ comment: { id: string; body: string } }> }, humanSteerSelected: true, humanSteerAttempted: false, agentSteerStatus: 403, + steerControlEnabled: true, continuationControlVisible: false, unsupportedStateRetained: false, steeredActivityCount: 1, queryReply: "Use Steer beside your saved message to send this direction to Morgan's active turn; waiting in the queue is separate from delivery.", steerObservedBranch: "same-run-acknowledged", steerStatus: 200, acknowledgedSameRun: true, steerAuditBound: true, idempotentRetryStatus: 200, ordinaryConsumerCount: 0, - actualBlueArtifact: true, blueRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true, - steerFailureCode: "", providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } }; + actualBlueArtifact: true, deliveryRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true, + steerFailureCode: "" as string | undefined, providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } }; } it("accepts real explicit same-turn delivery and actual blue result", () => expect(gradeActiveSteering(valid()).passed).toBe(true)); -it("accepts natural human-control guidance", () => { const o = valid(); o.state.queryReply = "Your direction is saved. Select the Steer action in Morgan's task to apply it during this execution."; expect(gradeActiveSteering(o).passed).toBe(true); }); it("accepts an actual visible provider rejection with retained one-time normal delivery", () => { const o = valid(); Object.assign(o.state, { steerObservedBranch: "provider-rejected-normal-continuation", steerStatus: 409, steerFailureCode: "steering_rejected", providerFailureVisible: true, failureRetainedMessage: true, supportedContinuationVisible: true, ordinaryConsumerCount: 1, continuationInputBound: true, acknowledgedSameRun: false, steerAuditBound: false }); o.state.queueAfterAction.entries = [{ comment: { id: "blue", body: "Switch to a blue design while you are working." } }]; o.runs.push({ id: "ordinary", status: "succeeded" }); @@ -22,10 +22,8 @@ it("accepts an actual visible provider rejection with retained one-time normal d it("rejects queue admission without actual same-run acknowledgement", () => { const o = valid(); o.state.acknowledgedSameRun = false; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects hidden normal replay after accepted same-turn delivery", () => { const o = valid(); o.state.ordinaryConsumerCount = 1; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects steering a foreign run", () => { const o = valid(); o.state.queueBefore.targetRunId = "other"; expect(gradeActiveSteering(o).passed).toBe(false); }); -it("rejects falsely claiming the queued direction was already implemented", () => { const o = valid(); o.state.queryReply = "I have implemented the blue design with Steer."; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects a forged human steering action", () => { const o = valid(); o.state.agentSteerStatus = 200; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects saved blue prose without a rendered blue deliverable", () => { const o = valid(); o.state.actualBlueArtifact = false; expect(gradeActiveSteering(o).passed).toBe(false); }); -it("rejects unavailable advertised capability", () => { const o = valid(); o.state.queueBefore.steeringDisposition = "unsupported"; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects wrong human ownership", () => { const o = valid(); o.state.queueBefore.entries[0]!.comment.authorUserId = "other"; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects absent actual evidence", () => expect(gradeActiveSteering(undefined).passed).toBe(false)); it.each(["rgb(0, 0, 255)", "rgb(42, 90, 180)", "rgba(15, 30, 98, 1)"])("accepts actual blue computed colors %s", color => expect(isBlueColor(color)).toBe(true)); @@ -37,3 +35,36 @@ it("rejects claiming the unobserved branch as qualified", () => { const o = vali it("rejects a red page with only default blue links", () => expect(hasVisibleBlueDesign([{ tag: "BODY", color: "rgb(0, 0, 0)", background: "rgb(255, 0, 0)" }, { tag: "A", color: "rgb(0, 0, 238)", background: "rgba(0, 0, 0, 0)" }])).toBe(false)); it("accepts a natural blue heading accent on a white page", () => expect(hasVisibleBlueDesign([{ tag: "BODY", color: "rgb(0, 0, 0)", background: "rgb(255, 255, 255)" }, { tag: "H1", color: "rgb(25, 70, 180)", background: "rgba(0, 0, 0, 0)" }])).toBe(true)); it("rejects transparent blue design elements", () => expect(hasVisibleBlueDesign([{ tag: "SECTION", color: "rgb(0, 0, 0)", background: "rgba(0, 0, 255, 0)" }])).toBe(false)); + +/** A runtime that cannot steer (the OpenCode harness advertises steering: false): the message stays visibly queued and one normal turn consumes it. */ +function unsupported() { + const o = valid(); + Object.assign(o.state, { steerObservedBranch: "unsupported-queued-normal-continuation", steerStatus: 409, steerFailureCode: "steering_unsupported", humanSteerSelected: false, humanSteerAttempted: true, + steerControlEnabled: false, continuationControlVisible: true, unsupportedStateRetained: true, steeredActivityCount: 0, acknowledgedSameRun: false, steerAuditBound: false, idempotentRetryStatus: 0, + ordinaryConsumerCount: 1, continuationInputBound: true }); + o.state.queueBefore.steeringDisposition = "unsupported"; + o.state.queueAfterAction.entries = [{ comment: { id: "blue", body: "Switch to a blue design while you are working." } }]; o.runs.push({ id: "ordinary", status: "succeeded" }); + return o; +} +it("accepts an honestly unsupported runtime that keeps the message queued for one normal continuation", () => expect(gradeActiveSteering(unsupported()).passed).toBe(true)); +it.each(["Your direction is saved. Select the Steer action in Morgan's task to apply it during this execution.", "I have implemented the blue design with Steer.", "Queued for Morgan's next turn."])( + "verdict does not depend on reply wording: %s", reply => { for (const o of [valid(), unsupported()]) { o.state.queryReply = reply; expect(gradeActiveSteering(o).passed).toBe(true); } }); +it.each([ + ["claims steering was recorded", (o: ReturnType) => { o.state.steeredActivityCount = 1; }], + ["offers an enabled Steer control it cannot honour", (o: ReturnType) => { o.state.steerControlEnabled = true; }], + ["hides the supported continuation control", (o: ReturnType) => { o.state.continuationControlVisible = false; }], + ["drops the queued message after the refused attempt", (o: ReturnType) => { o.state.queueAfterAction.entries = []; }], + ["stops reporting the capability after the attempt", (o: ReturnType) => { o.state.unsupportedStateRetained = false; }], + ["accepts the steer request without a runtime that can steer", (o: ReturnType) => { o.state.steerStatus = 200; }], + ["refuses with an unrelated conflict", (o: ReturnType) => { o.state.steerFailureCode = "queued_comment_stale_queue"; }], + ["never delivers the message", (o: ReturnType) => { o.state.ordinaryConsumerCount = 0; o.runs.pop(); }], + ["delivers it twice", (o: ReturnType) => { o.state.ordinaryConsumerCount = 2; o.runs.push({ id: "again", status: "succeeded" }); }], + ["continuation input lacks the message", (o: ReturnType) => { o.state.continuationInputBound = false; }], + ["lets the agent steer for the person", (o: ReturnType) => { o.state.agentSteerStatus = 200; }], + ["produces no rendered blue page", (o: ReturnType) => { o.state.actualBlueArtifact = false; }], +])("rejects an unsupported runtime that %s", (_label, mutate) => { const o = unsupported(); mutate(o); expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects a queue that never states a definitive steering capability", () => { const o = valid(); o.state.queueBefore.steeringDisposition = "temporarily_unavailable"; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects an advertised Steer control that the page does not enable", () => { const o = valid(); o.state.steerControlEnabled = false; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("accepts a delivery document that names the colour another way, since the rendered page proves it", () => { const o = valid(); o.state.deliveryRevisionBound = true; expect(gradeActiveSteering(o).passed).toBe(true); o.state.deliveryRevisionBound = false; expect(gradeActiveSteering(o).passed).toBe(false); }); +it.each([["available", 200, "same-run-acknowledged"], ["available", 409, "provider-rejected-normal-continuation"], ["unsupported", 409, "unsupported-queued-normal-continuation"], ["temporarily_unavailable", 409, "not-advertised"]] as const)( + "maps %s capability and status %s to its branch", (disposition, status, branch) => expect(steeringBranch(disposition, status)).toBe(branch)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts index ce890ad7084..b5980a43e92 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts @@ -10,6 +10,7 @@ import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.j import { renderRuntimeArtifact } from "./runtime-rendered-artifact.js"; import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; export function isBlueColor(color: string) { const parts = color.match(/^rgba?\(\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)/i); if (!parts) return false; @@ -23,43 +24,64 @@ export function hasVisibleBlueDesign(elements: Array<{ tag: string; role?: strin ? isBlueColor(element.background) || element.tag !== "BODY" && isBlueColor(element.color) : false); } const direction = "Switch to a blue design while you are working."; +/** Branch a run must take, decided by what the harness actually advertised and how the product answered. */ +export function steeringBranch(disposition: unknown, steerStatus: unknown) { + return disposition === "unsupported" ? "unsupported-queued-normal-continuation" : disposition !== "available" ? "not-advertised" + : steerStatus === 200 ? "same-run-acknowledged" : "provider-rejected-normal-continuation"; +} export function gradeActiveSteering(value: unknown) { const c = checks(), o = runtimeObservation(value); - c.add("actual-native-steering-context", !!o, "A genuine compatible active native run and saved human message are required."); if (!o) return c.result(); + c.add("actual-native-steering-context", !!o, "A genuine active native run and saved human message are required."); if (!o) return c.result(); const s = o.state, before = s.queueBefore as IssueQueuedCommentQueue, after = s.queueAfterAction as IssueQueuedCommentQueue; - const entry = before?.entries.find(row => row.comment.id === s.targetId); - c.add("saved-owned-message-compatible-active-target", !!entry && entry.comment.body === direction && entry.comment.authorUserId === s.humanId && !entry.comment.authorAgentId + const entry = before?.entries.find(row => row.comment.id === s.targetId), unsupported = before?.steeringDisposition === "unsupported"; + c.add("saved-owned-message-honest-active-target", !!entry && entry.comment.body === direction && entry.comment.authorUserId === s.humanId && !entry.comment.authorAgentId && before.entries.length === 1 && before.issueId === (s.task as Entity).id && before.targetRunId === s.workerRunId && before.protocol === "paperclip_runner_v1" - && before.steeringDisposition === "available" && s.workerWasReallyWaiting === true, - "The actual harness advertises steering for the exact saved human-owned message and running task, before the canonical request."); - c.add("explicit-human-control-and-truthful-guidance", s.humanSteerSelected === true && s.agentSteerStatus === 403 - && /steer|steering/i.test(String(s.queryReply)) && !/I(?:'ve| have)? (?:already )?(?:steered|delivered|implemented)|blue (?:design|page) (?:is|has been) (?:ready|complete)/i.test(String(s.queryReply)), - "Chat identifies the supported human Steer control; it cannot forge the action or claim queue admission already changed the active result."); - const accepted = s.steerStatus === 200; - c.add("delivery-distinct-from-queue-with-no-loss", s.steerObservedBranch === (accepted ? "same-run-acknowledged" : "provider-rejected-normal-continuation") && (accepted + && ["available", "unsupported"].includes(before.steeringDisposition) && s.workerWasReallyWaiting === true + && (unsupported ? s.steerControlEnabled === false && s.continuationControlVisible === true : s.steerControlEnabled === true), + "The running task's queue states the harness's real steering capability for the exact saved human message: an enabled Steer control when it can steer, otherwise no usable Steer control and a visible supported continuation control."); + const branch = steeringBranch(before?.steeringDisposition, s.steerStatus), accepted = branch === "same-run-acknowledged"; + c.add("human-only-control-without-forged-delivery", s.agentSteerStatus === 403 && (unsupported ? s.humanSteerAttempted === true : s.humanSteerSelected === true), + "Only the person's own action can steer the message; the agent's attempt is refused."); + const retained = after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true; + c.add("delivery-distinct-from-queue-with-no-loss", s.steerObservedBranch === branch && (accepted ? s.acknowledgedSameRun === true && after?.entries.length === 0 && s.steerAuditBound === true && s.idempotentRetryStatus === 200 && s.ordinaryConsumerCount === 0 - : ["steering_rejected", "steering_unsupported"].includes(String(s.steerFailureCode)) && s.steerStatus === 409 && s.providerFailureVisible === true && s.failureRetainedMessage === true && after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true - && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true), - "A durable same-run ACK is required for delivery; an actual provider rejection instead retains the exact message and visible failure until one normal continuation consumes it."); - c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.blueRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true + : branch === "unsupported-queued-normal-continuation" + ? s.steerStatus === 409 && s.steerFailureCode === "steering_unsupported" && s.steeredActivityCount === 0 && retained && s.unsupportedStateRetained === true + && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true + : branch === "provider-rejected-normal-continuation" && ["steering_rejected", "steering_unsupported"].includes(String(s.steerFailureCode)) && s.steerStatus === 409 && s.providerFailureVisible === true + && s.failureRetainedMessage === true && retained && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true), + "A durable same-run ACK is required for delivery; a runtime that cannot steer (or a provider rejection) instead keeps the exact message visibly queued, records no steering, and one normal continuation consumes it."); + c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.deliveryRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true && o.runs.length === (accepted ? 2 : 3) && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1, - "Actual delivered HTML and its task revision reflect blue under the correct consuming native run, with no duplicate work or hidden run."); + "Actual delivered HTML renders blue, and the consuming native run saved its task delivery document, with no duplicate work or hidden run."); return c.result(); } +/** What the person can actually do with the queued message: an enabled Steer control, or the supported continuation (Interrupt) control. */ +async function queuedSteeringControls(ctx: ChatCatalogContext, targetId: string, disposition: string) { + const steer = ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`), next = ctx.input.page.getByTestId(`task-chat-queued-interrupt-${targetId}`); + // The page reads the same queue the API returned; wait for the control that capability implies, then record both. + await expect(disposition === "available" ? steer : next).toBeEnabled(); + return { steerControlEnabled: await steer.count() > 0 && await steer.first().isEnabled(), continuationControlVisible: await next.count() > 0 && await next.first().isEnabled() }; +} export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-active", async ctx => { const state: Record = await busyRuntimeWorker(ctx), task = state.task as Entity, source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await submitTaskReply(ctx.input.page, direction); - const before = await pollUntil({ label: "Actual native same-turn steering advertised for saved human blue message", deadlineAt: Date.now() + 30_000, + const before = await pollUntil({ label: "Running task queue states its harness steering capability for the saved human blue message", deadlineAt: Date.now() + 30_000, load: () => ctx.input.api.get(`/api/issues/${task.id}/queued-comments`), - accept: q => q.entries.length === 1 && q.entries[0]?.comment.body === direction && q.targetRunId === state.workerRunId && q.protocol === "paperclip_runner_v1" && q.steeringDisposition === "available" }); + accept: q => q.entries.length === 1 && q.entries[0]?.comment.body === direction && q.targetRunId === state.workerRunId && q.protocol === "paperclip_runner_v1" + && ["available", "unsupported"].includes(q.steeringDisposition) }); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); if (before.entries[0]?.comment.authorUserId !== session.user.id || before.entries[0].comment.authorAgentId) throw new Error("Steering fixture lacks actual human message ownership"); - await expect(ctx.input.page.getByTestId(`task-chat-queued-steer-${before.entries[0].comment.id}`)).toBeEnabled(); + const targetId = before.entries[0].comment.id; + await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible(); + const controls = await queuedSteeringControls(ctx, targetId, before.steeringDisposition); + if (before.steeringDisposition === "available" ? !controls.steerControlEnabled : controls.steerControlEnabled || !controls.continuationControlVisible) + throw new Error("The queued message controls contradict the advertised steering capability"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); - return { ...state, humanId: session.user.id, queueBefore: before, targetId: before.entries[0].comment.id }; + return { ...state, humanId: session.user.id, queueBefore: before, targetId, ...controls }; }, gradeActiveSteering, { - authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, + authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.steer-active.v3", setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { @@ -74,30 +96,47 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act const mutation = { queueId: before.queueId, revision: before.revision, targetRunId: before.targetRunId }; state.agentSteerStatus = await probeAgentPermission(ctx, `${endpoint}/${targetId}/steer`, "POST", mutation); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); - const response = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `${endpoint}/${targetId}/steer`); - await ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`).click(); state.humanSteerSelected = true; - const result = await response; state.steerStatus = result.status(); - const after = await ctx.input.api.get(endpoint); state.queueAfterAction = after; - if (result.status() === 200) { - state.steerObservedBranch = "same-run-acknowledged"; - const run = await ctx.input.api.get<{ resultJson?: { queuedSteeringAcknowledgements?: Record } }>(`/api/heartbeat-runs/${state.workerRunId}`); - const ack = run.resultJson?.queuedSteeringAcknowledgements?.[targetId]; - state.acknowledgedSameRun = ack?.status === "acknowledged" && ack.queueId === before.queueId && typeof ack.turnId === "string" && ack.turnId.length > 0; - const retry = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.idempotentRetryStatus = retry.status(); await retry.dispose(); - } else { - if (result.status() !== 409) throw new Error("Steering failed outside its supported visible rejection contract"); - state.steerObservedBranch = "provider-rejected-normal-continuation"; - const body = await result.json() as { code?: string; details?: { code?: string } }; - state.steerFailureCode = body.code ?? body.details?.code; - if (!["steering_rejected", "steering_unsupported"].includes(String(state.steerFailureCode))) throw new Error("A non-provider queue conflict does not qualify as a provider steering rejection"); - await expect(ctx.input.page.getByText(/steer|temporarily unavailable|timed out/i).filter({ hasNot: ctx.input.page.locator("button") }).first()).toBeVisible(); - state.providerFailureVisible = true; state.failureRetainedMessage = after.entries.some(row => row.comment.id === targetId && row.comment.body === direction); + if ((state.queueBefore as IssueQueuedCommentQueue).steeringDisposition === "unsupported") { + // This harness cannot steer, and the page offers no Steer control. Attempt it once with the person's own session to + // prove the product refuses rather than claiming delivery, then leave the message for normal admission. + const attempt = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.humanSteerAttempted = true; + state.steerStatus = attempt.status(); const body = await attempt.json().catch(() => ({})) as { code?: string; details?: { code?: string } }; await attempt.dispose(); + state.steerFailureCode = body.code ?? body.details?.code; state.steerObservedBranch = steeringBranch("unsupported", state.steerStatus); + const after = await ctx.input.api.get(endpoint); state.queueAfterAction = after; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible(); - await expect(ctx.input.page.getByTestId("task-chat-composer-input").last()).toBeVisible(); state.supportedContinuationVisible = true; + const controls = await queuedSteeringControls(ctx, targetId, after.steeringDisposition); + state.unsupportedStateRetained = after.steeringDisposition === "unsupported" && after.targetRunId === state.workerRunId && !controls.steerControlEnabled && controls.continuationControlVisible; + } else { + const response = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `${endpoint}/${targetId}/steer`); + await ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`).click(); state.humanSteerSelected = true; + const result = await response; state.steerStatus = result.status(); + const after = await ctx.input.api.get(endpoint); state.queueAfterAction = after; + if (result.status() === 200) { + state.steerObservedBranch = "same-run-acknowledged"; + const run = await ctx.input.api.get<{ resultJson?: { queuedSteeringAcknowledgements?: Record } }>(`/api/heartbeat-runs/${state.workerRunId}`); + const ack = run.resultJson?.queuedSteeringAcknowledgements?.[targetId]; + state.acknowledgedSameRun = ack?.status === "acknowledged" && ack.queueId === before.queueId && typeof ack.turnId === "string" && ack.turnId.length > 0; + const retry = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.idempotentRetryStatus = retry.status(); await retry.dispose(); + } else { + if (result.status() !== 409) throw new Error("Steering failed outside its supported visible rejection contract"); + state.steerObservedBranch = "provider-rejected-normal-continuation"; + const body = await result.json() as { code?: string; details?: { code?: string } }; + state.steerFailureCode = body.code ?? body.details?.code; + if (!["steering_rejected", "steering_unsupported"].includes(String(state.steerFailureCode))) throw new Error("A non-provider queue conflict does not qualify as a provider steering rejection"); + await expect(ctx.input.page.getByText(/steer|temporarily unavailable|timed out/i).filter({ hasNot: ctx.input.page.locator("button") }).first()).toBeVisible(); + state.providerFailureVisible = true; state.failureRetainedMessage = after.entries.some(row => row.comment.id === targetId && row.comment.body === direction); + await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible(); + await expect(ctx.input.page.getByTestId("task-chat-composer-input").last()).toBeVisible(); state.supportedContinuationVisible = true; + } } await ctx.input.evidence("runtime-steering-observed-branch.json", { branch: state.steerObservedBranch, failureCode: state.steerFailureCode ?? null, status: state.steerStatus, issueId: task.id, runId: state.workerRunId, commentId: targetId, otherBranchLiveQualified: false }); await ctx.input.capture("actual-steering-delivery", "Selected human steering action and actual native delivery or retained rejection", "runtime-steer-active.png"); - await writeFile((state.gate as { gate: string }).gate, "The human brief is ready. Implement a small semantic website.html using the actual design direction you received, with a visible h1 and a labelled signup action. Register the real website.html file as a downloadable artifact work product. Save the actual chosen color and design in a non-plan website-delivery task document, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.", "utf8"); + const build = "Implement a small semantic website.html with a visible h1 and a labelled signup action, register the real website.html file as a downloadable artifact work product, and save the actual chosen color and design in a non-plan website-delivery task document."; + // Delivered same-turn direction is built now; a direction still queued reaches the next normal turn, which applies it to the same deliverable. + await writeFile((state.gate as { gate: string }).gate, state.steerStatus === 200 + ? `The human brief is ready. ${build} Use the actual design direction you received, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.` + : `The human brief is ready. ${build} Then finish this current turn normally. Keep human requirements queued by Paperclip intact for normal delivery; when one arrives, apply it to website.html, register the updated file, update the delivery document, and complete the task. Do not create tasks or publish externally.`, "utf8"); const finished = await pollUntil({ label: "Actual selected steering or preserved normal continuation produces its website", deadlineAt: Date.now() + 240_000, load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), accept: v => v.task.status === "done" && v.queue.entries.length === 0 && v.runs.length === (state.steerStatus === 200 ? 2 : 3) && v.runs.every(row => row.status === "succeeded"), @@ -115,12 +154,14 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act const o = await observeRuntime(ctx, state), task = state.task as Entity; state.steerAuditBound = o.activity.some(row => row.action === "issue.queued_comment_steered" && row.entityId === task.id && row.actorId === state.humanId && row.details?.commentId === state.targetId && row.details?.targetRunId === state.workerRunId && row.details?.duplicate === false); + state.steeredActivityCount = o.activity.filter(row => row.action === "issue.queued_comment_steered" && row.entityId === task.id).length; if (state.outputRunId) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, String(state.outputRunId), ["website.html"]); if (artifact?.contentVerified) { state.actualBlueArtifact = hasVisibleBlueDesign(await renderRuntimeArtifact(ctx, artifact.content)); } - state.blueRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id && /blue/i.test(row.body ?? "") + // The rendered page proves the colour; the document only has to be the consuming run's saved delivery record. + state.deliveryRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); } return o;