diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index b49ec36fcc9..925e40d2779 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5612,7 +5612,7 @@ "registered-artifact.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime.steer-active.v3", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts index 2deb1d60fa2..370a2a28894 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.test.ts @@ -1,18 +1,18 @@ import { expect, it } from "vitest"; -import { gradeActiveSteering, isBlueColor, hasVisibleBlueDesign } from "./runtime-active-steering.js"; +import { gradeActiveSteering, isBlueColor, hasVisibleBlueDesign, steeringBranch } from "./runtime-active-steering.js"; function valid() { return { scenarioId: "steering.steer-active", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [], runs: [{ id: "worker", status: "succeeded" }, { id: "source", status: "succeeded" }], state: { task: { id: "website" }, targetId: "blue", humanId: "human", workerRunId: "worker", workerWasReallyWaiting: true, queueBefore: { issueId: "website", targetRunId: "worker", protocol: "paperclip_runner_v1", steeringDisposition: "available", entries: [{ comment: { id: "blue", body: "Switch to a blue design while you are working.", authorUserId: "human", authorAgentId: null } }] }, - queueAfterAction: { entries: [] as Array<{ comment: { id: string; body: string } }> }, humanSteerSelected: true, agentSteerStatus: 403, + queueAfterAction: { entries: [] as Array<{ comment: { id: string; body: string } }> }, humanSteerSelected: true, humanSteerAttempted: false, agentSteerStatus: 403, + steerControlEnabled: true, continuationControlVisible: false, unsupportedStateRetained: false, steeredActivityCount: 1, queryReply: "Use Steer beside your saved message to send this direction to Morgan's active turn; waiting in the queue is separate from delivery.", steerObservedBranch: "same-run-acknowledged", steerStatus: 200, acknowledgedSameRun: true, steerAuditBound: true, idempotentRetryStatus: 200, ordinaryConsumerCount: 0, - actualBlueArtifact: true, blueRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true, - steerFailureCode: "", providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } }; + actualBlueArtifact: true, deliveryRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true, + steerFailureCode: "" as string | undefined, providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } }; } it("accepts real explicit same-turn delivery and actual blue result", () => expect(gradeActiveSteering(valid()).passed).toBe(true)); -it("accepts natural human-control guidance", () => { const o = valid(); o.state.queryReply = "Your direction is saved. Select the Steer action in Morgan's task to apply it during this execution."; expect(gradeActiveSteering(o).passed).toBe(true); }); it("accepts an actual visible provider rejection with retained one-time normal delivery", () => { const o = valid(); Object.assign(o.state, { steerObservedBranch: "provider-rejected-normal-continuation", steerStatus: 409, steerFailureCode: "steering_rejected", providerFailureVisible: true, failureRetainedMessage: true, supportedContinuationVisible: true, ordinaryConsumerCount: 1, continuationInputBound: true, acknowledgedSameRun: false, steerAuditBound: false }); o.state.queueAfterAction.entries = [{ comment: { id: "blue", body: "Switch to a blue design while you are working." } }]; o.runs.push({ id: "ordinary", status: "succeeded" }); @@ -22,10 +22,8 @@ it("accepts an actual visible provider rejection with retained one-time normal d it("rejects queue admission without actual same-run acknowledgement", () => { const o = valid(); o.state.acknowledgedSameRun = false; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects hidden normal replay after accepted same-turn delivery", () => { const o = valid(); o.state.ordinaryConsumerCount = 1; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects steering a foreign run", () => { const o = valid(); o.state.queueBefore.targetRunId = "other"; expect(gradeActiveSteering(o).passed).toBe(false); }); -it("rejects falsely claiming the queued direction was already implemented", () => { const o = valid(); o.state.queryReply = "I have implemented the blue design with Steer."; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects a forged human steering action", () => { const o = valid(); o.state.agentSteerStatus = 200; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects saved blue prose without a rendered blue deliverable", () => { const o = valid(); o.state.actualBlueArtifact = false; expect(gradeActiveSteering(o).passed).toBe(false); }); -it("rejects unavailable advertised capability", () => { const o = valid(); o.state.queueBefore.steeringDisposition = "unsupported"; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects wrong human ownership", () => { const o = valid(); o.state.queueBefore.entries[0]!.comment.authorUserId = "other"; expect(gradeActiveSteering(o).passed).toBe(false); }); it("rejects absent actual evidence", () => expect(gradeActiveSteering(undefined).passed).toBe(false)); it.each(["rgb(0, 0, 255)", "rgb(42, 90, 180)", "rgba(15, 30, 98, 1)"])("accepts actual blue computed colors %s", color => expect(isBlueColor(color)).toBe(true)); @@ -37,3 +35,36 @@ it("rejects claiming the unobserved branch as qualified", () => { const o = vali it("rejects a red page with only default blue links", () => expect(hasVisibleBlueDesign([{ tag: "BODY", color: "rgb(0, 0, 0)", background: "rgb(255, 0, 0)" }, { tag: "A", color: "rgb(0, 0, 238)", background: "rgba(0, 0, 0, 0)" }])).toBe(false)); it("accepts a natural blue heading accent on a white page", () => expect(hasVisibleBlueDesign([{ tag: "BODY", color: "rgb(0, 0, 0)", background: "rgb(255, 255, 255)" }, { tag: "H1", color: "rgb(25, 70, 180)", background: "rgba(0, 0, 0, 0)" }])).toBe(true)); it("rejects transparent blue design elements", () => expect(hasVisibleBlueDesign([{ tag: "SECTION", color: "rgb(0, 0, 0)", background: "rgba(0, 0, 255, 0)" }])).toBe(false)); + +/** A runtime that cannot steer (the OpenCode harness advertises steering: false): the message stays visibly queued and one normal turn consumes it. */ +function unsupported() { + const o = valid(); + Object.assign(o.state, { steerObservedBranch: "unsupported-queued-normal-continuation", steerStatus: 409, steerFailureCode: "steering_unsupported", humanSteerSelected: false, humanSteerAttempted: true, + steerControlEnabled: false, continuationControlVisible: true, unsupportedStateRetained: true, steeredActivityCount: 0, acknowledgedSameRun: false, steerAuditBound: false, idempotentRetryStatus: 0, + ordinaryConsumerCount: 1, continuationInputBound: true }); + o.state.queueBefore.steeringDisposition = "unsupported"; + o.state.queueAfterAction.entries = [{ comment: { id: "blue", body: "Switch to a blue design while you are working." } }]; o.runs.push({ id: "ordinary", status: "succeeded" }); + return o; +} +it("accepts an honestly unsupported runtime that keeps the message queued for one normal continuation", () => expect(gradeActiveSteering(unsupported()).passed).toBe(true)); +it.each(["Your direction is saved. Select the Steer action in Morgan's task to apply it during this execution.", "I have implemented the blue design with Steer.", "Queued for Morgan's next turn."])( + "verdict does not depend on reply wording: %s", reply => { for (const o of [valid(), unsupported()]) { o.state.queryReply = reply; expect(gradeActiveSteering(o).passed).toBe(true); } }); +it.each([ + ["claims steering was recorded", (o: ReturnType) => { o.state.steeredActivityCount = 1; }], + ["offers an enabled Steer control it cannot honour", (o: ReturnType) => { o.state.steerControlEnabled = true; }], + ["hides the supported continuation control", (o: ReturnType) => { o.state.continuationControlVisible = false; }], + ["drops the queued message after the refused attempt", (o: ReturnType) => { o.state.queueAfterAction.entries = []; }], + ["stops reporting the capability after the attempt", (o: ReturnType) => { o.state.unsupportedStateRetained = false; }], + ["accepts the steer request without a runtime that can steer", (o: ReturnType) => { o.state.steerStatus = 200; }], + ["refuses with an unrelated conflict", (o: ReturnType) => { o.state.steerFailureCode = "queued_comment_stale_queue"; }], + ["never delivers the message", (o: ReturnType) => { o.state.ordinaryConsumerCount = 0; o.runs.pop(); }], + ["delivers it twice", (o: ReturnType) => { o.state.ordinaryConsumerCount = 2; o.runs.push({ id: "again", status: "succeeded" }); }], + ["continuation input lacks the message", (o: ReturnType) => { o.state.continuationInputBound = false; }], + ["lets the agent steer for the person", (o: ReturnType) => { o.state.agentSteerStatus = 200; }], + ["produces no rendered blue page", (o: ReturnType) => { o.state.actualBlueArtifact = false; }], +])("rejects an unsupported runtime that %s", (_label, mutate) => { const o = unsupported(); mutate(o); expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects a queue that never states a definitive steering capability", () => { const o = valid(); o.state.queueBefore.steeringDisposition = "temporarily_unavailable"; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("rejects an advertised Steer control that the page does not enable", () => { const o = valid(); o.state.steerControlEnabled = false; expect(gradeActiveSteering(o).passed).toBe(false); }); +it("accepts a delivery document that names the colour another way, since the rendered page proves it", () => { const o = valid(); o.state.deliveryRevisionBound = true; expect(gradeActiveSteering(o).passed).toBe(true); o.state.deliveryRevisionBound = false; expect(gradeActiveSteering(o).passed).toBe(false); }); +it.each([["available", 200, "same-run-acknowledged"], ["available", 409, "provider-rejected-normal-continuation"], ["unsupported", 409, "unsupported-queued-normal-continuation"], ["temporarily_unavailable", 409, "not-advertised"]] as const)( + "maps %s capability and status %s to its branch", (disposition, status, branch) => expect(steeringBranch(disposition, status)).toBe(branch)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts index ce890ad7084..b5980a43e92 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-active-steering.ts @@ -10,6 +10,7 @@ import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.j import { renderRuntimeArtifact } from "./runtime-rendered-artifact.js"; import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; export function isBlueColor(color: string) { const parts = color.match(/^rgba?\(\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)/i); if (!parts) return false; @@ -23,43 +24,64 @@ export function hasVisibleBlueDesign(elements: Array<{ tag: string; role?: strin ? isBlueColor(element.background) || element.tag !== "BODY" && isBlueColor(element.color) : false); } const direction = "Switch to a blue design while you are working."; +/** Branch a run must take, decided by what the harness actually advertised and how the product answered. */ +export function steeringBranch(disposition: unknown, steerStatus: unknown) { + return disposition === "unsupported" ? "unsupported-queued-normal-continuation" : disposition !== "available" ? "not-advertised" + : steerStatus === 200 ? "same-run-acknowledged" : "provider-rejected-normal-continuation"; +} export function gradeActiveSteering(value: unknown) { const c = checks(), o = runtimeObservation(value); - c.add("actual-native-steering-context", !!o, "A genuine compatible active native run and saved human message are required."); if (!o) return c.result(); + c.add("actual-native-steering-context", !!o, "A genuine active native run and saved human message are required."); if (!o) return c.result(); const s = o.state, before = s.queueBefore as IssueQueuedCommentQueue, after = s.queueAfterAction as IssueQueuedCommentQueue; - const entry = before?.entries.find(row => row.comment.id === s.targetId); - c.add("saved-owned-message-compatible-active-target", !!entry && entry.comment.body === direction && entry.comment.authorUserId === s.humanId && !entry.comment.authorAgentId + const entry = before?.entries.find(row => row.comment.id === s.targetId), unsupported = before?.steeringDisposition === "unsupported"; + c.add("saved-owned-message-honest-active-target", !!entry && entry.comment.body === direction && entry.comment.authorUserId === s.humanId && !entry.comment.authorAgentId && before.entries.length === 1 && before.issueId === (s.task as Entity).id && before.targetRunId === s.workerRunId && before.protocol === "paperclip_runner_v1" - && before.steeringDisposition === "available" && s.workerWasReallyWaiting === true, - "The actual harness advertises steering for the exact saved human-owned message and running task, before the canonical request."); - c.add("explicit-human-control-and-truthful-guidance", s.humanSteerSelected === true && s.agentSteerStatus === 403 - && /steer|steering/i.test(String(s.queryReply)) && !/I(?:'ve| have)? (?:already )?(?:steered|delivered|implemented)|blue (?:design|page) (?:is|has been) (?:ready|complete)/i.test(String(s.queryReply)), - "Chat identifies the supported human Steer control; it cannot forge the action or claim queue admission already changed the active result."); - const accepted = s.steerStatus === 200; - c.add("delivery-distinct-from-queue-with-no-loss", s.steerObservedBranch === (accepted ? "same-run-acknowledged" : "provider-rejected-normal-continuation") && (accepted + && ["available", "unsupported"].includes(before.steeringDisposition) && s.workerWasReallyWaiting === true + && (unsupported ? s.steerControlEnabled === false && s.continuationControlVisible === true : s.steerControlEnabled === true), + "The running task's queue states the harness's real steering capability for the exact saved human message: an enabled Steer control when it can steer, otherwise no usable Steer control and a visible supported continuation control."); + const branch = steeringBranch(before?.steeringDisposition, s.steerStatus), accepted = branch === "same-run-acknowledged"; + c.add("human-only-control-without-forged-delivery", s.agentSteerStatus === 403 && (unsupported ? s.humanSteerAttempted === true : s.humanSteerSelected === true), + "Only the person's own action can steer the message; the agent's attempt is refused."); + const retained = after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true; + c.add("delivery-distinct-from-queue-with-no-loss", s.steerObservedBranch === branch && (accepted ? s.acknowledgedSameRun === true && after?.entries.length === 0 && s.steerAuditBound === true && s.idempotentRetryStatus === 200 && s.ordinaryConsumerCount === 0 - : ["steering_rejected", "steering_unsupported"].includes(String(s.steerFailureCode)) && s.steerStatus === 409 && s.providerFailureVisible === true && s.failureRetainedMessage === true && after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true - && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true), - "A durable same-run ACK is required for delivery; an actual provider rejection instead retains the exact message and visible failure until one normal continuation consumes it."); - c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.blueRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true + : branch === "unsupported-queued-normal-continuation" + ? s.steerStatus === 409 && s.steerFailureCode === "steering_unsupported" && s.steeredActivityCount === 0 && retained && s.unsupportedStateRetained === true + && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true + : branch === "provider-rejected-normal-continuation" && ["steering_rejected", "steering_unsupported"].includes(String(s.steerFailureCode)) && s.steerStatus === 409 && s.providerFailureVisible === true + && s.failureRetainedMessage === true && retained && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true), + "A durable same-run ACK is required for delivery; a runtime that cannot steer (or a provider rejection) instead keeps the exact message visibly queued, records no steering, and one normal continuation consumes it."); + c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.deliveryRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true && o.runs.length === (accepted ? 2 : 3) && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1, - "Actual delivered HTML and its task revision reflect blue under the correct consuming native run, with no duplicate work or hidden run."); + "Actual delivered HTML renders blue, and the consuming native run saved its task delivery document, with no duplicate work or hidden run."); return c.result(); } +/** What the person can actually do with the queued message: an enabled Steer control, or the supported continuation (Interrupt) control. */ +async function queuedSteeringControls(ctx: ChatCatalogContext, targetId: string, disposition: string) { + const steer = ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`), next = ctx.input.page.getByTestId(`task-chat-queued-interrupt-${targetId}`); + // The page reads the same queue the API returned; wait for the control that capability implies, then record both. + await expect(disposition === "available" ? steer : next).toBeEnabled(); + return { steerControlEnabled: await steer.count() > 0 && await steer.first().isEnabled(), continuationControlVisible: await next.count() > 0 && await next.first().isEnabled() }; +} export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-active", async ctx => { const state: Record = await busyRuntimeWorker(ctx), task = state.task as Entity, source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await submitTaskReply(ctx.input.page, direction); - const before = await pollUntil({ label: "Actual native same-turn steering advertised for saved human blue message", deadlineAt: Date.now() + 30_000, + const before = await pollUntil({ label: "Running task queue states its harness steering capability for the saved human blue message", deadlineAt: Date.now() + 30_000, load: () => ctx.input.api.get(`/api/issues/${task.id}/queued-comments`), - accept: q => q.entries.length === 1 && q.entries[0]?.comment.body === direction && q.targetRunId === state.workerRunId && q.protocol === "paperclip_runner_v1" && q.steeringDisposition === "available" }); + accept: q => q.entries.length === 1 && q.entries[0]?.comment.body === direction && q.targetRunId === state.workerRunId && q.protocol === "paperclip_runner_v1" + && ["available", "unsupported"].includes(q.steeringDisposition) }); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); if (before.entries[0]?.comment.authorUserId !== session.user.id || before.entries[0].comment.authorAgentId) throw new Error("Steering fixture lacks actual human message ownership"); - await expect(ctx.input.page.getByTestId(`task-chat-queued-steer-${before.entries[0].comment.id}`)).toBeEnabled(); + const targetId = before.entries[0].comment.id; + await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible(); + const controls = await queuedSteeringControls(ctx, targetId, before.steeringDisposition); + if (before.steeringDisposition === "available" ? !controls.steerControlEnabled : controls.steerControlEnabled || !controls.continuationControlVisible) + throw new Error("The queued message controls contradict the advertised steering capability"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); - return { ...state, humanId: session.user.id, queueBefore: before, targetId: before.entries[0].comment.id }; + return { ...state, humanId: session.user.id, queueBefore: before, targetId, ...controls }; }, gradeActiveSteering, { - authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, + authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.steer-active.v3", setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { @@ -74,30 +96,47 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act const mutation = { queueId: before.queueId, revision: before.revision, targetRunId: before.targetRunId }; state.agentSteerStatus = await probeAgentPermission(ctx, `${endpoint}/${targetId}/steer`, "POST", mutation); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); - const response = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `${endpoint}/${targetId}/steer`); - await ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`).click(); state.humanSteerSelected = true; - const result = await response; state.steerStatus = result.status(); - const after = await ctx.input.api.get(endpoint); state.queueAfterAction = after; - if (result.status() === 200) { - state.steerObservedBranch = "same-run-acknowledged"; - const run = await ctx.input.api.get<{ resultJson?: { queuedSteeringAcknowledgements?: Record } }>(`/api/heartbeat-runs/${state.workerRunId}`); - const ack = run.resultJson?.queuedSteeringAcknowledgements?.[targetId]; - state.acknowledgedSameRun = ack?.status === "acknowledged" && ack.queueId === before.queueId && typeof ack.turnId === "string" && ack.turnId.length > 0; - const retry = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.idempotentRetryStatus = retry.status(); await retry.dispose(); - } else { - if (result.status() !== 409) throw new Error("Steering failed outside its supported visible rejection contract"); - state.steerObservedBranch = "provider-rejected-normal-continuation"; - const body = await result.json() as { code?: string; details?: { code?: string } }; - state.steerFailureCode = body.code ?? body.details?.code; - if (!["steering_rejected", "steering_unsupported"].includes(String(state.steerFailureCode))) throw new Error("A non-provider queue conflict does not qualify as a provider steering rejection"); - await expect(ctx.input.page.getByText(/steer|temporarily unavailable|timed out/i).filter({ hasNot: ctx.input.page.locator("button") }).first()).toBeVisible(); - state.providerFailureVisible = true; state.failureRetainedMessage = after.entries.some(row => row.comment.id === targetId && row.comment.body === direction); + if ((state.queueBefore as IssueQueuedCommentQueue).steeringDisposition === "unsupported") { + // This harness cannot steer, and the page offers no Steer control. Attempt it once with the person's own session to + // prove the product refuses rather than claiming delivery, then leave the message for normal admission. + const attempt = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.humanSteerAttempted = true; + state.steerStatus = attempt.status(); const body = await attempt.json().catch(() => ({})) as { code?: string; details?: { code?: string } }; await attempt.dispose(); + state.steerFailureCode = body.code ?? body.details?.code; state.steerObservedBranch = steeringBranch("unsupported", state.steerStatus); + const after = await ctx.input.api.get(endpoint); state.queueAfterAction = after; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible(); - await expect(ctx.input.page.getByTestId("task-chat-composer-input").last()).toBeVisible(); state.supportedContinuationVisible = true; + const controls = await queuedSteeringControls(ctx, targetId, after.steeringDisposition); + state.unsupportedStateRetained = after.steeringDisposition === "unsupported" && after.targetRunId === state.workerRunId && !controls.steerControlEnabled && controls.continuationControlVisible; + } else { + const response = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `${endpoint}/${targetId}/steer`); + await ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`).click(); state.humanSteerSelected = true; + const result = await response; state.steerStatus = result.status(); + const after = await ctx.input.api.get(endpoint); state.queueAfterAction = after; + if (result.status() === 200) { + state.steerObservedBranch = "same-run-acknowledged"; + const run = await ctx.input.api.get<{ resultJson?: { queuedSteeringAcknowledgements?: Record } }>(`/api/heartbeat-runs/${state.workerRunId}`); + const ack = run.resultJson?.queuedSteeringAcknowledgements?.[targetId]; + state.acknowledgedSameRun = ack?.status === "acknowledged" && ack.queueId === before.queueId && typeof ack.turnId === "string" && ack.turnId.length > 0; + const retry = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.idempotentRetryStatus = retry.status(); await retry.dispose(); + } else { + if (result.status() !== 409) throw new Error("Steering failed outside its supported visible rejection contract"); + state.steerObservedBranch = "provider-rejected-normal-continuation"; + const body = await result.json() as { code?: string; details?: { code?: string } }; + state.steerFailureCode = body.code ?? body.details?.code; + if (!["steering_rejected", "steering_unsupported"].includes(String(state.steerFailureCode))) throw new Error("A non-provider queue conflict does not qualify as a provider steering rejection"); + await expect(ctx.input.page.getByText(/steer|temporarily unavailable|timed out/i).filter({ hasNot: ctx.input.page.locator("button") }).first()).toBeVisible(); + state.providerFailureVisible = true; state.failureRetainedMessage = after.entries.some(row => row.comment.id === targetId && row.comment.body === direction); + await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible(); + await expect(ctx.input.page.getByTestId("task-chat-composer-input").last()).toBeVisible(); state.supportedContinuationVisible = true; + } } await ctx.input.evidence("runtime-steering-observed-branch.json", { branch: state.steerObservedBranch, failureCode: state.steerFailureCode ?? null, status: state.steerStatus, issueId: task.id, runId: state.workerRunId, commentId: targetId, otherBranchLiveQualified: false }); await ctx.input.capture("actual-steering-delivery", "Selected human steering action and actual native delivery or retained rejection", "runtime-steer-active.png"); - await writeFile((state.gate as { gate: string }).gate, "The human brief is ready. Implement a small semantic website.html using the actual design direction you received, with a visible h1 and a labelled signup action. Register the real website.html file as a downloadable artifact work product. Save the actual chosen color and design in a non-plan website-delivery task document, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.", "utf8"); + const build = "Implement a small semantic website.html with a visible h1 and a labelled signup action, register the real website.html file as a downloadable artifact work product, and save the actual chosen color and design in a non-plan website-delivery task document."; + // Delivered same-turn direction is built now; a direction still queued reaches the next normal turn, which applies it to the same deliverable. + await writeFile((state.gate as { gate: string }).gate, state.steerStatus === 200 + ? `The human brief is ready. ${build} Use the actual design direction you received, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.` + : `The human brief is ready. ${build} Then finish this current turn normally. Keep human requirements queued by Paperclip intact for normal delivery; when one arrives, apply it to website.html, register the updated file, update the delivery document, and complete the task. Do not create tasks or publish externally.`, "utf8"); const finished = await pollUntil({ label: "Actual selected steering or preserved normal continuation produces its website", deadlineAt: Date.now() + 240_000, load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), accept: v => v.task.status === "done" && v.queue.entries.length === 0 && v.runs.length === (state.steerStatus === 200 ? 2 : 3) && v.runs.every(row => row.status === "succeeded"), @@ -115,12 +154,14 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act const o = await observeRuntime(ctx, state), task = state.task as Entity; state.steerAuditBound = o.activity.some(row => row.action === "issue.queued_comment_steered" && row.entityId === task.id && row.actorId === state.humanId && row.details?.commentId === state.targetId && row.details?.targetRunId === state.workerRunId && row.details?.duplicate === false); + state.steeredActivityCount = o.activity.filter(row => row.action === "issue.queued_comment_steered" && row.entityId === task.id).length; if (state.outputRunId) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, String(state.outputRunId), ["website.html"]); if (artifact?.contentVerified) { state.actualBlueArtifact = hasVisibleBlueDesign(await renderRuntimeArtifact(ctx, artifact.content)); } - state.blueRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id && /blue/i.test(row.body ?? "") + // The rendered page proves the colour; the document only has to be the consuming run's saved delivery record. + state.deliveryRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); } return o;