mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-13 07:12:00 +01:00
Merge remote-tracking branch 'origin/chat-eval/fix-steering' into chat-eval/b3-combined
* origin/chat-eval/fix-steering: fix(evals): grade steering.steer-active honestly on runtimes that cannot steer fix(evals): gate steering.approval-delivery on the parked pending-input turn
This commit is contained in:
Dotta committed
commit
8d2e34eac1
6 files changed
+221
-97
No files matched your search
@@ -5446,9 +5446,9 @@
|
||||
"chat-catalog-domains/runtime-support.ts",
|
||||
"chat-scenarios/steering.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.v2",
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.approval-delivery.v3",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 3,
|
||||
"expectedRunCount": 4,
|
||||
"turnTimeoutMs": 120000,
|
||||
"attemptTimeoutMs": 900000,
|
||||
"authentication": "human-session",
|
||||
@@ -5618,7 +5618,7 @@
|
||||
"registered-artifact.ts",
|
||||
"chat-scenarios/steering.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.v2",
|
||||
"oracleVersion": "paperclip.chat-catalog.runtime.steer-active.v3",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 3,
|
||||
"minimumRunCount": 2,
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { gradeActiveSteering, isBlueColor, hasVisibleBlueDesign } from "./runtime-active-steering.js";
|
||||
import { gradeActiveSteering, isBlueColor, hasVisibleBlueDesign, steeringBranch } from "./runtime-active-steering.js";
|
||||
function valid() {
|
||||
return { scenarioId: "steering.steer-active", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [],
|
||||
runs: [{ id: "worker", status: "succeeded" }, { id: "source", status: "succeeded" }],
|
||||
state: { task: { id: "website" }, targetId: "blue", humanId: "human", workerRunId: "worker", workerWasReallyWaiting: true,
|
||||
queueBefore: { issueId: "website", targetRunId: "worker", protocol: "paperclip_runner_v1", steeringDisposition: "available", entries: [{ comment: { id: "blue", body: "Switch to a blue design while you are working.", authorUserId: "human", authorAgentId: null } }] },
|
||||
queueAfterAction: { entries: [] as Array<{ comment: { id: string; body: string } }> }, humanSteerSelected: true, agentSteerStatus: 403,
|
||||
queueAfterAction: { entries: [] as Array<{ comment: { id: string; body: string } }> }, humanSteerSelected: true, humanSteerAttempted: false, agentSteerStatus: 403,
|
||||
steerControlEnabled: true, continuationControlVisible: false, unsupportedStateRetained: false, steeredActivityCount: 1,
|
||||
queryReply: "Use Steer beside your saved message to send this direction to Morgan's active turn; waiting in the queue is separate from delivery.",
|
||||
steerObservedBranch: "same-run-acknowledged", steerStatus: 200, acknowledgedSameRun: true, steerAuditBound: true, idempotentRetryStatus: 200, ordinaryConsumerCount: 0,
|
||||
actualBlueArtifact: true, blueRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true,
|
||||
steerFailureCode: "", providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } };
|
||||
actualBlueArtifact: true, deliveryRevisionBound: true, workerTaskDone: true, queueEmptyAtEnd: true,
|
||||
steerFailureCode: "" as string | undefined, providerFailureVisible: false, failureRetainedMessage: false, supportedContinuationVisible: false, continuationInputBound: false } };
|
||||
}
|
||||
it("accepts real explicit same-turn delivery and actual blue result", () => expect(gradeActiveSteering(valid()).passed).toBe(true));
|
||||
it("accepts natural human-control guidance", () => { const o = valid(); o.state.queryReply = "Your direction is saved. Select the Steer action in Morgan's task to apply it during this execution."; expect(gradeActiveSteering(o).passed).toBe(true); });
|
||||
it("accepts an actual visible provider rejection with retained one-time normal delivery", () => {
|
||||
const o = valid(); Object.assign(o.state, { steerObservedBranch: "provider-rejected-normal-continuation", steerStatus: 409, steerFailureCode: "steering_rejected", providerFailureVisible: true, failureRetainedMessage: true, supportedContinuationVisible: true, ordinaryConsumerCount: 1, continuationInputBound: true, acknowledgedSameRun: false, steerAuditBound: false });
|
||||
o.state.queueAfterAction.entries = [{ comment: { id: "blue", body: "Switch to a blue design while you are working." } }]; o.runs.push({ id: "ordinary", status: "succeeded" });
|
||||
@@ -22,10 +22,8 @@ it("accepts an actual visible provider rejection with retained one-time normal d
|
||||
it("rejects queue admission without actual same-run acknowledgement", () => { const o = valid(); o.state.acknowledgedSameRun = false; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects hidden normal replay after accepted same-turn delivery", () => { const o = valid(); o.state.ordinaryConsumerCount = 1; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects steering a foreign run", () => { const o = valid(); o.state.queueBefore.targetRunId = "other"; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects falsely claiming the queued direction was already implemented", () => { const o = valid(); o.state.queryReply = "I have implemented the blue design with Steer."; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects a forged human steering action", () => { const o = valid(); o.state.agentSteerStatus = 200; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects saved blue prose without a rendered blue deliverable", () => { const o = valid(); o.state.actualBlueArtifact = false; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects unavailable advertised capability", () => { const o = valid(); o.state.queueBefore.steeringDisposition = "unsupported"; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects wrong human ownership", () => { const o = valid(); o.state.queueBefore.entries[0]!.comment.authorUserId = "other"; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects absent actual evidence", () => expect(gradeActiveSteering(undefined).passed).toBe(false));
|
||||
it.each(["rgb(0, 0, 255)", "rgb(42, 90, 180)", "rgba(15, 30, 98, 1)"])("accepts actual blue computed colors %s", color => expect(isBlueColor(color)).toBe(true));
|
||||
@@ -37,3 +35,36 @@ it("rejects claiming the unobserved branch as qualified", () => { const o = vali
|
||||
it("rejects a red page with only default blue links", () => expect(hasVisibleBlueDesign([{ tag: "BODY", color: "rgb(0, 0, 0)", background: "rgb(255, 0, 0)" }, { tag: "A", color: "rgb(0, 0, 238)", background: "rgba(0, 0, 0, 0)" }])).toBe(false));
|
||||
it("accepts a natural blue heading accent on a white page", () => expect(hasVisibleBlueDesign([{ tag: "BODY", color: "rgb(0, 0, 0)", background: "rgb(255, 255, 255)" }, { tag: "H1", color: "rgb(25, 70, 180)", background: "rgba(0, 0, 0, 0)" }])).toBe(true));
|
||||
it("rejects transparent blue design elements", () => expect(hasVisibleBlueDesign([{ tag: "SECTION", color: "rgb(0, 0, 0)", background: "rgba(0, 0, 255, 0)" }])).toBe(false));
|
||||
|
||||
/** A runtime that cannot steer (the OpenCode harness advertises steering: false): the message stays visibly queued and one normal turn consumes it. */
|
||||
function unsupported() {
|
||||
const o = valid();
|
||||
Object.assign(o.state, { steerObservedBranch: "unsupported-queued-normal-continuation", steerStatus: 409, steerFailureCode: "steering_unsupported", humanSteerSelected: false, humanSteerAttempted: true,
|
||||
steerControlEnabled: false, continuationControlVisible: true, unsupportedStateRetained: true, steeredActivityCount: 0, acknowledgedSameRun: false, steerAuditBound: false, idempotentRetryStatus: 0,
|
||||
ordinaryConsumerCount: 1, continuationInputBound: true });
|
||||
o.state.queueBefore.steeringDisposition = "unsupported";
|
||||
o.state.queueAfterAction.entries = [{ comment: { id: "blue", body: "Switch to a blue design while you are working." } }]; o.runs.push({ id: "ordinary", status: "succeeded" });
|
||||
return o;
|
||||
}
|
||||
it("accepts an honestly unsupported runtime that keeps the message queued for one normal continuation", () => expect(gradeActiveSteering(unsupported()).passed).toBe(true));
|
||||
it.each(["Your direction is saved. Select the Steer action in Morgan's task to apply it during this execution.", "I have implemented the blue design with Steer.", "Queued for Morgan's next turn."])(
|
||||
"verdict does not depend on reply wording: %s", reply => { for (const o of [valid(), unsupported()]) { o.state.queryReply = reply; expect(gradeActiveSteering(o).passed).toBe(true); } });
|
||||
it.each([
|
||||
["claims steering was recorded", (o: ReturnType<typeof unsupported>) => { o.state.steeredActivityCount = 1; }],
|
||||
["offers an enabled Steer control it cannot honour", (o: ReturnType<typeof unsupported>) => { o.state.steerControlEnabled = true; }],
|
||||
["hides the supported continuation control", (o: ReturnType<typeof unsupported>) => { o.state.continuationControlVisible = false; }],
|
||||
["drops the queued message after the refused attempt", (o: ReturnType<typeof unsupported>) => { o.state.queueAfterAction.entries = []; }],
|
||||
["stops reporting the capability after the attempt", (o: ReturnType<typeof unsupported>) => { o.state.unsupportedStateRetained = false; }],
|
||||
["accepts the steer request without a runtime that can steer", (o: ReturnType<typeof unsupported>) => { o.state.steerStatus = 200; }],
|
||||
["refuses with an unrelated conflict", (o: ReturnType<typeof unsupported>) => { o.state.steerFailureCode = "queued_comment_stale_queue"; }],
|
||||
["never delivers the message", (o: ReturnType<typeof unsupported>) => { o.state.ordinaryConsumerCount = 0; o.runs.pop(); }],
|
||||
["delivers it twice", (o: ReturnType<typeof unsupported>) => { o.state.ordinaryConsumerCount = 2; o.runs.push({ id: "again", status: "succeeded" }); }],
|
||||
["continuation input lacks the message", (o: ReturnType<typeof unsupported>) => { o.state.continuationInputBound = false; }],
|
||||
["lets the agent steer for the person", (o: ReturnType<typeof unsupported>) => { o.state.agentSteerStatus = 200; }],
|
||||
["produces no rendered blue page", (o: ReturnType<typeof unsupported>) => { o.state.actualBlueArtifact = false; }],
|
||||
])("rejects an unsupported runtime that %s", (_label, mutate) => { const o = unsupported(); mutate(o); expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects a queue that never states a definitive steering capability", () => { const o = valid(); o.state.queueBefore.steeringDisposition = "temporarily_unavailable"; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("rejects an advertised Steer control that the page does not enable", () => { const o = valid(); o.state.steerControlEnabled = false; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it("accepts a delivery document that names the colour another way, since the rendered page proves it", () => { const o = valid(); o.state.deliveryRevisionBound = true; expect(gradeActiveSteering(o).passed).toBe(true); o.state.deliveryRevisionBound = false; expect(gradeActiveSteering(o).passed).toBe(false); });
|
||||
it.each([["available", 200, "same-run-acknowledged"], ["available", 409, "provider-rejected-normal-continuation"], ["unsupported", 409, "unsupported-queued-normal-continuation"], ["temporarily_unavailable", 409, "not-advertised"]] as const)(
|
||||
"maps %s capability and status %s to its branch", (disposition, status, branch) => expect(steeringBranch(disposition, status)).toBe(branch));
|
||||
@@ -10,6 +10,7 @@ import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.j
|
||||
import { renderRuntimeArtifact } from "./runtime-rendered-artifact.js";
|
||||
import { runtimeQueueCases } from "./runtime-queue.js";
|
||||
import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
export function isBlueColor(color: string) {
|
||||
const parts = color.match(/^rgba?\(\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)\s*[, ]\s*(\d+(?:\.\d+)?)/i);
|
||||
if (!parts) return false;
|
||||
@@ -23,43 +24,64 @@ export function hasVisibleBlueDesign(elements: Array<{ tag: string; role?: strin
|
||||
? isBlueColor(element.background) || element.tag !== "BODY" && isBlueColor(element.color) : false);
|
||||
}
|
||||
const direction = "Switch to a blue design while you are working.";
|
||||
/** Branch a run must take, decided by what the harness actually advertised and how the product answered. */
|
||||
export function steeringBranch(disposition: unknown, steerStatus: unknown) {
|
||||
return disposition === "unsupported" ? "unsupported-queued-normal-continuation" : disposition !== "available" ? "not-advertised"
|
||||
: steerStatus === 200 ? "same-run-acknowledged" : "provider-rejected-normal-continuation";
|
||||
}
|
||||
export function gradeActiveSteering(value: unknown) {
|
||||
const c = checks(), o = runtimeObservation(value);
|
||||
c.add("actual-native-steering-context", !!o, "A genuine compatible active native run and saved human message are required."); if (!o) return c.result();
|
||||
c.add("actual-native-steering-context", !!o, "A genuine active native run and saved human message are required."); if (!o) return c.result();
|
||||
const s = o.state, before = s.queueBefore as IssueQueuedCommentQueue, after = s.queueAfterAction as IssueQueuedCommentQueue;
|
||||
const entry = before?.entries.find(row => row.comment.id === s.targetId);
|
||||
c.add("saved-owned-message-compatible-active-target", !!entry && entry.comment.body === direction && entry.comment.authorUserId === s.humanId && !entry.comment.authorAgentId
|
||||
const entry = before?.entries.find(row => row.comment.id === s.targetId), unsupported = before?.steeringDisposition === "unsupported";
|
||||
c.add("saved-owned-message-honest-active-target", !!entry && entry.comment.body === direction && entry.comment.authorUserId === s.humanId && !entry.comment.authorAgentId
|
||||
&& before.entries.length === 1 && before.issueId === (s.task as Entity).id && before.targetRunId === s.workerRunId && before.protocol === "paperclip_runner_v1"
|
||||
&& before.steeringDisposition === "available" && s.workerWasReallyWaiting === true,
|
||||
"The actual harness advertises steering for the exact saved human-owned message and running task, before the canonical request.");
|
||||
c.add("explicit-human-control-and-truthful-guidance", s.humanSteerSelected === true && s.agentSteerStatus === 403
|
||||
&& /steer|steering/i.test(String(s.queryReply)) && !/I(?:'ve| have)? (?:already )?(?:steered|delivered|implemented)|blue (?:design|page) (?:is|has been) (?:ready|complete)/i.test(String(s.queryReply)),
|
||||
"Chat identifies the supported human Steer control; it cannot forge the action or claim queue admission already changed the active result.");
|
||||
const accepted = s.steerStatus === 200;
|
||||
c.add("delivery-distinct-from-queue-with-no-loss", s.steerObservedBranch === (accepted ? "same-run-acknowledged" : "provider-rejected-normal-continuation") && (accepted
|
||||
&& ["available", "unsupported"].includes(before.steeringDisposition) && s.workerWasReallyWaiting === true
|
||||
&& (unsupported ? s.steerControlEnabled === false && s.continuationControlVisible === true : s.steerControlEnabled === true),
|
||||
"The running task's queue states the harness's real steering capability for the exact saved human message: an enabled Steer control when it can steer, otherwise no usable Steer control and a visible supported continuation control.");
|
||||
const branch = steeringBranch(before?.steeringDisposition, s.steerStatus), accepted = branch === "same-run-acknowledged";
|
||||
c.add("human-only-control-without-forged-delivery", s.agentSteerStatus === 403 && (unsupported ? s.humanSteerAttempted === true : s.humanSteerSelected === true),
|
||||
"Only the person's own action can steer the message; the agent's attempt is refused.");
|
||||
const retained = after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true;
|
||||
c.add("delivery-distinct-from-queue-with-no-loss", s.steerObservedBranch === branch && (accepted
|
||||
? s.acknowledgedSameRun === true && after?.entries.length === 0 && s.steerAuditBound === true && s.idempotentRetryStatus === 200 && s.ordinaryConsumerCount === 0
|
||||
: ["steering_rejected", "steering_unsupported"].includes(String(s.steerFailureCode)) && s.steerStatus === 409 && s.providerFailureVisible === true && s.failureRetainedMessage === true && after?.entries.some(row => row.comment.id === s.targetId && row.comment.body === direction) === true
|
||||
&& s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true),
|
||||
"A durable same-run ACK is required for delivery; an actual provider rejection instead retains the exact message and visible failure until one normal continuation consumes it.");
|
||||
c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.blueRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true
|
||||
: branch === "unsupported-queued-normal-continuation"
|
||||
? s.steerStatus === 409 && s.steerFailureCode === "steering_unsupported" && s.steeredActivityCount === 0 && retained && s.unsupportedStateRetained === true
|
||||
&& s.ordinaryConsumerCount === 1 && s.continuationInputBound === true
|
||||
: branch === "provider-rejected-normal-continuation" && ["steering_rejected", "steering_unsupported"].includes(String(s.steerFailureCode)) && s.steerStatus === 409 && s.providerFailureVisible === true
|
||||
&& s.failureRetainedMessage === true && retained && s.supportedContinuationVisible === true && s.ordinaryConsumerCount === 1 && s.continuationInputBound === true),
|
||||
"A durable same-run ACK is required for delivery; a runtime that cannot steer (or a provider rejection) instead keeps the exact message visibly queued, records no steering, and one normal continuation consumes it.");
|
||||
c.add("actual-blue-output-correct-native-owner", s.actualBlueArtifact === true && s.deliveryRevisionBound === true && s.workerTaskDone === true && s.queueEmptyAtEnd === true
|
||||
&& o.runs.length === (accepted ? 2 : 3) && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1,
|
||||
"Actual delivered HTML and its task revision reflect blue under the correct consuming native run, with no duplicate work or hidden run.");
|
||||
"Actual delivered HTML renders blue, and the consuming native run saved its task delivery document, with no duplicate work or hidden run.");
|
||||
return c.result();
|
||||
}
|
||||
/** What the person can actually do with the queued message: an enabled Steer control, or the supported continuation (Interrupt) control. */
|
||||
async function queuedSteeringControls(ctx: ChatCatalogContext, targetId: string, disposition: string) {
|
||||
const steer = ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`), next = ctx.input.page.getByTestId(`task-chat-queued-interrupt-${targetId}`);
|
||||
// The page reads the same queue the API returned; wait for the control that capability implies, then record both.
|
||||
await expect(disposition === "available" ? steer : next).toBeEnabled();
|
||||
return { steerControlEnabled: await steer.count() > 0 && await steer.first().isEnabled(), continuationControlVisible: await next.count() > 0 && await next.first().isEnabled() };
|
||||
}
|
||||
export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-active", async ctx => {
|
||||
const state: Record<string, unknown> = await busyRuntimeWorker(ctx), task = state.task as Entity, source = ctx.input.page.url();
|
||||
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
|
||||
await submitTaskReply(ctx.input.page, direction);
|
||||
const before = await pollUntil({ label: "Actual native same-turn steering advertised for saved human blue message", deadlineAt: Date.now() + 30_000,
|
||||
const before = await pollUntil({ label: "Running task queue states its harness steering capability for the saved human blue message", deadlineAt: Date.now() + 30_000,
|
||||
load: () => ctx.input.api.get<IssueQueuedCommentQueue>(`/api/issues/${task.id}/queued-comments`),
|
||||
accept: q => q.entries.length === 1 && q.entries[0]?.comment.body === direction && q.targetRunId === state.workerRunId && q.protocol === "paperclip_runner_v1" && q.steeringDisposition === "available" });
|
||||
accept: q => q.entries.length === 1 && q.entries[0]?.comment.body === direction && q.targetRunId === state.workerRunId && q.protocol === "paperclip_runner_v1"
|
||||
&& ["available", "unsupported"].includes(q.steeringDisposition) });
|
||||
const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session");
|
||||
if (before.entries[0]?.comment.authorUserId !== session.user.id || before.entries[0].comment.authorAgentId) throw new Error("Steering fixture lacks actual human message ownership");
|
||||
await expect(ctx.input.page.getByTestId(`task-chat-queued-steer-${before.entries[0].comment.id}`)).toBeEnabled();
|
||||
const targetId = before.entries[0].comment.id;
|
||||
await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible();
|
||||
const controls = await queuedSteeringControls(ctx, targetId, before.steeringDisposition);
|
||||
if (before.steeringDisposition === "available" ? !controls.steerControlEnabled : controls.steerControlEnabled || !controls.continuationControlVisible)
|
||||
throw new Error("The queued message controls contradict the advertised steering capability");
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
return { ...state, humanId: session.user.id, queueBefore: before, targetId: before.entries[0].comment.id };
|
||||
return { ...state, humanId: session.user.id, queueBefore: before, targetId, ...controls };
|
||||
}, gradeActiveSteering, {
|
||||
authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000,
|
||||
authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, oracleVersion: "paperclip.chat-catalog.runtime.steer-active.v3",
|
||||
setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
|
||||
sourceFiles: ["chat-catalog-domains/runtime-active-steering.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/steering.yaml"],
|
||||
async act(ctx, state) {
|
||||
@@ -74,30 +96,47 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act
|
||||
const mutation = { queueId: before.queueId, revision: before.revision, targetRunId: before.targetRunId };
|
||||
state.agentSteerStatus = await probeAgentPermission(ctx, `${endpoint}/${targetId}/steer`, "POST", mutation);
|
||||
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
|
||||
const response = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `${endpoint}/${targetId}/steer`);
|
||||
await ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`).click(); state.humanSteerSelected = true;
|
||||
const result = await response; state.steerStatus = result.status();
|
||||
const after = await ctx.input.api.get<IssueQueuedCommentQueue>(endpoint); state.queueAfterAction = after;
|
||||
if (result.status() === 200) {
|
||||
state.steerObservedBranch = "same-run-acknowledged";
|
||||
const run = await ctx.input.api.get<{ resultJson?: { queuedSteeringAcknowledgements?: Record<string, { status?: string; queueId?: string; turnId?: string }> } }>(`/api/heartbeat-runs/${state.workerRunId}`);
|
||||
const ack = run.resultJson?.queuedSteeringAcknowledgements?.[targetId];
|
||||
state.acknowledgedSameRun = ack?.status === "acknowledged" && ack.queueId === before.queueId && typeof ack.turnId === "string" && ack.turnId.length > 0;
|
||||
const retry = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.idempotentRetryStatus = retry.status(); await retry.dispose();
|
||||
} else {
|
||||
if (result.status() !== 409) throw new Error("Steering failed outside its supported visible rejection contract");
|
||||
state.steerObservedBranch = "provider-rejected-normal-continuation";
|
||||
const body = await result.json() as { code?: string; details?: { code?: string } };
|
||||
state.steerFailureCode = body.code ?? body.details?.code;
|
||||
if (!["steering_rejected", "steering_unsupported"].includes(String(state.steerFailureCode))) throw new Error("A non-provider queue conflict does not qualify as a provider steering rejection");
|
||||
await expect(ctx.input.page.getByText(/steer|temporarily unavailable|timed out/i).filter({ hasNot: ctx.input.page.locator("button") }).first()).toBeVisible();
|
||||
state.providerFailureVisible = true; state.failureRetainedMessage = after.entries.some(row => row.comment.id === targetId && row.comment.body === direction);
|
||||
if ((state.queueBefore as IssueQueuedCommentQueue).steeringDisposition === "unsupported") {
|
||||
// This harness cannot steer, and the page offers no Steer control. Attempt it once with the person's own session to
|
||||
// prove the product refuses rather than claiming delivery, then leave the message for normal admission.
|
||||
const attempt = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.humanSteerAttempted = true;
|
||||
state.steerStatus = attempt.status(); const body = await attempt.json().catch(() => ({})) as { code?: string; details?: { code?: string } }; await attempt.dispose();
|
||||
state.steerFailureCode = body.code ?? body.details?.code; state.steerObservedBranch = steeringBranch("unsupported", state.steerStatus);
|
||||
const after = await ctx.input.api.get<IssueQueuedCommentQueue>(endpoint); state.queueAfterAction = after;
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
|
||||
await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible();
|
||||
await expect(ctx.input.page.getByTestId("task-chat-composer-input").last()).toBeVisible(); state.supportedContinuationVisible = true;
|
||||
const controls = await queuedSteeringControls(ctx, targetId, after.steeringDisposition);
|
||||
state.unsupportedStateRetained = after.steeringDisposition === "unsupported" && after.targetRunId === state.workerRunId && !controls.steerControlEnabled && controls.continuationControlVisible;
|
||||
} else {
|
||||
const response = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `${endpoint}/${targetId}/steer`);
|
||||
await ctx.input.page.getByTestId(`task-chat-queued-steer-${targetId}`).click(); state.humanSteerSelected = true;
|
||||
const result = await response; state.steerStatus = result.status();
|
||||
const after = await ctx.input.api.get<IssueQueuedCommentQueue>(endpoint); state.queueAfterAction = after;
|
||||
if (result.status() === 200) {
|
||||
state.steerObservedBranch = "same-run-acknowledged";
|
||||
const run = await ctx.input.api.get<{ resultJson?: { queuedSteeringAcknowledgements?: Record<string, { status?: string; queueId?: string; turnId?: string }> } }>(`/api/heartbeat-runs/${state.workerRunId}`);
|
||||
const ack = run.resultJson?.queuedSteeringAcknowledgements?.[targetId];
|
||||
state.acknowledgedSameRun = ack?.status === "acknowledged" && ack.queueId === before.queueId && typeof ack.turnId === "string" && ack.turnId.length > 0;
|
||||
const retry = await ctx.input.page.request.post(`${endpoint}/${targetId}/steer`, { data: mutation }); state.idempotentRetryStatus = retry.status(); await retry.dispose();
|
||||
} else {
|
||||
if (result.status() !== 409) throw new Error("Steering failed outside its supported visible rejection contract");
|
||||
state.steerObservedBranch = "provider-rejected-normal-continuation";
|
||||
const body = await result.json() as { code?: string; details?: { code?: string } };
|
||||
state.steerFailureCode = body.code ?? body.details?.code;
|
||||
if (!["steering_rejected", "steering_unsupported"].includes(String(state.steerFailureCode))) throw new Error("A non-provider queue conflict does not qualify as a provider steering rejection");
|
||||
await expect(ctx.input.page.getByText(/steer|temporarily unavailable|timed out/i).filter({ hasNot: ctx.input.page.locator("button") }).first()).toBeVisible();
|
||||
state.providerFailureVisible = true; state.failureRetainedMessage = after.entries.some(row => row.comment.id === targetId && row.comment.body === direction);
|
||||
await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${targetId}`)).toBeVisible();
|
||||
await expect(ctx.input.page.getByTestId("task-chat-composer-input").last()).toBeVisible(); state.supportedContinuationVisible = true;
|
||||
}
|
||||
}
|
||||
await ctx.input.evidence("runtime-steering-observed-branch.json", { branch: state.steerObservedBranch, failureCode: state.steerFailureCode ?? null, status: state.steerStatus, issueId: task.id, runId: state.workerRunId, commentId: targetId, otherBranchLiveQualified: false });
|
||||
await ctx.input.capture("actual-steering-delivery", "Selected human steering action and actual native delivery or retained rejection", "runtime-steer-active.png");
|
||||
await writeFile((state.gate as { gate: string }).gate, "The human brief is ready. Implement a small semantic website.html using the actual design direction you received, with a visible h1 and a labelled signup action. Register the real website.html file as a downloadable artifact work product. Save the actual chosen color and design in a non-plan website-delivery task document, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.", "utf8");
|
||||
const build = "Implement a small semantic website.html with a visible h1 and a labelled signup action, register the real website.html file as a downloadable artifact work product, and save the actual chosen color and design in a non-plan website-delivery task document.";
|
||||
// Delivered same-turn direction is built now; a direction still queued reaches the next normal turn, which applies it to the same deliverable.
|
||||
await writeFile((state.gate as { gate: string }).gate, state.steerStatus === 200
|
||||
? `The human brief is ready. ${build} Use the actual design direction you received, then complete normally. Preserve saved human requirements for normal admission; do not create tasks or publish externally.`
|
||||
: `The human brief is ready. ${build} Then finish this current turn normally. Keep human requirements queued by Paperclip intact for normal delivery; when one arrives, apply it to website.html, register the updated file, update the delivery document, and complete the task. Do not create tasks or publish externally.`, "utf8");
|
||||
const finished = await pollUntil({ label: "Actual selected steering or preserved normal continuation produces its website", deadlineAt: Date.now() + 240_000,
|
||||
load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get<IssueQueuedCommentQueue>(endpoint), task: await ctx.input.api.get<Entity>(`/api/issues/${task.id}`) }),
|
||||
accept: v => v.task.status === "done" && v.queue.entries.length === 0 && v.runs.length === (state.steerStatus === 200 ? 2 : 3) && v.runs.every(row => row.status === "succeeded"),
|
||||
@@ -115,12 +154,14 @@ export const runtimeActiveSteeringCases = [runtimeDefinition("steering.steer-act
|
||||
const o = await observeRuntime(ctx, state), task = state.task as Entity;
|
||||
state.steerAuditBound = o.activity.some(row => row.action === "issue.queued_comment_steered" && row.entityId === task.id && row.actorId === state.humanId
|
||||
&& row.details?.commentId === state.targetId && row.details?.targetRunId === state.workerRunId && row.details?.duplicate === false);
|
||||
state.steeredActivityCount = o.activity.filter(row => row.action === "issue.queued_comment_steered" && row.entityId === task.id).length;
|
||||
if (state.outputRunId) {
|
||||
const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, String(state.outputRunId), ["website.html"]);
|
||||
if (artifact?.contentVerified) {
|
||||
state.actualBlueArtifact = hasVisibleBlueDesign(await renderRuntimeArtifact(ctx, artifact.content));
|
||||
}
|
||||
state.blueRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id && /blue/i.test(row.body ?? "")
|
||||
// The rendered page proves the colour; the document only has to be the consuming run's saved delivery record.
|
||||
state.deliveryRevisionBound = o.revisions.some(row => row.createdByRunId === state.outputRunId && row.createdByAgentId === (state.worker as Entity).id
|
||||
&& o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id));
|
||||
}
|
||||
return o;
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { gradeApprovalQueue, acornNamingDirection, ordinaryApprovalEntries } from "./runtime-approval-queue.js";
|
||||
import { gradeApprovalQueue, ordinaryApprovalEntries, pendingInputYield } from "./runtime-approval-queue.js";
|
||||
function valid() {
|
||||
const payload = { questionSet: { questions: [{ id: "name", prompt: "What is the product name?" }] } };
|
||||
return { scenarioId: "steering.approval-delivery", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [],
|
||||
runs: [{ id: "prior", status: "succeeded" }, { id: "canonical", status: "succeeded" }, { id: "consumer", status: "succeeded" }],
|
||||
state: { task: { id: "website" }, worker: { id: "morgan" }, humanId: "human", workerRunId: "prior", workerWasReallyWaiting: true, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true,
|
||||
questionBefore: { id: "question", kind: "ask_user_questions", status: "pending", issueId: "website", createdByAgentId: "morgan", sourceRunId: "prior", payload },
|
||||
runs: [{ id: "asked", status: "succeeded" }, { id: "prior", status: "succeeded" }, { id: "canonical", status: "succeeded" }, { id: "consumer", status: "succeeded" }],
|
||||
state: { task: { id: "website" }, worker: { id: "morgan" }, humanId: "human", questionRunId: "asked", questionRunYielded: true, workerRunId: "prior", workerWasReallyWaiting: true, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true,
|
||||
questionBefore: { id: "question", kind: "ask_user_questions", status: "pending", issueId: "website", createdByAgentId: "morgan", sourceRunId: "asked", payload },
|
||||
questionAfter: { id: "question", status: "answered", resolvedByUserId: "human" as string | null, resolvedByAgentId: null as string | null, payload, result: { answers: [{ questionId: "name", otherText: "Acorn" }] } },
|
||||
responseRetainedAcrossOrdinaryMutations: true, responseUneditable: true, queryReply: "Use Acorn for the existing naming question; submit that bound answer through its supported response control.",
|
||||
actualOrdinaryEdit: true, actualOrdinaryReorder: true, ordinaryReorderRestricted: false, actualOrdinaryDiscard: true, humanAnswerSubmitted: true, canonicalAnswerAttributed: false,
|
||||
@@ -13,13 +13,12 @@ function valid() {
|
||||
}
|
||||
it("accepts real human response retained separately from mutated ordinary comments", () => expect(gradeApprovalQueue(valid()).passed).toBe(true));
|
||||
it("accepts a permitted actual canonical agent response with native source attribution", () => { const o = valid(); Object.assign(o.state, { canonicalAnswerAttributed: true, humanAnswerSubmitted: false, queryReply: "I answered Morgan's naming question with Acorn; the response is saved for normal continuation." }); Object.assign(o.state.questionAfter, { resolvedByUserId: null, resolvedByAgentId: "ceo" }); expect(gradeApprovalQueue(o).passed).toBe(true); });
|
||||
it("accepts natural correct-answer guidance", () => { const o = valid(); o.state.queryReply = "The product name is Acorn. Use the question's response card so this decision stays attached to the naming request."; expect(gradeApprovalQueue(o).passed).toBe(true); });
|
||||
it("rejects a fabricated question without actual prior native provenance", () => { const o = valid(); o.state.questionBefore.createdByAgentId = "other"; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects an ordinary comment masquerading as bound response", () => { const o = valid(); o.state.responseQueue.entries[0]!.source.kind = "comment"; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects a canonical response recording a different name", () => { const o = valid(); Object.assign(o.state, { canonicalAnswerAttributed: true, humanAnswerSubmitted: false }); Object.assign(o.state.questionAfter, { resolvedByUserId: null, resolvedByAgentId: "ceo" }); o.state.questionAfter.result.answers[0]!.otherText = "Birch"; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects wrong accepted answer", () => { const o = valid(); o.state.questionAfter.result.answers[0]!.otherText = "Other"; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects revoking the decision by discarding ordinary text", () => { const o = valid(); o.state.responseRetainedAcrossOrdinaryMutations = false; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects an editable interaction response", () => { const o = valid(); o.state.responseQueue.entries[0]!.canEdit = true; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects source claim of answering without an attributed actual answer", () => { const o = valid(); o.state.queryReply = "I have answered with Acorn."; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects unsupported actor answering for the human", () => { const o = valid(); o.state.questionAfter.resolvedByUserId = "other"; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects ordinary message operations performed on a different queue", () => { const o = valid(); o.state.actualOrdinaryReorder = false; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects duplicate decision consumers", () => { const o = valid(); o.state.consumingRunCount = 2; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
@@ -27,8 +26,6 @@ it("rejects native input lacking the bound question ID and answer", () => { cons
|
||||
it("rejects loss of earlier settled work", () => { const o = valid(); o.state.initialWorkRetained = false; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects missing actual response evidence", () => expect(gradeApprovalQueue(undefined).passed).toBe(false));
|
||||
|
||||
it.each(["Use Birch, not Acorn, as the name", "Acorn is wrong; use Birch for the product name", "The requested name was Acorn; Use Birch instead to answer the question"])("rejects contradictory named direction without rescue by later human input", reply => { const o = valid(); o.state.queryReply = reply; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it.each(["Use Acorn, not Birch, as the product name", "The answer is Acorn; submit it on the naming question", "Name it Acorn and retain the actual answer on the response card"])("accepts natural correct naming direction", reply => expect(acornNamingDirection(reply)).toBe(true));
|
||||
it("restricts every mutating ID to ordinary entries in a mixed two-comment plus response queue", () => {
|
||||
const ordinary = { comment: { id: "ordinary" } }, response = { source: { kind: "interaction" }, comment: { id: "decision" } };
|
||||
const ids = ordinaryApprovalEntries({ entries: [ordinary, response, { comment: { id: "other" } }] } as Parameters<typeof ordinaryApprovalEntries>[0]).map(row => row.comment.id);
|
||||
@@ -36,4 +33,21 @@ it("restricts every mutating ID to ordinary entries in a mixed two-comment plus
|
||||
});
|
||||
it("accepts the real mixed-queue UI reorder refusal while preserving immutable decision placement", () => { const o = valid(); o.state.actualOrdinaryReorder = false; o.state.ordinaryReorderRestricted = true; expect(gradeApprovalQueue(o).passed).toBe(true); });
|
||||
|
||||
it("accepts explicit rejection of a different name while selecting the actual answer", () => expect(acornNamingDirection("Do not use Birch; the product name is Acorn.")).toBe(true));
|
||||
|
||||
// The canonical reply is read by a person, not matched by words: the verdict rests on the saved answer path.
|
||||
it.each(["Use Acorn for the existing naming question; submit that bound answer through its supported response control.", "Open Morgan's naming card and send Acorn there so the decision stays attached to it.", "Got it."])("verdict does not depend on reply wording: %s", reply => { const o = valid(); o.state.queryReply = reply; expect(gradeApprovalQueue(o).passed).toBe(true); });
|
||||
it("rejects a question whose run never parked on it", () => { const o = valid(); o.state.questionRunYielded = false; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects a question attributed to the later active turn instead of the turn that asked", () => { const o = valid(); o.state.questionBefore.sourceRunId = "prior"; o.state.questionRunId = "prior"; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects an answer with no actual recording path", () => { const o = valid(); o.state.humanAnswerSubmitted = false; expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
it("rejects a missing parked-question turn in the run ledger", () => { const o = valid(); o.runs.shift(); expect(gradeApprovalQueue(o).passed).toBe(false); });
|
||||
|
||||
// Shape recorded by the retained runner-opencode attempt (gha-42926276473): request_human_input parks the turn.
|
||||
const parked = { id: "asked", status: "succeeded", resultJson: { nativeResult: { reportedWorkDisposition: "yielded", evidence: [{ ref: "interaction:question" }],
|
||||
artifacts: [{ ref: "interaction:question", kind: "issue_thread_interaction" }], continuation: { kind: "response_wake", idempotencyKey: "interaction-response:question" } } } };
|
||||
it("recognizes the product's parked pending-input turn", () => expect(pendingInputYield(parked, "question")).toBe(true));
|
||||
it.each([
|
||||
["still running", { ...parked, status: "running" }],
|
||||
["completed instead of yielded", { ...parked, resultJson: { nativeResult: { ...parked.resultJson.nativeResult, reportedWorkDisposition: "completed" } } }],
|
||||
["parked on another interaction", { ...parked, resultJson: { nativeResult: { ...parked.resultJson.nativeResult, continuation: { kind: "response_wake", idempotencyKey: "interaction-response:other" } } } }],
|
||||
["no result", { id: "asked", status: "succeeded", resultJson: null }],
|
||||
])("rejects a run that is not parked on the question (%s)", (_label, run) => expect(pendingInputYield(run, "question")).toBe(false));
|
||||
@@ -8,11 +8,17 @@ import { checks, type Entity } from "./internal-support.js";
|
||||
import { prepareRuntimeWorkerGate, readRuntimeTaskGate, releaseRuntimeWorker } from "./runtime-busy-worker.js";
|
||||
import { runtimeQueueCases } from "./runtime-queue.js";
|
||||
import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js";
|
||||
export function acornNamingDirection(reply: string) {
|
||||
if (/\b(?:not|instead of|rather than|avoid|reject)\s+(?:the name\s+)?Acorn\b|\bAcorn\s+(?:is|was|would be)\s+(?:the\s+)?(?:wrong|incorrect|invalid|rejected)\b/i.test(reply)) return false;
|
||||
const selectedNames = [...reply.matchAll(/\b(?:[Uu]se|[Cc]hoose|[Nn]ame it|[Cc]all it|[Aa]nswer with)\s+["']?([A-Z][a-zA-Z0-9-]*)/g)].filter(match => !/\b(?:not|don't|avoid)\s*$/i.test(reply.slice(Math.max(0, (match.index ?? 0) - 12), match.index))).map(match => match[1]!);
|
||||
if (selectedNames.some(name => !["Acorn", "The", "Submit", "That", "Its", "This"].includes(name))) return false;
|
||||
return /\bAcorn\b/i.test(reply) && /question|answer|respond|decision|nam(?:e|ing)/i.test(reply);
|
||||
type PendingInputRun = { id?: string; status?: string; resultJson?: { nativeResult?: { reportedWorkDisposition?: string; continuation?: { kind?: string; idempotencyKey?: string };
|
||||
evidence?: Array<{ ref?: string }>; artifacts?: Array<{ ref?: string }> } } | null };
|
||||
/**
|
||||
* A native worker that saves a durable wake-based question parks its turn: the run
|
||||
* succeeds with a yielded disposition whose continuation is the question's response
|
||||
* wake. That saved yield, not a still-running process, is the product's pending-input state.
|
||||
*/
|
||||
export function pendingInputYield(run: PendingInputRun | null | undefined, questionId: string) {
|
||||
const result = run?.resultJson?.nativeResult, ref = `interaction:${questionId}`;
|
||||
return run?.status === "succeeded" && result?.reportedWorkDisposition === "yielded" && result.continuation?.kind === "response_wake"
|
||||
&& result.continuation.idempotencyKey === `interaction-response:${questionId}` && [...(result.evidence ?? []), ...(result.artifacts ?? [])].some(row => row.ref === ref);
|
||||
}
|
||||
export function ordinaryApprovalEntries(queue: Pick<IssueQueuedCommentQueue, "entries">) { return queue.entries.filter(row => !row.source); }
|
||||
export function gradeApprovalQueue(value: unknown) {
|
||||
@@ -20,54 +26,71 @@ export function gradeApprovalQueue(value: unknown) {
|
||||
c.add("real-question-response-and-ordinary-queue-evidence", !!o, "The actual native question, human response, separate queue mutations and once-only consuming native input are required."); if (!o) return c.result();
|
||||
const s = o.state, question = s.questionBefore as IssueThreadInteraction, answered = s.questionAfter as IssueThreadInteraction;
|
||||
c.add("genuine-prior-worker-question-and-saved-comments", !!question && question.kind === "ask_user_questions" && question.status === "pending" && question.issueId === (s.task as Entity).id
|
||||
&& question.createdByAgentId === (s.worker as Entity).id && question.sourceRunId === s.workerRunId && s.workerWasReallyWaiting === true
|
||||
&& question.createdByAgentId === (s.worker as Entity).id && question.sourceRunId === s.questionRunId && s.questionRunYielded === true
|
||||
&& typeof s.workerRunId === "string" && s.workerRunId !== s.questionRunId && s.workerWasReallyWaiting === true
|
||||
&& s.ordinaryQueueOwned === true && s.ordinaryQueuedBeforeAnswer === true,
|
||||
"Morgan's actual native turn asks the naming question and remains active while the real human's separate ordinary comments are pending.");
|
||||
"Morgan's actual native turn saves the naming question and parks on it; a later real Morgan turn is still active while the human's separate ordinary comments wait.");
|
||||
const results = answered?.result as { answers?: Array<{ questionId?: string; otherText?: string }> } | null;
|
||||
c.add("actual-bound-human-answer-preserved", answered?.id === question.id && answered.status === "answered" && ((answered.resolvedByUserId === s.humanId && !answered.resolvedByAgentId && s.humanAnswerSubmitted === true) || (answered.resolvedByAgentId === o.leadId && s.canonicalAnswerAttributed === true))
|
||||
&& JSON.stringify(answered.payload) === JSON.stringify(question.payload) && results?.answers?.some(row => row.questionId === "name" && row.otherText === "Acorn") === true
|
||||
&& s.responseRetainedAcrossOrdinaryMutations === true && s.responseUneditable === true,
|
||||
"Acorn remains the same immutable typed interaction response, distinct from the editable/deletable ordinary comment queue.");
|
||||
c.add("human-guidance-and-real-ordinary-mutations", acornNamingDirection(String(s.queryReply))
|
||||
&& (s.canonicalAnswerAttributed === true || !/I(?:'ve| have)? (?:already )?(?:answered|submitted|accepted)/i.test(String(s.queryReply)))
|
||||
&& s.actualOrdinaryEdit === true && (s.actualOrdinaryReorder === true || s.ordinaryReorderRestricted === true) && s.actualOrdinaryDiscard === true && (s.humanAnswerSubmitted === true || s.canonicalAnswerAttributed === true),
|
||||
"Canonical chat identifies the intended answer and actual human response control; editing, ordering and discarding ordinary messages cannot impersonate or cancel it.");
|
||||
c.add("bound-answer-path-and-real-ordinary-mutations", (s.humanAnswerSubmitted === true) !== (s.canonicalAnswerAttributed === true)
|
||||
&& s.actualOrdinaryEdit === true && (s.actualOrdinaryReorder === true || s.ordinaryReorderRestricted === true) && s.actualOrdinaryDiscard === true,
|
||||
"Exactly one actual path records Acorn on the bound question (the human's response control, or an attributed canonical response); editing, ordering and discarding ordinary messages cannot impersonate or cancel it.");
|
||||
const response = s.responseQueue as IssueQueuedCommentQueue;
|
||||
c.add("response-own-delivery-semantics-and-once-only-admission", response?.entries.length === 1 && response.entries[0]?.source?.kind === "interaction"
|
||||
&& response.entries[0].source.interactionId === question.id && response.entries[0].canEdit === false && response.entries[0].canDiscard === false
|
||||
&& s.consumingRunCount === 1 && s.responseNativeInputBound === true && s.queueEmptyAfterDelivery === true
|
||||
&& o.runs.length === 3 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && s.actualAcornResult === true && s.initialWorkRetained === true,
|
||||
&& o.runs.length === 4 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && s.actualAcornResult === true && s.initialWorkRetained === true,
|
||||
"After the original owner settles, exactly one eligible native continuation receives the bound answer and saves actual Acorn work without replaying or losing it.");
|
||||
return c.result();
|
||||
}
|
||||
export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-delivery", async ctx => {
|
||||
const prepared = await prepareRuntimeWorkerGate(ctx, { taskBound: true }), root = `/api/companies/${ctx.input.fixtures.company.id}`;
|
||||
const task = await ctx.input.api.post<Entity>(`${root}/issues`, { title: "Website naming task", status: "todo", assigneeAgentId: prepared.worker.id,
|
||||
description: `We need a product name before continuing the website. During this actual native run, create exactly one pending ask_user_questions interaction using the supported public issue interaction API, authenticated as yourself with this actual run identity. Use idempotencyKey naming-${ctx.input.nonce}, title Website product name, continuationPolicy wake_assignee, and payload {version:1,supersedeOnUserComment:false,questionSet:{schema:"paperclip.question_set.v1",title:"Website product name",submitLabel:"Submit answers",questions:[{id:"name",prompt:"What is the product name?",answerMode:"text",required:true}]}}. This is an ordinary durable task question, not an artificial provider runtime-request receipt. Do not choose a name or create another interaction. After the real question is saved, run ${prepared.gate.command} once and wait for its actual output. Then save the returned pending-work note as a task document and finish this current turn normally, leaving queued human answers for normal continuation. On a later normal continuation, read the real submitted name response, save a non-plan website-delivery document using that name and a short semantic website outline, and complete the same task.` });
|
||||
const waiting = await pollUntil({ label: "Real native naming question while worker waits in its own tool process", deadlineAt: Date.now() + 180_000,
|
||||
load: async () => ({ runs: await ctx.allRuns(), ready: await readRuntimeTaskGate(ctx, prepared.gate, task.id, prepared.worker.id), questions: await ctx.input.api.get<IssueThreadInteraction[]>(`/api/issues/${task.id}/interactions`) }),
|
||||
accept: v => v.runs.length === 1 && v.runs[0]?.agentId === prepared.worker.id && v.runs[0].status === "running" && v.ready === "waiting"
|
||||
&& v.questions.length === 1 && v.questions[0]?.kind === "ask_user_questions" && v.questions[0].status === "pending" && v.questions[0].createdByAgentId === prepared.worker.id && v.questions[0].sourceRunId === v.runs[0].id,
|
||||
reject: v => v.runs.length > 1 || v.runs.some(row => ["failed", "cancelled", "succeeded", "timed_out"].includes(row.status)) ? "The actual question fixture did not reach its running pending-input boundary" : undefined });
|
||||
description: `We need a product name before continuing the website. If this task has no naming question yet, create exactly one pending ask_user_questions interaction on it, authenticated as yourself with this actual run identity. Use idempotencyKey naming-${ctx.input.nonce}, title Website product name, continuationPolicy wake_assignee, and payload {version:1,supersedeOnUserComment:false,questionSet:{schema:"paperclip.question_set.v1",title:"Website product name",submitLabel:"Submit answers",questions:[{id:"name",prompt:"What is the product name?",answerMode:"text",required:true}]}}. This is an ordinary durable task question, not an artificial provider runtime-request receipt. Do not choose a name or create another interaction. If a human message wakes you while that naming question is still pending, run ${prepared.gate.command} once and wait for its actual output, then follow that output, leaving queued human answers for normal continuation. On a later normal continuation that delivers the submitted name response, save a non-plan website-delivery document using that name and a short semantic website outline, and complete the same task.` });
|
||||
const interactions = () => ctx.input.api.get<IssueThreadInteraction[]>(`/api/issues/${task.id}/interactions`);
|
||||
// The product parks a native turn as soon as its durable wake-based question is
|
||||
// saved, so the pending-input boundary is the saved yield, not a running process.
|
||||
const asked = await pollUntil({ label: "Real native naming question parks Morgan's turn on pending input", deadlineAt: Date.now() + 180_000,
|
||||
load: async () => { const runs = await ctx.allRuns(), questions = await interactions();
|
||||
const run = runs.length === 1 && runs[0]?.status === "succeeded" ? await ctx.input.api.get<PendingInputRun & { id: string }>(`/api/heartbeat-runs/${runs[0].id}`) : null;
|
||||
return { runs, questions, run }; },
|
||||
accept: v => v.runs.length === 1 && v.runs[0]?.agentId === prepared.worker.id && v.questions.length === 1 && v.questions[0]?.kind === "ask_user_questions"
|
||||
&& v.questions[0].status === "pending" && v.questions[0].createdByAgentId === prepared.worker.id && v.questions[0].sourceRunId === v.runs[0].id && pendingInputYield(v.run, v.questions[0].id),
|
||||
reject: v => v.runs.length > 1 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) || (v.run && !(v.questions.length === 1 && pendingInputYield(v.run, v.questions[0]!.id)))
|
||||
? "The actual question fixture did not park its native turn on one pending naming question" : undefined });
|
||||
const question = asked.questions[0]!, questionRunId = asked.runs[0]!.id;
|
||||
const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
|
||||
const form = ctx.input.page.getByTestId("task-chat-composer-takeover"); await expect(form).toBeVisible(); await form.getByRole("button", { name: /^Dismiss / }).click();
|
||||
// An ordinary human message while the question stays open is normal discussion; it wakes Morgan into a new turn.
|
||||
await submitTaskReply(ctx.input.page, "Keep the hero section simple while the name is pending.");
|
||||
const busy = await pollUntil({ label: "Real Morgan turn active in its own tool process while the naming question stays pending", deadlineAt: Date.now() + 180_000,
|
||||
load: async () => ({ runs: await ctx.allRuns(), ready: await readRuntimeTaskGate(ctx, prepared.gate, task.id, prepared.worker.id), questions: await interactions() }),
|
||||
accept: v => v.runs.length === 2 && v.runs.some(row => row.id !== questionRunId && row.agentId === prepared.worker.id && row.status === "running") && v.ready === "waiting"
|
||||
&& v.questions.length === 1 && v.questions[0]?.id === question.id && v.questions[0].status === "pending",
|
||||
reject: v => v.runs.length > 2 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) || v.runs.some(row => row.id !== questionRunId && row.status === "succeeded")
|
||||
|| v.questions.length !== 1 || v.questions[0]?.status !== "pending" ? "The discussion turn did not stay active behind the pending naming question" : undefined });
|
||||
const workerRunId = busy.runs.find(row => row.id !== questionRunId)!.id;
|
||||
const messages = ["Preserve the original layout work.", "Use accessible labels."];
|
||||
for (const body of messages) await submitTaskReply(ctx.input.page, body);
|
||||
const queue = await pollUntil({ label: "Actual ordinary human comments remain separate behind running Morgan", deadlineAt: Date.now() + 20_000,
|
||||
load: () => ctx.input.api.get<IssueQueuedCommentQueue>(`/api/issues/${task.id}/queued-comments`), accept: q => q.entries.length === 2 && q.entries.every(row => !row.source && messages.includes(row.comment.body)) });
|
||||
load: () => ctx.input.api.get<IssueQueuedCommentQueue>(`/api/issues/${task.id}/queued-comments`),
|
||||
accept: q => q.targetRunId === workerRunId && q.entries.length === 2 && q.entries.every(row => !row.source && messages.includes(row.comment.body)) });
|
||||
const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session");
|
||||
if (queue.entries.some(row => row.comment.authorUserId !== session.user.id || !row.canEdit || !row.canDiscard)) throw new Error("Ordinary queue is not actually owned by the signed-in human");
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
return { worker: prepared.worker, gate: prepared.gate, task, workerRunId: waiting.runs[0]!.id, workerWasReallyWaiting: true, questionBefore: waiting.questions[0]!, ordinaryQueueBefore: queue,
|
||||
return { worker: prepared.worker, gate: prepared.gate, task, questionRunId, questionRunYielded: true, workerRunId, workerWasReallyWaiting: true, questionBefore: question, ordinaryQueueBefore: queue,
|
||||
humanId: session.user.id, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true };
|
||||
}, gradeApprovalQueue, {
|
||||
setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
|
||||
authentication: "human-session", expectedRunCount: 3, attemptTimeoutMs: 900_000,
|
||||
authentication: "human-session", expectedRunCount: 4, attemptTimeoutMs: 900_000, oracleVersion: "paperclip.chat-catalog.runtime.approval-delivery.v3",
|
||||
sourceFiles: ["chat-catalog-domains/runtime-approval-queue.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"],
|
||||
async act(ctx, state) {
|
||||
const canonicalPhase = await pollUntil({ label: "Canonical naming answer guidance while real worker remains active", deadlineAt: Date.now() + 150_000,
|
||||
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
|
||||
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") });
|
||||
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 3 && v.runs.some(row => row.id === state.workerRunId && row.status === "running") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") });
|
||||
state.queryReply = (await observeRuntime(ctx, state)).reply;
|
||||
const task = state.task as Entity, question = state.questionBefore as IssueThreadInteraction, source = ctx.input.page.url(), endpoint = `/api/issues/${task.id}/queued-comments`;
|
||||
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
|
||||
@@ -126,12 +149,12 @@ export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-d
|
||||
await expect(ctx.input.page.getByTestId(`task-chat-queued-discard-${question.id}`)).toBeDisabled();
|
||||
await ctx.input.capture("actual-immutable-answer-queue", "Bound Acorn answer remains distinct from edited, reordered and removed ordinary comments", "runtime-approval-queue.png");
|
||||
state.initialMarker = `Earlier website work ${ctx.input.nonce}`;
|
||||
await writeFile((state.gate as { gate: string }).gate, `The current pending-input turn is settling. Save a task document with the exact reference ${state.initialMarker} preserving earlier website work, then finish this turn normally. Do not manufacture or pre-consume a name decision; leave the actual accepted response for normal Paperclip admission.`, "utf8");
|
||||
await writeFile((state.gate as { gate: string }).gate, `This turn is settling while the naming question waits. Save a task document with the exact reference ${state.initialMarker} preserving earlier website work, then finish this turn normally. Do not manufacture or pre-consume a name decision; leave the actual accepted response for normal Paperclip admission.`, "utf8");
|
||||
const done = await pollUntil({ label: "One normal successor receives actual immutable name response", deadlineAt: Date.now() + 240_000,
|
||||
load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get<IssueQueuedCommentQueue>(endpoint), task: await ctx.input.api.get<Entity>(`/api/issues/${task.id}`) }),
|
||||
accept: v => v.runs.length === 3 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done",
|
||||
reject: v => v.runs.length > 3 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Accepted decision delivery failed or replayed work" : undefined });
|
||||
const consumers = done.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId); state.consumingRunCount = consumers.length; state.consumerRunId = consumers[0]?.id;
|
||||
accept: v => v.runs.length === 4 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done",
|
||||
reject: v => v.runs.length > 4 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Accepted decision delivery failed or replayed work" : undefined });
|
||||
const consumers = done.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId && row.id !== state.questionRunId); state.consumingRunCount = consumers.length; state.consumerRunId = consumers[0]?.id;
|
||||
const full = consumers[0] && await ctx.input.api.get<{ contextSnapshot?: { interactionId?: string }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumers[0].id}`);
|
||||
state.responseNativeInputBound = full?.contextSnapshot?.interactionId === question.id && full.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(question.id) === true && full.runnerProfileJson.nativeExecutionInput.task.prompt.includes("Acorn");
|
||||
state.queueEmptyAfterDelivery = done.queue.entries.length === 0;
|
||||
|
||||
@@ -7,11 +7,13 @@ const prepare = vi.hoisted(() => vi.fn());
|
||||
const poll = vi.hoisted(() => vi.fn());
|
||||
vi.mock("./runtime-busy-worker.js", async original => ({ ...await original<object>(), prepareRuntimeWorkerGate: prepare }));
|
||||
vi.mock("../api.js", async original => ({ ...await original<object>(), pollUntil: poll }));
|
||||
import { taskBoundRuntimeGate } from "./runtime-busy-worker.js";
|
||||
import { readRuntimeTaskGate, taskBoundRuntimeGate } from "./runtime-busy-worker.js";
|
||||
import { runtimeApprovalQueueCases } from "./runtime-approval-queue.js";
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); });
|
||||
async function fixture() {
|
||||
afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); poll.mockReset(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); });
|
||||
const parkedResult = { nativeResult: { reportedWorkDisposition: "yielded", evidence: [{ ref: "interaction:question" }],
|
||||
continuation: { kind: "response_wake", idempotencyKey: "interaction-response:question" } } };
|
||||
async function fixture(run: { status: string; resultJson: unknown } = { status: "succeeded", resultJson: parkedResult }) {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "approval-owned-task-")); roots.push(root); vi.stubEnv("PAPERCLIP_RUNNER_E2E_TEMP_ROOT", root);
|
||||
const cwd = path.join(root, "actual-isolated-task"); await mkdir(cwd);
|
||||
const gate = taskBoundRuntimeGate("approval-owned");
|
||||
@@ -20,26 +22,39 @@ async function fixture() {
|
||||
const workspace = { id: "workspace", companyId: "company", cwd };
|
||||
const post = vi.fn(async (_url: string, _body: unknown) => task);
|
||||
prepare.mockImplementation(async (_ctx, options) => { expect(options).toEqual({ taskBound: true }); return { gate, worker: { id: "morgan" } }; });
|
||||
const ctx = { input: { nonce: "approval-owned", fixtures: { company: { id: "company" } }, api: { post, get: async (url: string) => url.endsWith("/workspace") ? { workspace } : url.endsWith("/interactions") ? [{ id: "question", kind: "ask_user_questions", status: "pending", issueId: "task", createdByAgentId: "morgan", sourceRunId: "run" }] : task } }, allRuns: async () => [{ id: "run", agentId: "morgan", status: "running" }] } as unknown as ChatCatalogContext;
|
||||
const get = async (url: string) => url.endsWith("/workspace") ? { workspace }
|
||||
: url.endsWith("/interactions") ? [{ id: "question", kind: "ask_user_questions", status: "pending", issueId: "task", createdByAgentId: "morgan", sourceRunId: "run" }]
|
||||
: url === "/api/heartbeat-runs/run" ? { id: "run", ...run } : task;
|
||||
const ctx = { input: { nonce: "approval-owned", fixtures: { company: { id: "company" } }, api: { post, get } }, allRuns: async () => [{ id: "run", agentId: "morgan", status: run.status }] } as unknown as ChatCatalogContext;
|
||||
return { ctx, root, cwd, task, workspace, gate, post };
|
||||
}
|
||||
it("approval setup consumes actual assigned-workspace readiness using a cwd-local bounded command", async () => {
|
||||
it("approval setup gates the question on the product's parked pending-input turn and scripts a cwd-local bounded wait", async () => {
|
||||
const f = await fixture();
|
||||
poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(true); throw new Error("verified active boundary"); });
|
||||
await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("verified active boundary");
|
||||
poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(true); expect(options.reject(value)).toBeUndefined(); throw new Error("verified pending-input boundary"); });
|
||||
await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("verified pending-input boundary");
|
||||
expect(f.post.mock.calls[0]![1]).toMatchObject({ assigneeAgentId: "morgan", status: "todo" });
|
||||
const body = f.post.mock.calls[0]![1] as { description: string };
|
||||
expect(body.description).toContain(`run ${f.gate.command} once`);
|
||||
expect(body.description).not.toContain(f.root);
|
||||
});
|
||||
it.each([
|
||||
["a question turn that is still running", { status: "running", resultJson: null }, false],
|
||||
["a turn that completed without parking on the question", { status: "succeeded", resultJson: { nativeResult: { ...parkedResult.nativeResult, reportedWorkDisposition: "completed" } } }, true],
|
||||
])("approval setup does not accept %s", async (_label, run, rejected) => {
|
||||
const f = await fixture(run);
|
||||
poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(false); expect(Boolean(options.reject(value))).toBe(rejected); throw new Error("checked"); });
|
||||
await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("checked");
|
||||
});
|
||||
it("the active-turn gate binds the actual assigned task workspace", async () => {
|
||||
const f = await fixture();
|
||||
await expect(readRuntimeTaskGate(f.ctx, f.gate, "task", "morgan")).resolves.toBe("waiting");
|
||||
expect(f.gate.boundTaskId).toBe("task"); expect(f.gate.boundWorkspaceId).toBe("workspace");
|
||||
expect(f.gate.ready).toBe(path.join(await realpath(f.cwd), `${f.gate.gateName}.waiting`));
|
||||
});
|
||||
it.each(["company", "assignee", "workspace"])("approval setup rejects foreign %s readiness before accepting the question boundary", async invalid => {
|
||||
it.each(["company", "assignee", "workspace"])("the active-turn gate rejects foreign %s readiness", async invalid => {
|
||||
const f = await fixture();
|
||||
if (invalid === "company") f.workspace.companyId = "foreign";
|
||||
if (invalid === "assignee") f.task.assigneeAgentId = "other";
|
||||
if (invalid === "workspace") f.task.executionWorkspaceId = "other";
|
||||
const accept = vi.fn(); poll.mockImplementationOnce(async options => { await options.load(); accept(); });
|
||||
await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow(/actual company, assignee and workspace/);
|
||||
expect(accept).not.toHaveBeenCalled();
|
||||
await expect(readRuntimeTaskGate(f.ctx, f.gate, "task", "morgan")).rejects.toThrow(/actual company, assignee and workspace/);
|
||||
});
|
||||
Reference in new issue
Block a user