diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 79a751d53d5..7e25421f14b 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5451,9 +5451,10 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.approval-delivery.v5", + "oracleVersion": "paperclip.chat-catalog.runtime.approval-delivery.v6", "coverageGaps": [], - "expectedRunCount": 4, + "expectedRunCount": 5, + "minimumRunCount": 4, "turnTimeoutMs": 120000, "attemptTimeoutMs": 900000, "authentication": "human-session", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts index 36502c97ff9..c5b42ff9483 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeApprovalQueue, leadRelayEntries, ordinaryApprovalEntries, pendingInputYield } from "./runtime-approval-queue.js"; +import { approvalResponseListing, classifyApprovalSuccessors, gradeApprovalQueue, leadRelayEntries, ordinaryApprovalEntries, pendingInputYield } from "./runtime-approval-queue.js"; function valid() { const payload = { questionSet: { questions: [{ id: "name", prompt: "What is the product name?" }] } }; return { scenarioId: "steering.approval-delivery", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [], @@ -9,16 +9,17 @@ function valid() { questionAfter: { id: "question", status: "answered", resolvedByUserId: "human" as string | null, resolvedByAgentId: null as string | null, payload, result: { answers: [{ questionId: "name", otherText: "Acorn" }] } }, responseRetainedAcrossOrdinaryMutations: true, responseUneditable: true, queryReply: "Use Acorn for the existing naming question; submit that bound answer through its supported response control.", actualOrdinaryEdit: true, actualOrdinaryReorder: true, ordinaryReorderRestricted: false, actualOrdinaryDiscard: true, humanAnswerSubmitted: true, canonicalAnswerAttributed: false, - responseQueue: { entries: [{ source: { kind: "interaction", interactionId: "question" }, canEdit: false, canDiscard: false }] }, consumingRunCount: 1, responseNativeInputBound: true, queueEmptyAfterDelivery: true, actualAcornResult: true, initialWorkRetained: true } }; + responseQueue: { entries: [{ source: { kind: "interaction", interactionId: "question" }, canEdit: false, canDiscard: false }] } as { entries: unknown[] }, responseListing: "listed", responseListedImmutable: true, + consumingRunCount: 1, relayRunCount: 0, unexpectedSuccessorRunCount: 0, responseNativeInputBound: true, queueEmptyAfterDelivery: true, actualAcornResult: true, initialWorkRetained: true } }; } it("accepts real human response retained separately from mutated ordinary comments", () => expect(gradeApprovalQueue(valid()).passed).toBe(true)); it("accepts a permitted actual canonical agent response with native source attribution", () => { const o = valid(); Object.assign(o.state, { canonicalAnswerAttributed: true, humanAnswerSubmitted: false, queryReply: "I answered Morgan's naming question with Acorn; the response is saved for normal continuation." }); Object.assign(o.state.questionAfter, { resolvedByUserId: null, resolvedByAgentId: "ceo" }); expect(gradeApprovalQueue(o).passed).toBe(true); }); it("rejects a fabricated question without actual prior native provenance", () => { const o = valid(); o.state.questionBefore.createdByAgentId = "other"; expect(gradeApprovalQueue(o).passed).toBe(false); }); -it("rejects an ordinary comment masquerading as bound response", () => { const o = valid(); o.state.responseQueue.entries[0]!.source.kind = "comment"; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects an ordinary comment masquerading as bound response", () => { const o = valid(); (o.state.responseQueue.entries[0] as { source: { kind: string } }).source.kind = "comment"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects a canonical response recording a different name", () => { const o = valid(); Object.assign(o.state, { canonicalAnswerAttributed: true, humanAnswerSubmitted: false }); Object.assign(o.state.questionAfter, { resolvedByUserId: null, resolvedByAgentId: "ceo" }); o.state.questionAfter.result.answers[0]!.otherText = "Birch"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects wrong accepted answer", () => { const o = valid(); o.state.questionAfter.result.answers[0]!.otherText = "Other"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects revoking the decision by discarding ordinary text", () => { const o = valid(); o.state.responseRetainedAcrossOrdinaryMutations = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); -it("rejects an editable interaction response", () => { const o = valid(); o.state.responseQueue.entries[0]!.canEdit = true; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects an editable interaction response", () => { const o = valid(); (o.state.responseQueue.entries[0] as { canEdit: boolean }).canEdit = true; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects unsupported actor answering for the human", () => { const o = valid(); o.state.questionAfter.resolvedByUserId = "other"; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects ordinary message operations performed on a different queue", () => { const o = valid(); o.state.actualOrdinaryReorder = false; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects duplicate decision consumers", () => { const o = valid(); o.state.consumingRunCount = 2; expect(gradeApprovalQueue(o).passed).toBe(false); }); @@ -52,14 +53,47 @@ it.each([ ["no result", { id: "asked", status: "succeeded", resultJson: null }], ])("rejects a run that is not parked on the question (%s)", (_label, run) => expect(pendingInputYield(run, "question")).toBe(false)); -// Retained runner-opencode attempt (gha-43077703607): the CEO cannot answer a board-only question form, so it relayed the -// person's answer as its own comment on the task. That comment queues beside the person's messages and the bound response. -it("accepts the chat lead's own relayed comment beside the bound response", () => { const o = valid(); - o.state.responseQueue.entries.push({ comment: { id: "relay", authorAgentId: "ceo", authorUserId: null } } as never); expect(gradeApprovalQueue(o).passed).toBe(true); }); +// Retained runner-opencode attempt (gha-43120714598): the chat lead relayed the person's answer as its own comment on the task. +// The composer lists one wake at a time and prefers that editable comment, so the answer's own wake waited unlisted behind it. +// After the gate, the lead's relay may take one extra worker turn; the answer must still reach exactly one bound turn. +function behindRelay() { const o = valid(); + Object.assign(o.state, { responseQueue: { entries: [{ comment: { id: "relay", authorAgentId: "ceo", authorUserId: null } }] }, responseListing: "behind_lead_relay", responseListedImmutable: undefined, relayRunCount: 1 }); + o.runs.push({ id: "relay-turn", status: "succeeded" }); return o; } +it("accepts the answer waiting unlisted behind the chat lead's own relay when it is then delivered once", () => expect(gradeApprovalQueue(behindRelay()).passed).toBe(true)); +it("accepts the answer behind the relay when the relay never needed its own turn", () => { const o = behindRelay(); o.state.relayRunCount = 0; o.runs.pop(); expect(gradeApprovalQueue(o).passed).toBe(true); }); +it("rejects an answer that waited behind the relay but was never delivered to a bound turn", () => { const o = behindRelay(); o.state.consumingRunCount = 0; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects an answer behind the relay delivered to two turns", () => { const o = behindRelay(); o.state.consumingRunCount = 2; o.runs.push({ id: "replay", status: "succeeded" }); expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a person's message still queued behind the relay", () => { const o = behindRelay(); + o.state.responseQueue.entries.push({ comment: { id: "left", authorUserId: "human", authorAgentId: null } }); expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a worker turn that is neither the bound answer nor the lead's relay", () => { const o = behindRelay(); o.state.unexpectedSuccessorRunCount = 1; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects more relay-only turns than one relay wake can produce", () => { const o = behindRelay(); o.state.relayRunCount = 2; o.runs.push({ id: "relay-2", status: "succeeded" }); expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a recorded listing that disagrees with the queue actually observed", () => { const o = behindRelay(); o.state.responseListing = "listed"; expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("rejects a listed answer whose card controls were never shown as locked", () => { const o = valid(); o.state.responseListedImmutable = undefined as never; expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects a person's ordinary message left beside the bound response", () => { const o = valid(); - o.state.responseQueue.entries.push({ comment: { id: "left", authorUserId: "human", authorAgentId: null } } as never); expect(gradeApprovalQueue(o).passed).toBe(false); }); + o.state.responseQueue.entries.push({ comment: { id: "left", authorUserId: "human", authorAgentId: null } }); expect(gradeApprovalQueue(o).passed).toBe(false); }); it("rejects a second bound response entry", () => { const o = valid(); - o.state.responseQueue.entries.push({ source: { kind: "interaction", interactionId: "question" }, canEdit: false, canDiscard: false } as never); expect(gradeApprovalQueue(o).passed).toBe(false); }); + o.state.responseQueue.entries.push({ source: { kind: "interaction", interactionId: "question" }, canEdit: false, canDiscard: false }); expect(gradeApprovalQueue(o).passed).toBe(false); }); +it("classifies the queue listing from the observed entries", () => { + const answer = { source: { kind: "interaction", interactionId: "question" }, canEdit: false, canDiscard: false, comment: { id: "question" } }; + const relay = { comment: { id: "relay", authorAgentId: "ceo", authorUserId: null } }, mine = { comment: { id: "mine", authorUserId: "human", authorAgentId: null } }; + const listing = (entries: unknown[]) => approvalResponseListing({ entries } as Parameters[0], "question", "ceo"); + expect(listing([answer])).toBe("listed"); + expect(listing([relay])).toBe("behind_lead_relay"); + expect(listing([relay, mine])).toBeNull(); + expect(listing([{ ...answer, canEdit: true }])).toBeNull(); + expect(listing([{ ...answer, source: { kind: "interaction", interactionId: "other" } }])).toBeNull(); + expect(listing([])).toBeNull(); +}); +it("splits later worker turns into the bound answer, the lead's relay and anything else", () => { + const split = classifyApprovalSuccessors([ + { id: "bound", contextSnapshot: { interactionId: "question" } }, + { id: "relay", contextSnapshot: { wakeCommentIds: ["relay-comment"] } }, + { id: "mixed", contextSnapshot: { wakeCommentIds: ["relay-comment", "person-comment"] } }, + { id: "other-answer", contextSnapshot: { interactionId: "other" } }, + { id: "bare", contextSnapshot: {} }, + ], "question", new Set(["relay-comment"])); + expect(split).toEqual({ bound: ["bound"], relay: ["relay"], other: ["mixed", "other-answer", "bare"] }); +}); it("keeps only the person's own messages as editable ordinary entries", () => { const queue = { entries: [{ comment: { id: "mine", authorUserId: "human", authorAgentId: null } }, { comment: { id: "relay", authorUserId: null, authorAgentId: "ceo" } }, { source: { kind: "interaction" }, comment: { id: "decision" } }] } as unknown as Parameters[0]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts index e88988421ac..238385b735a 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts @@ -27,6 +27,31 @@ export function ordinaryApprovalEntries(queue: Pick, leadId: string) { return queue.entries.filter(row => !row.source && row.comment.authorAgentId === leadId); } +/** + * The composer lists one pending wake at a time and prefers editable ordinary messages over a saved card response + * (#19331): a receipt must never hide someone's later messages. So while the chat lead's own relay comment waits, the + * queue shows that relay and the bound answer keeps its own, unlisted wake. Either listing is a valid view of the + * same saved answer; what the person relies on is that the answer is retained and later delivered exactly once. + */ +export function approvalResponseListing(queue: Pick, questionId: string, leadId: string) { + const bound = queue.entries.filter(row => row.source); + if (bound.length === 1 && queue.entries.length === 1 && bound[0]!.source?.kind === "interaction" && bound[0]!.source.interactionId === questionId + && bound[0]!.canEdit === false && bound[0]!.canDiscard === false) return "listed" as const; + if (bound.length === 0 && queue.entries.length > 0 && queue.entries.length === leadRelayEntries(queue, leadId).length) return "behind_lead_relay" as const; + return null; +} +type SuccessorRun = { id: string; contextSnapshot?: { interactionId?: unknown; wakeCommentIds?: unknown } | null }; +/** Worker turns after the original owner: the one bound to the answer, turns that only carry the lead's relay, and anything else. */ +export function classifyApprovalSuccessors(runs: SuccessorRun[], questionId: string, leadCommentIds: ReadonlySet) { + const bound: string[] = [], relay: string[] = [], other: string[] = []; + for (const run of runs) { + const context = run.contextSnapshot ?? {}, comments = Array.isArray(context.wakeCommentIds) ? context.wakeCommentIds.map(String) : []; + if (context.interactionId === questionId) bound.push(run.id); + else if (!context.interactionId && comments.length > 0 && comments.every(id => leadCommentIds.has(id))) relay.push(run.id); + else other.push(run.id); + } + return { bound, relay, other }; +} export function gradeApprovalQueue(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("real-question-response-and-ordinary-queue-evidence", !!o, "The actual native question, human response, separate queue mutations and once-only consuming native input are required."); if (!o) return c.result(); @@ -44,14 +69,14 @@ export function gradeApprovalQueue(value: unknown) { c.add("bound-answer-path-and-real-ordinary-mutations", (s.humanAnswerSubmitted === true) !== (s.canonicalAnswerAttributed === true) && s.actualOrdinaryEdit === true && (s.actualOrdinaryReorder === true || s.ordinaryReorderRestricted === true) && s.actualOrdinaryDiscard === true, "Exactly one actual path records Acorn on the bound question (the human's response control, or an attributed canonical response); editing, ordering and discarding ordinary messages cannot impersonate or cancel it."); - const response = s.responseQueue as IssueQueuedCommentQueue; - const bound = response?.entries.filter(row => row.source) ?? []; - c.add("response-own-delivery-semantics-and-once-only-admission", !!response && bound.length === 1 && bound[0]?.source?.kind === "interaction" - && bound[0].source.interactionId === question.id && bound[0].canEdit === false && bound[0].canDiscard === false - && response.entries.length === 1 + leadRelayEntries(response, o.leadId).length - && s.consumingRunCount === 1 && s.responseNativeInputBound === true && s.queueEmptyAfterDelivery === true - && o.runs.length === 4 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && s.actualAcornResult === true && s.initialWorkRetained === true, - "After the original owner settles, exactly one eligible native continuation receives the bound answer and saves actual Acorn work without replaying or losing it."); + const response = s.responseQueue as IssueQueuedCommentQueue, listing = response ? approvalResponseListing(response, question.id, o.leadId) : null; + c.add("response-kept-apart-from-ordinary-queue", !!listing && listing === s.responseListing && (listing === "listed" ? s.responseListedImmutable === true : true), + "Once the person's messages are gone the queue holds either the uneditable, undiscardable Acorn response alone, or only the chat lead's own relay ahead of it; never a person's message or an editable copy of the answer."); + const relayRuns = Number(s.relayRunCount); + c.add("response-own-delivery-semantics-and-once-only-admission", s.consumingRunCount === 1 && s.unexpectedSuccessorRunCount === 0 && Number.isInteger(relayRuns) && relayRuns >= 0 && relayRuns <= 1 + && s.responseNativeInputBound === true && s.queueEmptyAfterDelivery === true + && o.runs.length === 4 + relayRuns && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && s.actualAcornResult === true && s.initialWorkRetained === true, + "After the original owner settles, exactly one native continuation is bound to the answer and receives it; the only other worker turn allowed carries just the lead's relay; Acorn work is saved and earlier work kept."); return c.result(); } export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-delivery", async ctx => { @@ -93,7 +118,7 @@ export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-d humanId: session.user.id, ordinaryQueueOwned: true, ordinaryQueuedBeforeAnswer: true }; }, gradeApprovalQueue, { setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, - authentication: "human-session", expectedRunCount: 4, attemptTimeoutMs: 900_000, oracleVersion: "paperclip.chat-catalog.runtime.approval-delivery.v5", + authentication: "human-session", expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 900_000, oracleVersion: "paperclip.chat-catalog.runtime.approval-delivery.v6", sourceFiles: ["chat-catalog-domains/runtime-approval-queue.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/steering.yaml"], async act(ctx, state) { const canonicalPhase = await pollUntil({ label: "Canonical naming answer guidance while real worker remains active", deadlineAt: Date.now() + 150_000, @@ -169,36 +194,50 @@ export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-d await verifyResponse(); } state.actualOrdinaryDiscard = true; state.responseRetainedAcrossOrdinaryMutations = retained; - const response = await pollUntil({ label: "Typed immutable naming answer remains queued after ordinary comments are removed", deadlineAt: Date.now() + 20_000, - load: () => ctx.input.api.get(endpoint), accept: q => q.entries.filter(row => row.source).length === 1 && q.entries.find(row => row.source)?.source?.interactionId === question.id - && ordinaryApprovalEntries(q, humanId).length === 0 && q.entries.length === 1 + leadRelayEntries(q, leadId).length }) + const response = await pollUntil({ label: "Typed immutable naming answer stays apart from the queue after ordinary comments are removed", deadlineAt: Date.now() + 20_000, + load: () => ctx.input.api.get(endpoint), accept: q => ordinaryApprovalEntries(q, humanId).length === 0 && approvalResponseListing(q, question.id, leadId) !== null }) .catch(async error => { const observed = await ctx.input.api.get(endpoint); await ctx.input.evidence("runtime-approval-queue-after-discard.json", { questionId: question.id, targetRunId: observed.targetRunId ?? null, entries: observed.entries.map(row => ({ id: row.comment.id, source: row.source?.kind ?? null, interactionId: row.source?.interactionId ?? null, person: row.comment.authorUserId === humanId, lead: row.comment.authorAgentId === leadId })), interaction: (await ctx.input.api.get(`/api/issues/${task.id}/interactions`)).filter(row => row.id === question.id).map(row => ({ status: row.status, dismissed: !!(row as { dismissedAt?: unknown }).dismissedAt })) }); throw error; - }); state.responseQueue = response; + }); state.responseQueue = response; state.responseListing = approvalResponseListing(response, question.id, leadId); + // Try to rewrite the answer through the queue's edit route; whichever wake is listed, the saved answer must not change. const immutable = await ctx.input.page.request.patch(`${endpoint}/${question.id}`, { data: { queueId: response.queueId, revision: response.revision, body: "Replace the accepted name with an unauthorized comment edit" }, headers: { Origin: new URL(ctx.input.page.url()).origin } }); - state.responseUneditable = immutable.status() >= 400 && immutable.status() < 500 && immutable.status() !== 401; await immutable.dispose(); await verifyResponse(); - await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${question.id}`)).toBeVisible(); - await expect(ctx.input.page.getByTestId(`task-chat-queued-discard-${question.id}`)).toBeDisabled(); + const refused = immutable.status() >= 400 && immutable.status() < 500 && immutable.status() !== 401; await immutable.dispose(); await verifyResponse(); + state.responseUneditable = refused && retained; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + if (state.responseListing === "listed") { + await expect(ctx.input.page.getByTestId(`task-chat-queued-message-${question.id}`)).toBeVisible(); + await expect(ctx.input.page.getByTestId(`task-chat-queued-discard-${question.id}`)).toBeDisabled(); state.responseListedImmutable = true; + } await ctx.input.capture("actual-immutable-answer-queue", "Bound Acorn answer remains distinct from edited, reordered and removed ordinary comments", "runtime-approval-queue.png"); state.initialMarker = `Earlier website work ${ctx.input.nonce}`; await writeFile((state.gate as { gate: string }).gate, `This turn is settling while the naming question waits. Save a task document with the exact reference ${state.initialMarker} preserving earlier website work, then finish this turn normally. Do not manufacture or pre-consume a name decision; leave the actual accepted response for normal Paperclip admission.`, "utf8"); - const done = await pollUntil({ label: "One normal successor receives actual immutable name response", deadlineAt: Date.now() + 240_000, - load: async () => ({ runs: await ctx.allRuns(), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), - accept: v => v.runs.length === 4 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", - reject: v => v.runs.length > 4 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Accepted decision delivery failed or replayed work" : undefined }); - const consumers = done.runs.filter(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId && row.id !== state.questionRunId); state.consumingRunCount = consumers.length; state.consumerRunId = consumers[0]?.id; - const full = consumers[0] && await ctx.input.api.get<{ contextSnapshot?: { interactionId?: string }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumers[0].id}`); + type FullRun = { id: string; contextSnapshot?: { interactionId?: string; wakeCommentIds?: unknown }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }; + const worker = (state.worker as Entity).id; + const successors = async () => { + const runs = await ctx.allRuns(), comments = await ctx.input.api.get>(`/api/issues/${task.id}/comments`); + const later = await Promise.all(runs.filter(row => row.agentId === worker && row.id !== state.workerRunId && row.id !== state.questionRunId) + .map(row => ctx.input.api.get(`/api/heartbeat-runs/${row.id}`))); + return { runs, later, split: classifyApprovalSuccessors(later, question.id, new Set(comments.filter(row => row.authorAgentId === leadId).map(row => row.id))) }; + }; + const done = await pollUntil({ label: "One normal successor receives actual immutable name response", deadlineAt: Date.now() + 300_000, + load: async () => ({ ...(await successors()), queue: await ctx.input.api.get(endpoint), task: await ctx.input.api.get(`/api/issues/${task.id}`) }), + accept: v => v.split.bound.length === 1 && v.runs.every(row => row.status === "succeeded") && v.queue.entries.length === 0 && v.task.status === "done", + reject: v => v.runs.length > 5 || v.split.bound.length > 1 || v.split.other.length > 0 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Accepted decision delivery failed or replayed work" : undefined }); + state.consumingRunCount = done.split.bound.length; state.consumerRunId = done.split.bound[0]; state.relayRunCount = done.split.relay.length; + state.relayRunIds = done.split.relay; state.unexpectedSuccessorRunCount = done.split.other.length; + const full = done.later.find(row => row.id === state.consumerRunId); state.responseNativeInputBound = full?.contextSnapshot?.interactionId === question.id && full.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(question.id) === true && full.runnerProfileJson.nativeExecutionInput.task.prompt.includes("Acorn"); state.queueEmptyAfterDelivery = done.queue.entries.length === 0; await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, async observe(ctx, state) { const o = await observeRuntime(ctx, state); const question = state.questionBefore as IssueThreadInteraction; state.questionAfter = (await ctx.input.api.get(`/api/issues/${(state.task as Entity).id}/interactions`)).find(row => row.id === question.id); - state.actualAcornResult = o.revisions.some(row => row.createdByRunId === state.consumerRunId && row.createdByAgentId === (state.worker as Entity).id && /Acorn/i.test(row.body ?? "") && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); + const successorIds = [state.consumerRunId, ...((state.relayRunIds as string[] | undefined) ?? [])]; + state.actualAcornResult = o.revisions.some(row => successorIds.includes(row.createdByRunId) && row.createdByAgentId === (state.worker as Entity).id && /Acorn/i.test(row.body ?? "") && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); state.initialWorkRetained = o.revisions.some(row => row.createdByRunId === state.workerRunId && (row.body ?? "").includes(String(state.initialMarker))); return o; }, async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Entity; await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" });