From a09aa6c9e2746e2b9c2310ad1fb597ac8114ea41 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 00:04:38 -0600 Subject: [PATCH] test(evals): keep discard-queued-message's worker busy through a slow reply and grade structure The baseline failed because the lead's first reply took ten minutes while the busy worker's brief wait ended after 300 s, so the queued message could no longer be discarded in time and the worker never got its brief. Let this case opt into a 590 s wait (OpenCode's bash tool caps at 600 s), give the attempt and final continuation room for slow turns, and leave other cases' 300 s default. Replace keyword checks on the replies and the direction comment with saved evidence: the reply links the task holding the queued request, the request is still pending after the reply, the consumed contact label was saved and then dropped from the latest plan. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 4 +- .../runtime-busy-worker.test.ts | 8 +++- .../runtime-busy-worker.ts | 10 +++-- .../runtime-discard.test.ts | 15 +++++++ .../chat-catalog-domains/runtime-discard.ts | 41 +++++++++++-------- .../runtime-queue-controls.ts | 4 +- 6 files changed, 55 insertions(+), 27 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 44757cbdddc..3e39ff022e2 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5423,12 +5423,12 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime-discard.v3", "coverageGaps": [], "expectedRunCount": 5, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, - "attemptTimeoutMs": 1020000, + "attemptTimeoutMs": 1800000, "authentication": "human-session", "setupPolicy": { "readiness": "preserve-scenario", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.test.ts index 3384b06f5df..046f4fafc05 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.test.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { tmpdir } from "node:os"; import { spawn } from "node:child_process"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; -import { readRuntimeTaskGate, releaseRuntimeWorker, taskBoundRuntimeGate, prepareRuntimeWorkerGate } from "./runtime-busy-worker.js"; +import { MAX_RUNTIME_BRIEF_WAIT_MS, readRuntimeTaskGate, releaseRuntimeWorker, taskBoundRuntimeGate, prepareRuntimeWorkerGate } from "./runtime-busy-worker.js"; const roots: string[] = []; afterEach(async () => { vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); async function fixture() { @@ -50,7 +50,11 @@ it("retains neutral cleanup release and refuses timeout widening", async () => { const f = await fixture(), gate = taskBoundRuntimeGate("cleanup"); await readRuntimeTaskGate(f.ctx, gate, "task", "worker"); await releaseRuntimeWorker({ gate }); expect(await readFile(gate.gate, "utf8")).toBe("The human ended this disposable fixture. Preserve any saved work and finish normally."); - expect(() => taskBoundRuntimeGate("too-long", 300001)).toThrow("bounded"); + expect(() => taskBoundRuntimeGate("too-long", MAX_RUNTIME_BRIEF_WAIT_MS + 1)).toThrow("bounded"); + // A case may opt into a wait that still ends inside OpenCode's 600 s bash ceiling. + expect(MAX_RUNTIME_BRIEF_WAIT_MS).toBeLessThan(600_000); + expect(taskBoundRuntimeGate("long-chat", MAX_RUNTIME_BRIEF_WAIT_MS).script).toContain(`Date.now()+${MAX_RUNTIME_BRIEF_WAIT_MS}`); + expect(taskBoundRuntimeGate("default").script).toContain("Date.now()+300000"); }); it("funds source and worker before task-bound history, without exposing an agent-home script or admitting work", async () => { const f = await fixture(); vi.stubEnv("PAPERCLIP_HOME", path.join(f.root, "paperclip-home")); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "owned-test"); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts index 3b8bdcd1166..cefe8ed956e 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts @@ -9,8 +9,10 @@ import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import type { Entity } from "./internal-support.js"; export interface RuntimeBriefGate { gate: string; ready: string; scriptPath: string; command: string; taskBound: boolean; gateName?: string; script?: string; boundTaskId?: string; boundWorkspaceId?: string } +/** OpenCode's bash tool rejects timeouts above 600 s, so a single wait must end before that ceiling. */ +export const MAX_RUNTIME_BRIEF_WAIT_MS = 590_000; export function taskBoundRuntimeGate(nonce: string, timeoutMs = 300_000): RuntimeBriefGate { - if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 300_000) throw new Error("Task brief wait must remain bounded"); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > MAX_RUNTIME_BRIEF_WAIT_MS) throw new Error("Task brief wait must remain bounded"); const gateName = `runtime-brief-${createHash("sha256").update(nonce).digest("hex").slice(0, 24)}.txt`; const script = `const fs=require("node:fs");const gate=${JSON.stringify(gateName)};fs.writeFileSync(gate+".waiting","waiting",{flag:"wx"});const end=Date.now()+${timeoutMs};const timer=setInterval(()=>{if(fs.existsSync(gate)){console.log(fs.readFileSync(gate,"utf8"));clearInterval(timer)}else if(Date.now()>end){clearInterval(timer);process.exitCode=1}},100);`; const quote = (value: string) => "'" + value.replace(/'/g, "'\\''") + "'"; @@ -43,14 +45,14 @@ export async function readRuntimeTaskGate(ctx: ChatCatalogContext, gate: Runtime throw error; } } -export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; taskBound?: boolean } = {}) { +export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; taskBound?: boolean; briefWaitMs?: number } = {}) { if (ctx.input.execution.environment.id !== "local") throw new Error("Owned file-gate fixture requires a real local environment"); const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root) throw new Error("Busy fixture requires its isolated attempt root"); const worker = options.worker ?? await runtimeWorker(ctx); const isolated = await realpath(root); const previousRuns = await ctx.allRuns(); let gate: RuntimeBriefGate; - if (options.taskBound) gate = taskBoundRuntimeGate(`${ctx.input.nonce}-${previousRuns.length}`); + if (options.taskBound) gate = taskBoundRuntimeGate(`${ctx.input.nonce}-${previousRuns.length}`, options.briefWaitMs); else { const candidate = path.resolve(resolveDefaultAgentWorkspaceDir(worker.id)); if (!candidate.startsWith(`${isolated}${path.sep}`)) throw new Error("Busy fixture escaped its isolated company home"); @@ -66,7 +68,7 @@ export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: await ctx.input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); return { worker, gate, previousRuns, company }; } -export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number } = {}) { +export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number; briefWaitMs?: number } = {}) { const { worker, gate, previousRuns, company } = await prepareRuntimeWorkerGate(ctx, { ...options, taskBound: true }); const task = await ctx.input.api.post(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, description: `The website brief is being supplied by the human. Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts index 3541301e050..64b2b1688ee 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts @@ -9,12 +9,27 @@ function valid() { { position: 1, comment: { id: "pending-contact", body: "Also add a contact section to the website.", authorUserId: "human", authorAgentId: null } }] }, queueAfterDiscard: { issueId: "website", entries: [structuredClone(neighbour)] }, discardVisible: true, discardedCommentStatus: 404, discardedNeverDelivered: true, queryReply: "Use the human Discard control on your pending contact request in the task queue.", agentDiscardStatus: 403, + queryLinkedTask: true, targetPendingAfterReply: true, consumedLabelSaved: true, consumedBeforeCancellation: true, consumedStillReadable: true, lateDiscardStatus: 409, latePhaseAttributed: true, lateReply: "The later request was already consumed. I sent new direction to remove that section; its original history remains intact.", cancelDirectionRunBound: true, finalQueueEmpty: true, finalPlanWithoutContact: true, initialRevisionRetained: true } }; } it("accepts genuine pending discard and distinct consumed-input cancellation as new direction", () => expect(gradeDiscard(valid()).passed).toBe(true)); it("accepts natural human-control and subsequent consumed-input guidance", () => { const o = valid(); o.state.queryReply = "That waiting message is yours. Open its queue menu and remove it with Discard."; o.state.lateReply = "Morgan processed the second request before you cancelled it. A follow-up now asks for the contact section to be removed; the delivered input stays in history."; expect(gradeDiscard(o).passed).toBe(true); }); +it("accepts the retained live reply and a reworded equivalent without wording checks", () => { + for (const reply of [ + "I found the contact-section request: it's the queued comment on [RUN-1](/RUN/issues/RUN-1), still pending (not yet delivered to Morgan). I couldn't cancel it from this agent: Paperclip returned 403. Since you're the comment author, cancel it from that queued comment's controls on RUN-1.", + "Only you can pull that one back. It's sitting unsent on [RUN-1](/RUN/issues/RUN-1); open it and use the menu next to the message.", + ]) { const o = valid(); o.state.queryReply = reply; o.state.lateReply = "Done: Morgan got new instructions to drop that section. Your earlier message is kept as it was."; expect(gradeDiscard(o).passed).toBe(true); } +}); +it("rejects guidance that does not link the task holding the queued request", () => { const o = valid(); o.state.queryLinkedTask = false; expect(gradeDiscard(o).passed).toBe(false); }); +it("rejects a reply after which the pending request had already vanished", () => { const o = valid(); o.state.targetPendingAfterReply = false; expect(gradeDiscard(o).passed).toBe(false); }); +it("rejects an empty guidance or later reply", () => { + const a = valid(); a.state.queryReply = " "; expect(gradeDiscard(a).passed).toBe(false); + const b = valid(); b.state.lateReply = ""; expect(gradeDiscard(b).passed).toBe(false); +}); +it("rejects a saved plan that still carries the cancelled contact label", () => { const o = valid(); o.state.finalPlanWithoutContact = false; expect(gradeDiscard(o).passed).toBe(false); }); +it("rejects a vacuous removal where the consumed contact request was never saved", () => { const o = valid(); o.state.consumedLabelSaved = false; expect(gradeDiscard(o).passed).toBe(false); }); it("rejects missing cancellation evidence", () => expect(gradeDiscard(undefined).passed).toBe(false)); it("rejects discarding a similarly named request owned by another person", () => { const o = valid(); o.state.queueBefore.entries[1]!.comment.authorUserId = "other"; expect(gradeDiscard(o).passed).toBe(false); }); it("rejects removing unrelated queue content", () => { const o = valid(); o.state.queueAfterDiscard.entries = []; expect(gradeDiscard(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts index 11e89d2d337..47658767b12 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts @@ -5,9 +5,9 @@ import type { IssueComment, IssueQueuedCommentQueue } from "../../../packages/sh import { sendChatMessage } from "../chat-flow.js"; import { pollUntil } from "../api.js"; import { submitTaskReply } from "../user-actions.js"; -import { checks, type Entity } from "./internal-support.js"; +import { checks, hasLink, type Entity } from "./internal-support.js"; import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; -import { releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { MAX_RUNTIME_BRIEF_WAIT_MS, releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { humanQueuedRuntime, probeConsumedHumanQueue } from "./runtime-queue-controls.js"; import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; @@ -27,21 +27,21 @@ export function gradeDiscard(value: unknown) { && o.activity.some(row => row.action === "issue.queued_comment_discarded" && row.actorType === "user" && row.actorId === s.humanId && row.entityId === (s.task as Entity).id && row.details?.commentId === s.targetId) && s.discardedNeverDelivered === true, "Actual absent pending comment, retained human audit history and consuming-run wake IDs prove this pending instruction will not be delivered."); - c.add("chat-guides-supported-human-discard", /queue|pending|waiting/i.test(String(s.queryReply)) && /discard|remove|cancel/i.test(String(s.queryReply)) - && /you|human|control|menu|button/i.test(String(s.queryReply)) && !/I(?:'ve| have)? (?:already )?(?:deleted|discarded|cancelled)/i.test(String(s.queryReply)) && s.agentDiscardStatus === 403, - "The canonical reply guides the person to their actual discard control instead of impersonating a user-owned mutation."); + c.add("chat-guides-supported-human-discard", String(s.queryReply ?? "").trim().length > 0 && s.queryLinkedTask === true + && s.targetPendingAfterReply === true && s.agentDiscardStatus === 403, + "The reply links the task whose queue holds the request, the request is still pending after the reply, and the agent cannot impersonate the person's discard."); c.add("already-delivered-input-cannot-be-erased", s.consumedBeforeCancellation === true && s.consumedStillReadable === true - && s.lateDiscardStatus === 409 && s.latePhaseAttributed === true && /already|deliver|consum|processed|history/i.test(String(s.lateReply)) - && /new|direction|follow.up|remove|change/i.test(String(s.lateReply)) && !/erased.*(?:input|history)|(?:unsent|undelivered|never delivered).*request/i.test(String(s.lateReply)), - "The separately attributed later answer reports that the later request was consumed and treats cancellation as new direction."); - c.add("new-direction-reaches-exactly-one-native-continuation", s.cancelDirectionRunBound === true && s.finalQueueEmpty === true && s.finalPlanWithoutContact === true + && s.lateDiscardStatus === 409 && s.latePhaseAttributed === true && String(s.lateReply ?? "").trim().length > 0, + "The consumed request stays readable and cannot be discarded late; the later answer is attributed to its own run."); + c.add("new-direction-reaches-exactly-one-native-continuation", s.cancelDirectionRunBound === true && s.finalQueueEmpty === true + && s.consumedLabelSaved === true && s.finalPlanWithoutContact === true && s.initialRevisionRetained === true && o.runs.length === 5 && o.runs.every(row => row.status === "succeeded") && o.tasks.length === 1 && o.tasks[0]?.id === (s.task as Entity).id && o.tasks[0]?.status === "done", "A distinct saved instruction reaches one real later worker, updates the saved plan and preserves prior work/history without replaying the discarded request."); return c.result(); } -export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, { - authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, +export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText, { briefWaitMs: MAX_RUNTIME_BRIEF_WAIT_MS }), gradeDiscard, { + authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_800_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v3", sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"], setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) { @@ -49,9 +49,11 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => chatCatalogNativePhaseIsTimely(ctx) && (v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "running") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded")) }); - state.queryReply = (await observeRuntime(ctx, state)).reply; + const answered = await observeRuntime(ctx, state); const source = ctx.input.page.url(), task = state.task as Entity, root = `/api/issues/${task.id}`, queuePath = `${root}/queued-comments`; const initial = await ctx.input.api.get(queuePath), targetId = String(state.targetId); + state.queryReply = answered.reply; state.queryLinkedTask = hasLink(answered, "issues", task); + state.targetPendingAfterReply = initial.entries.some(row => row.comment.id === targetId && row.comment.body === contactText); state.agentDiscardStatus = await probeAgentPermission(ctx, `${queuePath}/${targetId}`, "DELETE", { queueId: initial.queueId, revision: initial.revision }); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await ctx.input.page.getByTestId(`task-chat-queued-discard-${targetId}`).click(); @@ -75,6 +77,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m load: async () => ({ runs: await ctx.allRuns(), task: await ctx.input.api.get(root), queue: await ctx.input.api.get(queuePath) }), accept: v => v.runs.length === 3 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0 }); const consumer = consumed.runs.find(row => row.agentId === (state.worker as Entity).id && row.id !== state.workerRunId)!; + state.consumerRunId = consumer.id; const detail = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${consumer.id}`); state.consumedBeforeCancellation = detail.contextSnapshot?.wakeCommentIds?.includes(second.comment.id) === true && detail.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(secondText) === true; @@ -84,7 +87,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m const message = `I later queued a different contact request on Website ${task.identifier ?? task.id}, comment ${second.comment.id}, and Morgan has already consumed it. Cancel that later contact requirement now by sending Morgan new direction to remove the contact section from the saved website plan while retaining the accessible-font requirement. Explain what happened to the already-delivered input; do not erase or rewrite it.`; const priorIds = consumed.runs.map(row => row.id); await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx)); - const final = await pollUntil({ label: "One real new-direction continuation after already-delivered cancellation", deadlineAt: Date.now() + 300_000, + const final = await pollUntil({ label: "One real new-direction continuation after already-delivered cancellation", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 600_000), load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), task: await ctx.input.api.get(root), queue: await ctx.input.api.get(queuePath) }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0, reject: v => v.runs.length > 5 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The declared later cancellation failed or duplicated execution" : undefined }); @@ -96,7 +99,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m state.lateReply = replies.map(row => row.body).join("\n"); const comments = await ctx.input.api.get(`${root}/comments?order=asc`), direction = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === lead?.id); const workerInput = worker && await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[]; wakeCommentId?: string }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${worker.id}`); - state.cancelDirectionRunBound = later.length === 2 && !!worker && direction.length === 1 && /remov|without|cancel|omit|no contact/i.test(direction[0]!.body) + state.cancelDirectionRunBound = later.length === 2 && !!worker && direction.length === 1 && [...(workerInput?.contextSnapshot?.wakeCommentIds ?? []), workerInput?.contextSnapshot?.wakeCommentId].includes(direction[0]!.id) && workerInput?.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(direction[0]!.body) === true; state.finalWorkerRunId = worker?.id; state.consumedStillReadable = comments.some(row => row.id === second.comment.id && row.body === secondText && !row.deletedAt); @@ -105,10 +108,14 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m async observe(ctx, state) { const o = await observeRuntime(ctx, state); state.initialRevisionRetained = o.revisions.some(row => row.createdByRunId === state.workerRunId && row.createdByAgentId === (state.worker as Entity).id); - const latest = o.revisions.filter(row => row.createdByRunId === state.finalWorkerRunId && row.createdByAgentId === (state.worker as Entity).id + // The consumed request asked for a contact section labelled with the attempt nonce. Its + // consuming run must have saved that label, and the cancellation run's latest plan must drop it. + const label = ctx.input.nonce, worker = (state.worker as Entity).id; + state.consumedLabelSaved = o.revisions.some(row => row.createdByRunId === state.consumerRunId && row.createdByAgentId === worker && (row.body ?? "").includes(label)); + const latest = o.revisions.filter(row => row.createdByRunId === state.finalWorkerRunId && row.createdByAgentId === worker && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); - state.finalPlanWithoutContact = latest.some(row => /accessib|readable font/i.test(row.body ?? "") - && (!/contact/i.test(row.body ?? "") || /contact.*(?:removed|omitted|cancelled)|(?:removed|omitted|cancelled|without|no)\b.*contact/i.test(row.body ?? ""))); + state.finalPlanWithoutContact = latest.length > 0 && latest.every(row => !(row.body ?? "").includes(label)) + && o.documents.every(doc => !(o.revisions.find(row => row.id === doc.latestRevisionId)?.body ?? "").includes(label)); return o; }, async cleanup(ctx, state) { try { await cleanupIdentityProbes(ctx); } finally { await releaseRuntimeWorker(state); } }, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts index b90532837fc..41d74f3dc21 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts @@ -18,8 +18,8 @@ type State = Record; function endpoint(state: State) { return `/api/issues/${(state.task as Entity).id}/queued-comments`; } async function queue(ctx: ChatCatalogContext, state: State) { return ctx.input.api.get(endpoint(state)); } function taskUrl(ctx: ChatCatalogContext, state: State) { const task = state.task as Entity; return `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`; } -export async function humanQueuedRuntime(ctx: ChatCatalogContext, targetText = redText): Promise { - const state: State = await busyRuntimeWorker(ctx); const source = ctx.input.page.url(); +export async function humanQueuedRuntime(ctx: ChatCatalogContext, targetText = redText, options: { briefWaitMs?: number } = {}): Promise { + const state: State = await busyRuntimeWorker(ctx, options); const source = ctx.input.page.url(); await ctx.input.page.goto(taskUrl(ctx, state), { waitUntil: "domcontentloaded" }); await submitTaskReply(ctx.input.page, otherText); await submitTaskReply(ctx.input.page, targetText); const before = await pollUntil({ label: "Two real user-owned messages saved behind Morgan's active turn", deadlineAt: Date.now() + 30_000,