From b4dbd098da66550103e8613965adb486bb73965b Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 00:53:38 -0600 Subject: [PATCH] test(evals): run discard-queued-message under its original 300 s worker wait Restore the default brief wait, the 17-minute attempt and the 300 s final wait so the rerun tests the final-denial product thesis under the original conditions. The structural grading stays. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../chat-catalog-domains/runtime-busy-worker.test.ts | 8 ++------ .../chat-catalog-domains/runtime-busy-worker.ts | 10 ++++------ .../runner-e2e/chat-catalog-domains/runtime-discard.ts | 8 ++++---- .../chat-catalog-domains/runtime-queue-controls.ts | 4 ++-- 5 files changed, 13 insertions(+), 19 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 3e39ff022e2..7ba2b9ecee9 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5428,7 +5428,7 @@ "expectedRunCount": 5, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, - "attemptTimeoutMs": 1800000, + "attemptTimeoutMs": 1020000, "authentication": "human-session", "setupPolicy": { "readiness": "preserve-scenario", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.test.ts index 046f4fafc05..3384b06f5df 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.test.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { tmpdir } from "node:os"; import { spawn } from "node:child_process"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; -import { MAX_RUNTIME_BRIEF_WAIT_MS, readRuntimeTaskGate, releaseRuntimeWorker, taskBoundRuntimeGate, prepareRuntimeWorkerGate } from "./runtime-busy-worker.js"; +import { readRuntimeTaskGate, releaseRuntimeWorker, taskBoundRuntimeGate, prepareRuntimeWorkerGate } from "./runtime-busy-worker.js"; const roots: string[] = []; afterEach(async () => { vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); async function fixture() { @@ -50,11 +50,7 @@ it("retains neutral cleanup release and refuses timeout widening", async () => { const f = await fixture(), gate = taskBoundRuntimeGate("cleanup"); await readRuntimeTaskGate(f.ctx, gate, "task", "worker"); await releaseRuntimeWorker({ gate }); expect(await readFile(gate.gate, "utf8")).toBe("The human ended this disposable fixture. Preserve any saved work and finish normally."); - expect(() => taskBoundRuntimeGate("too-long", MAX_RUNTIME_BRIEF_WAIT_MS + 1)).toThrow("bounded"); - // A case may opt into a wait that still ends inside OpenCode's 600 s bash ceiling. - expect(MAX_RUNTIME_BRIEF_WAIT_MS).toBeLessThan(600_000); - expect(taskBoundRuntimeGate("long-chat", MAX_RUNTIME_BRIEF_WAIT_MS).script).toContain(`Date.now()+${MAX_RUNTIME_BRIEF_WAIT_MS}`); - expect(taskBoundRuntimeGate("default").script).toContain("Date.now()+300000"); + expect(() => taskBoundRuntimeGate("too-long", 300001)).toThrow("bounded"); }); it("funds source and worker before task-bound history, without exposing an agent-home script or admitting work", async () => { const f = await fixture(); vi.stubEnv("PAPERCLIP_HOME", path.join(f.root, "paperclip-home")); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "owned-test"); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts index cefe8ed956e..3b8bdcd1166 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts @@ -9,10 +9,8 @@ import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import type { Entity } from "./internal-support.js"; export interface RuntimeBriefGate { gate: string; ready: string; scriptPath: string; command: string; taskBound: boolean; gateName?: string; script?: string; boundTaskId?: string; boundWorkspaceId?: string } -/** OpenCode's bash tool rejects timeouts above 600 s, so a single wait must end before that ceiling. */ -export const MAX_RUNTIME_BRIEF_WAIT_MS = 590_000; export function taskBoundRuntimeGate(nonce: string, timeoutMs = 300_000): RuntimeBriefGate { - if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > MAX_RUNTIME_BRIEF_WAIT_MS) throw new Error("Task brief wait must remain bounded"); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 300_000) throw new Error("Task brief wait must remain bounded"); const gateName = `runtime-brief-${createHash("sha256").update(nonce).digest("hex").slice(0, 24)}.txt`; const script = `const fs=require("node:fs");const gate=${JSON.stringify(gateName)};fs.writeFileSync(gate+".waiting","waiting",{flag:"wx"});const end=Date.now()+${timeoutMs};const timer=setInterval(()=>{if(fs.existsSync(gate)){console.log(fs.readFileSync(gate,"utf8"));clearInterval(timer)}else if(Date.now()>end){clearInterval(timer);process.exitCode=1}},100);`; const quote = (value: string) => "'" + value.replace(/'/g, "'\\''") + "'"; @@ -45,14 +43,14 @@ export async function readRuntimeTaskGate(ctx: ChatCatalogContext, gate: Runtime throw error; } } -export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; taskBound?: boolean; briefWaitMs?: number } = {}) { +export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; taskBound?: boolean } = {}) { if (ctx.input.execution.environment.id !== "local") throw new Error("Owned file-gate fixture requires a real local environment"); const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root) throw new Error("Busy fixture requires its isolated attempt root"); const worker = options.worker ?? await runtimeWorker(ctx); const isolated = await realpath(root); const previousRuns = await ctx.allRuns(); let gate: RuntimeBriefGate; - if (options.taskBound) gate = taskBoundRuntimeGate(`${ctx.input.nonce}-${previousRuns.length}`, options.briefWaitMs); + if (options.taskBound) gate = taskBoundRuntimeGate(`${ctx.input.nonce}-${previousRuns.length}`); else { const candidate = path.resolve(resolveDefaultAgentWorkspaceDir(worker.id)); if (!candidate.startsWith(`${isolated}${path.sep}`)) throw new Error("Busy fixture escaped its isolated company home"); @@ -68,7 +66,7 @@ export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: await ctx.input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); return { worker, gate, previousRuns, company }; } -export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number; briefWaitMs?: number } = {}) { +export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number } = {}) { const { worker, gate, previousRuns, company } = await prepareRuntimeWorkerGate(ctx, { ...options, taskBound: true }); const task = await ctx.input.api.post(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, description: `The website brief is being supplied by the human. Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts index 47658767b12..1a879746ef2 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts @@ -7,7 +7,7 @@ import { pollUntil } from "../api.js"; import { submitTaskReply } from "../user-actions.js"; import { checks, hasLink, type Entity } from "./internal-support.js"; import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; -import { MAX_RUNTIME_BRIEF_WAIT_MS, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { humanQueuedRuntime, probeConsumedHumanQueue } from "./runtime-queue-controls.js"; import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; @@ -40,8 +40,8 @@ export function gradeDiscard(value: unknown) { "A distinct saved instruction reaches one real later worker, updates the saved plan and preserves prior work/history without replaying the discarded request."); return c.result(); } -export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText, { briefWaitMs: MAX_RUNTIME_BRIEF_WAIT_MS }), gradeDiscard, { - authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_800_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v3", +export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, { + authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v3", sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"], setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) { @@ -87,7 +87,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m const message = `I later queued a different contact request on Website ${task.identifier ?? task.id}, comment ${second.comment.id}, and Morgan has already consumed it. Cancel that later contact requirement now by sending Morgan new direction to remove the contact section from the saved website plan while retaining the accessible-font requirement. Explain what happened to the already-delivered input; do not erase or rewrite it.`; const priorIds = consumed.runs.map(row => row.id); await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx)); - const final = await pollUntil({ label: "One real new-direction continuation after already-delivered cancellation", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 600_000), + const final = await pollUntil({ label: "One real new-direction continuation after already-delivered cancellation", deadlineAt: Date.now() + 300_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), task: await ctx.input.api.get(root), queue: await ctx.input.api.get(queuePath) }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0, reject: v => v.runs.length > 5 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The declared later cancellation failed or duplicated execution" : undefined }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts index 41d74f3dc21..b90532837fc 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts @@ -18,8 +18,8 @@ type State = Record; function endpoint(state: State) { return `/api/issues/${(state.task as Entity).id}/queued-comments`; } async function queue(ctx: ChatCatalogContext, state: State) { return ctx.input.api.get(endpoint(state)); } function taskUrl(ctx: ChatCatalogContext, state: State) { const task = state.task as Entity; return `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`; } -export async function humanQueuedRuntime(ctx: ChatCatalogContext, targetText = redText, options: { briefWaitMs?: number } = {}): Promise { - const state: State = await busyRuntimeWorker(ctx, options); const source = ctx.input.page.url(); +export async function humanQueuedRuntime(ctx: ChatCatalogContext, targetText = redText): Promise { + const state: State = await busyRuntimeWorker(ctx); const source = ctx.input.page.url(); await ctx.input.page.goto(taskUrl(ctx, state), { waitUntil: "domcontentloaded" }); await submitTaskReply(ctx.input.page, otherText); await submitTaskReply(ctx.input.page, targetText); const before = await pollUntil({ label: "Two real user-owned messages saved behind Morgan's active turn", deadlineAt: Date.now() + 30_000,