test(evals): run discard-queued-message under its original 300 s worker wait

Restore the default brief wait, the 17-minute attempt and the 300 s final wait so
the rerun tests the final-denial product thesis under the original conditions.
The structural grading stays.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-08 03:34:16 -06:00
1 parent a09aa6c9e2
commit b4dbd098da
5 files changed
+13 -19

No files matched your search

@@ -5428,7 +5428,7 @@
"expectedRunCount": 5,
"expectedHumanMessageCount": 2,
"turnTimeoutMs": 120000,
"attemptTimeoutMs": 1800000,
"attemptTimeoutMs": 1020000,
"authentication": "human-session",
"setupPolicy": {
"readiness": "preserve-scenario",
@@ -4,7 +4,7 @@ import path from "node:path";
import { tmpdir } from "node:os";
import { spawn } from "node:child_process";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import { MAX_RUNTIME_BRIEF_WAIT_MS, readRuntimeTaskGate, releaseRuntimeWorker, taskBoundRuntimeGate, prepareRuntimeWorkerGate } from "./runtime-busy-worker.js";
import { readRuntimeTaskGate, releaseRuntimeWorker, taskBoundRuntimeGate, prepareRuntimeWorkerGate } from "./runtime-busy-worker.js";
const roots: string[] = [];
afterEach(async () => { vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); });
async function fixture() {
@@ -50,11 +50,7 @@ it("retains neutral cleanup release and refuses timeout widening", async () => {
const f = await fixture(), gate = taskBoundRuntimeGate("cleanup"); await readRuntimeTaskGate(f.ctx, gate, "task", "worker");
await releaseRuntimeWorker({ gate });
expect(await readFile(gate.gate, "utf8")).toBe("The human ended this disposable fixture. Preserve any saved work and finish normally.");
expect(() => taskBoundRuntimeGate("too-long", MAX_RUNTIME_BRIEF_WAIT_MS + 1)).toThrow("bounded");
// A case may opt into a wait that still ends inside OpenCode's 600 s bash ceiling.
expect(MAX_RUNTIME_BRIEF_WAIT_MS).toBeLessThan(600_000);
expect(taskBoundRuntimeGate("long-chat", MAX_RUNTIME_BRIEF_WAIT_MS).script).toContain(`Date.now()+${MAX_RUNTIME_BRIEF_WAIT_MS}`);
expect(taskBoundRuntimeGate("default").script).toContain("Date.now()+300000");
expect(() => taskBoundRuntimeGate("too-long", 300001)).toThrow("bounded");
});
it("funds source and worker before task-bound history, without exposing an agent-home script or admitting work", async () => {
const f = await fixture(); vi.stubEnv("PAPERCLIP_HOME", path.join(f.root, "paperclip-home")); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "owned-test");
@@ -9,10 +9,8 @@ import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import type { Entity } from "./internal-support.js";
export interface RuntimeBriefGate { gate: string; ready: string; scriptPath: string; command: string; taskBound: boolean;
gateName?: string; script?: string; boundTaskId?: string; boundWorkspaceId?: string }
/** OpenCode's bash tool rejects timeouts above 600 s, so a single wait must end before that ceiling. */
export const MAX_RUNTIME_BRIEF_WAIT_MS = 590_000;
export function taskBoundRuntimeGate(nonce: string, timeoutMs = 300_000): RuntimeBriefGate {
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > MAX_RUNTIME_BRIEF_WAIT_MS) throw new Error("Task brief wait must remain bounded");
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 300_000) throw new Error("Task brief wait must remain bounded");
const gateName = `runtime-brief-${createHash("sha256").update(nonce).digest("hex").slice(0, 24)}.txt`;
const script = `const fs=require("node:fs");const gate=${JSON.stringify(gateName)};fs.writeFileSync(gate+".waiting","waiting",{flag:"wx"});const end=Date.now()+${timeoutMs};const timer=setInterval(()=>{if(fs.existsSync(gate)){console.log(fs.readFileSync(gate,"utf8"));clearInterval(timer)}else if(Date.now()>end){clearInterval(timer);process.exitCode=1}},100);`;
const quote = (value: string) => "'" + value.replace(/'/g, "'\\''") + "'";
@@ -45,14 +43,14 @@ export async function readRuntimeTaskGate(ctx: ChatCatalogContext, gate: Runtime
throw error;
}
}
export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; taskBound?: boolean; briefWaitMs?: number } = {}) {
export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; taskBound?: boolean } = {}) {
if (ctx.input.execution.environment.id !== "local") throw new Error("Owned file-gate fixture requires a real local environment");
const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root) throw new Error("Busy fixture requires its isolated attempt root");
const worker = options.worker ?? await runtimeWorker(ctx);
const isolated = await realpath(root);
const previousRuns = await ctx.allRuns();
let gate: RuntimeBriefGate;
if (options.taskBound) gate = taskBoundRuntimeGate(`${ctx.input.nonce}-${previousRuns.length}`, options.briefWaitMs);
if (options.taskBound) gate = taskBoundRuntimeGate(`${ctx.input.nonce}-${previousRuns.length}`);
else {
const candidate = path.resolve(resolveDefaultAgentWorkspaceDir(worker.id));
if (!candidate.startsWith(`${isolated}${path.sep}`)) throw new Error("Busy fixture escaped its isolated company home");
@@ -68,7 +66,7 @@ export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options:
await ctx.input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
return { worker, gate, previousRuns, company };
}
export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number; briefWaitMs?: number } = {}) {
export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; observationDeadlineAt?: number } = {}) {
const { worker, gate, previousRuns, company } = await prepareRuntimeWorkerGate(ctx, { ...options, taskBound: true });
const task = await ctx.input.api.post<Entity>(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id,
description: `The website brief is being supplied by the human. Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` });
@@ -7,7 +7,7 @@ import { pollUntil } from "../api.js";
import { submitTaskReply } from "../user-actions.js";
import { checks, hasLink, type Entity } from "./internal-support.js";
import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js";
import { MAX_RUNTIME_BRIEF_WAIT_MS, releaseRuntimeWorker } from "./runtime-busy-worker.js";
import { releaseRuntimeWorker } from "./runtime-busy-worker.js";
import { humanQueuedRuntime, probeConsumedHumanQueue } from "./runtime-queue-controls.js";
import { runtimeQueueCases } from "./runtime-queue.js";
import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js";
@@ -40,8 +40,8 @@ export function gradeDiscard(value: unknown) {
"A distinct saved instruction reaches one real later worker, updates the saved plan and preserves prior work/history without replaying the discarded request.");
return c.result();
}
export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText, { briefWaitMs: MAX_RUNTIME_BRIEF_WAIT_MS }), gradeDiscard, {
authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_800_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v3",
export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, {
authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v3",
sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"],
setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
async act(ctx, state) {
@@ -87,7 +87,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m
const message = `I later queued a different contact request on Website ${task.identifier ?? task.id}, comment ${second.comment.id}, and Morgan has already consumed it. Cancel that later contact requirement now by sending Morgan new direction to remove the contact section from the saved website plan while retaining the accessible-font requirement. Explain what happened to the already-delivered input; do not erase or rewrite it.`;
const priorIds = consumed.runs.map(row => row.id);
await sendChatMessage(ctx.input.page, message, chatCatalogNativePhaseRemainingBudget(ctx));
const final = await pollUntil({ label: "One real new-direction continuation after already-delivered cancellation", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 600_000),
const final = await pollUntil({ label: "One real new-direction continuation after already-delivered cancellation", deadlineAt: Date.now() + 300_000,
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), task: await ctx.input.api.get<Entity>(root), queue: await ctx.input.api.get<IssueQueuedCommentQueue>(queuePath) }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 5 && v.runs.every(row => row.status === "succeeded") && v.task.status === "done" && v.queue.entries.length === 0,
reject: v => v.runs.length > 5 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The declared later cancellation failed or duplicated execution" : undefined });
@@ -18,8 +18,8 @@ type State = Record<string, unknown>;
function endpoint(state: State) { return `/api/issues/${(state.task as Entity).id}/queued-comments`; }
async function queue(ctx: ChatCatalogContext, state: State) { return ctx.input.api.get<IssueQueuedCommentQueue>(endpoint(state)); }
function taskUrl(ctx: ChatCatalogContext, state: State) { const task = state.task as Entity; return `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`; }
export async function humanQueuedRuntime(ctx: ChatCatalogContext, targetText = redText, options: { briefWaitMs?: number } = {}): Promise<State> {
const state: State = await busyRuntimeWorker(ctx, options); const source = ctx.input.page.url();
export async function humanQueuedRuntime(ctx: ChatCatalogContext, targetText = redText): Promise<State> {
const state: State = await busyRuntimeWorker(ctx); const source = ctx.input.page.url();
await ctx.input.page.goto(taskUrl(ctx, state), { waitUntil: "domcontentloaded" });
await submitTaskReply(ctx.input.page, otherText); await submitTaskReply(ctx.input.page, targetText);
const before = await pollUntil({ label: "Two real user-owned messages saved behind Morgan's active turn", deadlineAt: Date.now() + 30_000,