test(evals): preserve real waiting prerequisite states

tests/runner-e2e/chat-catalog-domains/internal-project-defaults.ts

@@ -7,6 +7,17 @@ import { fixtureWorker, makeDefinition } from "./internal-definition.js";
 import { startRunBriefGate } from "./internal-run-brief.js";
 import { checks, entity, goal, observation, observeInternal, project, sameTask, task, type Entity, type InternalObservation } from "./internal-support.js";
 interface DefaultsObservation extends InternalObservation { futureContext: { project?: Entity; goal?: Entity }; futureComments: Entity[]; futureProjectReads?: { runId: string; projectId: string; callId: string }[] }
+interface PreparedAgent extends Entity { lifecycleState?: string; adapterConfig?: { timeoutSec?: number } }
+export function gradeProjectDefaultsSetup(value: { lead: PreparedAgent; future: PreparedAgent; held: PreparedAgent; task: Entity & { executionRunId?: string }; runs: Entity[]; activeRunId: string; gateCalls: { authorized?: boolean }[] }) {
+  const c = checks();
+  const ready = (agent: PreparedAgent) => agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status ?? "");
+  c.add("lead-ready", ready(value.lead) && value.lead.adapterConfig?.timeoutSec === 120, "The canonical lead is independently ready with its original 120-second native timeout.");
+  c.add("future-worker-ready", ready(value.future) && value.future.adapterConfig?.timeoutSec === 120, "The future-context worker is independently ready with its original 120-second native timeout.");
+  c.add("existing-work-actively-owned", value.held.lifecycleState === "ready" && value.held.status === "running" && value.held.adapterConfig?.timeoutSec === 420
+    && value.task.assigneeAgentId === value.held.id && value.task.executionRunId === value.activeRunId && value.runs.some(run => run.id === value.activeRunId && run.agentId === value.held.id && run.status === "running")
+    && value.gateCalls.some(call => call.authorized), "Only the deliberately waiting prerequisite has the declared 420-second timeout, spanning the subsequent bounded 150+180-second stages with exact live ownership and authorized gate proof.");
+  return c.result();
+}
 function canonical(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; if (value && typeof value === "object") return `{${Object.entries(value).filter(([, v]) => v !== undefined).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`; return JSON.stringify(value) ?? "null"; }
 /** Public native history intentionally redacts API bodies but retains exact input digests, call identity and HTTP status. */
 export function projectReadCalls(events: unknown[], runId: string, projectId: string) {
@@ -16,7 +27,10 @@ export function projectReadCalls(events: unknown[], runId: string, projectId: st
   // handler (codex-session-server-requests.ts). Never parse assistant prose as proof.
   for (const value of events) {
     if (!value || typeof value !== "object") continue;
-    const event = value as { runId?: string; eventType?: string; itemId?: string; payload?: { kind?: string; item?: Record<string, unknown> } };
+    const row = value as { runId?: string; sourceEventId?: string; payload?: { prpEvent?: { schema?: string; runId?: string; sourceEventId?: string } } };
+    const envelope = row.payload?.prpEvent;
+    if (envelope && (envelope.schema !== "paperclip.prp.event.v1" || envelope.runId !== row.runId || envelope.sourceEventId !== row.sourceEventId)) continue;
+    const event = (envelope ?? value) as { runId?: string; eventType?: string; itemId?: string; payload?: { kind?: string; item?: Record<string, unknown> } };
     if (event.runId !== runId || event.payload?.kind !== "dynamicToolCall") continue;
     const item = event.payload.item; if (!item || typeof item.id !== "string" || event.itemId !== item.id) continue;
     if (event.eventType === "item.started" && item.type === "tool_use" && item.name === "call_api") {
@@ -63,6 +77,10 @@ function defaults(): ChatCatalogCase {
   let gate: Awaited<ReturnType<typeof startRunBriefGate>> | undefined;
   const base = makeDefinition("projects.project-defaults", async ctx => {
     const worker = await fixtureWorker(ctx); const futureWorker = await fixtureWorker(ctx); await ctx.input.api.patch(`/api/agents/${futureWorker.id}`, { name: "Riley" });
+    const original = await ctx.input.api.get<PreparedAgent>(`/api/agents/${worker.id}`);
+    await ctx.input.api.patch(`/api/agents/${worker.id}`, { adapterConfig: { ...original.adapterConfig, timeoutSec: 420 } });
+    await pollUntil({ label: "Declared long-wait prerequisite configuration ready", deadlineAt: Date.now() + 45000,
+      load: () => ctx.input.api.get<PreparedAgent>(`/api/agents/${worker.id}`), accept: row => row.lifecycleState === "ready" && row.adapterConfig?.timeoutSec === 420 });
     const p = await project(ctx); const g = await goal(ctx, "Growth: reach 1,000 signups");
     const checklist = await task(ctx, "Accessibility checklist", p.id, { description: "All Launch work must support keyboard navigation, sufficient color contrast, and explicitly labeled controls." });
     const root = `/api/companies/${ctx.input.fixtures.company.id}`;
@@ -73,9 +91,15 @@ function defaults(): ChatCatalogCase {
     const held = await pollUntil({ label: "Real prior active Launch checkout", deadlineAt: Date.now() + 150000,
       load: async () => ({ task: await ctx.input.api.get<Entity & { executionRunId?: string }>(`/api/issues/${t.id}`), runs: await ctx.allRuns() }),
       accept: row => gate!.calls.length > 0 && row.runs.length === 1 && row.runs[0]?.status === "running" && row.task.executionRunId === row.runs[0]?.id });
+    await ctx.input.evidence("chat-catalog-project-defaults-prerequisite-timeout.json", { heldWorkerId: worker.id, heldWorkerTimeoutSec: 420, canonicalTimeoutSec: 120, futureWorkerTimeoutSec: 120, canonicalStageBoundMs: 150000, futureProbeBoundMs: 180000, attemptBoundMs: 660000 });
     return { project: p, goal: g, checklist, task: held.task, worker, futureWorker, activeRunId: held.runs[0]!.id, originalEnv: (p as Entity & { env?: unknown }).env ?? null };
   }, gradeProjectDefaults);
   return { ...base, expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 660000, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 },
+    setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "future-worker-ready", "existing-work-actively-owned"] },
+    async verifySetup(ctx, state) { const [lead, future, held, t, runs] = await Promise.all([
+      ctx.input.api.get<PreparedAgent>(`/api/agents/${ctx.input.fixtures.agent.id}`), ctx.input.api.get<PreparedAgent>(`/api/agents/${entity(state, "futureWorker").id}`), ctx.input.api.get<PreparedAgent>(`/api/agents/${entity(state, "worker").id}`),
+      ctx.input.api.get<Entity & { executionRunId?: string }>(`/api/issues/${entity(state, "task").id}`), ctx.allRuns() ]);
+      return gradeProjectDefaultsSetup({ lead, future, held, task: t, runs, activeRunId: String(state.activeRunId), gateCalls: gate?.calls ?? [] }); },
     sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-project-defaults.ts", "chat-catalog-domains/internal-run-brief.ts", "../e2e/fetch-allowed-port.ts", "run-observations.ts"],
     async act(ctx, state) {
       const settled = await pollUntil({ label: "Canonical project-defaults turn settles", deadlineAt: Date.now() + 150000, load: () => observeInternal(ctx, state),

tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts

@@ -1,10 +1,19 @@
 import { expect, it } from "vitest";
 import { createHash } from "node:crypto";
-import { gradeProjectDefaults, projectReadCalls } from "./internal-project-defaults.js";
+import { gradeProjectDefaults, gradeProjectDefaultsSetup, projectReadCalls } from "./internal-project-defaults.js";
 import { gradeRoutineAudit, routineHistoricalContext } from "./internal-routine-audit.js";
 import { gradeWorkTimeline } from "./internal-work-timeline.js";
 import { gradeArchivePartial } from "./internal-archive-partial.js";
 function base(scenarioId: string) { return { scenarioId, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: {} as Record<string, unknown>, tasks: [] as any[], projects: [] as any[], goals: [] as any[], comments: [] as any[], activity: [] as any[], documents: [], revisions: [], links: [] as string[], reply: "", humanQueries: [], runs: [] as any[] }; }
+it("preserves the exact active prerequisite while independently proving ready canonical and future agents", () => {
+  const ready = { id: "ceo", lifecycleState: "ready", status: "idle", adapterConfig: { timeoutSec: 120 } };
+  const prepared = { lead: ready, future: { ...ready, id: "riley" }, held: { ...ready, id: "morgan", status: "running", adapterConfig: { timeoutSec: 420 } }, task: { id: "task", assigneeAgentId: "morgan", executionRunId: "held" }, runs: [{ id: "held", agentId: "morgan", status: "running" }], activeRunId: "held", gateCalls: [{ authorized: true }] };
+  expect(gradeProjectDefaultsSetup(prepared).passed).toBe(true);
+  expect(gradeProjectDefaultsSetup({ ...prepared, held: { ...prepared.held, status: "idle" } }).passed).toBe(false);
+  expect(gradeProjectDefaultsSetup({ ...prepared, future: { ...prepared.future, status: "error" } }).passed).toBe(false);
+  expect(gradeProjectDefaultsSetup({ ...prepared, held: { ...prepared.held, adapterConfig: { timeoutSec: 120 } } }).passed).toBe(false);
+  expect(gradeProjectDefaultsSetup({ ...prepared, gateCalls: [] }).passed).toBe(false);
+});
 function defaults() {
   const o = base("projects.project-defaults"); const task = { id: "active", status: "in_progress", projectId: "p", assigneeAgentId: "morgan", executionRunId: "prior" }; const future = { id: "future", status: "done", projectId: "p" }; const p = { id: "p", description: "Keyboard navigation; adequate contrast; labeled controls", goalIds: ["growth"], env: null };
   Object.assign(o.state, { project: { id: "p" }, goal: { id: "growth", title: "Growth" }, task, futureTask: future, futureWorker: { id: "riley" }, activeRunId: "prior", originalEnv: null, defaultsReply: "New work uses the saved defaults. Existing active work remains unchanged." });
@@ -22,6 +31,9 @@ it("accepts consumed supported same-project API reads without requiring a partic
   const events = [{ runId: "new", eventType: "item.started", itemId: "read-p", payload: { kind: "dynamicToolCall", item: { type: "tool_use", id: "read-p", name: "call_api", input: { operationId: "GET /api/projects/{id}", apiPayloadRedacted: true, apiPayloadSha256: digest } } } },
     { runId: "new", eventType: "item.completed", itemId: "read-p", payload: { kind: "dynamicToolCall", item: { type: "tool_result", id: "read-p", tool_use_id: "read-p", result: { apiOperationId: "GET /api/projects/{id}", ok: true, status: 200 } } } }];
   Object.assign(o, { futureProjectReads: projectReadCalls(events, "new", "p") }); expect(gradeProjectDefaults(o).passed).toBe(true);
+  const stored = events.map((event, index) => ({ runId: "new", sourceEventId: `source-${index}`, payload: { prpEvent: { ...event, schema: "paperclip.prp.event.v1", sourceEventId: `source-${index}` } } }));
+  expect(projectReadCalls(stored, "new", "p")).toEqual(projectReadCalls(events, "new", "p"));
+  stored[0]!.payload.prpEvent.sourceEventId = "foreign-event"; expect(projectReadCalls(stored, "new", "p")).toEqual([]);
   expect(projectReadCalls(events, "new", "another-project")).toEqual([]); expect(projectReadCalls(events, "another-run", "p")).toEqual([]);
   events[1]!.itemId = "other-call"; expect(projectReadCalls(events, "new", "p")).toEqual([]);
   expect(projectReadCalls([{ runId: "new", eventType: "item.completed", payload: { kind: "agentMessage", text: JSON.stringify(events) } }], "new", "p")).toEqual([]);

tests/runner-e2e/chat-catalog-domains/internal-website-delivery.test.ts

@@ -1,7 +1,14 @@
 import { createHash } from "node:crypto";
 import { describe, expect, it } from "vitest";
-import { gradeWebsiteDelivery } from "./internal-website-delivery.js";
+import { gradeWebsiteDelivery, websitePrerequisiteSettled } from "./internal-website-delivery.js";
 const hash = (body: string) => createHash("sha256").update(body).digest("hex");
+it("keeps generated-but-undelivered work honestly blocked until the canonical delivery request", () => {
+  const staged = { task: { id: "task", status: "blocked" }, runs: [{ agentId: "morgan", status: "succeeded" }] };
+  expect(websitePrerequisiteSettled(staged, "morgan", false)).toBe(true);
+  expect(websitePrerequisiteSettled(staged, "morgan", true)).toBe(false);
+  expect(websitePrerequisiteSettled({ ...staged, runs: [{ agentId: "morgan", status: "failed" }] }, "morgan", false)).toBe(false);
+  expect(websitePrerequisiteSettled({ ...staged, runs: [{ agentId: "unrelated", status: "succeeded" }] }, "morgan", false)).toBe(false);
+});
 function valid(revising = false) {
   const task = { id: "task", companyId: "company", title: "Website task", status: "done", assigneeAgentId: "ceo" };
   const prior = "<html><body><h1>Bring your research team notes together in one accessible place</h1><p>Research teams stay organized.</p><a href='https://example.test/signup'>Signup</a><footer>ORCHID</footer></body></html>";

tests/runner-e2e/chat-catalog-domains/internal-website-delivery.ts

@@ -8,11 +8,15 @@ import type { WorkspaceFileContent } from "../../../packages/shared/src/types/wo
 import { pollUntil } from "../api.js";
 import { readRegisteredArtifacts } from "../registered-artifact.js";
 import { fixtureWorker, makeDefinition } from "./internal-definition.js";
-import { checks, entity, hasLink, observation, observeInternal, project, task, type InternalObservation } from "./internal-support.js";
+import { checks, entity, hasLink, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js";
 interface Downloaded { id: string; sha256: string; body: string }
 interface DeliveryObservation extends InternalObservation { attachments: IssueAttachment[]; products: IssueWorkProduct[]; downloaded: Downloaded[] }
 const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
 const headline = (body: string) => /<h1\b[^>]*>([\s\S]*?)<\/h1>/i.exec(body)?.[1]?.replace(/<[^>]+>/g, "").trim() ?? "";
+export function websitePrerequisiteSettled(value: { task: Entity; runs: { agentId: string; status: string }[] }, workerId: string, revising: boolean) {
+  return value.runs.length === 1 && value.runs[0]?.agentId === workerId && value.runs[0]?.status === "succeeded"
+    && value.task.status === (revising ? "done" : "blocked");
+}
 function downloadHref(o: InternalObservation, attachment: IssueAttachment) {
   const paths = [attachment.contentPath, attachment.openPath, attachment.downloadPath].filter((path): path is string => !!path);
   return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && paths.some(path => new URL(path, o.origin).pathname === url.pathname); } catch { return false; } });
@@ -57,10 +61,10 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
     const bodyFact = "Acorn Notes keeps quiet research teams organized."; const signupHref = "https://example.test/acorn-signup";
     const producedBody = `<!doctype html><html><head><meta charset="utf-8"><title>Acorn Notes</title></head><body><h1>Bring all of your research team notes together in one calm and accessible place</h1><p>${bodyFact}</p><a href="${signupHref}">Join the list</a><footer>${marker}</footer></body></html>\n`;
     const t = await task(ctx, "Website task", p.id, { status: "todo", assigneeAgentId: worker.id,
-      description: `Generate the website file ${filename} in this task's authorized project workspace, with exactly these UTF-8 bytes (JSON string): ${JSON.stringify(producedBody)}. Use ordinary filesystem tools and independently verify the saved file. ${revising ? `Deliver ${filename} through the normal product artifact registration and upload path, with that exact filename as its work-product title.` : "Keep the file in the authorized workspace for the subsequent delivery request; do not register or upload it yet."} Complete this initial build prerequisite after verifying the actual file. Do not create other tasks.` });
+      description: `Generate the website file ${filename} in this task's authorized project workspace, with exactly these UTF-8 bytes (JSON string): ${JSON.stringify(producedBody)}. Use ordinary filesystem tools and independently verify the saved file. ${revising ? `Deliver ${filename} through the normal product artifact registration and upload path, with that exact filename as its work-product title. Complete this initial build prerequisite after verifying the actual file.` : "This is the generate-only prerequisite stage: keep the verified file in the authorized workspace and pause the task as blocked awaiting the person's subsequent explicit delivery request. Do not claim finished delivery, register, or upload it yet."} Do not create other tasks.` });
     const ready = await pollUntil({ label: "Real prior website generation", deadlineAt: Date.now() + 240_000,
       load: async () => ({ task: await ctx.input.api.get<typeof t>(`/api/issues/${t.id}`), runs: await ctx.allRuns() }),
-      accept: value => value.runs.length === 1 && value.runs[0]?.agentId === worker.id && value.runs[0]?.status === "succeeded" && value.task.status === "done",
+      accept: value => websitePrerequisiteSettled(value, worker.id, revising),
       reject: value => value.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "The real prerequisite website build did not finish" : undefined });
     const produced = await ctx.input.api.get<WorkspaceFileContent>(`/api/issues/${t.id}/file-resources/content?workspace=execution&path=${encodeURIComponent(filename)}`);
     const actual = produced.content.encoding === "base64" ? Buffer.from(produced.content.data, "base64").toString("utf8") : produced.content.data;
@@ -71,8 +75,10 @@ function websiteDelivery(revising: boolean): ChatCatalogCase {
       if (original.content !== actual) throw new Error("The prior registered delivery does not match the generated website");
       originalAttachment = (await ctx.input.api.get<IssueAttachment[]>(`/api/issues/${t.id}/attachments`)).find(file => file.id === original.attachmentId)!; originalProductId = original.workProductId;
     } else if ((await ctx.input.api.get<IssueAttachment[]>(`/api/issues/${t.id}/attachments`)).length) throw new Error("The undelivered prerequisite unexpectedly uploaded the requested outcome");
-    // The responsible human hands the finished source to the CEO for the next ordinary delivery/revision request.
-    const handedOff = await ctx.input.api.patch<typeof t>(`/api/issues/${t.id}`, { assigneeAgentId: ctx.input.fixtures.agent.id });
+    // The delivered revision source can be handed over without starting work. The
+    // undelivered prerequisite stays honestly blocked with its original worker;
+    // assigning a nonterminal task here could admit an undeclared extra run.
+    const handedOff = revising ? await ctx.input.api.patch<typeof t>(`/api/issues/${t.id}`, { assigneeAgentId: ctx.input.fixtures.agent.id }) : ready.task;
     const state = { project: p, task: handedOff, worker, priorRunId, filename, producedBody: actual, producedHash: hash(actual), marker, bodyFact, signupHref, originalAttachment, originalProductId };
     await ctx.input.evidence("chat-catalog-actual-prior-website.json", state); return state;
   }, gradeWebsiteDelivery);