fix(evals): bind approval wait to actual task workspace

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/runtime-approval-queue.ts

@@ -1,11 +1,11 @@
 import { expect } from "@playwright/test";
-import { readFile, writeFile } from "node:fs/promises";
+import { writeFile } from "node:fs/promises";
 import type { ActivityEvent } from "../../../packages/shared/src/types/activity.js";
 import type { IssueQueuedCommentQueue, IssueThreadInteraction } from "../../../packages/shared/src/types/issue.js";
 import { pollUntil } from "../api.js";
 import { submitTaskReply } from "../user-actions.js";
 import { checks, type Entity } from "./internal-support.js";
-import { prepareRuntimeWorkerGate, releaseRuntimeWorker } from "./runtime-busy-worker.js";
+import { prepareRuntimeWorkerGate, readRuntimeTaskGate, releaseRuntimeWorker } from "./runtime-busy-worker.js";
 import { runtimeQueueCases } from "./runtime-queue.js";
 import { runtimeDefinition, observeRuntime, runtimeObservation } from "./runtime-support.js";
 export function acornNamingDirection(reply: string) {
@@ -41,11 +41,11 @@ export function gradeApprovalQueue(value: unknown) {
   return c.result();
 }
 export const runtimeApprovalQueueCases = [runtimeDefinition("steering.approval-delivery", async ctx => {
-  const prepared = await prepareRuntimeWorkerGate(ctx), root = `/api/companies/${ctx.input.fixtures.company.id}`;
+  const prepared = await prepareRuntimeWorkerGate(ctx, { taskBound: true }), root = `/api/companies/${ctx.input.fixtures.company.id}`;
   const task = await ctx.input.api.post<Entity>(`${root}/issues`, { title: "Website naming task", status: "todo", assigneeAgentId: prepared.worker.id,
-    description: `We need a product name before continuing the website. During this actual native run, create exactly one pending ask_user_questions interaction using the supported public issue interaction API, authenticated as yourself with this actual run identity. Use idempotencyKey naming-${ctx.input.nonce}, title Website product name, continuationPolicy wake_assignee, and payload {version:1,supersedeOnUserComment:false,questionSet:{schema:"paperclip.question_set.v1",title:"Website product name",submitLabel:"Submit answers",questions:[{id:"name",prompt:"What is the product name?",answerMode:"text",required:true}]}}. This is an ordinary durable task question, not an artificial provider runtime-request receipt. Do not choose a name or create another interaction. After the real question is saved, run node ${JSON.stringify(prepared.gate.scriptPath)} once and wait for its actual output. Then save the returned pending-work note as a task document and finish this current turn normally, leaving queued human answers for normal continuation. On a later normal continuation, read the real submitted name response, save a non-plan website-delivery document using that name and a short semantic website outline, and complete the same task.` });
+    description: `We need a product name before continuing the website. During this actual native run, create exactly one pending ask_user_questions interaction using the supported public issue interaction API, authenticated as yourself with this actual run identity. Use idempotencyKey naming-${ctx.input.nonce}, title Website product name, continuationPolicy wake_assignee, and payload {version:1,supersedeOnUserComment:false,questionSet:{schema:"paperclip.question_set.v1",title:"Website product name",submitLabel:"Submit answers",questions:[{id:"name",prompt:"What is the product name?",answerMode:"text",required:true}]}}. This is an ordinary durable task question, not an artificial provider runtime-request receipt. Do not choose a name or create another interaction. After the real question is saved, run ${prepared.gate.command} once and wait for its actual output. Then save the returned pending-work note as a task document and finish this current turn normally, leaving queued human answers for normal continuation. On a later normal continuation, read the real submitted name response, save a non-plan website-delivery document using that name and a short semantic website outline, and complete the same task.` });
   const waiting = await pollUntil({ label: "Real native naming question while worker waits in its own tool process", deadlineAt: Date.now() + 180_000,
-    load: async () => ({ runs: await ctx.allRuns(), ready: await readFile(prepared.gate.ready, "utf8").catch(() => ""), questions: await ctx.input.api.get<IssueThreadInteraction[]>(`/api/issues/${task.id}/interactions`) }),
+    load: async () => ({ runs: await ctx.allRuns(), ready: await readRuntimeTaskGate(ctx, prepared.gate, task.id, prepared.worker.id), questions: await ctx.input.api.get<IssueThreadInteraction[]>(`/api/issues/${task.id}/interactions`) }),
     accept: v => v.runs.length === 1 && v.runs[0]?.agentId === prepared.worker.id && v.runs[0].status === "running" && v.ready === "waiting"
       && v.questions.length === 1 && v.questions[0]?.kind === "ask_user_questions" && v.questions[0].status === "pending" && v.questions[0].createdByAgentId === prepared.worker.id && v.questions[0].sourceRunId === v.runs[0].id,
     reject: v => v.runs.length > 1 || v.runs.some(row => ["failed", "cancelled", "succeeded", "timed_out"].includes(row.status)) ? "The actual question fixture did not reach its running pending-input boundary" : undefined });

tests/runner-e2e/chat-catalog-domains/runtime-approval-workspace.test.ts

@@ -0,0 +1,45 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+const prepare = vi.hoisted(() => vi.fn());
+const poll = vi.hoisted(() => vi.fn());
+vi.mock("./runtime-busy-worker.js", async original => ({ ...await original<object>(), prepareRuntimeWorkerGate: prepare }));
+vi.mock("../api.js", async original => ({ ...await original<object>(), pollUntil: poll }));
+import { taskBoundRuntimeGate } from "./runtime-busy-worker.js";
+import { runtimeApprovalQueueCases } from "./runtime-approval-queue.js";
+const roots: string[] = [];
+afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); });
+async function fixture() {
+  const root = await mkdtemp(path.join(tmpdir(), "approval-owned-task-")); roots.push(root); vi.stubEnv("PAPERCLIP_RUNNER_E2E_TEMP_ROOT", root);
+  const cwd = path.join(root, "actual-isolated-task"); await mkdir(cwd);
+  const gate = taskBoundRuntimeGate("approval-owned");
+  await writeFile(path.join(cwd, `${gate.gateName}.waiting`), "waiting");
+  const task = { id: "task", companyId: "company", assigneeAgentId: "morgan", executionWorkspaceId: "workspace" };
+  const workspace = { id: "workspace", companyId: "company", cwd };
+  const post = vi.fn(async (_url: string, _body: unknown) => task);
+  prepare.mockImplementation(async (_ctx, options) => { expect(options).toEqual({ taskBound: true }); return { gate, worker: { id: "morgan" } }; });
+  const ctx = { input: { nonce: "approval-owned", fixtures: { company: { id: "company" } }, api: { post, get: async (url: string) => url.endsWith("/workspace") ? { workspace } : url.endsWith("/interactions") ? [{ id: "question", kind: "ask_user_questions", status: "pending", issueId: "task", createdByAgentId: "morgan", sourceRunId: "run" }] : task } }, allRuns: async () => [{ id: "run", agentId: "morgan", status: "running" }] } as unknown as ChatCatalogContext;
+  return { ctx, root, cwd, task, workspace, gate, post };
+}
+it("approval setup consumes actual assigned-workspace readiness using a cwd-local bounded command", async () => {
+  const f = await fixture();
+  poll.mockImplementationOnce(async options => { const value = await options.load(); expect(options.accept(value)).toBe(true); throw new Error("verified active boundary"); });
+  await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow("verified active boundary");
+  expect(f.post.mock.calls[0]![1]).toMatchObject({ assigneeAgentId: "morgan", status: "todo" });
+  const body = f.post.mock.calls[0]![1] as { description: string };
+  expect(body.description).toContain(`run ${f.gate.command} once`);
+  expect(body.description).not.toContain(f.root);
+  expect(f.gate.boundTaskId).toBe("task"); expect(f.gate.boundWorkspaceId).toBe("workspace");
+  expect(f.gate.ready).toBe(path.join(await realpath(f.cwd), `${f.gate.gateName}.waiting`));
+});
+it.each(["company", "assignee", "workspace"])("approval setup rejects foreign %s readiness before accepting the question boundary", async invalid => {
+  const f = await fixture();
+  if (invalid === "company") f.workspace.companyId = "foreign";
+  if (invalid === "assignee") f.task.assigneeAgentId = "other";
+  if (invalid === "workspace") f.task.executionWorkspaceId = "other";
+  const accept = vi.fn(); poll.mockImplementationOnce(async options => { await options.load(); accept(); });
+  await expect(runtimeApprovalQueueCases[0]!.setup(f.ctx)).rejects.toThrow(/actual company, assignee and workspace/);
+  expect(accept).not.toHaveBeenCalled();
+});