diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index f1c892cdd42..7e29d3539cf 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -675,8 +675,10 @@ before execution. The idle chat cell asks the agent to delegate one welcome note to a named worker and report its result without another user message. A bounded local file read in -the managed project workspace delays completion until the source chat is positively -observed idle, with a three-minute handoff setup budget and a four-minute worker +the worker task's own isolated task directory delays completion until the source +chat is positively observed idle. The wait command names only cwd-relative files, +because native providers cannot read agent homes or unconfigured project roots. The +setup uses a three-minute handoff setup budget and a four-minute worker wait limit. The brief is then released, the worker must save the output and reach Done. Its output must include the start time supplied only in that brief, allowing ordinary numeric and written forms. That check runs after completion observation diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index d20a9377361..b1aa9050765 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1934,7 +1934,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [localEnvironment], tasks: [...firstTaskTasks.filter(task => task.id === "interview-plan-accept"), ...chatCompletionTasks], expectedMatrixSize: 10, - definitionMetadata: { version: 27, runGrading: "evidenced-nonexecution-and-refusal", resultNavigation: "loaded-task-header", instructionSetup: "read-before-write-base-hash", requirementEvidence: "recorded-user-comments-and-resolved-answers", busyReferenceWait: "committed-conversation-document-response-held", busyBoundary: "source-tool-in-flight-and-public-deferred-wake", workerReference: "rsvp-code-in-saved-note", judge: "completion-quality-v14-observed-rendered-result-access", judgeMaxDollarsPerRequest: 0.5, instructions: "production", correlation: "authoritative-task-facts-required-in-reply-run", restartBoundary: "done-and-source-provider-at-reference-gate", idleBoundaryTimeoutMs: 180_000, workerBriefTimeoutMs: 240_000, workerBriefWorkspace: "managed-project", workerBriefEvidence: "released-start-time", grading: "post-completion-reply-and-result-access", semanticReview: "required-separately", chatBoundary: "worker-gated-until-source-idle", observationWindowMs: 120_000, scheduling: "explicit-only" }, + definitionMetadata: { version: 28, runGrading: "evidenced-nonexecution-and-refusal", resultNavigation: "loaded-task-header", instructionSetup: "read-before-write-base-hash", requirementEvidence: "recorded-user-comments-and-resolved-answers", busyReferenceWait: "committed-conversation-document-response-held", busyBoundary: "source-tool-in-flight-and-public-deferred-wake", workerReference: "rsvp-code-in-saved-note", judge: "completion-quality-v14-observed-rendered-result-access", judgeMaxDollarsPerRequest: 0.5, instructions: "production", correlation: "authoritative-task-facts-required-in-reply-run", restartBoundary: "done-and-source-provider-at-reference-gate", idleBoundaryTimeoutMs: 180_000, workerBriefTimeoutMs: 240_000, workerBriefWorkspace: "run-isolated-task-directory", workerBriefEvidence: "released-start-time", grading: "post-completion-reply-and-result-access", semanticReview: "required-separately", chatBoundary: "worker-gated-until-source-idle", observationWindowMs: 120_000, scheduling: "explicit-only" }, }, ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []), diff --git a/tests/runner-e2e/chat-stories.test.ts b/tests/runner-e2e/chat-stories.test.ts index 3979b9f1219..aeb650fa2e8 100644 --- a/tests/runner-e2e/chat-stories.test.ts +++ b/tests/runner-e2e/chat-stories.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { assertInterruptedChat, prepareChatBrief, publishChatBrief } from "./chat-stories.js"; +import { assertInterruptedChat, prepareChatBrief, publishChatBrief, workspaceChatBrief } from "./chat-stories.js"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -38,6 +38,28 @@ describe("active chat follow-up oracle", () => { await rm(root, { recursive: true, force: true }); } }); + it("waits for a brief inside the agent's own working directory without host paths", async () => { + const root = await mkdtemp(path.join(tmpdir(), "chat-brief-relative-")); + try { + const brief = workspaceChatBrief("fixture-1", 5_000); + // A sandboxed agent can read only its own workspace. The command must + // not name the fixture host directory, and must resolve from its cwd. + expect(brief.command).not.toContain(root); + expect(brief.command).not.toContain(path.sep + "tmp"); + const { gate, ready } = brief.at(root); + const command = promisify(execFile)("/bin/sh", ["-c", brief.command], { cwd: root, timeout: 5_000 }); + try { + await expect.poll(() => readFile(ready, "utf8").catch(() => "")).toBe("waiting"); + } finally { + await publishChatBrief(gate, "BRIEF relative result"); + expect((await command).stdout.trim()).toBe("BRIEF relative result"); + } + expect(() => workspaceChatBrief("../escape")).toThrow(); + expect(() => workspaceChatBrief("ok", 0)).toThrow("bounded"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); it("accepts either steering or one queued successor, with the saved correction", () => { expect(() => assertInterruptedChat(valid)).not.toThrow(); expect(() => assertInterruptedChat({ ...valid, runs: [run, { ...run, id: "successor" }], comments: [...valid.comments.slice(0, 2), { ...valid.comments[2]!, createdByRunId: "successor" }] })).not.toThrow(); diff --git a/tests/runner-e2e/chat-stories.ts b/tests/runner-e2e/chat-stories.ts index 73aa77a4f88..3caded16223 100644 --- a/tests/runner-e2e/chat-stories.ts +++ b/tests/runner-e2e/chat-stories.ts @@ -79,6 +79,23 @@ export async function prepareChatBrief(workspacePath: string, nonce: string, tim return { gate, ready, scriptPath }; } +/** + * A bounded brief wait that resolves only from the agent's working directory. + * Sandboxed native providers can read their own task directory but not agent + * homes or unrelated project roots, so the command names no host path. The + * fixture publishes to `at()` once that directory is known. + */ +export function workspaceChatBrief(nonce: string, timeoutMs = 120_000) { + if (!/^[A-Za-z0-9_-]+$/.test(nonce)) throw new Error("Brief nonce must be a plain file name segment"); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 300_000) throw new Error("Brief wait must be bounded between 1 and 300000ms"); + const name = `chat-brief-${nonce}.txt`; + const script = `const fs=require("node:fs"),g="${name}",end=Date.now()+${Math.floor(timeoutMs)};fs.writeFileSync(g+".waiting","waiting");const t=setInterval(()=>{if(fs.existsSync(g)){console.log(fs.readFileSync(g,"utf8"));clearInterval(t)}else if(Date.now()>end){clearInterval(t);process.exitCode=1}},100)`; + return { + command: `node -e '${script}'`, + at: (workspacePath: string) => ({ gate: path.join(workspacePath, name), ready: path.join(workspacePath, `${name}.waiting`) }), + }; +} + export async function runChatInterruption(context: Context & { refreshIssue(): Promise }) { const { input, marker, idle, allRuns, comments } = context; // Unprojected host /tmp files are intentionally hidden from native Codex. diff --git a/tests/runner-e2e/completion-update-flow.ts b/tests/runner-e2e/completion-update-flow.ts index 956f8971ada..9e6978ad53a 100644 --- a/tests/runner-e2e/completion-update-flow.ts +++ b/tests/runner-e2e/completion-update-flow.ts @@ -1,11 +1,11 @@ import { expect, type Page } from "@playwright/test"; -import { readFile } from "node:fs/promises"; +import { mkdir, readFile } from "node:fs/promises"; import path from "node:path"; -import { resolveManagedProjectWorkspaceDir, resolveDefaultAgentWorkspaceDir } from "../../server/src/home-paths.js"; +import { resolvePaperclipInstanceRoot } from "../../server/src/home-paths.js"; import { pollUntil, type RunnerApi } from "./api.js"; import { sendChatMessage, readChatOutputDocument, collectChatRunEvidence, type ChatFlowInput, type ChatRun } from "./chat-flow.js"; import { arm as armDocumentGate, clear as clearDocumentGate, release as releaseDocumentGate, waitUntilHeld } from "./context-comment-gate.js"; -import { prepareChatBrief, publishChatBrief } from "./chat-stories.js"; +import { publishChatBrief, workspaceChatBrief } from "./chat-stories.js"; import { completionDelivery, completionOutputUsesReleasedBrief, type CompletionObservation } from "./completion-updates.js"; type Row = Record; @@ -132,23 +132,35 @@ export async function runChatCompletionUpdate(context: { const restart = execution.task.id === "handoff-completion-restart"; const taskCount = multiple ? 2 : 1; const userMessages: string[] = []; - const completionWait = restart ? await prepareChatBrief(resolveDefaultAgentWorkspaceDir(f.agent.id), `${input.nonce}-completion`, 240_000) : null; + // Native runs see only their own task directory: a chat in its isolated + // conversation directory and a task in a workspace-free project in its + // isolated task directory. Agent homes and unconfigured project roots are + // outside the provider sandbox, so each wait resolves from the run's cwd. + const taskDirectory = (issueId: string) => { + const directory = path.join(resolvePaperclipInstanceRoot(), "isolated-workspaces", f.company.id, issueId); + const relative = path.relative(path.dirname(input.workspacePath), directory); + if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("Completion fixture escaped isolated instance"); + return directory; + }; + const publishBrief = async (brief: ReturnType, issueId: string, content: string) => { + const directory = taskDirectory(issueId); + await mkdir(directory, { recursive: true }); + await publishChatBrief(brief.at(directory).gate, content); + }; + const runCwd = (runs: Row[], issueId: string) => + runs.filter(r => r.contextSnapshot?.issueId === issueId).map(r => r.contextSnapshot?.paperclipWorkspace?.cwd ?? null); + const completionWait = restart ? workspaceChatBrief(`${input.nonce}-completion`, 240_000) : null; if (completionWait) { - const instruction = `For automatic delegated-work completion updates, first run node ${completionWait.scriptPath} and read the organizer's reference before posting your final reply. Do not run this script for the initial task handoff or other user messages. Reading this reference does not authorize additional tasks or changes to the completed work.`; + const instruction = `For automatic delegated-work completion updates, first run this exact command from your current working directory and read the organizer's reference before posting your final reply:\n\n${completionWait.command}\n\nDo not run this command for the initial task handoff or other user messages. Reading this reference does not authorize additional tasks or changes to the completed work.`; await api.saveAgentInstructions(f.agent.id, instruction); } const config = execution.profile.buildAgent({ environmentId: f.environment.id, environmentFixtureId: "local", workspacePath: input.workspacePath, secretRefs: f.secretRefs, executionId: input.nonce }); const worker = await api.post(`${company}/agents`, { ...config, name: "Riley Writer", role: "engineer", reportsTo: f.agent.id }); const project = await api.post(`${company}/projects`, { name: "Garden welcome", description: "A non-code neighborhood garden meetup. No repository needed." }); - // A project task runs in its managed project workspace. The agent-home path - // used by projectless chats is outside the native Codex workspace projection. - const workspace = resolveManagedProjectWorkspaceDir({ companyId: f.company.id, projectId: project.id }); - const relative = path.relative(path.dirname(input.workspacePath), workspace); - if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("Completion fixture escaped isolated instance"); - const wait = await prepareChatBrief(workspace, input.nonce, 240_000); + const wait = workspaceChatBrief(input.nonce, 240_000); const reference = marker; const brief = `The free Friday meetup starts at 10:30 in the community garden. The saved note must include RSVP code ${reference}.`; - const instructions = `For the welcome-note assignment, run node ${wait.scriptPath} to read the organizer's brief before writing the final note. Save a two-sentence welcome note as a Paperclip document on your assigned task using the brief's details and explicitly include its RSVP code in the note. Then complete your task. Do not edit or comment on another task.`; + const instructions = `For the welcome-note assignment, run this exact command from your task's working directory to read the organizer's brief before writing the final note:\n\n${wait.command}\n\nSave a two-sentence welcome note as a Paperclip document on your assigned task using the brief's details and explicitly include its RSVP code in the note. Then complete your task. Do not edit or comment on another task.`; await api.saveAgentInstructions(worker.id, instructions); expect(await api.get(`/api/agents/${worker.id}/instructions-bundle/file?path=AGENTS.md`)).toMatchObject({ content: instructions }); const prompt = `Create ${multiple ? "two separate tasks titled Welcome note A and Welcome note B" : "one task"} in the Garden welcome project (${project.id}) assigned to Riley Writer to write a two-sentence welcome note for our free Friday garden meetup. Riley has the organizer's brief. Save the finished note on that task and include RSVP code ${marker} in the note so attendees know which code to give the organizer. Please tell me here when the work is finished and give me access to the result. You may start the handoff now; no further approval is needed. Let Riley write the note.`; @@ -165,7 +177,8 @@ export async function runChatCompletionUpdate(context: { delegated = tasks.filter(t => t.assigneeAgentId === worker.id); task = delegated[0]; const runs = await context.allRuns(); - return { source, tasks, runs, ready: await readFile(wait.ready, "utf8").catch(() => "") }; + const ready = task ? await readFile(wait.at(taskDirectory(task.id)).ready, "utf8").catch(() => "") : ""; + return { source, tasks, runs, ready, workerCwds: task ? runCwd(runs, task.id) : [] }; }, accept: state => delegated.length === taskCount && state.ready === "waiting" && state.source.conversationState === "waiting" && state.runs.some(r => r.contextSnapshot?.issueId === task!.id && r.status === "running") && @@ -190,7 +203,7 @@ export async function runChatCompletionUpdate(context: { expect(reference.body).toContain(`REFERENCE${marker}`); await input.evidence("completion-busy-document-gate.json", { sourceRun: busyRun, document: reference, responseHeld: true }); } - await publishChatBrief(wait.gate, brief); + for (const item of delegated) await publishBrief(wait, item.id, brief); await pollUntil({ label: "delegated welcome note completed", deadlineAt: Date.now() + 180_000, intervalMs: 1000, load: () => api.get(`/api/issues/${task!.id}`), accept: t => t.status === "done" }); if (busyRun) { @@ -216,7 +229,11 @@ export async function runChatCompletionUpdate(context: { } if (restart) { const reportingRun = await pollUntil({ label: "completion reply is waiting at its reference gate before publication", deadlineAt: Date.now() + 120_000, intervalMs: 1000, - load: async () => ({ runs: await context.allRuns(), ready: await readFile(completionWait!.ready, "utf8").catch(() => "") }), + load: async () => { + const runs = await context.allRuns(); + return { runs, ready: await readFile(completionWait!.at(taskDirectory(context.issue().id)).ready, "utf8").catch(() => ""), + sourceCwds: runCwd(runs, context.issue().id) }; + }, accept: state => state.ready === "waiting" && state.runs.some(r => r.contextSnapshot?.issueId === context.issue().id && r.contextSnapshot?.wakeReason === "chat_task_completed" && r.status === "running") }); const beforeRestart = await api.get(`/api/issues/${context.issue().id}/comments?order=asc`); @@ -227,7 +244,7 @@ export async function runChatCompletionUpdate(context: { // neither task records nor completion events are fabricated by the fixture. await input.evidence("completion-restart-boundary.json", { worker: await api.get(`/api/issues/${task!.id}`), runs: reportingRun.runs, sourceReferenceGateReady: true }); await input.restart(); - await publishChatBrief(completionWait!.gate, "The organizer is ready to receive the saved result. Report the completed work now."); + await publishBrief(completionWait!, context.issue().id, "The organizer is ready to receive the saved result. Report the completed work now."); await page.goto(`/${f.company.issuePrefix}/chats/${f.agent.id}`, { waitUntil: "commit" }); } for (const [index, item] of delegated.entries()) { @@ -253,9 +270,12 @@ export async function runChatCompletionUpdate(context: { expect(replies, `one correlated completion reply for ${item.id}`).toHaveLength(1); } } finally { - await publishChatBrief(wait.gate, `Reference: ${reference}`); + // Release every wait a failed attempt may have left running. + const release = (work: Promise) => work.catch(() => {}); + for (const item of delegated) await release(publishBrief(wait, item.id, `Reference: ${reference}`)); if (busy) await releaseDocumentGate(context.issue().id); - if (completionWait) await publishChatBrief(completionWait.gate, "The organizer is ready to receive the saved result."); + const sourceId = (context.issue() as { id: string } | undefined)?.id; + if (completionWait && sourceId) await release(publishBrief(completionWait, sourceId, "The organizer is ready to receive the saved result.")); await context.refreshIssue(); } }