From 1d6ee1a642a650b6f31704b283eb7a6c2d35202c Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 8 Sep 2026 23:59:55 -0500 Subject: [PATCH] test: read full run context and compare exact warm file bytes Co-Authored-By: Paperclip --- doc/sandbox-work-folders.md | 12 +++ tests/runner-e2e/catalog.ts | 30 +++++- tests/runner-e2e/warm-workspace.test.ts | 128 ++++++++++++++++++------ tests/runner-e2e/warm-workspace.ts | 23 ++++- 4 files changed, 158 insertions(+), 35 deletions(-) diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index 21ee8ad0cd3..63e64a48727 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -357,3 +357,15 @@ binary. An older binary is replaced with the app’s compatible artifact through a temporary file and atomic rename, preserving image symlink targets and the previous launcher on interrupted uploads. The durable checkpoint contract stays at version 2 so existing native session backups remain restorable. + +Warm continuity qualification reads the full run record before selecting the +persistence contract; company run listings omit the scoped-folder manifest. +Each completed scoped turn must have its own successful final save before the +harness reads cached bytes. Legacy host-workspace fallback is used only when +the full run record has no scoped manifest. + +The three-turn fixture supplies a shell script that compares exact bytes before +appending and after writing. Missing or changed prior content fails without +repair; repeating the first turn cannot truncate existing work. This avoids +model-generated byte-count arithmetic while retaining independent per-turn +persistence checks and the same-sandbox, same-provider continuity requirements. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index db8184ffe06..04b366df3a0 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -776,6 +776,26 @@ function warmWorkspaceLine(turn: 1 | 2 | 3, nonce: string) { return `T${turn}-${nonce}`; } +function warmWorkspaceScript(turn: 1 | 2 | 3, nonce: string) { + if (!/^[a-zA-Z0-9-]+$/.test(nonce)) { + throw new Error("Invalid warm fixture nonce"); + } + const lines = Array.from({ length: turn }, (_, index) => + warmWorkspaceLine((index + 1) as 1 | 2 | 3, nonce), + ); + const expected = (values: string[]) => + `printf '%s\\n' ${values.map((line) => `'${line}'`).join(" ")}`; + return [ + "set -eu", + `target="\${PAPERCLIP_TASK_DIR:-$PWD}/daytona-warm-${nonce}.txt"`, + turn === 1 + ? '[ ! -e "$target" ]' + : `${expected(lines.slice(0, -1))} | cmp - "$target"`, + `${expected([lines.at(-1)!])} >> "$target"`, + `${expected(lines)} | cmp - "$target"`, + ].join("\n"); +} + function warmTurnInstructions(turn: 1 | 2 | 3, nonce: string) { const file = `"\${PAPERCLIP_TASK_DIR:-$PWD}/daytona-warm-${nonce}.txt"`; const lines = Array.from({ length: turn }, (_, index) => @@ -789,9 +809,13 @@ function warmTurnInstructions(turn: 1 | 2 | 3, nonce: string) { return [ `This is warm Daytona continuity turn ${turn} of 3. Use ${file} for every read and write. Expand this shell path inside each tool invocation; do not rely on a directory change in a previous shell call. The host-supplied task folder takes precedence over the current directory.`, turn === 1 - ? `Create ${file} with exactly this one line followed by a newline: ${lines[0]}` - : `Before changing anything, read ${file} and verify its content is exactly ${lines.slice(0, -1).join("\\n")} followed by a newline. Then append exactly ${lines.at(-1)} followed by a newline.`, - `After the write, verify ${file} contains exactly these lines, once each and in order: ${lines.join(" | ")}.`, + ? `The script below creates ${file} with exactly this one line followed by a newline: ${lines[0]}` + : `The script below first checks ${file} contains exactly ${lines.slice(0, -1).join("\\n")} followed by a newline, then appends exactly ${lines.at(-1)} followed by a newline.`, + `The final comparison requires ${file} to contain exactly these lines, once each and in order: ${lines.join(" | ")}.`, + "Execute this exact shell script once for the file operation and verification. It compares every byte, including the final newline; do not add guessed byte counts. If it fails, report the failure and do not repair the file or claim success. Only after it succeeds perform the runner-specific completion below.", + "```sh", + warmWorkspaceScript(turn, nonce), + "```", `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"${finalTurn ? "done" : "needs_review"}",summary:"${marker}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[{commandOrCheck:${JSON.stringify(`read ${file}`)},status:"passed"}]}. Wait for that tool call to succeed, then emit exactly ${marker} once as the complete user-facing final response.`, legacyCompletion, `In a legacy runner, the PATCH comment is the complete visible response. After its 2xx response, finish silently: do not print, echo, or emit ${marker} again as assistant text.`, diff --git a/tests/runner-e2e/warm-workspace.test.ts b/tests/runner-e2e/warm-workspace.test.ts index b0b824fbb76..cd3f7e435ee 100644 --- a/tests/runner-e2e/warm-workspace.test.ts +++ b/tests/runner-e2e/warm-workspace.test.ts @@ -40,23 +40,33 @@ async function fixture() { status: () => 200, text: async () => "T1-nonce\n", }; + const fullRun = { + id: "run", + companyId: "company", + agentId: "agent", + contextSnapshot: { paperclipWorkFolders: binding } as Record< + string, + unknown + > | null, + }; const api = { - get: vi.fn().mockResolvedValue([saved]), + get: vi + .fn() + .mockImplementation(async (url: string) => + url === "/api/heartbeat-runs/run" ? fullRun : [saved], + ), request: { get: vi.fn().mockResolvedValue(response) }, }; return { input: { api, - run: { - id: "run", - companyId: "company", - agentId: "agent", - contextSnapshot: { paperclipWorkFolders: binding }, - }, + run: { id: "run", companyId: "company", agentId: "agent" }, issueId: "task", workspacePath, filename: "daytona-warm-nonce.txt", }, + fullRun, + binding, saved, }; } @@ -75,16 +85,13 @@ describe("warm workspace persistence observation", () => { daytonaWarmContinuityTask.buildPrompt("nonce"), ...daytonaWarmContinuityTask.buildFollowupMessages!("nonce"), ]; - for (const [index, prompt] of prompts.entries()) { - const shellPath = prompt.match( - /Use ("[^"\n]+") for every read and write\./, - )?.[1]; - expect(shellPath).toBeTruthy(); - await promisify(execFile)( - "sh", - ["-c", `printf 'T${index + 1}-nonce\\n' >> ${shellPath}`], - { cwd: input.workspacePath, env }, - ); + for (const prompt of prompts) { + const script = prompt.match(/```sh\n([\s\S]*?)\n```/)?.[1]; + expect(script).toBeTruthy(); + await promisify(execFile)("sh", ["-c", script!], { + cwd: input.workspacePath, + env, + }); } expect( await readFile( @@ -102,7 +109,48 @@ describe("warm workspace persistence observation", () => { }, ); - it("reads the saved scoped file without requiring a mirrored host file", async () => { + it.each([undefined, "wrong\n", "T1-nonce", "T1-nonce\n\n"])( + "rejects missing or changed prior content without repairing it: %j", + async (prior) => { + const { input } = await fixture(); + const filename = path.join(input.workspacePath, input.filename); + if (prior !== undefined) await writeFile(filename, prior); + const prompt = + daytonaWarmContinuityTask.buildFollowupMessages!("nonce")[0]!; + const script = prompt.match(/```sh\n([\s\S]*?)\n```/)![1]!; + await expect( + promisify(execFile)("sh", ["-c", script], { + cwd: input.workspacePath, + env: { ...process.env, PAPERCLIP_TASK_DIR: input.workspacePath }, + }), + ).rejects.toThrow(); + if (prior === undefined) { + await expect(readFile(filename)).rejects.toMatchObject({ + code: "ENOENT", + }); + } else { + expect(await readFile(filename, "utf8")).toBe(prior); + } + }, + ); + + it("rejects repeating the initial write without truncating saved work", async () => { + const { input } = await fixture(); + const filename = path.join(input.workspacePath, input.filename); + await writeFile(filename, "T1-nonce\n"); + const script = daytonaWarmContinuityTask + .buildPrompt("nonce") + .match(/```sh\n([\s\S]*?)\n```/)![1]!; + await expect( + promisify(execFile)("sh", ["-c", script], { + cwd: input.workspacePath, + env: { ...process.env, PAPERCLIP_TASK_DIR: input.workspacePath }, + }), + ).rejects.toThrow(); + expect(await readFile(filename, "utf8")).toBe("T1-nonce\n"); + }); + + it("hydrates an abbreviated run before reading scoped bytes without a host mirror", async () => { const { input } = await fixture(); expect(await readWarmWorkspaceFile(input)).toEqual({ source: "task-cache", @@ -131,6 +179,27 @@ describe("warm workspace persistence observation", () => { ); }); + it("does not infer an unscoped run from another abbreviated API response", async () => { + const { input } = await fixture(); + input.api.get.mockResolvedValue(input.run); + await writeFile( + path.join(input.workspacePath, input.filename), + "stale host copy", + ); + await expect(readWarmWorkspaceFile(input)).rejects.toThrow( + "Full run context is required", + ); + expect(input.api.request.get).not.toHaveBeenCalled(); + }); + + it("rejects a full response for a different run", async () => { + const { input, fullRun } = await fixture(); + fullRun.id = "different-run"; + await expect(readWarmWorkspaceFile(input)).rejects.toThrow(); + expect(input.api.get).toHaveBeenCalledTimes(1); + expect(input.api.request.get).not.toHaveBeenCalled(); + }); + it.each([ { runId: "older-run" }, { state: "failed" }, @@ -141,7 +210,7 @@ describe("warm workspace persistence observation", () => { "rejects incomplete or unrelated checkpoint evidence: %j", async (override) => { const { input, saved } = await fixture(); - input.api.get.mockResolvedValue([{ ...saved, ...override }]); + Object.assign(saved, override); await expect(readWarmWorkspaceFile(input)).rejects.toThrow( "successful final file save", ); @@ -152,26 +221,25 @@ describe("warm workspace persistence observation", () => { it.each(["runId", "companyId", "agentId", "taskId"] as const)( "rejects a manifest with a different %s", async (field) => { - const { input } = await fixture(); - input.run.contextSnapshot.paperclipWorkFolders[field] = "other"; + const { input, binding } = await fixture(); + binding[field] = "other"; await expect(readWarmWorkspaceFile(input)).rejects.toThrow(); - expect(input.api.get).not.toHaveBeenCalled(); + expect(input.api.get).toHaveBeenCalledTimes(1); }, ); it("keeps the host workspace contract when the run has no scoped manifest", async () => { - const { input } = await fixture(); + const { input, fullRun } = await fixture(); + fullRun.contextSnapshot = null; await writeFile( path.join(input.workspacePath, input.filename), "T1-local\n", ); - expect( - await readWarmWorkspaceFile({ - ...input, - run: { ...input.run, contextSnapshot: null }, - }), - ).toEqual({ source: "host-workspace", content: "T1-local\n" }); - expect(input.api.get).not.toHaveBeenCalled(); + expect(await readWarmWorkspaceFile(input)).toEqual({ + source: "host-workspace", + content: "T1-local\n", + }); + expect(input.api.get).toHaveBeenCalledTimes(1); expect(input.api.request.get).not.toHaveBeenCalled(); }); }); diff --git a/tests/runner-e2e/warm-workspace.ts b/tests/runner-e2e/warm-workspace.ts index eb857ccb7c2..542bd6574dd 100644 --- a/tests/runner-e2e/warm-workspace.ts +++ b/tests/runner-e2e/warm-workspace.ts @@ -16,7 +16,6 @@ export async function readWarmWorkspaceFile(input: { id: string; companyId: string; agentId: string; - contextSnapshot?: Record | null; }; issueId: string; workspacePath: string; @@ -27,7 +26,27 @@ export async function readWarmWorkspaceFile(input: { /^[a-zA-Z0-9-]+\.txt$/.test(filename), "Invalid warm fixture filename", ); - const manifest = run.contextSnapshot?.paperclipWorkFolders; + // Company run listings summarize context. Absence there cannot establish + // that the run uses the legacy host-workspace contract. + const fullRun = await api.get< + typeof run & { + contextSnapshot: Record | null; + } + >(`/api/heartbeat-runs/${encodeURIComponent(run.id)}`); + assert.equal(fullRun.id, run.id); + assert.equal(fullRun.companyId, run.companyId); + assert.equal(fullRun.agentId, run.agentId); + assert( + Object.hasOwn(fullRun, "contextSnapshot"), + "Full run context is required", + ); + assert( + fullRun.contextSnapshot === null || + (typeof fullRun.contextSnapshot === "object" && + !Array.isArray(fullRun.contextSnapshot)), + "Invalid full run context", + ); + const manifest = fullRun.contextSnapshot?.paperclipWorkFolders; if (manifest === undefined || manifest === null) { return { source: "host-workspace",