From 97b89981d30ee540f634c9c906ba836b0a6610ba Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:54:15 -0600 Subject: [PATCH] test(chaos): wait for the closing save before reading a remote R1 workspace's files (#20348) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart chaos harness grades Daytona R1 tasks by reading their files through `GET /issues/:id/file-resources/content` > - The server serves a remote workspace's files either from its live lease or, once the closing save finishes, from the saved snapshot > - The closing save takes about a minute for a Paperclip-sized workspace, and during it the workspace is dirty with a writer, so reads return 422 `workspace_unavailable` > - The harness read once at `done` and once at rest, both inside that window, so every Daytona R1 task failed its file checks even when the agent did the work > - This pull request retries a 422 read for up to 4 minutes, until the saved snapshot is readable, and returns a 404 at once > - The benefit is that Daytona R1 grades the agent's actual files ## Linked Issues or Issue Description Follow-up to #20288. **What happened?** Daytona R1 wave w-20261109T191837-s9950 (K1 at +180 s, native runs) recovered both tasks without a Retry (R1, R3 and R6 pass), but both failed `T:module_present`. The server log shows every content read returning 422, from the moment the task was `done` (19:23:04) through rest (19:23:10). Track 3's R1 waves lost 10 of 16 tasks the same way. **Expected behavior** The oracle reads the files once the workspace's closing save makes them readable. **Actual behavior** It gave up on the first 422. ## What Changed - `remoteWorkspaceFile` retries a 422 every 2 s for up to 4 minutes (`REMOTE_FILE_WAIT_MS`). Any other failure, including 404 for a file the agent never wrote, returns `null` at once. - README: notes the wait. ## Verification - `npx vitest run --config tests/restart-chaos/vitest.config.ts`: 260 passed. New tests: a server that answers 422 twice and then the files (red before), and a missing file that returns within 10 s. - `node scripts/typecheck-test-projects.mjs tests/restart-chaos`: clean. ## Risks - A remote workspace whose closing save never completes now costs up to 4 minutes of grading per file read before failing, which is visible in the wave's duration. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/README.md | 4 +- tests/restart-chaos/observer.ts | 12 ++---- tests/restart-chaos/remote-oracle.test.ts | 39 +++++++++++++++++++ tests/restart-chaos/remote-workspace-files.ts | 30 ++++++++++++++ 4 files changed, 74 insertions(+), 11 deletions(-) create mode 100644 tests/restart-chaos/remote-workspace-files.ts diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 10484cdc381..d1189d1aadf 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -201,8 +201,8 @@ With `--env daytona --storage s3`, each task's workspace is a remote workspace: and restores it through the store, so the faults also hit workspace save, restore and copyback. R1 (`--workspace-repo`) runs on Daytona only with `--storage s3`: the project points at the checkout's GitHub `origin` over https at its `origin/master` commit, so the sandbox clones the repository and its -snapshots are Paperclip-sized. Its oracle reads the task's files through the board API and runs its -commands on a local copy. +snapshots are Paperclip-sized. Its oracle reads the task's files through the board API, retrying for up +to 4 minutes while the closing save makes the workspace readable, and runs its commands on a local copy. Every Retry the grader needs is clicked in the real board UI: the harness opens the issue in a headless Chromium, waits for the visible Retry button and clicks it. A task whose Retry button is missing fails diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 94397449e30..9cb756a5325 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -15,6 +15,7 @@ import type { import { restartProcessStartMatches } from "../runner-e2e/process-tree-owner.js"; import type { UiRetryClick } from "./board-ui.js"; import { runOracleCommand } from "./oracle-sandbox.js"; +import { captureRemoteWorkspaceFiles, readRemoteWorkspaceFile } from "./remote-workspace-files.js"; import { taskFaultSchedule } from "./scheduler.js"; import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js"; @@ -692,20 +693,13 @@ export class TaskObserver { private async remoteWorkspaceFile(issueId: string, relative: string): Promise { const captured = this.record.remoteFiles?.[relative]; - if (typeof captured === "string") return captured; - return await this.api.getText(`/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&download=true`, 15_000) - .catch(() => null); + return typeof captured === "string" ? captured : await readRemoteWorkspaceFile(this.api, issueId, relative); } private async captureRemoteFilesAtDone(sample: TaskSample): Promise { const wanted = this.definition.collect.files(this.plan.nonce); if (!wanted.length || this.lab.environmentKind === "local" || sample.issueStatus !== "done" || !this.record.issueId) return; - const captured = this.record.remoteFiles ??= {}; - for (const relative of wanted) { - if (typeof captured[relative] === "string") continue; - captured[relative] = await this.api.getText(`/api/issues/${this.record.issueId}/file-resources/content?path=${encodeURIComponent(relative)}&download=true`, 15_000) - .catch(() => null); - } + await captureRemoteWorkspaceFiles(this.api, this.record.issueId, wanted, this.record.remoteFiles ??= {}); } async workspacePath(): Promise { diff --git a/tests/restart-chaos/remote-oracle.test.ts b/tests/restart-chaos/remote-oracle.test.ts index 2e7c81fc180..ec544bbd581 100644 --- a/tests/restart-chaos/remote-oracle.test.ts +++ b/tests/restart-chaos/remote-oracle.test.ts @@ -31,6 +31,7 @@ const plan: TaskPlan = { taskId: "t01", index: 0, workload: "R1", adapter: "code holdMs: 0, followUpAfterMs: null, nonce }; let server: Server; let sandboxReleased = false; +let unavailableReads = 0; let baseUrl: string; beforeAll(async () => { @@ -38,6 +39,7 @@ beforeAll(async () => { const url = new URL(incoming.url ?? "/", "http://x"); if (url.pathname === "/api/issues/i1/file-resources/content" && url.searchParams.get("download") === "true") { if (incoming.headers.origin !== baseUrl) { outgoing.writeHead(403).end(); return; } + if (unavailableReads > 0) { unavailableReads -= 1; outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; } if (sandboxReleased) { outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; } const body = remoteFiles[url.searchParams.get("path") ?? ""]; outgoing.writeHead(body ? 200 : 404, { "content-type": "text/plain" }).end(body ?? ""); @@ -105,5 +107,42 @@ describe("oracle for a remote workspace", () => { expect(artifacts.commands.node_test).toMatchObject({ exitCode: 0 }); } finally { sandboxReleased = false; } }, 60_000); + + it("waits for the closing save to make a remote workspace readable again before grading its files", async () => { + const api = { + baseUrl, + async get(route: string) { + if (route.endsWith("/comments")) return []; + return { id: "i1", status: "done", currentExecutionWorkspace: { cwd: "/home/daytona/workspaces/not-on-this-host" } }; + }, + async post() { return { id: "i1", identifier: "CHA-1" }; }, + getText: (route: string) => new ChaosApi(baseUrl).getText(route), + } as unknown as ChaosApi; + const observer = new TaskObserver(plan, api, lab, new Timeline(null), { serverReady: true, faultInProgress: false }); + await observer.launch(); + sandboxReleased = false; + unavailableReads = 2; + const artifacts = await observer.collectArtifacts(); + expect(unavailableReads).toBe(0); + expect(artifacts.files).toEqual(remoteFiles); + }, 60_000); + + it("does not wait on a file the agent never wrote", async () => { + const api = { + baseUrl, + async get(route: string) { + if (route.endsWith("/comments")) return []; + return { id: "i1", status: "done", currentExecutionWorkspace: { cwd: "/home/daytona/workspaces/not-on-this-host" } }; + }, + async post() { return { id: "i1", identifier: "CHA-1" }; }, + getText: (route: string) => new ChaosApi(baseUrl).getText(route.replace(/slug-/g, "missing-")), + } as unknown as ChaosApi; + const observer = new TaskObserver(plan, api, lab, new Timeline(null), { serverReady: true, faultInProgress: false }); + await observer.launch(); + const started = Date.now(); + const artifacts = await observer.collectArtifacts(); + expect(Object.values(artifacts.files)).toEqual([null, null]); + expect(Date.now() - started).toBeLessThan(10_000); + }, 60_000); }); diff --git a/tests/restart-chaos/remote-workspace-files.ts b/tests/restart-chaos/remote-workspace-files.ts new file mode 100644 index 00000000000..897090f1281 --- /dev/null +++ b/tests/restart-chaos/remote-workspace-files.ts @@ -0,0 +1,30 @@ +import { ChaosApiError, type ChaosApi } from "./api.js"; + +export const REMOTE_FILE_WAIT_MS = 4 * 60_000; +export const REMOTE_FILE_POLL_MS = 2_000; + +function contentRoute(issueId: string, relative: string): string { + return `/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&download=true`; +} + +export async function readRemoteWorkspaceFile(api: Pick, issueId: string, relative: string, + options: { waitMs?: number; pollMs?: number } = {}): Promise { + const deadline = Date.now() + (options.waitMs ?? REMOTE_FILE_WAIT_MS); + for (;;) { + let unavailable = false; + const text = await api.getText(contentRoute(issueId, relative), 15_000).catch((error: unknown) => { + unavailable = error instanceof ChaosApiError && error.status === 422; + return null; + }); + if (text !== null || !unavailable || Date.now() >= deadline) return text; + await new Promise(resolve => setTimeout(resolve, options.pollMs ?? REMOTE_FILE_POLL_MS)); + } +} + +export async function captureRemoteWorkspaceFiles(api: Pick, issueId: string, wanted: readonly string[], + captured: Record): Promise { + for (const relative of wanted) { + if (typeof captured[relative] === "string") continue; + captured[relative] = await readRemoteWorkspaceFile(api, issueId, relative, { waitMs: 0 }); + } +}