From 0c7a628fd1e9d8e52804a8717edd74ec8a5e83ed Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:34:35 -0600 Subject: [PATCH] test(chaos): read remote R1 files from the execution workspace and record why a read failed (#20440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart chaos harness grades Daytona R1 tasks by reading their files through `GET /issues/:id/file-resources/content` > - In replays w-20261110T010031-s9913 and w-20261110T012332-s9950 the tasks recovered (R1, R3, R6 pass), but all 506 content reads returned 422, even after the closing save left the workspace clean (`snapshotId` set, no writer, `location: null`) > - The harness discarded each error, so the waves could not show which 422 the server returned > - This pull request asks for the execution workspace explicitly and records the last error per file in the task record and the timeline > - The benefit is that the next wave names the exact cause instead of a blind `module_present` failure ## Linked Issues or Issue Description Follow-up to #20348. **What happened?** Every file read for a finished Daytona R1 task returned 422, and the response was discarded. **Expected behavior** Failed reads leave their reason in the evidence. **Actual behavior** Only the outcome, a `null` file, was kept. ## What Changed - File reads add `workspace=execution`, so the server reads the issue's execution workspace and not the `auto` choice. - `readRemoteWorkspaceFile` and `captureRemoteWorkspaceFiles` report each failure through a callback. The observer keeps the last message per file in `record.remoteFileErrors` and logs `remote_file_read_failed` once per distinct message. - The remote-workspace file reader stays in `remote-workspace-files.ts`; `observer.ts` is at 993 lines. ## Verification - `npx vitest run --config tests/restart-chaos/vitest.config.ts`: 263 passed. The test server now requires `workspace=execution`, so the existing remote tests were red before, and a new test records the 422 `workspace_unavailable` message. - `node scripts/typecheck-test-projects.mjs tests/restart-chaos`: clean. ## Risks - If a Daytona task's files only exist in a project workspace rather than its execution workspace, an explicit `execution` selector reports that clearly instead of falling back. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/observer.ts | 15 ++++++++-- tests/restart-chaos/remote-oracle.test.ts | 28 +++++++++++++++++++ tests/restart-chaos/remote-workspace-files.ts | 9 +++--- tests/restart-chaos/types.ts | 1 + 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 7ed6e1fe0b9..0b3a27c5abf 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -694,13 +694,24 @@ export class TaskObserver { private async remoteWorkspaceFile(issueId: string, relative: string): Promise { const captured = this.record.remoteFiles?.[relative]; - return typeof captured === "string" ? captured : await readRemoteWorkspaceFile(this.api, issueId, relative); + const text = typeof captured === "string" ? captured + : await readRemoteWorkspaceFile(this.api, issueId, relative, { onError: message => this.noteRemoteFileError(relative, message) }); + if (text !== null) delete this.record.remoteFileErrors?.[relative]; + return text; } private async captureRemoteFilesAtDone(sample: TaskSample): Promise { const wanted = this.definition.collect.files(this.plan.nonce); if (!wanted.length || this.lab.environmentKind === "local" || sample.issueStatus !== "done" || !this.record.issueId) return; - await captureRemoteWorkspaceFiles(this.api, this.record.issueId, wanted, this.record.remoteFiles ??= {}); + await captureRemoteWorkspaceFiles(this.api, this.record.issueId, wanted, this.record.remoteFiles ??= {}, + (relative, message) => this.noteRemoteFileError(relative, message)); + } + + private noteRemoteFileError(relative: string, message: string): void { + const errors = this.record.remoteFileErrors ??= {}; + if (errors[relative] === message) return; + errors[relative] = message; + this.timeline.add("remote_file_read_failed", { taskId: this.plan.taskId, path: relative, error: message }); } async workspacePath(): Promise { diff --git a/tests/restart-chaos/remote-oracle.test.ts b/tests/restart-chaos/remote-oracle.test.ts index ec544bbd581..6e9ac351d92 100644 --- a/tests/restart-chaos/remote-oracle.test.ts +++ b/tests/restart-chaos/remote-oracle.test.ts @@ -38,6 +38,7 @@ beforeAll(async () => { server = createServer((incoming, outgoing) => { const url = new URL(incoming.url ?? "/", "http://x"); if (url.pathname === "/api/issues/i1/file-resources/content" && url.searchParams.get("download") === "true") { + if (url.searchParams.get("workspace") !== "execution") { outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "no_local_workspace" })); return; } if (incoming.headers.origin !== baseUrl) { outgoing.writeHead(403).end(); return; } if (unavailableReads > 0) { unavailableReads -= 1; outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; } if (sandboxReleased) { outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; } @@ -144,5 +145,32 @@ describe("oracle for a remote workspace", () => { expect(Object.values(artifacts.files)).toEqual([null, null]); expect(Date.now() - started).toBeLessThan(10_000); }, 60_000); + + it("records why a remote file could not be read", async () => { + const api = { + baseUrl, + async get(route: string) { + if (route.endsWith("/comments") || route.endsWith("/runs")) return []; + return { id: "i1", status: "done", currentExecutionWorkspace: { cwd: "/home/daytona/workspaces/not-on-this-host" } }; + }, + async post() { return { id: "i1", identifier: "CHA-1" }; }, + getText: (route: string) => new ChaosApi(baseUrl).getText(route), + } as unknown as ChaosApi; + const timeline = new Timeline(null); + const observer = new TaskObserver(plan, api, lab, timeline, { serverReady: true, faultInProgress: false }); + await observer.launch(); + sandboxReleased = true; + try { + await observer.poll(observer.record.createdAt! + 1_000); + await observer.poll(observer.record.createdAt! + 5_000); + } finally { sandboxReleased = false; } + expect(observer.record.remoteFileErrors?.[modulePath]).toMatch(/422.*workspace_unavailable/); + expect(timeline.entries.filter(entry => entry.kind === "remote_file_read_failed" && entry.path === modulePath)).toHaveLength(1); + await observer.poll(observer.record.createdAt! + 9_000); + expect(observer.record.remoteFiles?.[modulePath]).toBe(remoteFiles[modulePath]); + const artifacts = await observer.collectArtifacts(); + expect(artifacts.files[modulePath]).toBe(remoteFiles[modulePath]); + expect(observer.record.remoteFileErrors?.[modulePath]).toBeUndefined(); + }, 60_000); }); diff --git a/tests/restart-chaos/remote-workspace-files.ts b/tests/restart-chaos/remote-workspace-files.ts index 897090f1281..2b62c6acfec 100644 --- a/tests/restart-chaos/remote-workspace-files.ts +++ b/tests/restart-chaos/remote-workspace-files.ts @@ -4,16 +4,17 @@ export const REMOTE_FILE_WAIT_MS = 4 * 60_000; export const REMOTE_FILE_POLL_MS = 2_000; function contentRoute(issueId: string, relative: string): string { - return `/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&download=true`; + return `/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&workspace=execution&download=true`; } export async function readRemoteWorkspaceFile(api: Pick, issueId: string, relative: string, - options: { waitMs?: number; pollMs?: number } = {}): Promise { + options: { waitMs?: number; pollMs?: number; onError?: (message: string) => void } = {}): Promise { const deadline = Date.now() + (options.waitMs ?? REMOTE_FILE_WAIT_MS); for (;;) { let unavailable = false; const text = await api.getText(contentRoute(issueId, relative), 15_000).catch((error: unknown) => { unavailable = error instanceof ChaosApiError && error.status === 422; + options.onError?.((error instanceof Error ? error.message : String(error)).slice(0, 400)); return null; }); if (text !== null || !unavailable || Date.now() >= deadline) return text; @@ -22,9 +23,9 @@ export async function readRemoteWorkspaceFile(api: Pick, is } export async function captureRemoteWorkspaceFiles(api: Pick, issueId: string, wanted: readonly string[], - captured: Record): Promise { + captured: Record, onError?: (relative: string, message: string) => void): Promise { for (const relative of wanted) { if (typeof captured[relative] === "string") continue; - captured[relative] = await readRemoteWorkspaceFile(api, issueId, relative, { waitMs: 0 }); + captured[relative] = await readRemoteWorkspaceFile(api, issueId, relative, { waitMs: 0, ...(onError ? { onError: message => onError(relative, message) } : {}) }); } } diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 5e5def6b5c1..787bef559e5 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -326,6 +326,7 @@ export interface TaskRecord { preFaultSnapshotError?: string | null; remoteWorkspace?: boolean; remoteFiles?: Record; + remoteFileErrors?: Record; /** Pre-fault files missing at rest; null when not checked. */ lostFiles: string[] | null; /** B5: distinct bench progress values the observer read during polling. */