test(chaos): read remote R1 files from the execution workspace and record why a read failed (#20440)

## Thinking Path

> - Paperclip is the open source app people use to manage AI agents for
work
> - The restart chaos harness grades Daytona R1 tasks by reading their
files through `GET /issues/:id/file-resources/content`
> - In replays w-20261110T010031-s9913 and w-20261110T012332-s9950 the
tasks recovered (R1, R3, R6 pass), but all 506 content reads returned
422, even after the closing save left the workspace clean (`snapshotId`
set, no writer, `location: null`)
> - The harness discarded each error, so the waves could not show which
422 the server returned
> - This pull request asks for the execution workspace explicitly and
records the last error per file in the task record and the timeline
> - The benefit is that the next wave names the exact cause instead of a
blind `module_present` failure

## Linked Issues or Issue Description

Follow-up to #20348.

**What happened?**
Every file read for a finished Daytona R1 task returned 422, and the
response was discarded.

**Expected behavior**
Failed reads leave their reason in the evidence.

**Actual behavior**
Only the outcome, a `null` file, was kept.

## What Changed

- File reads add `workspace=execution`, so the server reads the issue's
execution workspace and not the `auto` choice.
- `readRemoteWorkspaceFile` and `captureRemoteWorkspaceFiles` report
each failure through a callback. The observer keeps the last message per
file in `record.remoteFileErrors` and logs `remote_file_read_failed`
once per distinct message.
- The remote-workspace file reader stays in `remote-workspace-files.ts`;
`observer.ts` is at 993 lines.

## Verification

- `npx vitest run --config tests/restart-chaos/vitest.config.ts`: 263
passed. The test server now requires `workspace=execution`, so the
existing remote tests were red before, and a new test records the 422
`workspace_unavailable` message.
- `node scripts/typecheck-test-projects.mjs tests/restart-chaos`: clean.

## Risks

- If a Daytona task's files only exist in a project workspace rather
than its execution workspace, an explicit `execution` selector reports
that clearly instead of falling back.

## Model Used

- Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code
execution in Claude Code.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` / `Closes
#` / `Refs #` OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change (e.g. `docs/...`, `fix/...`)
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [ ] All Paperclip CI gates are green
- [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

tests/restart-chaos/observer.ts

@@ -694,13 +694,24 @@ export class TaskObserver {
 
   private async remoteWorkspaceFile(issueId: string, relative: string): Promise<string | null> {
     const captured = this.record.remoteFiles?.[relative];
-    return typeof captured === "string" ? captured : await readRemoteWorkspaceFile(this.api, issueId, relative);
+    const text = typeof captured === "string" ? captured
+      : await readRemoteWorkspaceFile(this.api, issueId, relative, { onError: message => this.noteRemoteFileError(relative, message) });
+    if (text !== null) delete this.record.remoteFileErrors?.[relative];
+    return text;
   }
 
   private async captureRemoteFilesAtDone(sample: TaskSample): Promise<void> {
     const wanted = this.definition.collect.files(this.plan.nonce);
     if (!wanted.length || this.lab.environmentKind === "local" || sample.issueStatus !== "done" || !this.record.issueId) return;
-    await captureRemoteWorkspaceFiles(this.api, this.record.issueId, wanted, this.record.remoteFiles ??= {});
+    await captureRemoteWorkspaceFiles(this.api, this.record.issueId, wanted, this.record.remoteFiles ??= {},
+      (relative, message) => this.noteRemoteFileError(relative, message));
+  }
+
+  private noteRemoteFileError(relative: string, message: string): void {
+    const errors = this.record.remoteFileErrors ??= {};
+    if (errors[relative] === message) return;
+    errors[relative] = message;
+    this.timeline.add("remote_file_read_failed", { taskId: this.plan.taskId, path: relative, error: message });
   }
 
   async workspacePath(): Promise<string | null> {

tests/restart-chaos/remote-oracle.test.ts

@@ -38,6 +38,7 @@ beforeAll(async () => {
   server = createServer((incoming, outgoing) => {
     const url = new URL(incoming.url ?? "/", "http://x");
     if (url.pathname === "/api/issues/i1/file-resources/content" && url.searchParams.get("download") === "true") {
+      if (url.searchParams.get("workspace") !== "execution") { outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "no_local_workspace" })); return; }
       if (incoming.headers.origin !== baseUrl) { outgoing.writeHead(403).end(); return; }
       if (unavailableReads > 0) { unavailableReads -= 1; outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; }
       if (sandboxReleased) { outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; }
@@ -144,5 +145,32 @@ describe("oracle for a remote workspace", () => {
     expect(Object.values(artifacts.files)).toEqual([null, null]);
     expect(Date.now() - started).toBeLessThan(10_000);
   }, 60_000);
+
+  it("records why a remote file could not be read", async () => {
+    const api = {
+      baseUrl,
+      async get(route: string) {
+        if (route.endsWith("/comments") || route.endsWith("/runs")) return [];
+        return { id: "i1", status: "done", currentExecutionWorkspace: { cwd: "/home/daytona/workspaces/not-on-this-host" } };
+      },
+      async post() { return { id: "i1", identifier: "CHA-1" }; },
+      getText: (route: string) => new ChaosApi(baseUrl).getText(route),
+    } as unknown as ChaosApi;
+    const timeline = new Timeline(null);
+    const observer = new TaskObserver(plan, api, lab, timeline, { serverReady: true, faultInProgress: false });
+    await observer.launch();
+    sandboxReleased = true;
+    try {
+      await observer.poll(observer.record.createdAt! + 1_000);
+      await observer.poll(observer.record.createdAt! + 5_000);
+    } finally { sandboxReleased = false; }
+    expect(observer.record.remoteFileErrors?.[modulePath]).toMatch(/422.*workspace_unavailable/);
+    expect(timeline.entries.filter(entry => entry.kind === "remote_file_read_failed" && entry.path === modulePath)).toHaveLength(1);
+    await observer.poll(observer.record.createdAt! + 9_000);
+    expect(observer.record.remoteFiles?.[modulePath]).toBe(remoteFiles[modulePath]);
+    const artifacts = await observer.collectArtifacts();
+    expect(artifacts.files[modulePath]).toBe(remoteFiles[modulePath]);
+    expect(observer.record.remoteFileErrors?.[modulePath]).toBeUndefined();
+  }, 60_000);
 });
 

tests/restart-chaos/remote-workspace-files.ts

@@ -4,16 +4,17 @@ export const REMOTE_FILE_WAIT_MS = 4 * 60_000;
 export const REMOTE_FILE_POLL_MS = 2_000;
 
 function contentRoute(issueId: string, relative: string): string {
-  return `/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&download=true`;
+  return `/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&workspace=execution&download=true`;
 }
 
 export async function readRemoteWorkspaceFile(api: Pick<ChaosApi, "getText">, issueId: string, relative: string,
-  options: { waitMs?: number; pollMs?: number } = {}): Promise<string | null> {
+  options: { waitMs?: number; pollMs?: number; onError?: (message: string) => void } = {}): Promise<string | null> {
   const deadline = Date.now() + (options.waitMs ?? REMOTE_FILE_WAIT_MS);
   for (;;) {
     let unavailable = false;
     const text = await api.getText(contentRoute(issueId, relative), 15_000).catch((error: unknown) => {
       unavailable = error instanceof ChaosApiError && error.status === 422;
+      options.onError?.((error instanceof Error ? error.message : String(error)).slice(0, 400));
       return null;
     });
     if (text !== null || !unavailable || Date.now() >= deadline) return text;
@@ -22,9 +23,9 @@ export async function readRemoteWorkspaceFile(api: Pick<ChaosApi, "getText">, is
 }
 
 export async function captureRemoteWorkspaceFiles(api: Pick<ChaosApi, "getText">, issueId: string, wanted: readonly string[],
-  captured: Record<string, string | null>): Promise<void> {
+  captured: Record<string, string | null>, onError?: (relative: string, message: string) => void): Promise<void> {
   for (const relative of wanted) {
     if (typeof captured[relative] === "string") continue;
-    captured[relative] = await readRemoteWorkspaceFile(api, issueId, relative, { waitMs: 0 });
+    captured[relative] = await readRemoteWorkspaceFile(api, issueId, relative, { waitMs: 0, ...(onError ? { onError: message => onError(relative, message) } : {}) });
   }
 }

tests/restart-chaos/types.ts

@@ -326,6 +326,7 @@ export interface TaskRecord {
   preFaultSnapshotError?: string | null;
   remoteWorkspace?: boolean;
   remoteFiles?: Record<string, string | null>;
+  remoteFileErrors?: Record<string, string>;
   /** Pre-fault files missing at rest; null when not checked. */
   lostFiles: string[] | null;
   /** B5: distinct bench progress values the observer read during polling. */