diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 7ed6e1fe0b9..0b3a27c5abf 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -694,13 +694,24 @@ export class TaskObserver { private async remoteWorkspaceFile(issueId: string, relative: string): Promise { const captured = this.record.remoteFiles?.[relative]; - return typeof captured === "string" ? captured : await readRemoteWorkspaceFile(this.api, issueId, relative); + const text = typeof captured === "string" ? captured + : await readRemoteWorkspaceFile(this.api, issueId, relative, { onError: message => this.noteRemoteFileError(relative, message) }); + if (text !== null) delete this.record.remoteFileErrors?.[relative]; + return text; } private async captureRemoteFilesAtDone(sample: TaskSample): Promise { const wanted = this.definition.collect.files(this.plan.nonce); if (!wanted.length || this.lab.environmentKind === "local" || sample.issueStatus !== "done" || !this.record.issueId) return; - await captureRemoteWorkspaceFiles(this.api, this.record.issueId, wanted, this.record.remoteFiles ??= {}); + await captureRemoteWorkspaceFiles(this.api, this.record.issueId, wanted, this.record.remoteFiles ??= {}, + (relative, message) => this.noteRemoteFileError(relative, message)); + } + + private noteRemoteFileError(relative: string, message: string): void { + const errors = this.record.remoteFileErrors ??= {}; + if (errors[relative] === message) return; + errors[relative] = message; + this.timeline.add("remote_file_read_failed", { taskId: this.plan.taskId, path: relative, error: message }); } async workspacePath(): Promise { diff --git a/tests/restart-chaos/remote-oracle.test.ts b/tests/restart-chaos/remote-oracle.test.ts index ec544bbd581..6e9ac351d92 100644 --- a/tests/restart-chaos/remote-oracle.test.ts +++ b/tests/restart-chaos/remote-oracle.test.ts @@ -38,6 +38,7 @@ beforeAll(async () => { server = createServer((incoming, outgoing) => { const url = new URL(incoming.url ?? "/", "http://x"); if (url.pathname === "/api/issues/i1/file-resources/content" && url.searchParams.get("download") === "true") { + if (url.searchParams.get("workspace") !== "execution") { outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "no_local_workspace" })); return; } if (incoming.headers.origin !== baseUrl) { outgoing.writeHead(403).end(); return; } if (unavailableReads > 0) { unavailableReads -= 1; outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; } if (sandboxReleased) { outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; } @@ -144,5 +145,32 @@ describe("oracle for a remote workspace", () => { expect(Object.values(artifacts.files)).toEqual([null, null]); expect(Date.now() - started).toBeLessThan(10_000); }, 60_000); + + it("records why a remote file could not be read", async () => { + const api = { + baseUrl, + async get(route: string) { + if (route.endsWith("/comments") || route.endsWith("/runs")) return []; + return { id: "i1", status: "done", currentExecutionWorkspace: { cwd: "/home/daytona/workspaces/not-on-this-host" } }; + }, + async post() { return { id: "i1", identifier: "CHA-1" }; }, + getText: (route: string) => new ChaosApi(baseUrl).getText(route), + } as unknown as ChaosApi; + const timeline = new Timeline(null); + const observer = new TaskObserver(plan, api, lab, timeline, { serverReady: true, faultInProgress: false }); + await observer.launch(); + sandboxReleased = true; + try { + await observer.poll(observer.record.createdAt! + 1_000); + await observer.poll(observer.record.createdAt! + 5_000); + } finally { sandboxReleased = false; } + expect(observer.record.remoteFileErrors?.[modulePath]).toMatch(/422.*workspace_unavailable/); + expect(timeline.entries.filter(entry => entry.kind === "remote_file_read_failed" && entry.path === modulePath)).toHaveLength(1); + await observer.poll(observer.record.createdAt! + 9_000); + expect(observer.record.remoteFiles?.[modulePath]).toBe(remoteFiles[modulePath]); + const artifacts = await observer.collectArtifacts(); + expect(artifacts.files[modulePath]).toBe(remoteFiles[modulePath]); + expect(observer.record.remoteFileErrors?.[modulePath]).toBeUndefined(); + }, 60_000); }); diff --git a/tests/restart-chaos/remote-workspace-files.ts b/tests/restart-chaos/remote-workspace-files.ts index 897090f1281..2b62c6acfec 100644 --- a/tests/restart-chaos/remote-workspace-files.ts +++ b/tests/restart-chaos/remote-workspace-files.ts @@ -4,16 +4,17 @@ export const REMOTE_FILE_WAIT_MS = 4 * 60_000; export const REMOTE_FILE_POLL_MS = 2_000; function contentRoute(issueId: string, relative: string): string { - return `/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&download=true`; + return `/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&workspace=execution&download=true`; } export async function readRemoteWorkspaceFile(api: Pick, issueId: string, relative: string, - options: { waitMs?: number; pollMs?: number } = {}): Promise { + options: { waitMs?: number; pollMs?: number; onError?: (message: string) => void } = {}): Promise { const deadline = Date.now() + (options.waitMs ?? REMOTE_FILE_WAIT_MS); for (;;) { let unavailable = false; const text = await api.getText(contentRoute(issueId, relative), 15_000).catch((error: unknown) => { unavailable = error instanceof ChaosApiError && error.status === 422; + options.onError?.((error instanceof Error ? error.message : String(error)).slice(0, 400)); return null; }); if (text !== null || !unavailable || Date.now() >= deadline) return text; @@ -22,9 +23,9 @@ export async function readRemoteWorkspaceFile(api: Pick, is } export async function captureRemoteWorkspaceFiles(api: Pick, issueId: string, wanted: readonly string[], - captured: Record): Promise { + captured: Record, onError?: (relative: string, message: string) => void): Promise { for (const relative of wanted) { if (typeof captured[relative] === "string") continue; - captured[relative] = await readRemoteWorkspaceFile(api, issueId, relative, { waitMs: 0 }); + captured[relative] = await readRemoteWorkspaceFile(api, issueId, relative, { waitMs: 0, ...(onError ? { onError: message => onError(relative, message) } : {}) }); } } diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 5e5def6b5c1..787bef559e5 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -326,6 +326,7 @@ export interface TaskRecord { preFaultSnapshotError?: string | null; remoteWorkspace?: boolean; remoteFiles?: Record; + remoteFileErrors?: Record; /** Pre-fault files missing at rest; null when not checked. */ lostFiles: string[] | null; /** B5: distinct bench progress values the observer read during polling. */