diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 10484cdc381..d1189d1aadf 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -201,8 +201,8 @@ With `--env daytona --storage s3`, each task's workspace is a remote workspace: and restores it through the store, so the faults also hit workspace save, restore and copyback. R1 (`--workspace-repo`) runs on Daytona only with `--storage s3`: the project points at the checkout's GitHub `origin` over https at its `origin/master` commit, so the sandbox clones the repository and its -snapshots are Paperclip-sized. Its oracle reads the task's files through the board API and runs its -commands on a local copy. +snapshots are Paperclip-sized. Its oracle reads the task's files through the board API, retrying for up +to 4 minutes while the closing save makes the workspace readable, and runs its commands on a local copy. Every Retry the grader needs is clicked in the real board UI: the harness opens the issue in a headless Chromium, waits for the visible Retry button and clicks it. A task whose Retry button is missing fails diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 94397449e30..9cb756a5325 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -15,6 +15,7 @@ import type { import { restartProcessStartMatches } from "../runner-e2e/process-tree-owner.js"; import type { UiRetryClick } from "./board-ui.js"; import { runOracleCommand } from "./oracle-sandbox.js"; +import { captureRemoteWorkspaceFiles, readRemoteWorkspaceFile } from "./remote-workspace-files.js"; import { taskFaultSchedule } from "./scheduler.js"; import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js"; @@ -692,20 +693,13 @@ export class TaskObserver { private async remoteWorkspaceFile(issueId: string, relative: string): Promise { const captured = this.record.remoteFiles?.[relative]; - if (typeof captured === "string") return captured; - return await this.api.getText(`/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&download=true`, 15_000) - .catch(() => null); + return typeof captured === "string" ? captured : await readRemoteWorkspaceFile(this.api, issueId, relative); } private async captureRemoteFilesAtDone(sample: TaskSample): Promise { const wanted = this.definition.collect.files(this.plan.nonce); if (!wanted.length || this.lab.environmentKind === "local" || sample.issueStatus !== "done" || !this.record.issueId) return; - const captured = this.record.remoteFiles ??= {}; - for (const relative of wanted) { - if (typeof captured[relative] === "string") continue; - captured[relative] = await this.api.getText(`/api/issues/${this.record.issueId}/file-resources/content?path=${encodeURIComponent(relative)}&download=true`, 15_000) - .catch(() => null); - } + await captureRemoteWorkspaceFiles(this.api, this.record.issueId, wanted, this.record.remoteFiles ??= {}); } async workspacePath(): Promise { diff --git a/tests/restart-chaos/remote-oracle.test.ts b/tests/restart-chaos/remote-oracle.test.ts index 2e7c81fc180..ec544bbd581 100644 --- a/tests/restart-chaos/remote-oracle.test.ts +++ b/tests/restart-chaos/remote-oracle.test.ts @@ -31,6 +31,7 @@ const plan: TaskPlan = { taskId: "t01", index: 0, workload: "R1", adapter: "code holdMs: 0, followUpAfterMs: null, nonce }; let server: Server; let sandboxReleased = false; +let unavailableReads = 0; let baseUrl: string; beforeAll(async () => { @@ -38,6 +39,7 @@ beforeAll(async () => { const url = new URL(incoming.url ?? "/", "http://x"); if (url.pathname === "/api/issues/i1/file-resources/content" && url.searchParams.get("download") === "true") { if (incoming.headers.origin !== baseUrl) { outgoing.writeHead(403).end(); return; } + if (unavailableReads > 0) { unavailableReads -= 1; outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; } if (sandboxReleased) { outgoing.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ code: "workspace_unavailable" })); return; } const body = remoteFiles[url.searchParams.get("path") ?? ""]; outgoing.writeHead(body ? 200 : 404, { "content-type": "text/plain" }).end(body ?? ""); @@ -105,5 +107,42 @@ describe("oracle for a remote workspace", () => { expect(artifacts.commands.node_test).toMatchObject({ exitCode: 0 }); } finally { sandboxReleased = false; } }, 60_000); + + it("waits for the closing save to make a remote workspace readable again before grading its files", async () => { + const api = { + baseUrl, + async get(route: string) { + if (route.endsWith("/comments")) return []; + return { id: "i1", status: "done", currentExecutionWorkspace: { cwd: "/home/daytona/workspaces/not-on-this-host" } }; + }, + async post() { return { id: "i1", identifier: "CHA-1" }; }, + getText: (route: string) => new ChaosApi(baseUrl).getText(route), + } as unknown as ChaosApi; + const observer = new TaskObserver(plan, api, lab, new Timeline(null), { serverReady: true, faultInProgress: false }); + await observer.launch(); + sandboxReleased = false; + unavailableReads = 2; + const artifacts = await observer.collectArtifacts(); + expect(unavailableReads).toBe(0); + expect(artifacts.files).toEqual(remoteFiles); + }, 60_000); + + it("does not wait on a file the agent never wrote", async () => { + const api = { + baseUrl, + async get(route: string) { + if (route.endsWith("/comments")) return []; + return { id: "i1", status: "done", currentExecutionWorkspace: { cwd: "/home/daytona/workspaces/not-on-this-host" } }; + }, + async post() { return { id: "i1", identifier: "CHA-1" }; }, + getText: (route: string) => new ChaosApi(baseUrl).getText(route.replace(/slug-/g, "missing-")), + } as unknown as ChaosApi; + const observer = new TaskObserver(plan, api, lab, new Timeline(null), { serverReady: true, faultInProgress: false }); + await observer.launch(); + const started = Date.now(); + const artifacts = await observer.collectArtifacts(); + expect(Object.values(artifacts.files)).toEqual([null, null]); + expect(Date.now() - started).toBeLessThan(10_000); + }, 60_000); }); diff --git a/tests/restart-chaos/remote-workspace-files.ts b/tests/restart-chaos/remote-workspace-files.ts new file mode 100644 index 00000000000..897090f1281 --- /dev/null +++ b/tests/restart-chaos/remote-workspace-files.ts @@ -0,0 +1,30 @@ +import { ChaosApiError, type ChaosApi } from "./api.js"; + +export const REMOTE_FILE_WAIT_MS = 4 * 60_000; +export const REMOTE_FILE_POLL_MS = 2_000; + +function contentRoute(issueId: string, relative: string): string { + return `/api/issues/${issueId}/file-resources/content?path=${encodeURIComponent(relative)}&download=true`; +} + +export async function readRemoteWorkspaceFile(api: Pick, issueId: string, relative: string, + options: { waitMs?: number; pollMs?: number } = {}): Promise { + const deadline = Date.now() + (options.waitMs ?? REMOTE_FILE_WAIT_MS); + for (;;) { + let unavailable = false; + const text = await api.getText(contentRoute(issueId, relative), 15_000).catch((error: unknown) => { + unavailable = error instanceof ChaosApiError && error.status === 422; + return null; + }); + if (text !== null || !unavailable || Date.now() >= deadline) return text; + await new Promise(resolve => setTimeout(resolve, options.pollMs ?? REMOTE_FILE_POLL_MS)); + } +} + +export async function captureRemoteWorkspaceFiles(api: Pick, issueId: string, wanted: readonly string[], + captured: Record): Promise { + for (const relative of wanted) { + if (typeof captured[relative] === "string") continue; + captured[relative] = await readRemoteWorkspaceFile(api, issueId, relative, { waitMs: 0 }); + } +}