From 8b29ca5489f2713b153cdd97d855be603ec49488 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 23:19:31 -0500 Subject: [PATCH] fix(evals): enable real workspace view before lifecycle journeys --- .../chat-catalog-domains/runtime-preview.ts | 3 +- .../runtime-workspace-closure.ts | 3 +- ...untime-workspace-view-prerequisite.test.ts | 29 +++++++++++++++++++ .../runtime-workspaces.ts | 9 ++++-- 4 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-workspace-view-prerequisite.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-preview.ts b/tests/runner-e2e/chat-catalog-domains/runtime-preview.ts index 5aeb8ff33fc..77420768e4b 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-preview.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-preview.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; import { checks, type Entity } from "./internal-support.js"; -import { savedTaskWorkspace } from "./runtime-workspaces.js"; +import { enableTaskWorkspaceView, savedTaskWorkspace } from "./runtime-workspaces.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; import type { WorkspaceRuntimeService } from "../../../packages/shared/src/types/workspace-runtime.js"; import type { WorkspaceOperation } from "../../../packages/shared/src/types/workspace-operation.js"; @@ -38,6 +38,7 @@ export function gradePreview(value: unknown) { } export const runtimePreviewCases = [runtimeDefinition("workspaces.preview-service", async ctx => { if (ctx.input.execution.environment.id !== "local") throw new Error("Owned loopback website preview fixture requires the actual local target"); + await enableTaskWorkspaceView(ctx); const state = await savedTaskWorkspace(ctx, { html: true }); const cwd = await realpath(state.workspaceBefore.workspace.cwd); const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root || !cwd.startsWith(`${await realpath(root)}${path.sep}`)) throw new Error("Preview fixture escaped the isolated task directory"); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-workspace-closure.ts b/tests/runner-e2e/chat-catalog-domains/runtime-workspace-closure.ts index d4f7804027c..9e6dac472b8 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-workspace-closure.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-workspace-closure.ts @@ -4,7 +4,7 @@ import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; import { openStandingChat } from "./identity-support.js"; import { checks, sameTask, type Entity } from "./internal-support.js"; -import { savedTaskWorkspace } from "./runtime-workspaces.js"; +import { enableTaskWorkspaceView, savedTaskWorkspace } from "./runtime-workspaces.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; type Artifact = Awaited>[number]; interface Workspace extends Entity { sourceIssueId?: string | null; rootIssueId?: string | null; cwd?: string; repoUrl?: string | null; branchName?: string | null; closedAt?: string | null } @@ -33,6 +33,7 @@ export function gradeWorkspaceClosure(value: unknown) { return c.result(); } export const runtimeWorkspaceClosureCases = [runtimeDefinition("workspaces.close-and-reopen", async ctx => { + await enableTaskWorkspaceView(ctx); const state = await savedTaskWorkspace(ctx, { artifact: true }); const id = state.workspaceBefore.workspace.id; const other = await savedTaskWorkspace(ctx, { workerName: "Riley" }); await ctx.input.api.patch(`/api/issues/${other.task.id}`, { title: "Unrelated retained notes" }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-workspace-view-prerequisite.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-workspace-view-prerequisite.test.ts new file mode 100644 index 00000000000..64ae36a49c5 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-workspace-view-prerequisite.test.ts @@ -0,0 +1,29 @@ +import { expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { enableTaskWorkspaceView } from "./runtime-workspaces.js"; +import { runtimeWorkspaceClosureCases } from "./runtime-workspace-closure.js"; +import { runtimePreviewCases } from "./runtime-preview.js"; + +function context(enabled: unknown) { + const api = { patch: vi.fn(async () => ({})), get: vi.fn(async () => ({ enableIsolatedWorkspaces: enabled })) }; + return { api, ctx: { input: { api, execution: { environment: { id: "local" } } } } as unknown as ChatCatalogContext }; +} +it("enables and independently reads back the supported public workspace view", async () => { + const { ctx, api } = context(true); + await enableTaskWorkspaceView(ctx); + expect(api.patch).toHaveBeenCalledExactlyOnceWith("/api/instance/settings/experimental", { enableIsolatedWorkspaces: true }); + expect(api.get).toHaveBeenCalledExactlyOnceWith("/api/instance/settings/experimental"); + expect(api.patch.mock.invocationCallOrder[0]).toBeLessThan(api.get.mock.invocationCallOrder[0]!); +}); +it.each([false, undefined, "true"])("both actual setup callbacks reject unenabled %s before any native prerequisite", async enabled => { + for (const definition of [runtimeWorkspaceClosureCases[0]!, runtimePreviewCases[0]!]) { + const { ctx, api } = context(enabled); + await expect(definition.setup(ctx)).rejects.toThrow("supported workspace view is not enabled"); + expect(api.patch).toHaveBeenCalledTimes(1); expect(api.get).toHaveBeenCalledTimes(1); + } +}); +it("does not rescue a denied public feature-setting operation", async () => { + const { ctx, api } = context(true); api.patch.mockRejectedValueOnce(new Error("403 Forbidden")); + await expect(runtimePreviewCases[0]!.setup(ctx)).rejects.toThrow("403 Forbidden"); + expect(api.get).not.toHaveBeenCalled(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts index 61a0011ed9a..c0263333d4c 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts @@ -5,6 +5,11 @@ import { checks, sameTask, type Entity } from "./internal-support.js"; import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; interface WorkspaceView { workspace: { id: string; companyId: string; projectId?: string | null; name: string; cwd: string }; repositories: unknown[] } +export async function enableTaskWorkspaceView(ctx: ChatCatalogContext) { + await ctx.input.api.patch("/api/instance/settings/experimental", { enableIsolatedWorkspaces: true }); + const settings = await ctx.input.api.get<{ enableIsolatedWorkspaces: boolean }>("/api/instance/settings/experimental"); + if (settings.enableIsolatedWorkspaces !== true) throw new Error("The supported workspace view is not enabled"); +} export async function savedTaskWorkspace(ctx: ChatCatalogContext, options: { artifact?: boolean; workerName?: string; html?: boolean } = {}) { const { execution, fixtures, workspacePath, nonce, api } = ctx.input; const base = execution.profile.buildAgent({ environmentId: fixtures.environment.id, environmentFixtureId: execution.environment.id, @@ -64,9 +69,7 @@ export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-file // The actual detail route is gated, independently of workspace realization. // Enable its supported UI fixture before the query, never after a bad link. if (!ctx.input.fixtures.company.issuePrefix) throw new Error("Workspace fixture has no public company prefix"); - await ctx.input.api.patch("/api/instance/settings/experimental", { enableIsolatedWorkspaces: true }); - const settings = await ctx.input.api.get<{ enableIsolatedWorkspaces: boolean }>("/api/instance/settings/experimental"); - if (!settings.enableIsolatedWorkspaces) throw new Error("The supported workspace view is not enabled"); + await enableTaskWorkspaceView(ctx); return { ...await savedTaskWorkspace(ctx), companyPrefix: ctx.input.fixtures.company.issuePrefix }; }, gradeLocateFiles, { sourceFiles: ["chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/workspaces.yaml", "../../ui/src/components/IsolatedWorkspacesRouteGate.tsx", "../../ui/src/pages/ExecutionWorkspaceDetail.tsx"],