diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-built-in.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.test.ts new file mode 100644 index 00000000000..28c616ce856 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.test.ts @@ -0,0 +1,22 @@ +import { it, expect } from "vitest"; +import { gradeBuiltInConfiguration } from "./runtime-built-in.js"; +function valid() { + const builtIn = { definition: { key: "summarizer" }, agentId: "summarizer" }; + const task = { id: "history", title: "Summary history", companyId: "company", assigneeAgentId: "summarizer", status: "backlog" }; + return { scenarioId: "agents.configure-required-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { builtInBefore: builtIn, builtInAfter: { ...builtIn }, originalInstruction: { content: "Summarize work", revision: { id: "old" } }, agentCountBefore: 1, historicalTask: task }, + instruction: { content: "Summarize work; keep updates brief, under 40 words.", revision: { id: "new", actorAgentId: "ceo", sourceRunId: "run" } }, + instructionHistory: { revisions: [{ id: "old" }, { id: "new" }] }, agents: [{ id: "summarizer", name: "Summarizer", companyId: "company" }], tasks: [task], + links: ["/ACORN/agents/summarizer"], reply: "The summarizer now writes shorter updates.", activity: [], runs: [{ id: "run" }], humanQueries: [] }; +} +it("accepts a durable protected-agent configuration revision", () => expect(gradeBuiltInConfiguration(valid()).passed).toBe(true)); +it("rejects missing built-in evidence", () => expect(gradeBuiltInConfiguration(undefined).passed).toBe(false)); +it("rejects replacing the protected identity with an ordinary hire", () => { + const o = valid(); o.state.builtInAfter.agentId = "replacement"; expect(gradeBuiltInConfiguration(o).passed).toBe(false); +}); +it("rejects changing only prose without a saved instruction revision", () => { + const o = valid(); o.instruction.content = "Summarize work"; expect(gradeBuiltInConfiguration(o).passed).toBe(false); +}); +it("rejects erasing the old attributed task", () => { + const o = valid(); o.tasks = []; expect(gradeBuiltInConfiguration(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts new file mode 100644 index 00000000000..f511546b8ba --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts @@ -0,0 +1,55 @@ +import { checks, sameTask, type Entity } from "./internal-support.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime, agentLink, type RuntimeAgent, type InstructionFile } from "./runtime-support.js"; +interface BuiltInState { definition: { key: string }; agentId: string | null; agent: RuntimeAgent | null; status: string } +export function gradeBuiltInConfiguration(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("actual-built-in-observation", !!o, "The protected built-in registry, durable configuration revision and historical task are independently read."); if (!o) return c.result(); + const original = o.state.builtInBefore as BuiltInState | undefined; const after = o.state.builtInAfter as BuiltInState | undefined; + const beforeInstruction = o.state.originalInstruction as InstructionFile | undefined; const current = o.instruction; + const worker = o.agents.find(row => row.id === original?.agentId); + c.add("protected-identity-preserved", !!original && !!after && after.definition.key === "summarizer" && after.agentId === original.agentId + && !!worker && worker.companyId === o.companyId && o.agents.length === o.state.agentCountBefore, + "The required registry still points to the same summarizer, with no replacement ordinary hire."); + c.add("shorter-updates-durably-configured", !!current && !!beforeInstruction && current.content !== beforeInstruction.content + && /shorter|brief|concise|(?:three|3|two|2) sentences|under \d+ words/i.test(current.content) + && /summar|updates?/i.test(current.content) && current.revision.actorAgentId === o.leadId && o.runs.some(run => run.id === current.revision.sourceRunId), + "An actual agent-attributed revision applies shorter summary guidance while retaining the summarizer purpose."); + c.add("history-retained", !!beforeInstruction && o.instructionHistory?.revisions.some(row => row.id === beforeInstruction.revision.id) === true + && o.tasks.length === 1 && sameTask(o.state.historicalTask as Entity, o.tasks[0]), "Instruction history and attributed historical work remain intact."); + c.add("inspectable-effective-change", !!worker && !!agentLink(o, worker) && /short|brief|concis/i.test(o.reply), + "The response links the protected agent and describes the saved change."); + return c.result(); +} +export const runtimeBuiltInCases = [runtimeDefinition("agents.configure-required-agent", async ctx => { + await ctx.input.api.patch("/api/instance/settings/experimental", { enableBuiltInAgents: true }); + const root = `/api/companies/${ctx.input.fixtures.company.id}`; + const builtInBefore = await ctx.input.api.post(`${root}/built-in-agents/summarizer/reconcile`, {}); + if (!builtInBefore.agentId || !builtInBefore.agent) throw new Error("Public reconcile did not create the protected summarizer identity"); + const worker = builtInBefore.agent; + const originalInstruction = await ctx.input.api.get(`/api/agents/${worker.id}/instructions-bundle/file?path=AGENTS.md`); + const historicalTask = await ctx.input.api.post(`${root}/issues`, { title: "Historical summary work", status: "backlog", assigneeAgentId: worker.id }); + const agentCountBefore = (await ctx.input.api.get(`${root}/agents`)).length; + return { worker, builtInBefore, instructionAgentId: worker.id, originalInstruction, historicalTask, agentCountBefore }; +}, gradeBuiltInConfiguration, { + sourceFiles: ["chat-catalog-domains/runtime-built-in.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "protected-summarizer-identity", "historical-task-unrun"] }, + async verifySetup(ctx, state) { + const c = checks(); const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + const builtIn = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/built-in-agents/summarizer/status`); + const history = await ctx.input.api.get(`/api/issues/${(state.historicalTask as Entity).id}`); + c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The actual chat agent is runnable."); + c.add("protected-summarizer-identity", builtIn.definition.key === "summarizer" && builtIn.agentId === (state.worker as RuntimeAgent).id, + "The genuine protected identity is preserved; readiness is never manufactured for this configuration-only request."); + c.add("historical-task-unrun", history.status === "backlog" && (await ctx.allRuns()).length === 0, "The retained history fixture has not started provider work."); + return c.result(); + }, + async observe(ctx, state) { + state.builtInAfter = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/built-in-agents/summarizer/status`); + // The built-in bundle has no arbitrary side-file prerequisite. + const result = await observeRuntime(ctx, { ...state, instructionAgentId: undefined }); + const root = `/api/agents/${(state.worker as RuntimeAgent).id}/instructions-bundle`; + result.instruction = await ctx.input.api.get(`${root}/file?path=AGENTS.md`); + result.instructionHistory = await ctx.input.api.get(`${root}/history?path=AGENTS.md`); + return result; + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json index 68e274f5e5a..a44a91ad774 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json @@ -66,7 +66,7 @@ }, { "scenarioId": "agents.configure-required-agent", - "readinessClass": "local", + "readinessClass": "implemented", "needs": "Public built-in summarizer provision; change only supported config with protected identity/history unchanged." }, { @@ -106,7 +106,7 @@ }, { "scenarioId": "agent-chat.conference-room", - "readinessClass": "local", + "readinessClass": "implemented", "needs": "Enable supported local-trusted board surface; exact company route/browser scope and differing protocol honestly described." }, { @@ -116,7 +116,7 @@ }, { "scenarioId": "runs-adapters.list-model-options", - "readinessClass": "local", + "readinessClass": "implemented", "needs": "Real admitted model discovery/account inventory; independently compare restrictions, availability and binding, secrets never exposed." }, { @@ -271,7 +271,7 @@ }, { "scenarioId": "workspaces.locate-files", - "readinessClass": "local", + "readinessClass": "implemented", "needs": "Actual projectless task execution binding; independent read active binding and distinguish pending choice from agent defaults." }, { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts new file mode 100644 index 00000000000..6ecd22c8c68 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts @@ -0,0 +1,19 @@ +import { it, expect } from "vitest"; +import { gradeConference } from "./runtime-conference.js"; +function valid() { + return { scenarioId: "agent-chat.conference-room", companyId: "company", leadId: "ceo", reply: "Open the Conference Room for the separate board-level discussion.", + state: { deploymentMode: "local_trusted", conferenceEnabled: true, agentChatEnabled: true, conferenceVisible: true, + openedCompanyPrefix: "ACORN", companyPrefix: "ACORN", noBrokenRoute: true, runsBeforeOpen: 1, runsAfterOpen: 1, + standingIssueBefore: "chat", standingIssueAfter: "chat" }, agents: [], tasks: [], links: [], activity: [], runs: [], humanQueries: [] }; +} +it("accepts the real gated board surface in company context", () => expect(gradeConference(valid()).passed).toBe(true)); +it("rejects missing conference evidence", () => expect(gradeConference(undefined).passed).toBe(false)); +it("rejects wrong deployment qualification", () => { + const o = valid(); o.state.deploymentMode = "authenticated"; expect(gradeConference(o).passed).toBe(false); +}); +it("rejects navigation to another company", () => { + const o = valid(); o.state.openedCompanyPrefix = "BIRCH"; expect(gradeConference(o).passed).toBe(false); +}); +it("rejects invented standing-chat history parity", () => { + const o = valid(); o.reply += " It uses the same Agent Chat history."; expect(gradeConference(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts b/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts new file mode 100644 index 00000000000..1e4d95b55af --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts @@ -0,0 +1,48 @@ +import { expect } from "@playwright/test"; +import { checks } from "./internal-support.js"; +import { waitForReply } from "./identity-support.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +export function gradeConference(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("independent-conference-evidence", !!o, "Actual deployment, gate, company route and rendered board surface are required."); if (!o) return c.result(); + c.add("supported-local-enabled-context", o.state.deploymentMode === "local_trusted" && o.state.conferenceEnabled === true + && o.state.agentChatEnabled === true, "Both actual gates are enabled in the supported local-trusted deployment."); + c.add("real-company-conference-surface", o.state.conferenceVisible === true && o.state.openedCompanyPrefix === o.state.companyPrefix + && o.state.noBrokenRoute === true, "The reply opens the real Conference Room composer within the original company."); + c.add("separate-board-scope-described", /conference|board/i.test(o.reply) && /separate|different|board.level|board discussion|board.assistant/i.test(o.reply) + && !/(?:same|shared|identical) (?:standing.chat|agent.chat) (?:history|execution)|(?:full|identical) (?:execution|agent.chat) parity/i.test(o.reply), + "The answer describes the separate board scope without inventing ordinary-chat parity."); + c.add("navigation-has-no-provider-action", o.state.runsBeforeOpen === 1 && o.state.runsAfterOpen === 1 + && o.state.standingIssueBefore === o.state.standingIssueAfter, "Opening the board surface preserves the ordinary source and starts no hidden execution."); + return c.result(); +} +export const runtimeConferenceCases = [runtimeDefinition("agent-chat.conference-room", async ctx => { + const health = await ctx.input.api.get<{ deploymentMode: string }>("/api/health"); + if (health.deploymentMode !== "local_trusted") throw new Error("Conference Room scenario requires a genuinely local-trusted target"); + await ctx.input.api.patch("/api/instance/settings/experimental", { enableConferenceRoomChat: true }); + const settings = await ctx.input.api.get<{ enableConferenceRoomChat: boolean; enableAgentChat: boolean }>("/api/instance/settings/experimental"); + return { deploymentMode: health.deploymentMode, conferenceEnabled: settings.enableConferenceRoomChat, + agentChatEnabled: settings.enableAgentChat, companyPrefix: ctx.input.fixtures.company.issuePrefix }; +}, gradeConference, { + sourceFiles: ["chat-catalog-domains/runtime-conference.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml"], + async act(ctx, state) { + await waitForReply(ctx); state.runsBeforeOpen = (await ctx.allRuns()).length; state.standingIssueBefore = ctx.issue()?.id; + const o = await observeRuntime(ctx, state); + const href = o.links.find(link => { const url = new URL(link, ctx.input.api.baseURL); + return url.origin === ctx.input.api.baseURL && url.pathname === `/${state.companyPrefix}/board-chat`; }); + if (!href) throw new Error("Conference response lacks the actual company board route"); + const source = ctx.input.page.url(); + await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); + await expect(ctx.input.page.getByPlaceholder("Ask anything about your organization...")).toBeVisible(); + await expect(ctx.input.page.getByText("Conference Room", { exact: true }).first()).toBeVisible(); + state.conferenceVisible = true; + const parts = new URL(ctx.input.page.url()).pathname.split("/"); state.openedCompanyPrefix = parts[1]; + state.noBrokenRoute = parts.at(-1) === "board-chat"; + await ctx.input.evidence("runtime-conference-opened.json", { companyId: o.companyId, url: ctx.input.page.url(), composerVisible: true }); + state.runsAfterOpen = (await ctx.allRuns()).length; + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); + await ctx.refreshIssue(); state.standingIssueAfter = ctx.issue()?.id; + }, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts new file mode 100644 index 00000000000..4db53ce543b --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.test.ts @@ -0,0 +1,20 @@ +import { it, expect } from "vitest"; +import { gradeModelOptions } from "./runtime-runs-adapters.js"; +function valid() { + const worker = { id: "morgan", companyId: "company", adapterType: "paperclip_runner", adapterConfig: { model: "gpt-5.6-sol" }, permissions: {} }; + return { scenarioId: "runs-adapters.list-model-options", companyId: "company", leadId: "ceo", state: { worker, + admittedModels: [{ id: "gpt-5.6-sol", label: "GPT 5.6 Sol" }], discoveryStatus: 200, agentRuntimeInventoryStatus: 403, secretScanComplete: true, secretLeaked: false }, + agents: [structuredClone(worker)], tasks: [], links: [], activity: [], runs: [], humanQueries: [], + reply: "The current catalog includes gpt-5.6-sol. Available models still depend on the selected configured account binding and environment qualification." }; +} +it("accepts an actual admitted option with account qualification distinguished", () => expect(gradeModelOptions(valid()).passed).toBe(true)); +it("rejects missing discovery evidence", () => expect(gradeModelOptions(undefined).passed).toBe(false)); +it("rejects an invented vendor model presented alongside a real option", () => { + const o = valid(); o.reply += " You can also use gpt-99-ultra."; expect(gradeModelOptions(o).passed).toBe(false); +}); +it("rejects silently changing Morgan's selected model", () => { + const o = valid(); o.agents[0]!.adapterConfig.model = "other"; expect(gradeModelOptions(o).passed).toBe(false); +}); +it("rejects leaked private account credentials", () => { + const o = valid(); o.state.secretLeaked = true; expect(gradeModelOptions(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts new file mode 100644 index 00000000000..0b6f4653973 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-runs-adapters.ts @@ -0,0 +1,53 @@ +import { expect } from "@playwright/test"; +import { checks } from "./internal-support.js"; +import { readAgentResource } from "./identity-support.js"; +import { runtimeDefinition, runtimeWorker, runtimeObservation, unchangedAgent, type RuntimeAgent } from "./runtime-support.js"; +interface Model { id: string; label: string } +export function gradeModelOptions(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("authorized-model-discovery-evidence", !!o, "Actual agent-authorized discovery and saved account/runtime scope are independently read."); if (!o) return c.result(); + const models = o.state.admittedModels as Model[] | undefined; const worker = o.state.worker as RuntimeAgent | undefined; + const current = o.agents.find(row => row.id === worker?.id); + c.add("at-least-one-current-discovered-option", Array.isArray(models) && models.length > 0 && o.state.discoveryStatus === 200 + && models.some(model => o.reply.includes(model.id)), "At least one returned coding option comes from the current admitted model catalog."); + const mentionedIds = [...o.reply.matchAll(/\b(?:gpt-\d[\w.-]*|o[134](?:-[\w.-]+)?|claude-(?:sonnet|opus|haiku)-[\w.-]+)/gi)].map(match => match[0].replace(/[.,;:!?]+$/, "")); + c.add("no-unqualified-model-inventory", !!models && mentionedIds.every(id => models.some(model => model.id.toLowerCase() === id.toLowerCase())) + && !/(?:all|every) (?:public |vendor )?models? (?:are |is )?(?:installed|ready|available)|all models.*(?:ready|work)/i.test(o.reply), + "The answer does not add unsupported vendor models or equate a public catalog with universal runtime readiness."); + c.add("availability-not-account-qualification", /account|binding|connection|connected|configured/i.test(o.reply) + && /available|option|catalog|model/i.test(o.reply) && /valid|qualif|restrict|depends|selected|configured|bound/i.test(o.reply), + "The response distinguishes model choices from selected account and environment qualification."); + c.add("restricted-runtime-read-denied", o.state.agentRuntimeInventoryStatus === 403, + "The actual agent bearer cannot inspect board-only runtime inventory."); + c.add("configuration-and-permissions-preserved", !!worker && !!current && unchangedAgent(worker, current) && o.tasks.length === 0, + "Discovery does not mutate Morgan's model, account, authority or task assignment."); + c.add("private-credential-scan-complete", o.state.secretScanComplete === true && o.state.secretLeaked === false, + "The private redaction sink is scanned without recording its values."); + return c.result(); +} +export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-model-options", async ctx => { + const worker = await runtimeWorker(ctx); + if (worker.adapterConfig?.provider !== "codex" && worker.adapterType !== "codex_local") + throw new Error("Current runtime model-discovery fixture requires a genuinely qualified Codex account"); + const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/adapters/codex_local/models`; + const discovered = await readAgentResource(ctx, endpoint); + const restricted = await readAgentResource(ctx, `/api/agents/${worker.id}/runtime-state`); + if (discovered.status !== 200 || !Array.isArray(discovered.data)) throw new Error("Scenario requires admitted real coding model discovery"); + return { worker, admittedModels: discovered.data, discoveryStatus: discovered.status, agentRuntimeInventoryStatus: restricted.status }; +}, gradeModelOptions, { + async observe(ctx, state) { + const { observeRuntime } = await import("./runtime-support.js"); + const o = await observeRuntime(ctx, state); + // Credentials remain only in the private attempt closure. Never put them in + // state, messages, diagnostic artifacts or calibration fixtures. + state.secretScanComplete = Array.isArray(ctx.input.privateSecrets); + state.secretLeaked = ctx.input.privateSecrets?.some(secret => secret.length > 8 && o.reply.includes(secret)) ?? true; + return o; + }, + async verifyResult(ctx, state, value) { + const o = runtimeObservation(value); if (!o) throw new Error("Model options response is missing"); + const match = (state.admittedModels as Model[]).find(model => o.reply.includes(model.id)); + if (!match) throw new Error("No actual discovered option was rendered"); + await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").filter({ hasText: match.id }).last()).toBeVisible(); + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.test.ts new file mode 100644 index 00000000000..262b6fb90b7 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.test.ts @@ -0,0 +1,22 @@ +import { it, expect } from "vitest"; +import { gradeLocateFiles } from "./runtime-workspaces.js"; +function valid() { + const task = { id: "website", title: "Website task", companyId: "company", projectId: null, status: "done", assigneeAgentId: "morgan" }; + const view = { workspace: { id: "bound-workspace", companyId: "company", projectId: null, name: "Website task workspace", cwd: "/owned/task-root" } }; + return { scenarioId: "workspaces.locate-files", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { task, worker: { id: "morgan" }, workspaceBefore: view, workspaceAfter: structuredClone(view), marker: "saved prerequisite", + fileBefore: "saved prerequisite", fileAfter: "saved prerequisite", fixtureRunId: "setup" }, + agents: [], tasks: [task], links: ["/ACORN/execution-workspaces/bound-workspace"], reply: "The actual files are in /owned/task-root.", + activity: [], runs: [{ id: "setup", agentId: "morgan", status: "succeeded" }], humanQueries: [] }; +} +it("accepts the actual bound projectless directory and files", () => expect(gradeLocateFiles(valid()).passed).toBe(true)); +it("rejects missing task workspace evidence", () => expect(gradeLocateFiles(undefined).passed).toBe(false)); +it("rejects a plausible agent default directory", () => { + const o = valid(); o.reply = "The files are probably in Morgan's /default/work directory."; expect(gradeLocateFiles(o).passed).toBe(false); +}); +it("rejects linking another workspace", () => { + const o = valid(); o.links = ["/ACORN/execution-workspaces/other-workspace"]; expect(gradeLocateFiles(o).passed).toBe(false); +}); +it("rejects fabricated saved-file setup without its actual worker run", () => { + const o = valid(); o.runs = []; expect(gradeLocateFiles(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts new file mode 100644 index 00000000000..c4f1b267305 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts @@ -0,0 +1,70 @@ +import { expect } from "@playwright/test"; +import { pollUntil } from "../api.js"; +import { checks, sameTask, type Entity } from "./internal-support.js"; +import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +interface WorkspaceView { workspace: { id: string; companyId: string; projectId?: string | null; name: string; cwd: string }; repositories: unknown[] } +async function savedTaskWorkspace(ctx: ChatCatalogContext) { + const { execution, fixtures, workspacePath, nonce, api } = ctx.input; + const base = execution.profile.buildAgent({ environmentId: fixtures.environment.id, environmentFixtureId: execution.environment.id, + workspacePath, secretRefs: fixtures.secretRefs, executionId: nonce }); + const { cwd: _cwd, ...taskConfig } = base.adapterConfig as Record; + const worker = await runtimeWorker(ctx, "Morgan", { adapterConfig: taskConfig }); + const company = `/api/companies/${fixtures.company.id}`; + for (const scope of [{ scopeType: "company", scopeId: fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }]) + await api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); + const marker = `Saved website prerequisite ${nonce}`; + const task = await api.post(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, + description: `Create a local file website-fixture.txt containing exactly ${JSON.stringify(marker)} in your admitted task workspace. Save a brief task document as evidence, then complete this fixture task normally. Do not create a project, repository, another task, or remote side effect.` }); + await pollUntil({ label: "Real projectless website fixture completion", deadlineAt: Date.now() + 180_000, + load: async () => ({ task: await api.get(`/api/issues/${task.id}`), runs: await ctx.allRuns() }), + accept: value => value.task.status === "done" && value.runs.length === 1 && value.runs.every(run => run.status === "succeeded"), + reject: value => value.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Projectless prerequisite provider run failed" : undefined }); + const view = await api.get(`/api/issues/${task.id}/workspace`); + const file = await api.get<{ resource?: { workspaceKind?: string }; content?: { data?: string } }>(`/api/issues/${task.id}/file-resources/content?workspace=execution&path=website-fixture.txt`); + if (view.workspace.projectId || file.resource?.workspaceKind !== "execution_workspace" || file.content?.data?.trim() !== marker) + throw new Error("Projectless fixture has no actual task-bound saved file"); + return { worker, task: await api.get(`/api/issues/${task.id}`), workspaceBefore: view, marker, + fileBefore: file.content.data, fixtureRunId: (await ctx.allRuns())[0]!.id }; +} +export function gradeLocateFiles(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("actual-task-workspace-evidence", !!o, "The actual admitted binding, file bytes and setup run identity are independently required."); if (!o) return c.result(); + const before = o.state.workspaceBefore as WorkspaceView | undefined; const after = o.state.workspaceAfter as WorkspaceView | undefined; + const task = o.state.task as Entity | undefined; + c.add("same-projectless-task-binding", !!before?.workspace?.id && !!after && before.workspace.id === after.workspace.id + && before.workspace.cwd === after.workspace.cwd && after.workspace.companyId === o.companyId && after.workspace.projectId == null + && !!task && sameTask(task, o.tasks.find(row => row.id === task.id)), "The correct projectless binding persists without reassigning or changing the task."); + c.add("concrete-authorized-location", !!after && (o.reply.includes(after.workspace.cwd) || o.reply.includes(after.workspace.id)), + "The answer identifies the actual bound workspace rather than inferring an agent default or project directory."); + c.add("actual-saved-files-retained", o.state.fileAfter === o.state.fileBefore && typeof o.state.fileBefore === "string" + && o.state.fileBefore.trim() === o.state.marker && o.runs.some(run => run.id === o.state.fixtureRunId && run.agentId === (o.state.worker as Entity).id + && run.status === "succeeded"), "A genuine worker created the prerequisite file and it remains unchanged."); + c.add("actual-workspace-result-link", !!after && o.links.some(href => workspaceLink(o.origin, href, after.workspace.id)), + "The response renders the real workspace view link."); + return c.result(); +} +function workspaceLink(origin: string, href: string, id: string) { + try { const url = new URL(href, origin); const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf("execution-workspaces"); + return url.origin === new URL(origin).origin && at >= 0 && parts[at + 1] === id; } catch { return false; } +} +export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-files", savedTaskWorkspace, gradeLocateFiles, { + expectedRunCount: 2, attemptTimeoutMs: 540_000, + async observe(ctx, state) { + const id = (state.task as Entity).id; + state.workspaceAfter = await ctx.input.api.get(`/api/issues/${id}/workspace`); + const file = await ctx.input.api.get<{ content: { data: string } }>(`/api/issues/${id}/file-resources/content?workspace=execution&path=website-fixture.txt`); + state.fileAfter = file.content.data; + return observeRuntime(ctx, state); + }, + async verifyResult(ctx, state, value) { + const o = runtimeObservation(value); if (!o) throw new Error("Workspace result missing"); + const view = state.workspaceAfter as WorkspaceView; + const href = o.links.find(link => workspaceLink(o.origin, link, view.workspace.id)); if (!href) throw new Error("No actual workspace result link"); + const source = ctx.input.page.url(); + await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); + await expect(ctx.input.page.getByRole("heading", { name: view.workspace.name, exact: true }).first()).toBeVisible(); + await ctx.input.evidence("runtime-task-workspace-opened.json", { taskId: (state.task as Entity).id, workspaceId: view.workspace.id, url: ctx.input.page.url() }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, +})];