From b65cce861b5667f2424bdfacbdfd610fa697ba60 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 05:22:33 -0500 Subject: [PATCH] Correct evidenced held-B runtime prerequisites and resource checks --- .../runtime-agent-chat.ts | 6 ++ .../chat-catalog-domains/runtime-built-in.ts | 9 +- .../runtime-conference.test.ts | 4 + .../runtime-held-b-fixtures.test.ts | 88 +++++++++++++++++++ .../runtime-managed-profile.test.ts | 5 ++ .../chat-catalog-domains/runtime-recovery.ts | 21 +++-- .../runtime-resume.test.ts | 10 ++- .../chat-catalog-domains/runtime-resume.ts | 21 ++++- .../runtime-workspaces.test.ts | 15 +++- .../runtime-workspaces.ts | 39 ++++++-- 10 files changed, 196 insertions(+), 22 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-held-b-fixtures.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agent-chat.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agent-chat.ts index 1081b7ed8f7..0a4e454e9e1 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-agent-chat.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agent-chat.ts @@ -24,6 +24,12 @@ export function gradeReopenConversation(value: unknown) { } export const runtimeAgentChatCases = [runtimeDefinition("agent-chat.reopen-conversation", async ctx => { await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}`, { name: "Morgan" }); + // Genuine history runs before shared pre-canonical funding. Subscription + // usage is unpriced, so fund both actual scopes before that setup turn. + for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, + { scopeType: "agent", scopeId: ctx.input.fixtures.agent.id }]) + await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { + ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); const source = await openStandingChat(ctx); const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); const historyMarker = `Retained launch context ${ctx.input.nonce}`; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts index 94ca5fb87c7..06e23d3ff23 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-built-in.ts @@ -1,3 +1,4 @@ +import { pollUntil } from "../api.js"; import { checks, sameTask, type Entity } from "./internal-support.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type InstructionFile } from "./runtime-support.js"; interface BuiltInState { definition: { key: string }; agentId: string | null; agent: RuntimeAgent | null; status: string } @@ -34,7 +35,13 @@ export const runtimeBuiltInCases = [runtimeDefinition("agents.configure-required sourceFiles: ["chat-catalog-domains/runtime-built-in.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "protected-summarizer-identity", "historical-task-unrun"] }, async verifySetup(ctx, state) { - const c = checks(); const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + const c = checks(); + // Preserve the protected summarizer's real pause; only the ordinary source + // agent must complete its asynchronous public readiness verification. + const lead = await pollUntil({ label: "Required-agent configuration source readiness", deadlineAt: Date.now() + 45_000, + load: () => ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`), + accept: row => row.lifecycleState === "ready" && ["idle", "active"].includes(row.status ?? ""), + reject: row => row.lifecycleError ? "Source runtime verification failed" : undefined }); const builtIn = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/built-in-agents/summarizer/status`); const history = await ctx.input.api.get(`/api/issues/${(state.historicalTask as Entity).id}`); c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The actual chat agent is runnable."); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts index 4082e0e7238..317588fa072 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts @@ -18,3 +18,7 @@ it("rejects navigation to another company", () => { it("rejects invented standing-chat history parity", () => { const o = valid(); o.reply += " It uses the same Agent Chat history."; expect(gradeConference(o).passed).toBe(false); }); + +it("preserves failure when the actual model only asks launch questions and supplies no board navigation or scope", () => { + const o = valid(); o.reply = ""; expect(gradeConference(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-held-b-fixtures.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-held-b-fixtures.test.ts new file mode 100644 index 00000000000..fd75351f7de --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-held-b-fixtures.test.ts @@ -0,0 +1,88 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtemp, mkdir, writeFile, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const calls = vi.hoisted(() => ({ events: [] as string[] })); +vi.mock("../chat-flow.js", () => ({ sendChatMessage: vi.fn(async () => { calls.events.push("human-send"); }) })); +vi.mock("./identity-support.js", async importOriginal => ({ ...await importOriginal(), + openStandingChat: vi.fn(async () => ({ id: "source", conversationState: "waiting" })), + readAgentResource: vi.fn(async () => ({ status: 403 })), +})); +import { runtimeAgentChatCases } from "./runtime-agent-chat.js"; +import { runtimeBuiltInCases } from "./runtime-built-in.js"; +import { runtimeResumeCases } from "./runtime-resume.js"; +import { currentControllerFile } from "./runtime-recovery.js"; +const roots: string[] = []; +afterEach(async () => { calls.events.length = 0; vi.useRealTimers(); vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); + +it("funds both real history-turn scopes before the genuine setup message, without adding another native turn", async () => { + const ctx = { input: { nonce: "fixture", fixtures: { company: { id: "company" }, agent: { id: "lead" } }, page: {}, api: { + patch: async () => undefined, + post: async (url: string, payload: { scopeId: string; amount: number; unpricedUsagePolicy: string }) => { + expect(url).toBe("/api/companies/company/budgets/policies"); expect(payload.amount).toBe(500); expect(payload.unpricedUsagePolicy).toBe("allow"); + calls.events.push(`fund-${payload.scopeId}`); + }, + get: async (url: string) => url.endsWith("comments?order=asc") ? [{ id: "reply", authorAgentId: "lead", createdByRunId: "history" }] : { id: "lead", name: "Morgan" }, + } }, issue: () => ({ id: "source", conversationState: "waiting" }), refreshIssue: async () => undefined, + allRuns: async () => [{ id: "history", status: "succeeded" }] } as unknown as ChatCatalogContext; + await runtimeAgentChatCases[0]!.setup(ctx); + expect(calls.events).toEqual(["fund-company", "fund-lead", "human-send"]); +}); + +it("waits for actual source verification while preserving the protected summarizer's paused identity", async () => { + vi.useFakeTimers(); let leadReads = 0; + const get = vi.fn(async (url: string) => { + if (url === "/api/agents/lead") return ++leadReads === 1 ? { lifecycleState: "verifying", status: "paused" } : { lifecycleState: "ready", status: "idle" }; + if (url.endsWith("summarizer/status")) return { definition: { key: "summarizer" }, agentId: "summarizer", agent: { status: "paused" } }; + return { status: "backlog" }; + }); + const ctx = { input: { fixtures: { company: { id: "company" }, agent: { id: "lead" } }, api: { get } }, allRuns: async () => [] } as unknown as ChatCatalogContext; + const pending = runtimeBuiltInCases[0]!.verifySetup!(ctx, { worker: { id: "summarizer" }, historicalTask: { id: "history" } }); + await vi.advanceTimersByTimeAsync(2000); + expect((await pending).passed).toBe(true); expect(leadReads).toBe(2); + expect(get.mock.calls.some(([url]) => url === "/api/agents/summarizer")).toBe(false); +}); + +it("rejects a real source verification error instead of forcing ready state", async () => { + const ctx = { input: { fixtures: { agent: { id: "lead" } }, api: { get: async () => ({ lifecycleState: "error", lifecycleError: "actual failed validation" }) } } } as unknown as ChatCatalogContext; + await expect(runtimeBuiltInCases[0]!.verifySetup!(ctx, {})).rejects.toThrow("Source runtime verification failed"); +}); + +it("establishes the second budget context only after first saved resume, with explicit synthetic receipt provenance", async () => { + let imported = false; + const ctx = { scenario: { query: "Resume Morgan so they can pick up their work." }, input: { + nonce: "fixture", page: {}, fixtures: { company: { id: "company" }, agent: { id: "lead" } }, evidence: async (_name: string, body: { modelProviderUsage: string }) => { + expect(body.modelProviderUsage).toContain("Not model/provider usage"); calls.events.push("fixture-evidence"); + }, api: { + get: async (url: string) => url.endsWith("comments?order=asc") ? [{ id: "first-reply", authorAgentId: "lead", createdByRunId: "first", body: "Morgan is available." }] + : imported ? { id: "morgan", status: "paused", lifecycleState: "paused", lifecycleHolds: ["manual", "budget"] } + : { id: "morgan", status: "idle", lifecycleState: "ready" }, + post: async (url: string, payload: { amount?: number; provider?: string; costCents?: number; heartbeatRunId?: string }) => { + if (url.endsWith("/pause")) calls.events.push("pause-after-first"); + else if (url.endsWith("budgets/policies")) { expect(payload.amount).toBe(100); calls.events.push("positive-hard-cap"); } + else { expect(url).toContain("cost-events"); expect(payload.provider).toBe("synthetic-eval-import"); expect(payload.costCents).toBe(125); + expect(payload.heartbeatRunId).toBeUndefined(); imported = true; calls.events.push("public-synthetic-import"); return { id: "receipt" }; } + }, + }, + }, refreshIssue: async () => undefined, issue: () => ({ id: "source", conversationState: "waiting" }), allRuns: async () => [{ id: "first", status: "succeeded" }] } as unknown as ChatCatalogContext; + const state = { worker: { id: "morgan" } }; + await runtimeResumeCases[0]!.act!(ctx, state); + expect(calls.events).toEqual(["pause-after-first", "positive-hard-cap", "public-synthetic-import", "fixture-evidence", "human-send"]); +}); + +it("reads real prior bytes from the actual isolated source binding, rejecting wrong identity and escaping files", async () => { + const root = await mkdtemp(path.join(tmpdir(), "controller-binding-")); roots.push(root); + vi.stubEnv("PAPERCLIP_RUNNER_E2E_TEMP_ROOT", root); + const cwd = path.join(root, "isolated-workspaces", "company", "source"); await mkdir(cwd, { recursive: true }); + await writeFile(path.join(cwd, "website.html"), "

Real prior draft

"); + let binding = "bound"; + const ctx = { input: { fixtures: { company: { id: "company" } }, api: { get: async (url: string) => url.endsWith("/workspace") + ? { workspace: { id: binding, companyId: "company", cwd } } : { companyId: "company", executionWorkspaceId: "bound" } } }, + refreshIssue: async () => undefined, issue: () => ({ id: "source" }) } as unknown as ChatCatalogContext; + expect(await currentControllerFile(ctx, "website.html")).toBe("

Real prior draft

"); + binding = "other"; await expect(currentControllerFile(ctx, "website.html")).rejects.toThrow("actual same-company source binding"); binding = "bound"; + await writeFile(path.join(root, "foreign.html"), "wrong"); await symlink(path.join(root, "foreign.html"), path.join(cwd, "escape.html")); + await expect(currentControllerFile(ctx, "escape.html")).rejects.toThrow("escapes"); + await expect(currentControllerFile(ctx, "../foreign.html")).rejects.toThrow("escapes"); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts index b8c9baebc02..d179c972053 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts @@ -25,3 +25,8 @@ it("rejects declaring an unqualified profile ready", () => { const o = valid(); it("rejects substituting another runtime configuration", () => { const o = valid(); o.agents[0]!.adapterConfig.provider = "acpx"; expect(gradeManagedProfile(o).passed).toBe(false); }); it("rejects a fabricated supported management path", () => { const o = valid(); o.state.operatorContractAvailable = false; expect(gradeManagedProfile(o).passed).toBe(false); }); it("rejects claiming required qualification without an actual validation guard", () => { const o = valid(); o.state.qualificationProbeStatus = 201; expect(gradeManagedProfile(o).passed).toBe(false); }); + +it("preserves the actual permission-only answer as incomplete without locked environment and retention guidance", () => { + const o = valid(); o.reply = "Setup is blocked by a platform permission boundary: the managed-agent profile API returned 403 — Board access required. A Board actor must create or attest the Claude Managed Agent profile using the Anthropic agent, environment, and API-key secret IDs. Once completed, resume RUN-1 and I can continue."; + expect(gradeManagedProfile(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-recovery.ts b/tests/runner-e2e/chat-catalog-domains/runtime-recovery.ts index 8da558eda0f..5b4240965ec 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-recovery.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-recovery.ts @@ -1,7 +1,6 @@ import { expect } from "@playwright/test"; import { readFile, realpath } from "node:fs/promises"; import path from "node:path"; -import { resolveDefaultAgentWorkspaceDir } from "../../../server/src/home-paths.js"; import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; import { readRegisteredArtifacts } from "../registered-artifact.js"; @@ -15,12 +14,20 @@ async function settled(ctx: ChatCatalogContext, count: number) { accept: ({ issue, runs }) => issue?.conversationState === "waiting" && runs.length === count && runs.every(run => run.status === "succeeded"), reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "A continuity provider turn failed" : undefined }); } -async function currentFile(ctx: ChatCatalogContext, name: string) { +export async function currentControllerFile(ctx: ChatCatalogContext, name: string) { const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root) throw new Error("Continuity inspection requires the isolated attempt root"); - const home = await realpath(resolveDefaultAgentWorkspaceDir(ctx.input.fixtures.agent.id)); + await ctx.refreshIssue(); const source = ctx.issue(); + if (!source) throw new Error("Continuity file inspection requires its actual source"); + const issue = await ctx.input.api.get<{ companyId: string; executionWorkspaceId: string | null }>(`/api/issues/${source.id}`); + const { workspace } = await ctx.input.api.get<{ workspace: { id: string; companyId: string; cwd: string } }>(`/api/issues/${source.id}/workspace`); + if (issue.companyId !== ctx.input.fixtures.company.id || workspace.companyId !== issue.companyId + || workspace.id !== issue.executionWorkspaceId) throw new Error("Continuity workspace is not the actual same-company source binding"); + const home = await realpath(workspace.cwd); const isolated = await realpath(root); if (!home.startsWith(`${isolated}${path.sep}`) || path.basename(name) !== name) throw new Error("Continuity file escapes its owned fixture directory"); - return readFile(path.join(home, name), "utf8"); + const filename = await realpath(path.join(home, name)); + if (path.dirname(filename) !== home) throw new Error("Continuity file escapes its actual source binding"); + return readFile(filename, "utf8"); } export function gradeControllerRestart(value: unknown) { const c = checks(); const o = runtimeObservation(value); @@ -50,7 +57,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest `Start a small website draft in ${filename} in your current working directory, with a single header containing exactly ${JSON.stringify(headerMarker)}. Register that real file as a downloadable artifact work product titled ${filename}. This is a draft: do not add the contact section yet. Remember this instruction for when I ask to continue: keep the existing header and file, then add a contact section containing hello@acorn.example and register the final file as a new artifact from that continuation. Do this within our current conversation; no project, delegated task or external action is needed.`); const first = await settled(ctx, 1); const source = ctx.issue()!; const [initialArtifact] = await readRegisteredArtifacts(ctx.input.api, source.id, first.runs[0]!.id, [filename]); - const fileBeforeRestart = await currentFile(ctx, filename); + const fileBeforeRestart = await currentControllerFile(ctx, filename); if (fileBeforeRestart !== initialArtifact!.content) throw new Error("Initial downloadable artifact does not match the actual saved draft file"); const priorHistory = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); const url = ctx.input.page.url(); @@ -60,7 +67,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest await ctx.refreshIssue(); const afterHistory = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); return { headerMarker, filename, initialArtifact, initialRunId: first.runs[0]!.id, fileBeforeRestart, - fileAfterRestart: await currentFile(ctx, filename), restarted: true, sourceIssueBefore: source.id, sourceIssueAfter: ctx.issue()?.id, + fileAfterRestart: await currentControllerFile(ctx, filename), restarted: true, sourceIssueBefore: source.id, sourceIssueAfter: ctx.issue()?.id, priorHistoryRetained: priorHistory.every(prior => afterHistory.some(row => row.id === prior.id && row.body === prior.body && row.createdByRunId === prior.createdByRunId)) }; }, gradeControllerRestart, { setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 2, attemptTimeoutMs: 600_000, @@ -69,7 +76,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest const final = runs.find(run => run.id !== state.initialRunId && run.status === "succeeded"); if (source && final) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, source.id, final.id, [String(state.filename)]); - state.finalArtifact = artifact; state.finalFile = await currentFile(ctx, String(state.filename)); + state.finalArtifact = artifact; state.finalFile = await currentControllerFile(ctx, String(state.filename)); } return observeRuntime(ctx, state); }, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts index 080560755ed..3b70af35d45 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts @@ -4,7 +4,7 @@ function valid() { const worker = { id: "morgan", companyId: "company" }; return { scenarioId: "agents.resume-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { worker, firstResumedWorker: { ...worker, lifecycleState: "ready", status: "idle", lifecycleHolds: [] }, - firstReply: "Morgan is resumed and available for future work.", secondBudgetHardStop: true, secondBudgetAmount: 0, + firstReply: "Morgan is resumed and available for future work.", secondBudgetHardStop: true, secondBudgetAmount: 100, secondBudgetPolicyActive: true, syntheticReceiptId: "imported-receipt", syntheticReceiptOrigin: "public-imported-synthetic-fixture", secondBudgetIncidentOpen: true, secondReply: "The budget cap remains a hold; Morgan cannot pick up work before it is resolved." }, agents: [{ ...worker, status: "paused", lifecycleHolds: ["budget"] }], tasks: [], links: ["/ACORN/agents/morgan"], reply: "", activity: [{ id: "resume", action: "agent.resumed", entityId: "morgan", agentId: "ceo" }], @@ -25,3 +25,11 @@ it("rejects silently clearing the remaining budget gate", () => { it("rejects pretending a new worker execution already started", () => { const o = valid(); o.runs.push({ id: "worker", agentId: "morgan", status: "succeeded" }); expect(gradeResumeAgent(o).passed).toBe(false); }); + +it("rejects the unenforced zero-cap fixture and missing real incident", () => { + const o = valid(); o.state.secondBudgetAmount = 0; expect(gradeResumeAgent(o).passed).toBe(false); + o.state.secondBudgetAmount = 100; o.state.secondBudgetIncidentOpen = false; expect(gradeResumeAgent(o).passed).toBe(false); +}); +it("rejects a synthetic receipt misrepresented as provider spending", () => { + const o = valid(); o.state.syntheticReceiptOrigin = "actual-provider-usage"; expect(gradeResumeAgent(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts b/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts index fc967297f02..ff07e6f9c60 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts @@ -17,7 +17,9 @@ export function gradeResumeAgent(value: unknown) { && !/started (?:a |the )?(?:run|task|work)|already (?:working|running)/i.test(o.state.firstReply), "The unheld phase describes availability without inventing a new worker execution."); c.add("real-budget-gate-preserved", current?.lifecycleHolds?.includes("budget") === true && current.status === "paused" - && o.state.secondBudgetHardStop === true && o.state.secondBudgetAmount === 0, + && o.state.secondBudgetHardStop === true && o.state.secondBudgetAmount === 100 + && o.state.secondBudgetPolicyActive === true && o.state.syntheticReceiptOrigin === "public-imported-synthetic-fixture" + && typeof o.state.syntheticReceiptId === "string" && !!o.state.syntheticReceiptId && o.state.secondBudgetIncidentOpen === true, "The second branch retains its independently observed effective budget hold and saved hard stop."); c.add("held-reply-bound-to-second-request", typeof o.state.secondReply === "string" && /budget|spend|cap|limit/i.test(o.state.secondReply) && /hold|block|cannot|can't|remain|before|needs|not.*resum/i.test(o.state.secondReply) @@ -64,7 +66,17 @@ export const runtimeResumeCases = [runtimeDefinition("agents.resume-agent", asyn state.firstRunIds = first.runs.map(run => run.id); state.firstCommentIds = first.comments.map(row => row.id); await ctx.input.api.post(`/api/agents/${worker.id}/pause`, {}); await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { scopeType: "agent", scopeId: worker.id, - amount: 0, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "block" }); + amount: 100, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "allow" }); + // Zero disables enforcement. Import a truthful accounting fixture through + // the supported public path; this is not native/model-provider spending. + const receipt = await ctx.input.api.post<{ id: string }>(`/api/companies/${ctx.input.fixtures.company.id}/cost-events`, { + agentId: worker.id, provider: "synthetic-eval-import", biller: "synthetic-eval-import", model: "imported-fixture-receipt", + billingType: "metered_api", costStatus: "reported", costCents: 125, inputTokens: 0, outputTokens: 0, + occurredAt: new Date().toISOString(), idempotencyKey: `resume-held-prerequisite:${ctx.input.nonce}` }); + state.syntheticReceiptId = receipt.id; state.syntheticReceiptOrigin = "public-imported-synthetic-fixture"; + await ctx.input.evidence("runtime-resume-imported-budget-context.json", { agentId: worker.id, receiptId: receipt.id, + origin: state.syntheticReceiptOrigin, importedFixtureCostCents: 125, hardCapCents: 100, + modelProviderUsage: "Not model/provider usage; explicit public synthetic accounting fixture" }); await pollUntil({ label: "Independent effective budget hold before the second explicit request", deadlineAt: Date.now() + 45_000, load: () => ctx.input.api.get(`/api/agents/${worker.id}`), accept: row => row.lifecycleHolds?.includes("budget") === true }); await sendChatMessage(ctx.input.page, ctx.scenario.query); @@ -78,9 +90,12 @@ export const runtimeResumeCases = [runtimeDefinition("agents.resume-agent", asyn && !oldComments.includes(row.id) && phase.runs.some(run => run.id === row.createdByRunId && !(state.firstRunIds as string[]).includes(run.id)) && Date.parse(row.createdAt ?? "") >= Date.parse(secondHuman.createdAt ?? "")).map(row => row.body ?? "").join("\n") : ""; } - const budget = await ctx.input.api.get<{ policies: Array<{ scopeId: string; scopeType: string; amount: number; hardStopEnabled: boolean }> }>(`/api/companies/${ctx.input.fixtures.company.id}/budgets/overview`); + const budget = await ctx.input.api.get<{ policies: Array<{ scopeId: string; scopeType: string; amount: number; hardStopEnabled: boolean; isActive: boolean }>; activeIncidents: Array<{ scopeId: string; scopeType: string; thresholdType: string; status: string }> }>(`/api/companies/${ctx.input.fixtures.company.id}/budgets/overview`); const policy = budget.policies.find(row => row.scopeId === (state.worker as RuntimeAgent).id && row.scopeType === "agent"); state.secondBudgetAmount = policy?.amount; state.secondBudgetHardStop = policy?.hardStopEnabled; + state.secondBudgetPolicyActive = policy?.isActive; + state.secondBudgetIncidentOpen = budget.activeIncidents.some(row => row.scopeId === (state.worker as RuntimeAgent).id + && row.scopeType === "agent" && row.thresholdType === "hard" && row.status === "open"); return observeRuntime(ctx, state); }, })]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.test.ts index 283e0652216..318cffce7c5 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.test.ts @@ -4,7 +4,7 @@ function valid() { const task = { id: "website", title: "Website task", companyId: "company", projectId: null, status: "done", assigneeAgentId: "morgan" }; const view = { workspace: { id: "bound-workspace", companyId: "company", projectId: null, name: "Website task workspace", cwd: "/owned/task-root" } }; return { scenarioId: "workspaces.locate-files", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", - state: { task, worker: { id: "morgan" }, workspaceBefore: view, workspaceAfter: structuredClone(view), marker: "saved prerequisite", + state: { companyPrefix: "ACORN", task, worker: { id: "morgan" }, workspaceBefore: view, workspaceAfter: structuredClone(view), marker: "saved prerequisite", fileBefore: "saved prerequisite", fileAfter: "saved prerequisite", fixtureRunId: "setup" }, agents: [], tasks: [task], links: ["/ACORN/execution-workspaces/bound-workspace"], reply: "The actual files are in /owned/task-root.", activity: [], runs: [{ id: "setup", agentId: "morgan", status: "succeeded" }], humanQueries: [] }; @@ -25,3 +25,16 @@ it("accepts the actual directory and supported view directions without a mandato const o = valid(); o.links = []; o.reply = "The bound workspace uses /owned/task-root; open the Workspace view to inspect its files."; expect(gradeLocateFiles(o).passed).toBe(true); }); + +it("rejects a wrong saved workspace href even before its DOM anchor renders", () => { + const o = valid(); o.links = []; o.reply = "Workspace files are in /owned/task-root. [Open workspace](/ACORN/execution-workspaces/another)."; + expect(gradeLocateFiles(o).passed).toBe(false); +}); +it("rejects correct resource IDs routed through another company or an unsupported tab", () => { + const o = valid(); o.links = ["/BIRCH/execution-workspaces/bound-workspace"]; expect(gradeLocateFiles(o).passed).toBe(false); + o.links = ["/ACORN/execution-workspaces/bound-workspace/not-a-tab"]; expect(gradeLocateFiles(o).passed).toBe(false); +}); +it("accepts the supported same-resource issues tab without demanding a different heading", () => { + const o = valid(); o.links = ["/ACORN/execution-workspaces/bound-workspace/issues"]; + expect(gradeLocateFiles(o).passed).toBe(true); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts index 57057e19b68..61a0011ed9a 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-workspaces.ts @@ -44,18 +44,32 @@ export function gradeLocateFiles(value: unknown) { c.add("actual-saved-files-retained", o.state.fileAfter === o.state.fileBefore && typeof o.state.fileBefore === "string" && o.state.fileBefore.trim() === o.state.marker && o.runs.some(run => run.id === o.state.fixtureRunId && run.agentId === (o.state.worker as Entity).id && run.status === "succeeded"), "A genuine worker created the prerequisite file and it remains unchanged."); + const resultLinks = workspaceResultLinks(o.reply, o.links); c.add("supported-workspace-view-guidance", (/workspace|files? (?:view|tab|panel)/i.test(o.reply) - || !!after && o.links.some(href => workspaceLink(o.origin, href, after.workspace.id))) - && !!after && o.links.filter(href => { try { return new URL(href, o.origin).pathname.includes("/execution-workspaces/"); } catch { return false; } }) - .every(href => workspaceLink(o.origin, href, after.workspace.id)), + || !!after && resultLinks.some(href => workspaceLink(o.origin, href, after.workspace.id, String(o.state.companyPrefix)))) + && !!after && resultLinks.filter(href => { try { return new URL(href, o.origin).pathname.includes("/execution-workspaces/"); } catch { return false; } }) + .every(href => workspaceLink(o.origin, href, after.workspace.id, String(o.state.companyPrefix))), "The response points to the supported workspace view; any workspace link it does give must identify the actual bound workspace."); return c.result(); } -function workspaceLink(origin: string, href: string, id: string) { - try { const url = new URL(href, origin); const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf("execution-workspaces"); - return url.origin === new URL(origin).origin && at >= 0 && parts[at + 1] === id; } catch { return false; } +export function workspaceResultLinks(reply: string, rendered: string[]) { + return [...new Set([...rendered, ...Array.from(reply.matchAll(/\[[^\]]*\]\(([^)\s]+)\)/g), match => match[1]!)])]; } -export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-files", ctx => savedTaskWorkspace(ctx), gradeLocateFiles, { +export function workspaceLink(origin: string, href: string, id: string, prefix: string) { + try { const url = new URL(href, origin); const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf("execution-workspaces"); + return url.origin === new URL(origin).origin && at === 2 && parts[1] === prefix && parts[at + 1] === id + && (parts.length === 4 || parts.length === 5 && ["services", "configuration", "runtime-logs", "issues", "routines"].includes(parts[4]!)); } catch { return false; } +} +export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-files", async ctx => { + // The actual detail route is gated, independently of workspace realization. + // Enable its supported UI fixture before the query, never after a bad link. + if (!ctx.input.fixtures.company.issuePrefix) throw new Error("Workspace fixture has no public company prefix"); + await ctx.input.api.patch("/api/instance/settings/experimental", { enableIsolatedWorkspaces: true }); + const settings = await ctx.input.api.get<{ enableIsolatedWorkspaces: boolean }>("/api/instance/settings/experimental"); + if (!settings.enableIsolatedWorkspaces) throw new Error("The supported workspace view is not enabled"); + return { ...await savedTaskWorkspace(ctx), companyPrefix: ctx.input.fixtures.company.issuePrefix }; +}, gradeLocateFiles, { + sourceFiles: ["chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/workspaces.yaml", "../../ui/src/components/IsolatedWorkspacesRouteGate.tsx", "../../ui/src/pages/ExecutionWorkspaceDetail.tsx"], expectedRunCount: 2, attemptTimeoutMs: 540_000, async observe(ctx, state) { const id = (state.task as Entity).id; @@ -67,10 +81,17 @@ export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-file async verifyResult(ctx, state, value) { const o = runtimeObservation(value); if (!o) throw new Error("Workspace result missing"); const view = state.workspaceAfter as WorkspaceView; - const href = o.links.find(link => workspaceLink(o.origin, link, view.workspace.id)); + const prefix = state.companyPrefix; + if (typeof prefix !== "string" || !prefix) throw new Error("Workspace view lost its actual company prefix"); + const links = workspaceResultLinks(o.reply, o.links); + if (links.some(link => { try { return new URL(link, o.origin).pathname.includes("/execution-workspaces/") + && !workspaceLink(o.origin, link, view.workspace.id, prefix); } catch { return false; } })) + throw new Error("The actual reply supplied an incorrect workspace link"); + const href = links.find(link => workspaceLink(o.origin, link, view.workspace.id, prefix)); const source = ctx.input.page.url(); if (href) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); - else await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/execution-workspaces/${view.workspace.id}`, { waitUntil: "domcontentloaded" }); + else await ctx.input.page.goto(`/${prefix}/execution-workspaces/${view.workspace.id}`, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page).toHaveURL(new RegExp(`/${prefix}/execution-workspaces/${view.workspace.id}(?:[/?#]|$)`)); await expect(ctx.input.page.getByRole("heading", { name: view.workspace.name, exact: true }).first()).toBeVisible(); await ctx.input.evidence("runtime-task-workspace-opened.json", { taskId: (state.task as Entity).id, workspaceId: view.workspace.id, url: ctx.input.page.url() }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });