Correct evidenced held-B runtime prerequisites and resource checks

This commit is contained in:
Dotta committed 2026-10-27 05:22:33 -05:00
1 parent bd40f66ab8
commit b65cce861b
10 files changed
+196 -22

No files matched your search

@@ -24,6 +24,12 @@ export function gradeReopenConversation(value: unknown) {
}
export const runtimeAgentChatCases = [runtimeDefinition("agent-chat.reopen-conversation", async ctx => {
await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}`, { name: "Morgan" });
// Genuine history runs before shared pre-canonical funding. Subscription
// usage is unpriced, so fund both actual scopes before that setup turn.
for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id },
{ scopeType: "agent", scopeId: ctx.input.fixtures.agent.id }])
await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, {
...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
const source = await openStandingChat(ctx);
const lead = await ctx.input.api.get<Entity>(`/api/agents/${ctx.input.fixtures.agent.id}`);
const historyMarker = `Retained launch context ${ctx.input.nonce}`;
@@ -1,3 +1,4 @@
import { pollUntil } from "../api.js";
import { checks, sameTask, type Entity } from "./internal-support.js";
import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type InstructionFile } from "./runtime-support.js";
interface BuiltInState { definition: { key: string }; agentId: string | null; agent: RuntimeAgent | null; status: string }
@@ -34,7 +35,13 @@ export const runtimeBuiltInCases = [runtimeDefinition("agents.configure-required
sourceFiles: ["chat-catalog-domains/runtime-built-in.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"],
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "protected-summarizer-identity", "historical-task-unrun"] },
async verifySetup(ctx, state) {
const c = checks(); const lead = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${ctx.input.fixtures.agent.id}`);
const c = checks();
// Preserve the protected summarizer's real pause; only the ordinary source
// agent must complete its asynchronous public readiness verification.
const lead = await pollUntil({ label: "Required-agent configuration source readiness", deadlineAt: Date.now() + 45_000,
load: () => ctx.input.api.get<RuntimeAgent & { lifecycleError?: string | null }>(`/api/agents/${ctx.input.fixtures.agent.id}`),
accept: row => row.lifecycleState === "ready" && ["idle", "active"].includes(row.status ?? ""),
reject: row => row.lifecycleError ? "Source runtime verification failed" : undefined });
const builtIn = await ctx.input.api.get<BuiltInState>(`/api/companies/${ctx.input.fixtures.company.id}/built-in-agents/summarizer/status`);
const history = await ctx.input.api.get<Entity>(`/api/issues/${(state.historicalTask as Entity).id}`);
c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The actual chat agent is runnable.");
@@ -18,3 +18,7 @@ it("rejects navigation to another company", () => {
it("rejects invented standing-chat history parity", () => {
const o = valid(); o.reply += " It uses the same Agent Chat history."; expect(gradeConference(o).passed).toBe(false);
});
it("preserves failure when the actual model only asks launch questions and supplies no board navigation or scope", () => {
const o = valid(); o.reply = ""; expect(gradeConference(o).passed).toBe(false);
});
@@ -0,0 +1,88 @@
import { afterEach, expect, it, vi } from "vitest";
import { mkdtemp, mkdir, writeFile, rm, symlink } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
const calls = vi.hoisted(() => ({ events: [] as string[] }));
vi.mock("../chat-flow.js", () => ({ sendChatMessage: vi.fn(async () => { calls.events.push("human-send"); }) }));
vi.mock("./identity-support.js", async importOriginal => ({ ...await importOriginal<typeof import("./identity-support.js")>(),
openStandingChat: vi.fn(async () => ({ id: "source", conversationState: "waiting" })),
readAgentResource: vi.fn(async () => ({ status: 403 })),
}));
import { runtimeAgentChatCases } from "./runtime-agent-chat.js";
import { runtimeBuiltInCases } from "./runtime-built-in.js";
import { runtimeResumeCases } from "./runtime-resume.js";
import { currentControllerFile } from "./runtime-recovery.js";
const roots: string[] = [];
afterEach(async () => { calls.events.length = 0; vi.useRealTimers(); vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); });
it("funds both real history-turn scopes before the genuine setup message, without adding another native turn", async () => {
const ctx = { input: { nonce: "fixture", fixtures: { company: { id: "company" }, agent: { id: "lead" } }, page: {}, api: {
patch: async () => undefined,
post: async (url: string, payload: { scopeId: string; amount: number; unpricedUsagePolicy: string }) => {
expect(url).toBe("/api/companies/company/budgets/policies"); expect(payload.amount).toBe(500); expect(payload.unpricedUsagePolicy).toBe("allow");
calls.events.push(`fund-${payload.scopeId}`);
},
get: async (url: string) => url.endsWith("comments?order=asc") ? [{ id: "reply", authorAgentId: "lead", createdByRunId: "history" }] : { id: "lead", name: "Morgan" },
} }, issue: () => ({ id: "source", conversationState: "waiting" }), refreshIssue: async () => undefined,
allRuns: async () => [{ id: "history", status: "succeeded" }] } as unknown as ChatCatalogContext;
await runtimeAgentChatCases[0]!.setup(ctx);
expect(calls.events).toEqual(["fund-company", "fund-lead", "human-send"]);
});
it("waits for actual source verification while preserving the protected summarizer's paused identity", async () => {
vi.useFakeTimers(); let leadReads = 0;
const get = vi.fn(async (url: string) => {
if (url === "/api/agents/lead") return ++leadReads === 1 ? { lifecycleState: "verifying", status: "paused" } : { lifecycleState: "ready", status: "idle" };
if (url.endsWith("summarizer/status")) return { definition: { key: "summarizer" }, agentId: "summarizer", agent: { status: "paused" } };
return { status: "backlog" };
});
const ctx = { input: { fixtures: { company: { id: "company" }, agent: { id: "lead" } }, api: { get } }, allRuns: async () => [] } as unknown as ChatCatalogContext;
const pending = runtimeBuiltInCases[0]!.verifySetup!(ctx, { worker: { id: "summarizer" }, historicalTask: { id: "history" } });
await vi.advanceTimersByTimeAsync(2000);
expect((await pending).passed).toBe(true); expect(leadReads).toBe(2);
expect(get.mock.calls.some(([url]) => url === "/api/agents/summarizer")).toBe(false);
});
it("rejects a real source verification error instead of forcing ready state", async () => {
const ctx = { input: { fixtures: { agent: { id: "lead" } }, api: { get: async () => ({ lifecycleState: "error", lifecycleError: "actual failed validation" }) } } } as unknown as ChatCatalogContext;
await expect(runtimeBuiltInCases[0]!.verifySetup!(ctx, {})).rejects.toThrow("Source runtime verification failed");
});
it("establishes the second budget context only after first saved resume, with explicit synthetic receipt provenance", async () => {
let imported = false;
const ctx = { scenario: { query: "Resume Morgan so they can pick up their work." }, input: {
nonce: "fixture", page: {}, fixtures: { company: { id: "company" }, agent: { id: "lead" } }, evidence: async (_name: string, body: { modelProviderUsage: string }) => {
expect(body.modelProviderUsage).toContain("Not model/provider usage"); calls.events.push("fixture-evidence");
}, api: {
get: async (url: string) => url.endsWith("comments?order=asc") ? [{ id: "first-reply", authorAgentId: "lead", createdByRunId: "first", body: "Morgan is available." }]
: imported ? { id: "morgan", status: "paused", lifecycleState: "paused", lifecycleHolds: ["manual", "budget"] }
: { id: "morgan", status: "idle", lifecycleState: "ready" },
post: async (url: string, payload: { amount?: number; provider?: string; costCents?: number; heartbeatRunId?: string }) => {
if (url.endsWith("/pause")) calls.events.push("pause-after-first");
else if (url.endsWith("budgets/policies")) { expect(payload.amount).toBe(100); calls.events.push("positive-hard-cap"); }
else { expect(url).toContain("cost-events"); expect(payload.provider).toBe("synthetic-eval-import"); expect(payload.costCents).toBe(125);
expect(payload.heartbeatRunId).toBeUndefined(); imported = true; calls.events.push("public-synthetic-import"); return { id: "receipt" }; }
},
},
}, refreshIssue: async () => undefined, issue: () => ({ id: "source", conversationState: "waiting" }), allRuns: async () => [{ id: "first", status: "succeeded" }] } as unknown as ChatCatalogContext;
const state = { worker: { id: "morgan" } };
await runtimeResumeCases[0]!.act!(ctx, state);
expect(calls.events).toEqual(["pause-after-first", "positive-hard-cap", "public-synthetic-import", "fixture-evidence", "human-send"]);
});
it("reads real prior bytes from the actual isolated source binding, rejecting wrong identity and escaping files", async () => {
const root = await mkdtemp(path.join(tmpdir(), "controller-binding-")); roots.push(root);
vi.stubEnv("PAPERCLIP_RUNNER_E2E_TEMP_ROOT", root);
const cwd = path.join(root, "isolated-workspaces", "company", "source"); await mkdir(cwd, { recursive: true });
await writeFile(path.join(cwd, "website.html"), "<h1>Real prior draft</h1>");
let binding = "bound";
const ctx = { input: { fixtures: { company: { id: "company" } }, api: { get: async (url: string) => url.endsWith("/workspace")
? { workspace: { id: binding, companyId: "company", cwd } } : { companyId: "company", executionWorkspaceId: "bound" } } },
refreshIssue: async () => undefined, issue: () => ({ id: "source" }) } as unknown as ChatCatalogContext;
expect(await currentControllerFile(ctx, "website.html")).toBe("<h1>Real prior draft</h1>");
binding = "other"; await expect(currentControllerFile(ctx, "website.html")).rejects.toThrow("actual same-company source binding"); binding = "bound";
await writeFile(path.join(root, "foreign.html"), "wrong"); await symlink(path.join(root, "foreign.html"), path.join(cwd, "escape.html"));
await expect(currentControllerFile(ctx, "escape.html")).rejects.toThrow("escapes");
await expect(currentControllerFile(ctx, "../foreign.html")).rejects.toThrow("escapes");
});
@@ -25,3 +25,8 @@ it("rejects declaring an unqualified profile ready", () => { const o = valid();
it("rejects substituting another runtime configuration", () => { const o = valid(); o.agents[0]!.adapterConfig.provider = "acpx"; expect(gradeManagedProfile(o).passed).toBe(false); });
it("rejects a fabricated supported management path", () => { const o = valid(); o.state.operatorContractAvailable = false; expect(gradeManagedProfile(o).passed).toBe(false); });
it("rejects claiming required qualification without an actual validation guard", () => { const o = valid(); o.state.qualificationProbeStatus = 201; expect(gradeManagedProfile(o).passed).toBe(false); });
it("preserves the actual permission-only answer as incomplete without locked environment and retention guidance", () => {
const o = valid(); o.reply = "Setup is blocked by a platform permission boundary: the managed-agent profile API returned 403 — Board access required. A Board actor must create or attest the Claude Managed Agent profile using the Anthropic agent, environment, and API-key secret IDs. Once completed, resume RUN-1 and I can continue.";
expect(gradeManagedProfile(o).passed).toBe(false);
});
@@ -1,7 +1,6 @@
import { expect } from "@playwright/test";
import { readFile, realpath } from "node:fs/promises";
import path from "node:path";
import { resolveDefaultAgentWorkspaceDir } from "../../../server/src/home-paths.js";
import { pollUntil } from "../api.js";
import { sendChatMessage } from "../chat-flow.js";
import { readRegisteredArtifacts } from "../registered-artifact.js";
@@ -15,12 +14,20 @@ async function settled(ctx: ChatCatalogContext, count: number) {
accept: ({ issue, runs }) => issue?.conversationState === "waiting" && runs.length === count && runs.every(run => run.status === "succeeded"),
reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "A continuity provider turn failed" : undefined });
}
async function currentFile(ctx: ChatCatalogContext, name: string) {
export async function currentControllerFile(ctx: ChatCatalogContext, name: string) {
const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root) throw new Error("Continuity inspection requires the isolated attempt root");
const home = await realpath(resolveDefaultAgentWorkspaceDir(ctx.input.fixtures.agent.id));
await ctx.refreshIssue(); const source = ctx.issue();
if (!source) throw new Error("Continuity file inspection requires its actual source");
const issue = await ctx.input.api.get<{ companyId: string; executionWorkspaceId: string | null }>(`/api/issues/${source.id}`);
const { workspace } = await ctx.input.api.get<{ workspace: { id: string; companyId: string; cwd: string } }>(`/api/issues/${source.id}/workspace`);
if (issue.companyId !== ctx.input.fixtures.company.id || workspace.companyId !== issue.companyId
|| workspace.id !== issue.executionWorkspaceId) throw new Error("Continuity workspace is not the actual same-company source binding");
const home = await realpath(workspace.cwd);
const isolated = await realpath(root);
if (!home.startsWith(`${isolated}${path.sep}`) || path.basename(name) !== name) throw new Error("Continuity file escapes its owned fixture directory");
return readFile(path.join(home, name), "utf8");
const filename = await realpath(path.join(home, name));
if (path.dirname(filename) !== home) throw new Error("Continuity file escapes its actual source binding");
return readFile(filename, "utf8");
}
export function gradeControllerRestart(value: unknown) {
const c = checks(); const o = runtimeObservation(value);
@@ -50,7 +57,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest
`Start a small website draft in ${filename} in your current working directory, with a single header containing exactly ${JSON.stringify(headerMarker)}. Register that real file as a downloadable artifact work product titled ${filename}. This is a draft: do not add the contact section yet. Remember this instruction for when I ask to continue: keep the existing header and file, then add a contact section containing hello@acorn.example and register the final file as a new artifact from that continuation. Do this within our current conversation; no project, delegated task or external action is needed.`);
const first = await settled(ctx, 1); const source = ctx.issue()!;
const [initialArtifact] = await readRegisteredArtifacts(ctx.input.api, source.id, first.runs[0]!.id, [filename]);
const fileBeforeRestart = await currentFile(ctx, filename);
const fileBeforeRestart = await currentControllerFile(ctx, filename);
if (fileBeforeRestart !== initialArtifact!.content) throw new Error("Initial downloadable artifact does not match the actual saved draft file");
const priorHistory = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
const url = ctx.input.page.url();
@@ -60,7 +67,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest
await ctx.refreshIssue();
const afterHistory = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
return { headerMarker, filename, initialArtifact, initialRunId: first.runs[0]!.id, fileBeforeRestart,
fileAfterRestart: await currentFile(ctx, filename), restarted: true, sourceIssueBefore: source.id, sourceIssueAfter: ctx.issue()?.id,
fileAfterRestart: await currentControllerFile(ctx, filename), restarted: true, sourceIssueBefore: source.id, sourceIssueAfter: ctx.issue()?.id,
priorHistoryRetained: priorHistory.every(prior => afterHistory.some(row => row.id === prior.id && row.body === prior.body && row.createdByRunId === prior.createdByRunId)) };
}, gradeControllerRestart, {
setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 2, attemptTimeoutMs: 600_000,
@@ -69,7 +76,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest
const final = runs.find(run => run.id !== state.initialRunId && run.status === "succeeded");
if (source && final) {
const [artifact] = await readRegisteredArtifacts(ctx.input.api, source.id, final.id, [String(state.filename)]);
state.finalArtifact = artifact; state.finalFile = await currentFile(ctx, String(state.filename));
state.finalArtifact = artifact; state.finalFile = await currentControllerFile(ctx, String(state.filename));
}
return observeRuntime(ctx, state);
},
@@ -4,7 +4,7 @@ function valid() {
const worker = { id: "morgan", companyId: "company" };
return { scenarioId: "agents.resume-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
state: { worker, firstResumedWorker: { ...worker, lifecycleState: "ready", status: "idle", lifecycleHolds: [] },
firstReply: "Morgan is resumed and available for future work.", secondBudgetHardStop: true, secondBudgetAmount: 0,
firstReply: "Morgan is resumed and available for future work.", secondBudgetHardStop: true, secondBudgetAmount: 100, secondBudgetPolicyActive: true, syntheticReceiptId: "imported-receipt", syntheticReceiptOrigin: "public-imported-synthetic-fixture", secondBudgetIncidentOpen: true,
secondReply: "The budget cap remains a hold; Morgan cannot pick up work before it is resolved." },
agents: [{ ...worker, status: "paused", lifecycleHolds: ["budget"] }], tasks: [], links: ["/ACORN/agents/morgan"], reply: "",
activity: [{ id: "resume", action: "agent.resumed", entityId: "morgan", agentId: "ceo" }],
@@ -25,3 +25,11 @@ it("rejects silently clearing the remaining budget gate", () => {
it("rejects pretending a new worker execution already started", () => {
const o = valid(); o.runs.push({ id: "worker", agentId: "morgan", status: "succeeded" }); expect(gradeResumeAgent(o).passed).toBe(false);
});
it("rejects the unenforced zero-cap fixture and missing real incident", () => {
const o = valid(); o.state.secondBudgetAmount = 0; expect(gradeResumeAgent(o).passed).toBe(false);
o.state.secondBudgetAmount = 100; o.state.secondBudgetIncidentOpen = false; expect(gradeResumeAgent(o).passed).toBe(false);
});
it("rejects a synthetic receipt misrepresented as provider spending", () => {
const o = valid(); o.state.syntheticReceiptOrigin = "actual-provider-usage"; expect(gradeResumeAgent(o).passed).toBe(false);
});
@@ -17,7 +17,9 @@ export function gradeResumeAgent(value: unknown) {
&& !/started (?:a |the )?(?:run|task|work)|already (?:working|running)/i.test(o.state.firstReply),
"The unheld phase describes availability without inventing a new worker execution.");
c.add("real-budget-gate-preserved", current?.lifecycleHolds?.includes("budget") === true && current.status === "paused"
&& o.state.secondBudgetHardStop === true && o.state.secondBudgetAmount === 0,
&& o.state.secondBudgetHardStop === true && o.state.secondBudgetAmount === 100
&& o.state.secondBudgetPolicyActive === true && o.state.syntheticReceiptOrigin === "public-imported-synthetic-fixture"
&& typeof o.state.syntheticReceiptId === "string" && !!o.state.syntheticReceiptId && o.state.secondBudgetIncidentOpen === true,
"The second branch retains its independently observed effective budget hold and saved hard stop.");
c.add("held-reply-bound-to-second-request", typeof o.state.secondReply === "string" && /budget|spend|cap|limit/i.test(o.state.secondReply)
&& /hold|block|cannot|can't|remain|before|needs|not.*resum/i.test(o.state.secondReply)
@@ -64,7 +66,17 @@ export const runtimeResumeCases = [runtimeDefinition("agents.resume-agent", asyn
state.firstRunIds = first.runs.map(run => run.id); state.firstCommentIds = first.comments.map(row => row.id);
await ctx.input.api.post(`/api/agents/${worker.id}/pause`, {});
await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { scopeType: "agent", scopeId: worker.id,
amount: 0, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "block" });
amount: 100, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "allow" });
// Zero disables enforcement. Import a truthful accounting fixture through
// the supported public path; this is not native/model-provider spending.
const receipt = await ctx.input.api.post<{ id: string }>(`/api/companies/${ctx.input.fixtures.company.id}/cost-events`, {
agentId: worker.id, provider: "synthetic-eval-import", biller: "synthetic-eval-import", model: "imported-fixture-receipt",
billingType: "metered_api", costStatus: "reported", costCents: 125, inputTokens: 0, outputTokens: 0,
occurredAt: new Date().toISOString(), idempotencyKey: `resume-held-prerequisite:${ctx.input.nonce}` });
state.syntheticReceiptId = receipt.id; state.syntheticReceiptOrigin = "public-imported-synthetic-fixture";
await ctx.input.evidence("runtime-resume-imported-budget-context.json", { agentId: worker.id, receiptId: receipt.id,
origin: state.syntheticReceiptOrigin, importedFixtureCostCents: 125, hardCapCents: 100,
modelProviderUsage: "Not model/provider usage; explicit public synthetic accounting fixture" });
await pollUntil({ label: "Independent effective budget hold before the second explicit request", deadlineAt: Date.now() + 45_000,
load: () => ctx.input.api.get<RuntimeAgent>(`/api/agents/${worker.id}`), accept: row => row.lifecycleHolds?.includes("budget") === true });
await sendChatMessage(ctx.input.page, ctx.scenario.query);
@@ -78,9 +90,12 @@ export const runtimeResumeCases = [runtimeDefinition("agents.resume-agent", asyn
&& !oldComments.includes(row.id) && phase.runs.some(run => run.id === row.createdByRunId && !(state.firstRunIds as string[]).includes(run.id))
&& Date.parse(row.createdAt ?? "") >= Date.parse(secondHuman.createdAt ?? "")).map(row => row.body ?? "").join("\n") : "";
}
const budget = await ctx.input.api.get<{ policies: Array<{ scopeId: string; scopeType: string; amount: number; hardStopEnabled: boolean }> }>(`/api/companies/${ctx.input.fixtures.company.id}/budgets/overview`);
const budget = await ctx.input.api.get<{ policies: Array<{ scopeId: string; scopeType: string; amount: number; hardStopEnabled: boolean; isActive: boolean }>; activeIncidents: Array<{ scopeId: string; scopeType: string; thresholdType: string; status: string }> }>(`/api/companies/${ctx.input.fixtures.company.id}/budgets/overview`);
const policy = budget.policies.find(row => row.scopeId === (state.worker as RuntimeAgent).id && row.scopeType === "agent");
state.secondBudgetAmount = policy?.amount; state.secondBudgetHardStop = policy?.hardStopEnabled;
state.secondBudgetPolicyActive = policy?.isActive;
state.secondBudgetIncidentOpen = budget.activeIncidents.some(row => row.scopeId === (state.worker as RuntimeAgent).id
&& row.scopeType === "agent" && row.thresholdType === "hard" && row.status === "open");
return observeRuntime(ctx, state);
},
})];
@@ -4,7 +4,7 @@ function valid() {
const task = { id: "website", title: "Website task", companyId: "company", projectId: null, status: "done", assigneeAgentId: "morgan" };
const view = { workspace: { id: "bound-workspace", companyId: "company", projectId: null, name: "Website task workspace", cwd: "/owned/task-root" } };
return { scenarioId: "workspaces.locate-files", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
state: { task, worker: { id: "morgan" }, workspaceBefore: view, workspaceAfter: structuredClone(view), marker: "saved prerequisite",
state: { companyPrefix: "ACORN", task, worker: { id: "morgan" }, workspaceBefore: view, workspaceAfter: structuredClone(view), marker: "saved prerequisite",
fileBefore: "saved prerequisite", fileAfter: "saved prerequisite", fixtureRunId: "setup" },
agents: [], tasks: [task], links: ["/ACORN/execution-workspaces/bound-workspace"], reply: "The actual files are in /owned/task-root.",
activity: [], runs: [{ id: "setup", agentId: "morgan", status: "succeeded" }], humanQueries: [] };
@@ -25,3 +25,16 @@ it("accepts the actual directory and supported view directions without a mandato
const o = valid(); o.links = []; o.reply = "The bound workspace uses /owned/task-root; open the Workspace view to inspect its files.";
expect(gradeLocateFiles(o).passed).toBe(true);
});
it("rejects a wrong saved workspace href even before its DOM anchor renders", () => {
const o = valid(); o.links = []; o.reply = "Workspace files are in /owned/task-root. [Open workspace](/ACORN/execution-workspaces/another).";
expect(gradeLocateFiles(o).passed).toBe(false);
});
it("rejects correct resource IDs routed through another company or an unsupported tab", () => {
const o = valid(); o.links = ["/BIRCH/execution-workspaces/bound-workspace"]; expect(gradeLocateFiles(o).passed).toBe(false);
o.links = ["/ACORN/execution-workspaces/bound-workspace/not-a-tab"]; expect(gradeLocateFiles(o).passed).toBe(false);
});
it("accepts the supported same-resource issues tab without demanding a different heading", () => {
const o = valid(); o.links = ["/ACORN/execution-workspaces/bound-workspace/issues"];
expect(gradeLocateFiles(o).passed).toBe(true);
});
@@ -44,18 +44,32 @@ export function gradeLocateFiles(value: unknown) {
c.add("actual-saved-files-retained", o.state.fileAfter === o.state.fileBefore && typeof o.state.fileBefore === "string"
&& o.state.fileBefore.trim() === o.state.marker && o.runs.some(run => run.id === o.state.fixtureRunId && run.agentId === (o.state.worker as Entity).id
&& run.status === "succeeded"), "A genuine worker created the prerequisite file and it remains unchanged.");
const resultLinks = workspaceResultLinks(o.reply, o.links);
c.add("supported-workspace-view-guidance", (/workspace|files? (?:view|tab|panel)/i.test(o.reply)
|| !!after && o.links.some(href => workspaceLink(o.origin, href, after.workspace.id)))
&& !!after && o.links.filter(href => { try { return new URL(href, o.origin).pathname.includes("/execution-workspaces/"); } catch { return false; } })
.every(href => workspaceLink(o.origin, href, after.workspace.id)),
|| !!after && resultLinks.some(href => workspaceLink(o.origin, href, after.workspace.id, String(o.state.companyPrefix))))
&& !!after && resultLinks.filter(href => { try { return new URL(href, o.origin).pathname.includes("/execution-workspaces/"); } catch { return false; } })
.every(href => workspaceLink(o.origin, href, after.workspace.id, String(o.state.companyPrefix))),
"The response points to the supported workspace view; any workspace link it does give must identify the actual bound workspace.");
return c.result();
}
function workspaceLink(origin: string, href: string, id: string) {
try { const url = new URL(href, origin); const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf("execution-workspaces");
return url.origin === new URL(origin).origin && at >= 0 && parts[at + 1] === id; } catch { return false; }
export function workspaceResultLinks(reply: string, rendered: string[]) {
return [...new Set([...rendered, ...Array.from(reply.matchAll(/\[[^\]]*\]\(([^)\s]+)\)/g), match => match[1]!)])];
}
export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-files", ctx => savedTaskWorkspace(ctx), gradeLocateFiles, {
export function workspaceLink(origin: string, href: string, id: string, prefix: string) {
try { const url = new URL(href, origin); const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf("execution-workspaces");
return url.origin === new URL(origin).origin && at === 2 && parts[1] === prefix && parts[at + 1] === id
&& (parts.length === 4 || parts.length === 5 && ["services", "configuration", "runtime-logs", "issues", "routines"].includes(parts[4]!)); } catch { return false; }
}
export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-files", async ctx => {
// The actual detail route is gated, independently of workspace realization.
// Enable its supported UI fixture before the query, never after a bad link.
if (!ctx.input.fixtures.company.issuePrefix) throw new Error("Workspace fixture has no public company prefix");
await ctx.input.api.patch("/api/instance/settings/experimental", { enableIsolatedWorkspaces: true });
const settings = await ctx.input.api.get<{ enableIsolatedWorkspaces: boolean }>("/api/instance/settings/experimental");
if (!settings.enableIsolatedWorkspaces) throw new Error("The supported workspace view is not enabled");
return { ...await savedTaskWorkspace(ctx), companyPrefix: ctx.input.fixtures.company.issuePrefix };
}, gradeLocateFiles, {
sourceFiles: ["chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/workspaces.yaml", "../../ui/src/components/IsolatedWorkspacesRouteGate.tsx", "../../ui/src/pages/ExecutionWorkspaceDetail.tsx"],
expectedRunCount: 2, attemptTimeoutMs: 540_000,
async observe(ctx, state) {
const id = (state.task as Entity).id;
@@ -67,10 +81,17 @@ export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-file
async verifyResult(ctx, state, value) {
const o = runtimeObservation(value); if (!o) throw new Error("Workspace result missing");
const view = state.workspaceAfter as WorkspaceView;
const href = o.links.find(link => workspaceLink(o.origin, link, view.workspace.id));
const prefix = state.companyPrefix;
if (typeof prefix !== "string" || !prefix) throw new Error("Workspace view lost its actual company prefix");
const links = workspaceResultLinks(o.reply, o.links);
if (links.some(link => { try { return new URL(link, o.origin).pathname.includes("/execution-workspaces/")
&& !workspaceLink(o.origin, link, view.workspace.id, prefix); } catch { return false; } }))
throw new Error("The actual reply supplied an incorrect workspace link");
const href = links.find(link => workspaceLink(o.origin, link, view.workspace.id, prefix));
const source = ctx.input.page.url();
if (href) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();
else await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/execution-workspaces/${view.workspace.id}`, { waitUntil: "domcontentloaded" });
else await ctx.input.page.goto(`/${prefix}/execution-workspaces/${view.workspace.id}`, { waitUntil: "domcontentloaded" });
await expect(ctx.input.page).toHaveURL(new RegExp(`/${prefix}/execution-workspaces/${view.workspace.id}(?:[/?#]|$)`));
await expect(ctx.input.page.getByRole("heading", { name: view.workspace.name, exact: true }).first()).toBeVisible();
await ctx.input.evidence("runtime-task-workspace-opened.json", { taskId: (state.task as Entity).id, workspaceId: view.workspace.id, url: ctx.input.page.url() });
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });