mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 03:51:54 +01:00
Correct evidenced held-B runtime prerequisites and resource checks
This commit is contained in:
1 parent
bd40f66ab8
commit
b65cce861b
10 files changed
+196
-22
No files matched your search
@@ -24,6 +24,12 @@ export function gradeReopenConversation(value: unknown) {
|
||||
}
|
||||
export const runtimeAgentChatCases = [runtimeDefinition("agent-chat.reopen-conversation", async ctx => {
|
||||
await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}`, { name: "Morgan" });
|
||||
// Genuine history runs before shared pre-canonical funding. Subscription
|
||||
// usage is unpriced, so fund both actual scopes before that setup turn.
|
||||
for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id },
|
||||
{ scopeType: "agent", scopeId: ctx.input.fixtures.agent.id }])
|
||||
await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, {
|
||||
...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
|
||||
const source = await openStandingChat(ctx);
|
||||
const lead = await ctx.input.api.get<Entity>(`/api/agents/${ctx.input.fixtures.agent.id}`);
|
||||
const historyMarker = `Retained launch context ${ctx.input.nonce}`;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { pollUntil } from "../api.js";
|
||||
import { checks, sameTask, type Entity } from "./internal-support.js";
|
||||
import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent, type InstructionFile } from "./runtime-support.js";
|
||||
interface BuiltInState { definition: { key: string }; agentId: string | null; agent: RuntimeAgent | null; status: string }
|
||||
@@ -34,7 +35,13 @@ export const runtimeBuiltInCases = [runtimeDefinition("agents.configure-required
|
||||
sourceFiles: ["chat-catalog-domains/runtime-built-in.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"],
|
||||
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "protected-summarizer-identity", "historical-task-unrun"] },
|
||||
async verifySetup(ctx, state) {
|
||||
const c = checks(); const lead = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${ctx.input.fixtures.agent.id}`);
|
||||
const c = checks();
|
||||
// Preserve the protected summarizer's real pause; only the ordinary source
|
||||
// agent must complete its asynchronous public readiness verification.
|
||||
const lead = await pollUntil({ label: "Required-agent configuration source readiness", deadlineAt: Date.now() + 45_000,
|
||||
load: () => ctx.input.api.get<RuntimeAgent & { lifecycleError?: string | null }>(`/api/agents/${ctx.input.fixtures.agent.id}`),
|
||||
accept: row => row.lifecycleState === "ready" && ["idle", "active"].includes(row.status ?? ""),
|
||||
reject: row => row.lifecycleError ? "Source runtime verification failed" : undefined });
|
||||
const builtIn = await ctx.input.api.get<BuiltInState>(`/api/companies/${ctx.input.fixtures.company.id}/built-in-agents/summarizer/status`);
|
||||
const history = await ctx.input.api.get<Entity>(`/api/issues/${(state.historicalTask as Entity).id}`);
|
||||
c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The actual chat agent is runnable.");
|
||||
|
||||
@@ -18,3 +18,7 @@ it("rejects navigation to another company", () => {
|
||||
it("rejects invented standing-chat history parity", () => {
|
||||
const o = valid(); o.reply += " It uses the same Agent Chat history."; expect(gradeConference(o).passed).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves failure when the actual model only asks launch questions and supplies no board navigation or scope", () => {
|
||||
const o = valid(); o.reply = ""; expect(gradeConference(o).passed).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { mkdtemp, mkdir, writeFile, rm, symlink } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
const calls = vi.hoisted(() => ({ events: [] as string[] }));
|
||||
vi.mock("../chat-flow.js", () => ({ sendChatMessage: vi.fn(async () => { calls.events.push("human-send"); }) }));
|
||||
vi.mock("./identity-support.js", async importOriginal => ({ ...await importOriginal<typeof import("./identity-support.js")>(),
|
||||
openStandingChat: vi.fn(async () => ({ id: "source", conversationState: "waiting" })),
|
||||
readAgentResource: vi.fn(async () => ({ status: 403 })),
|
||||
}));
|
||||
import { runtimeAgentChatCases } from "./runtime-agent-chat.js";
|
||||
import { runtimeBuiltInCases } from "./runtime-built-in.js";
|
||||
import { runtimeResumeCases } from "./runtime-resume.js";
|
||||
import { currentControllerFile } from "./runtime-recovery.js";
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => { calls.events.length = 0; vi.useRealTimers(); vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); });
|
||||
|
||||
it("funds both real history-turn scopes before the genuine setup message, without adding another native turn", async () => {
|
||||
const ctx = { input: { nonce: "fixture", fixtures: { company: { id: "company" }, agent: { id: "lead" } }, page: {}, api: {
|
||||
patch: async () => undefined,
|
||||
post: async (url: string, payload: { scopeId: string; amount: number; unpricedUsagePolicy: string }) => {
|
||||
expect(url).toBe("/api/companies/company/budgets/policies"); expect(payload.amount).toBe(500); expect(payload.unpricedUsagePolicy).toBe("allow");
|
||||
calls.events.push(`fund-${payload.scopeId}`);
|
||||
},
|
||||
get: async (url: string) => url.endsWith("comments?order=asc") ? [{ id: "reply", authorAgentId: "lead", createdByRunId: "history" }] : { id: "lead", name: "Morgan" },
|
||||
} }, issue: () => ({ id: "source", conversationState: "waiting" }), refreshIssue: async () => undefined,
|
||||
allRuns: async () => [{ id: "history", status: "succeeded" }] } as unknown as ChatCatalogContext;
|
||||
await runtimeAgentChatCases[0]!.setup(ctx);
|
||||
expect(calls.events).toEqual(["fund-company", "fund-lead", "human-send"]);
|
||||
});
|
||||
|
||||
it("waits for actual source verification while preserving the protected summarizer's paused identity", async () => {
|
||||
vi.useFakeTimers(); let leadReads = 0;
|
||||
const get = vi.fn(async (url: string) => {
|
||||
if (url === "/api/agents/lead") return ++leadReads === 1 ? { lifecycleState: "verifying", status: "paused" } : { lifecycleState: "ready", status: "idle" };
|
||||
if (url.endsWith("summarizer/status")) return { definition: { key: "summarizer" }, agentId: "summarizer", agent: { status: "paused" } };
|
||||
return { status: "backlog" };
|
||||
});
|
||||
const ctx = { input: { fixtures: { company: { id: "company" }, agent: { id: "lead" } }, api: { get } }, allRuns: async () => [] } as unknown as ChatCatalogContext;
|
||||
const pending = runtimeBuiltInCases[0]!.verifySetup!(ctx, { worker: { id: "summarizer" }, historicalTask: { id: "history" } });
|
||||
await vi.advanceTimersByTimeAsync(2000);
|
||||
expect((await pending).passed).toBe(true); expect(leadReads).toBe(2);
|
||||
expect(get.mock.calls.some(([url]) => url === "/api/agents/summarizer")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects a real source verification error instead of forcing ready state", async () => {
|
||||
const ctx = { input: { fixtures: { agent: { id: "lead" } }, api: { get: async () => ({ lifecycleState: "error", lifecycleError: "actual failed validation" }) } } } as unknown as ChatCatalogContext;
|
||||
await expect(runtimeBuiltInCases[0]!.verifySetup!(ctx, {})).rejects.toThrow("Source runtime verification failed");
|
||||
});
|
||||
|
||||
it("establishes the second budget context only after first saved resume, with explicit synthetic receipt provenance", async () => {
|
||||
let imported = false;
|
||||
const ctx = { scenario: { query: "Resume Morgan so they can pick up their work." }, input: {
|
||||
nonce: "fixture", page: {}, fixtures: { company: { id: "company" }, agent: { id: "lead" } }, evidence: async (_name: string, body: { modelProviderUsage: string }) => {
|
||||
expect(body.modelProviderUsage).toContain("Not model/provider usage"); calls.events.push("fixture-evidence");
|
||||
}, api: {
|
||||
get: async (url: string) => url.endsWith("comments?order=asc") ? [{ id: "first-reply", authorAgentId: "lead", createdByRunId: "first", body: "Morgan is available." }]
|
||||
: imported ? { id: "morgan", status: "paused", lifecycleState: "paused", lifecycleHolds: ["manual", "budget"] }
|
||||
: { id: "morgan", status: "idle", lifecycleState: "ready" },
|
||||
post: async (url: string, payload: { amount?: number; provider?: string; costCents?: number; heartbeatRunId?: string }) => {
|
||||
if (url.endsWith("/pause")) calls.events.push("pause-after-first");
|
||||
else if (url.endsWith("budgets/policies")) { expect(payload.amount).toBe(100); calls.events.push("positive-hard-cap"); }
|
||||
else { expect(url).toContain("cost-events"); expect(payload.provider).toBe("synthetic-eval-import"); expect(payload.costCents).toBe(125);
|
||||
expect(payload.heartbeatRunId).toBeUndefined(); imported = true; calls.events.push("public-synthetic-import"); return { id: "receipt" }; }
|
||||
},
|
||||
},
|
||||
}, refreshIssue: async () => undefined, issue: () => ({ id: "source", conversationState: "waiting" }), allRuns: async () => [{ id: "first", status: "succeeded" }] } as unknown as ChatCatalogContext;
|
||||
const state = { worker: { id: "morgan" } };
|
||||
await runtimeResumeCases[0]!.act!(ctx, state);
|
||||
expect(calls.events).toEqual(["pause-after-first", "positive-hard-cap", "public-synthetic-import", "fixture-evidence", "human-send"]);
|
||||
});
|
||||
|
||||
it("reads real prior bytes from the actual isolated source binding, rejecting wrong identity and escaping files", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "controller-binding-")); roots.push(root);
|
||||
vi.stubEnv("PAPERCLIP_RUNNER_E2E_TEMP_ROOT", root);
|
||||
const cwd = path.join(root, "isolated-workspaces", "company", "source"); await mkdir(cwd, { recursive: true });
|
||||
await writeFile(path.join(cwd, "website.html"), "<h1>Real prior draft</h1>");
|
||||
let binding = "bound";
|
||||
const ctx = { input: { fixtures: { company: { id: "company" } }, api: { get: async (url: string) => url.endsWith("/workspace")
|
||||
? { workspace: { id: binding, companyId: "company", cwd } } : { companyId: "company", executionWorkspaceId: "bound" } } },
|
||||
refreshIssue: async () => undefined, issue: () => ({ id: "source" }) } as unknown as ChatCatalogContext;
|
||||
expect(await currentControllerFile(ctx, "website.html")).toBe("<h1>Real prior draft</h1>");
|
||||
binding = "other"; await expect(currentControllerFile(ctx, "website.html")).rejects.toThrow("actual same-company source binding"); binding = "bound";
|
||||
await writeFile(path.join(root, "foreign.html"), "wrong"); await symlink(path.join(root, "foreign.html"), path.join(cwd, "escape.html"));
|
||||
await expect(currentControllerFile(ctx, "escape.html")).rejects.toThrow("escapes");
|
||||
await expect(currentControllerFile(ctx, "../foreign.html")).rejects.toThrow("escapes");
|
||||
});
|
||||
@@ -25,3 +25,8 @@ it("rejects declaring an unqualified profile ready", () => { const o = valid();
|
||||
it("rejects substituting another runtime configuration", () => { const o = valid(); o.agents[0]!.adapterConfig.provider = "acpx"; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects a fabricated supported management path", () => { const o = valid(); o.state.operatorContractAvailable = false; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
it("rejects claiming required qualification without an actual validation guard", () => { const o = valid(); o.state.qualificationProbeStatus = 201; expect(gradeManagedProfile(o).passed).toBe(false); });
|
||||
|
||||
it("preserves the actual permission-only answer as incomplete without locked environment and retention guidance", () => {
|
||||
const o = valid(); o.reply = "Setup is blocked by a platform permission boundary: the managed-agent profile API returned 403 — Board access required. A Board actor must create or attest the Claude Managed Agent profile using the Anthropic agent, environment, and API-key secret IDs. Once completed, resume RUN-1 and I can continue.";
|
||||
expect(gradeManagedProfile(o).passed).toBe(false);
|
||||
});
|
||||
@@ -1,7 +1,6 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { readFile, realpath } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { resolveDefaultAgentWorkspaceDir } from "../../../server/src/home-paths.js";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { sendChatMessage } from "../chat-flow.js";
|
||||
import { readRegisteredArtifacts } from "../registered-artifact.js";
|
||||
@@ -15,12 +14,20 @@ async function settled(ctx: ChatCatalogContext, count: number) {
|
||||
accept: ({ issue, runs }) => issue?.conversationState === "waiting" && runs.length === count && runs.every(run => run.status === "succeeded"),
|
||||
reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "A continuity provider turn failed" : undefined });
|
||||
}
|
||||
async function currentFile(ctx: ChatCatalogContext, name: string) {
|
||||
export async function currentControllerFile(ctx: ChatCatalogContext, name: string) {
|
||||
const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root) throw new Error("Continuity inspection requires the isolated attempt root");
|
||||
const home = await realpath(resolveDefaultAgentWorkspaceDir(ctx.input.fixtures.agent.id));
|
||||
await ctx.refreshIssue(); const source = ctx.issue();
|
||||
if (!source) throw new Error("Continuity file inspection requires its actual source");
|
||||
const issue = await ctx.input.api.get<{ companyId: string; executionWorkspaceId: string | null }>(`/api/issues/${source.id}`);
|
||||
const { workspace } = await ctx.input.api.get<{ workspace: { id: string; companyId: string; cwd: string } }>(`/api/issues/${source.id}/workspace`);
|
||||
if (issue.companyId !== ctx.input.fixtures.company.id || workspace.companyId !== issue.companyId
|
||||
|| workspace.id !== issue.executionWorkspaceId) throw new Error("Continuity workspace is not the actual same-company source binding");
|
||||
const home = await realpath(workspace.cwd);
|
||||
const isolated = await realpath(root);
|
||||
if (!home.startsWith(`${isolated}${path.sep}`) || path.basename(name) !== name) throw new Error("Continuity file escapes its owned fixture directory");
|
||||
return readFile(path.join(home, name), "utf8");
|
||||
const filename = await realpath(path.join(home, name));
|
||||
if (path.dirname(filename) !== home) throw new Error("Continuity file escapes its actual source binding");
|
||||
return readFile(filename, "utf8");
|
||||
}
|
||||
export function gradeControllerRestart(value: unknown) {
|
||||
const c = checks(); const o = runtimeObservation(value);
|
||||
@@ -50,7 +57,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest
|
||||
`Start a small website draft in ${filename} in your current working directory, with a single header containing exactly ${JSON.stringify(headerMarker)}. Register that real file as a downloadable artifact work product titled ${filename}. This is a draft: do not add the contact section yet. Remember this instruction for when I ask to continue: keep the existing header and file, then add a contact section containing hello@acorn.example and register the final file as a new artifact from that continuation. Do this within our current conversation; no project, delegated task or external action is needed.`);
|
||||
const first = await settled(ctx, 1); const source = ctx.issue()!;
|
||||
const [initialArtifact] = await readRegisteredArtifacts(ctx.input.api, source.id, first.runs[0]!.id, [filename]);
|
||||
const fileBeforeRestart = await currentFile(ctx, filename);
|
||||
const fileBeforeRestart = await currentControllerFile(ctx, filename);
|
||||
if (fileBeforeRestart !== initialArtifact!.content) throw new Error("Initial downloadable artifact does not match the actual saved draft file");
|
||||
const priorHistory = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
|
||||
const url = ctx.input.page.url();
|
||||
@@ -60,7 +67,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest
|
||||
await ctx.refreshIssue();
|
||||
const afterHistory = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
|
||||
return { headerMarker, filename, initialArtifact, initialRunId: first.runs[0]!.id, fileBeforeRestart,
|
||||
fileAfterRestart: await currentFile(ctx, filename), restarted: true, sourceIssueBefore: source.id, sourceIssueAfter: ctx.issue()?.id,
|
||||
fileAfterRestart: await currentControllerFile(ctx, filename), restarted: true, sourceIssueBefore: source.id, sourceIssueAfter: ctx.issue()?.id,
|
||||
priorHistoryRetained: priorHistory.every(prior => afterHistory.some(row => row.id === prior.id && row.body === prior.body && row.createdByRunId === prior.createdByRunId)) };
|
||||
}, gradeControllerRestart, {
|
||||
setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 2, attemptTimeoutMs: 600_000,
|
||||
@@ -69,7 +76,7 @@ export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-rest
|
||||
const final = runs.find(run => run.id !== state.initialRunId && run.status === "succeeded");
|
||||
if (source && final) {
|
||||
const [artifact] = await readRegisteredArtifacts(ctx.input.api, source.id, final.id, [String(state.filename)]);
|
||||
state.finalArtifact = artifact; state.finalFile = await currentFile(ctx, String(state.filename));
|
||||
state.finalArtifact = artifact; state.finalFile = await currentControllerFile(ctx, String(state.filename));
|
||||
}
|
||||
return observeRuntime(ctx, state);
|
||||
},
|
||||
|
||||
@@ -4,7 +4,7 @@ function valid() {
|
||||
const worker = { id: "morgan", companyId: "company" };
|
||||
return { scenarioId: "agents.resume-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
|
||||
state: { worker, firstResumedWorker: { ...worker, lifecycleState: "ready", status: "idle", lifecycleHolds: [] },
|
||||
firstReply: "Morgan is resumed and available for future work.", secondBudgetHardStop: true, secondBudgetAmount: 0,
|
||||
firstReply: "Morgan is resumed and available for future work.", secondBudgetHardStop: true, secondBudgetAmount: 100, secondBudgetPolicyActive: true, syntheticReceiptId: "imported-receipt", syntheticReceiptOrigin: "public-imported-synthetic-fixture", secondBudgetIncidentOpen: true,
|
||||
secondReply: "The budget cap remains a hold; Morgan cannot pick up work before it is resolved." },
|
||||
agents: [{ ...worker, status: "paused", lifecycleHolds: ["budget"] }], tasks: [], links: ["/ACORN/agents/morgan"], reply: "",
|
||||
activity: [{ id: "resume", action: "agent.resumed", entityId: "morgan", agentId: "ceo" }],
|
||||
@@ -25,3 +25,11 @@ it("rejects silently clearing the remaining budget gate", () => {
|
||||
it("rejects pretending a new worker execution already started", () => {
|
||||
const o = valid(); o.runs.push({ id: "worker", agentId: "morgan", status: "succeeded" }); expect(gradeResumeAgent(o).passed).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects the unenforced zero-cap fixture and missing real incident", () => {
|
||||
const o = valid(); o.state.secondBudgetAmount = 0; expect(gradeResumeAgent(o).passed).toBe(false);
|
||||
o.state.secondBudgetAmount = 100; o.state.secondBudgetIncidentOpen = false; expect(gradeResumeAgent(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects a synthetic receipt misrepresented as provider spending", () => {
|
||||
const o = valid(); o.state.syntheticReceiptOrigin = "actual-provider-usage"; expect(gradeResumeAgent(o).passed).toBe(false);
|
||||
});
|
||||
@@ -17,7 +17,9 @@ export function gradeResumeAgent(value: unknown) {
|
||||
&& !/started (?:a |the )?(?:run|task|work)|already (?:working|running)/i.test(o.state.firstReply),
|
||||
"The unheld phase describes availability without inventing a new worker execution.");
|
||||
c.add("real-budget-gate-preserved", current?.lifecycleHolds?.includes("budget") === true && current.status === "paused"
|
||||
&& o.state.secondBudgetHardStop === true && o.state.secondBudgetAmount === 0,
|
||||
&& o.state.secondBudgetHardStop === true && o.state.secondBudgetAmount === 100
|
||||
&& o.state.secondBudgetPolicyActive === true && o.state.syntheticReceiptOrigin === "public-imported-synthetic-fixture"
|
||||
&& typeof o.state.syntheticReceiptId === "string" && !!o.state.syntheticReceiptId && o.state.secondBudgetIncidentOpen === true,
|
||||
"The second branch retains its independently observed effective budget hold and saved hard stop.");
|
||||
c.add("held-reply-bound-to-second-request", typeof o.state.secondReply === "string" && /budget|spend|cap|limit/i.test(o.state.secondReply)
|
||||
&& /hold|block|cannot|can't|remain|before|needs|not.*resum/i.test(o.state.secondReply)
|
||||
@@ -64,7 +66,17 @@ export const runtimeResumeCases = [runtimeDefinition("agents.resume-agent", asyn
|
||||
state.firstRunIds = first.runs.map(run => run.id); state.firstCommentIds = first.comments.map(row => row.id);
|
||||
await ctx.input.api.post(`/api/agents/${worker.id}/pause`, {});
|
||||
await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { scopeType: "agent", scopeId: worker.id,
|
||||
amount: 0, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "block" });
|
||||
amount: 100, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "allow" });
|
||||
// Zero disables enforcement. Import a truthful accounting fixture through
|
||||
// the supported public path; this is not native/model-provider spending.
|
||||
const receipt = await ctx.input.api.post<{ id: string }>(`/api/companies/${ctx.input.fixtures.company.id}/cost-events`, {
|
||||
agentId: worker.id, provider: "synthetic-eval-import", biller: "synthetic-eval-import", model: "imported-fixture-receipt",
|
||||
billingType: "metered_api", costStatus: "reported", costCents: 125, inputTokens: 0, outputTokens: 0,
|
||||
occurredAt: new Date().toISOString(), idempotencyKey: `resume-held-prerequisite:${ctx.input.nonce}` });
|
||||
state.syntheticReceiptId = receipt.id; state.syntheticReceiptOrigin = "public-imported-synthetic-fixture";
|
||||
await ctx.input.evidence("runtime-resume-imported-budget-context.json", { agentId: worker.id, receiptId: receipt.id,
|
||||
origin: state.syntheticReceiptOrigin, importedFixtureCostCents: 125, hardCapCents: 100,
|
||||
modelProviderUsage: "Not model/provider usage; explicit public synthetic accounting fixture" });
|
||||
await pollUntil({ label: "Independent effective budget hold before the second explicit request", deadlineAt: Date.now() + 45_000,
|
||||
load: () => ctx.input.api.get<RuntimeAgent>(`/api/agents/${worker.id}`), accept: row => row.lifecycleHolds?.includes("budget") === true });
|
||||
await sendChatMessage(ctx.input.page, ctx.scenario.query);
|
||||
@@ -78,9 +90,12 @@ export const runtimeResumeCases = [runtimeDefinition("agents.resume-agent", asyn
|
||||
&& !oldComments.includes(row.id) && phase.runs.some(run => run.id === row.createdByRunId && !(state.firstRunIds as string[]).includes(run.id))
|
||||
&& Date.parse(row.createdAt ?? "") >= Date.parse(secondHuman.createdAt ?? "")).map(row => row.body ?? "").join("\n") : "";
|
||||
}
|
||||
const budget = await ctx.input.api.get<{ policies: Array<{ scopeId: string; scopeType: string; amount: number; hardStopEnabled: boolean }> }>(`/api/companies/${ctx.input.fixtures.company.id}/budgets/overview`);
|
||||
const budget = await ctx.input.api.get<{ policies: Array<{ scopeId: string; scopeType: string; amount: number; hardStopEnabled: boolean; isActive: boolean }>; activeIncidents: Array<{ scopeId: string; scopeType: string; thresholdType: string; status: string }> }>(`/api/companies/${ctx.input.fixtures.company.id}/budgets/overview`);
|
||||
const policy = budget.policies.find(row => row.scopeId === (state.worker as RuntimeAgent).id && row.scopeType === "agent");
|
||||
state.secondBudgetAmount = policy?.amount; state.secondBudgetHardStop = policy?.hardStopEnabled;
|
||||
state.secondBudgetPolicyActive = policy?.isActive;
|
||||
state.secondBudgetIncidentOpen = budget.activeIncidents.some(row => row.scopeId === (state.worker as RuntimeAgent).id
|
||||
&& row.scopeType === "agent" && row.thresholdType === "hard" && row.status === "open");
|
||||
return observeRuntime(ctx, state);
|
||||
},
|
||||
})];
|
||||
@@ -4,7 +4,7 @@ function valid() {
|
||||
const task = { id: "website", title: "Website task", companyId: "company", projectId: null, status: "done", assigneeAgentId: "morgan" };
|
||||
const view = { workspace: { id: "bound-workspace", companyId: "company", projectId: null, name: "Website task workspace", cwd: "/owned/task-root" } };
|
||||
return { scenarioId: "workspaces.locate-files", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
|
||||
state: { task, worker: { id: "morgan" }, workspaceBefore: view, workspaceAfter: structuredClone(view), marker: "saved prerequisite",
|
||||
state: { companyPrefix: "ACORN", task, worker: { id: "morgan" }, workspaceBefore: view, workspaceAfter: structuredClone(view), marker: "saved prerequisite",
|
||||
fileBefore: "saved prerequisite", fileAfter: "saved prerequisite", fixtureRunId: "setup" },
|
||||
agents: [], tasks: [task], links: ["/ACORN/execution-workspaces/bound-workspace"], reply: "The actual files are in /owned/task-root.",
|
||||
activity: [], runs: [{ id: "setup", agentId: "morgan", status: "succeeded" }], humanQueries: [] };
|
||||
@@ -25,3 +25,16 @@ it("accepts the actual directory and supported view directions without a mandato
|
||||
const o = valid(); o.links = []; o.reply = "The bound workspace uses /owned/task-root; open the Workspace view to inspect its files.";
|
||||
expect(gradeLocateFiles(o).passed).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects a wrong saved workspace href even before its DOM anchor renders", () => {
|
||||
const o = valid(); o.links = []; o.reply = "Workspace files are in /owned/task-root. [Open workspace](/ACORN/execution-workspaces/another).";
|
||||
expect(gradeLocateFiles(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects correct resource IDs routed through another company or an unsupported tab", () => {
|
||||
const o = valid(); o.links = ["/BIRCH/execution-workspaces/bound-workspace"]; expect(gradeLocateFiles(o).passed).toBe(false);
|
||||
o.links = ["/ACORN/execution-workspaces/bound-workspace/not-a-tab"]; expect(gradeLocateFiles(o).passed).toBe(false);
|
||||
});
|
||||
it("accepts the supported same-resource issues tab without demanding a different heading", () => {
|
||||
const o = valid(); o.links = ["/ACORN/execution-workspaces/bound-workspace/issues"];
|
||||
expect(gradeLocateFiles(o).passed).toBe(true);
|
||||
});
|
||||
@@ -44,18 +44,32 @@ export function gradeLocateFiles(value: unknown) {
|
||||
c.add("actual-saved-files-retained", o.state.fileAfter === o.state.fileBefore && typeof o.state.fileBefore === "string"
|
||||
&& o.state.fileBefore.trim() === o.state.marker && o.runs.some(run => run.id === o.state.fixtureRunId && run.agentId === (o.state.worker as Entity).id
|
||||
&& run.status === "succeeded"), "A genuine worker created the prerequisite file and it remains unchanged.");
|
||||
const resultLinks = workspaceResultLinks(o.reply, o.links);
|
||||
c.add("supported-workspace-view-guidance", (/workspace|files? (?:view|tab|panel)/i.test(o.reply)
|
||||
|| !!after && o.links.some(href => workspaceLink(o.origin, href, after.workspace.id)))
|
||||
&& !!after && o.links.filter(href => { try { return new URL(href, o.origin).pathname.includes("/execution-workspaces/"); } catch { return false; } })
|
||||
.every(href => workspaceLink(o.origin, href, after.workspace.id)),
|
||||
|| !!after && resultLinks.some(href => workspaceLink(o.origin, href, after.workspace.id, String(o.state.companyPrefix))))
|
||||
&& !!after && resultLinks.filter(href => { try { return new URL(href, o.origin).pathname.includes("/execution-workspaces/"); } catch { return false; } })
|
||||
.every(href => workspaceLink(o.origin, href, after.workspace.id, String(o.state.companyPrefix))),
|
||||
"The response points to the supported workspace view; any workspace link it does give must identify the actual bound workspace.");
|
||||
return c.result();
|
||||
}
|
||||
function workspaceLink(origin: string, href: string, id: string) {
|
||||
try { const url = new URL(href, origin); const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf("execution-workspaces");
|
||||
return url.origin === new URL(origin).origin && at >= 0 && parts[at + 1] === id; } catch { return false; }
|
||||
export function workspaceResultLinks(reply: string, rendered: string[]) {
|
||||
return [...new Set([...rendered, ...Array.from(reply.matchAll(/\[[^\]]*\]\(([^)\s]+)\)/g), match => match[1]!)])];
|
||||
}
|
||||
export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-files", ctx => savedTaskWorkspace(ctx), gradeLocateFiles, {
|
||||
export function workspaceLink(origin: string, href: string, id: string, prefix: string) {
|
||||
try { const url = new URL(href, origin); const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf("execution-workspaces");
|
||||
return url.origin === new URL(origin).origin && at === 2 && parts[1] === prefix && parts[at + 1] === id
|
||||
&& (parts.length === 4 || parts.length === 5 && ["services", "configuration", "runtime-logs", "issues", "routines"].includes(parts[4]!)); } catch { return false; }
|
||||
}
|
||||
export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-files", async ctx => {
|
||||
// The actual detail route is gated, independently of workspace realization.
|
||||
// Enable its supported UI fixture before the query, never after a bad link.
|
||||
if (!ctx.input.fixtures.company.issuePrefix) throw new Error("Workspace fixture has no public company prefix");
|
||||
await ctx.input.api.patch("/api/instance/settings/experimental", { enableIsolatedWorkspaces: true });
|
||||
const settings = await ctx.input.api.get<{ enableIsolatedWorkspaces: boolean }>("/api/instance/settings/experimental");
|
||||
if (!settings.enableIsolatedWorkspaces) throw new Error("The supported workspace view is not enabled");
|
||||
return { ...await savedTaskWorkspace(ctx), companyPrefix: ctx.input.fixtures.company.issuePrefix };
|
||||
}, gradeLocateFiles, {
|
||||
sourceFiles: ["chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/workspaces.yaml", "../../ui/src/components/IsolatedWorkspacesRouteGate.tsx", "../../ui/src/pages/ExecutionWorkspaceDetail.tsx"],
|
||||
expectedRunCount: 2, attemptTimeoutMs: 540_000,
|
||||
async observe(ctx, state) {
|
||||
const id = (state.task as Entity).id;
|
||||
@@ -67,10 +81,17 @@ export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-file
|
||||
async verifyResult(ctx, state, value) {
|
||||
const o = runtimeObservation(value); if (!o) throw new Error("Workspace result missing");
|
||||
const view = state.workspaceAfter as WorkspaceView;
|
||||
const href = o.links.find(link => workspaceLink(o.origin, link, view.workspace.id));
|
||||
const prefix = state.companyPrefix;
|
||||
if (typeof prefix !== "string" || !prefix) throw new Error("Workspace view lost its actual company prefix");
|
||||
const links = workspaceResultLinks(o.reply, o.links);
|
||||
if (links.some(link => { try { return new URL(link, o.origin).pathname.includes("/execution-workspaces/")
|
||||
&& !workspaceLink(o.origin, link, view.workspace.id, prefix); } catch { return false; } }))
|
||||
throw new Error("The actual reply supplied an incorrect workspace link");
|
||||
const href = links.find(link => workspaceLink(o.origin, link, view.workspace.id, prefix));
|
||||
const source = ctx.input.page.url();
|
||||
if (href) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();
|
||||
else await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/execution-workspaces/${view.workspace.id}`, { waitUntil: "domcontentloaded" });
|
||||
else await ctx.input.page.goto(`/${prefix}/execution-workspaces/${view.workspace.id}`, { waitUntil: "domcontentloaded" });
|
||||
await expect(ctx.input.page).toHaveURL(new RegExp(`/${prefix}/execution-workspaces/${view.workspace.id}(?:[/?#]|$)`));
|
||||
await expect(ctx.input.page.getByRole("heading", { name: view.workspace.name, exact: true }).first()).toBeVisible();
|
||||
await ctx.input.evidence("runtime-task-workspace-opened.json", { taskId: (state.task as Entity).id, workspaceId: view.workspace.id, url: ctx.input.page.url() });
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
|
||||
Reference in new issue
Block a user