mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 03:51:54 +01:00
test(e2e): verify runtime surfaces and actual task workspace binding
This commit is contained in:
1 parent
f1762c5609
commit
875371a262
9 files changed
+313
-4
No files matched your search
@@ -0,0 +1,22 @@
|
||||
import { it, expect } from "vitest";
|
||||
import { gradeBuiltInConfiguration } from "./runtime-built-in.js";
|
||||
function valid() {
|
||||
const builtIn = { definition: { key: "summarizer" }, agentId: "summarizer" };
|
||||
const task = { id: "history", title: "Summary history", companyId: "company", assigneeAgentId: "summarizer", status: "backlog" };
|
||||
return { scenarioId: "agents.configure-required-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
|
||||
state: { builtInBefore: builtIn, builtInAfter: { ...builtIn }, originalInstruction: { content: "Summarize work", revision: { id: "old" } }, agentCountBefore: 1, historicalTask: task },
|
||||
instruction: { content: "Summarize work; keep updates brief, under 40 words.", revision: { id: "new", actorAgentId: "ceo", sourceRunId: "run" } },
|
||||
instructionHistory: { revisions: [{ id: "old" }, { id: "new" }] }, agents: [{ id: "summarizer", name: "Summarizer", companyId: "company" }], tasks: [task],
|
||||
links: ["/ACORN/agents/summarizer"], reply: "The summarizer now writes shorter updates.", activity: [], runs: [{ id: "run" }], humanQueries: [] };
|
||||
}
|
||||
it("accepts a durable protected-agent configuration revision", () => expect(gradeBuiltInConfiguration(valid()).passed).toBe(true));
|
||||
it("rejects missing built-in evidence", () => expect(gradeBuiltInConfiguration(undefined).passed).toBe(false));
|
||||
it("rejects replacing the protected identity with an ordinary hire", () => {
|
||||
const o = valid(); o.state.builtInAfter.agentId = "replacement"; expect(gradeBuiltInConfiguration(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects changing only prose without a saved instruction revision", () => {
|
||||
const o = valid(); o.instruction.content = "Summarize work"; expect(gradeBuiltInConfiguration(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects erasing the old attributed task", () => {
|
||||
const o = valid(); o.tasks = []; expect(gradeBuiltInConfiguration(o).passed).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { checks, sameTask, type Entity } from "./internal-support.js";
|
||||
import { runtimeDefinition, runtimeObservation, observeRuntime, agentLink, type RuntimeAgent, type InstructionFile } from "./runtime-support.js";
|
||||
interface BuiltInState { definition: { key: string }; agentId: string | null; agent: RuntimeAgent | null; status: string }
|
||||
export function gradeBuiltInConfiguration(value: unknown) {
|
||||
const c = checks(); const o = runtimeObservation(value);
|
||||
c.add("actual-built-in-observation", !!o, "The protected built-in registry, durable configuration revision and historical task are independently read."); if (!o) return c.result();
|
||||
const original = o.state.builtInBefore as BuiltInState | undefined; const after = o.state.builtInAfter as BuiltInState | undefined;
|
||||
const beforeInstruction = o.state.originalInstruction as InstructionFile | undefined; const current = o.instruction;
|
||||
const worker = o.agents.find(row => row.id === original?.agentId);
|
||||
c.add("protected-identity-preserved", !!original && !!after && after.definition.key === "summarizer" && after.agentId === original.agentId
|
||||
&& !!worker && worker.companyId === o.companyId && o.agents.length === o.state.agentCountBefore,
|
||||
"The required registry still points to the same summarizer, with no replacement ordinary hire.");
|
||||
c.add("shorter-updates-durably-configured", !!current && !!beforeInstruction && current.content !== beforeInstruction.content
|
||||
&& /shorter|brief|concise|(?:three|3|two|2) sentences|under \d+ words/i.test(current.content)
|
||||
&& /summar|updates?/i.test(current.content) && current.revision.actorAgentId === o.leadId && o.runs.some(run => run.id === current.revision.sourceRunId),
|
||||
"An actual agent-attributed revision applies shorter summary guidance while retaining the summarizer purpose.");
|
||||
c.add("history-retained", !!beforeInstruction && o.instructionHistory?.revisions.some(row => row.id === beforeInstruction.revision.id) === true
|
||||
&& o.tasks.length === 1 && sameTask(o.state.historicalTask as Entity, o.tasks[0]), "Instruction history and attributed historical work remain intact.");
|
||||
c.add("inspectable-effective-change", !!worker && !!agentLink(o, worker) && /short|brief|concis/i.test(o.reply),
|
||||
"The response links the protected agent and describes the saved change.");
|
||||
return c.result();
|
||||
}
|
||||
export const runtimeBuiltInCases = [runtimeDefinition("agents.configure-required-agent", async ctx => {
|
||||
await ctx.input.api.patch("/api/instance/settings/experimental", { enableBuiltInAgents: true });
|
||||
const root = `/api/companies/${ctx.input.fixtures.company.id}`;
|
||||
const builtInBefore = await ctx.input.api.post<BuiltInState>(`${root}/built-in-agents/summarizer/reconcile`, {});
|
||||
if (!builtInBefore.agentId || !builtInBefore.agent) throw new Error("Public reconcile did not create the protected summarizer identity");
|
||||
const worker = builtInBefore.agent;
|
||||
const originalInstruction = await ctx.input.api.get<InstructionFile>(`/api/agents/${worker.id}/instructions-bundle/file?path=AGENTS.md`);
|
||||
const historicalTask = await ctx.input.api.post<Entity>(`${root}/issues`, { title: "Historical summary work", status: "backlog", assigneeAgentId: worker.id });
|
||||
const agentCountBefore = (await ctx.input.api.get<RuntimeAgent[]>(`${root}/agents`)).length;
|
||||
return { worker, builtInBefore, instructionAgentId: worker.id, originalInstruction, historicalTask, agentCountBefore };
|
||||
}, gradeBuiltInConfiguration, {
|
||||
sourceFiles: ["chat-catalog-domains/runtime-built-in.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"],
|
||||
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "protected-summarizer-identity", "historical-task-unrun"] },
|
||||
async verifySetup(ctx, state) {
|
||||
const c = checks(); const lead = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${ctx.input.fixtures.agent.id}`);
|
||||
const builtIn = await ctx.input.api.get<BuiltInState>(`/api/companies/${ctx.input.fixtures.company.id}/built-in-agents/summarizer/status`);
|
||||
const history = await ctx.input.api.get<Entity>(`/api/issues/${(state.historicalTask as Entity).id}`);
|
||||
c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The actual chat agent is runnable.");
|
||||
c.add("protected-summarizer-identity", builtIn.definition.key === "summarizer" && builtIn.agentId === (state.worker as RuntimeAgent).id,
|
||||
"The genuine protected identity is preserved; readiness is never manufactured for this configuration-only request.");
|
||||
c.add("historical-task-unrun", history.status === "backlog" && (await ctx.allRuns()).length === 0, "The retained history fixture has not started provider work.");
|
||||
return c.result();
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
state.builtInAfter = await ctx.input.api.get<BuiltInState>(`/api/companies/${ctx.input.fixtures.company.id}/built-in-agents/summarizer/status`);
|
||||
// The built-in bundle has no arbitrary side-file prerequisite.
|
||||
const result = await observeRuntime(ctx, { ...state, instructionAgentId: undefined });
|
||||
const root = `/api/agents/${(state.worker as RuntimeAgent).id}/instructions-bundle`;
|
||||
result.instruction = await ctx.input.api.get<InstructionFile>(`${root}/file?path=AGENTS.md`);
|
||||
result.instructionHistory = await ctx.input.api.get(`${root}/history?path=AGENTS.md`);
|
||||
return result;
|
||||
},
|
||||
})];
|
||||
@@ -66,7 +66,7 @@
|
||||
},
|
||||
{
|
||||
"scenarioId": "agents.configure-required-agent",
|
||||
"readinessClass": "local",
|
||||
"readinessClass": "implemented",
|
||||
"needs": "Public built-in summarizer provision; change only supported config with protected identity/history unchanged."
|
||||
},
|
||||
{
|
||||
@@ -106,7 +106,7 @@
|
||||
},
|
||||
{
|
||||
"scenarioId": "agent-chat.conference-room",
|
||||
"readinessClass": "local",
|
||||
"readinessClass": "implemented",
|
||||
"needs": "Enable supported local-trusted board surface; exact company route/browser scope and differing protocol honestly described."
|
||||
},
|
||||
{
|
||||
@@ -116,7 +116,7 @@
|
||||
},
|
||||
{
|
||||
"scenarioId": "runs-adapters.list-model-options",
|
||||
"readinessClass": "local",
|
||||
"readinessClass": "implemented",
|
||||
"needs": "Real admitted model discovery/account inventory; independently compare restrictions, availability and binding, secrets never exposed."
|
||||
},
|
||||
{
|
||||
@@ -271,7 +271,7 @@
|
||||
},
|
||||
{
|
||||
"scenarioId": "workspaces.locate-files",
|
||||
"readinessClass": "local",
|
||||
"readinessClass": "implemented",
|
||||
"needs": "Actual projectless task execution binding; independent read active binding and distinguish pending choice from agent defaults."
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { it, expect } from "vitest";
|
||||
import { gradeConference } from "./runtime-conference.js";
|
||||
function valid() {
|
||||
return { scenarioId: "agent-chat.conference-room", companyId: "company", leadId: "ceo", reply: "Open the Conference Room for the separate board-level discussion.",
|
||||
state: { deploymentMode: "local_trusted", conferenceEnabled: true, agentChatEnabled: true, conferenceVisible: true,
|
||||
openedCompanyPrefix: "ACORN", companyPrefix: "ACORN", noBrokenRoute: true, runsBeforeOpen: 1, runsAfterOpen: 1,
|
||||
standingIssueBefore: "chat", standingIssueAfter: "chat" }, agents: [], tasks: [], links: [], activity: [], runs: [], humanQueries: [] };
|
||||
}
|
||||
it("accepts the real gated board surface in company context", () => expect(gradeConference(valid()).passed).toBe(true));
|
||||
it("rejects missing conference evidence", () => expect(gradeConference(undefined).passed).toBe(false));
|
||||
it("rejects wrong deployment qualification", () => {
|
||||
const o = valid(); o.state.deploymentMode = "authenticated"; expect(gradeConference(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects navigation to another company", () => {
|
||||
const o = valid(); o.state.openedCompanyPrefix = "BIRCH"; expect(gradeConference(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects invented standing-chat history parity", () => {
|
||||
const o = valid(); o.reply += " It uses the same Agent Chat history."; expect(gradeConference(o).passed).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { checks } from "./internal-support.js";
|
||||
import { waitForReply } from "./identity-support.js";
|
||||
import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js";
|
||||
export function gradeConference(value: unknown) {
|
||||
const c = checks(); const o = runtimeObservation(value);
|
||||
c.add("independent-conference-evidence", !!o, "Actual deployment, gate, company route and rendered board surface are required."); if (!o) return c.result();
|
||||
c.add("supported-local-enabled-context", o.state.deploymentMode === "local_trusted" && o.state.conferenceEnabled === true
|
||||
&& o.state.agentChatEnabled === true, "Both actual gates are enabled in the supported local-trusted deployment.");
|
||||
c.add("real-company-conference-surface", o.state.conferenceVisible === true && o.state.openedCompanyPrefix === o.state.companyPrefix
|
||||
&& o.state.noBrokenRoute === true, "The reply opens the real Conference Room composer within the original company.");
|
||||
c.add("separate-board-scope-described", /conference|board/i.test(o.reply) && /separate|different|board.level|board discussion|board.assistant/i.test(o.reply)
|
||||
&& !/(?:same|shared|identical) (?:standing.chat|agent.chat) (?:history|execution)|(?:full|identical) (?:execution|agent.chat) parity/i.test(o.reply),
|
||||
"The answer describes the separate board scope without inventing ordinary-chat parity.");
|
||||
c.add("navigation-has-no-provider-action", o.state.runsBeforeOpen === 1 && o.state.runsAfterOpen === 1
|
||||
&& o.state.standingIssueBefore === o.state.standingIssueAfter, "Opening the board surface preserves the ordinary source and starts no hidden execution.");
|
||||
return c.result();
|
||||
}
|
||||
export const runtimeConferenceCases = [runtimeDefinition("agent-chat.conference-room", async ctx => {
|
||||
const health = await ctx.input.api.get<{ deploymentMode: string }>("/api/health");
|
||||
if (health.deploymentMode !== "local_trusted") throw new Error("Conference Room scenario requires a genuinely local-trusted target");
|
||||
await ctx.input.api.patch("/api/instance/settings/experimental", { enableConferenceRoomChat: true });
|
||||
const settings = await ctx.input.api.get<{ enableConferenceRoomChat: boolean; enableAgentChat: boolean }>("/api/instance/settings/experimental");
|
||||
return { deploymentMode: health.deploymentMode, conferenceEnabled: settings.enableConferenceRoomChat,
|
||||
agentChatEnabled: settings.enableAgentChat, companyPrefix: ctx.input.fixtures.company.issuePrefix };
|
||||
}, gradeConference, {
|
||||
sourceFiles: ["chat-catalog-domains/runtime-conference.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml"],
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx); state.runsBeforeOpen = (await ctx.allRuns()).length; state.standingIssueBefore = ctx.issue()?.id;
|
||||
const o = await observeRuntime(ctx, state);
|
||||
const href = o.links.find(link => { const url = new URL(link, ctx.input.api.baseURL);
|
||||
return url.origin === ctx.input.api.baseURL && url.pathname === `/${state.companyPrefix}/board-chat`; });
|
||||
if (!href) throw new Error("Conference response lacks the actual company board route");
|
||||
const source = ctx.input.page.url();
|
||||
await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();
|
||||
await expect(ctx.input.page.getByPlaceholder("Ask anything about your organization...")).toBeVisible();
|
||||
await expect(ctx.input.page.getByText("Conference Room", { exact: true }).first()).toBeVisible();
|
||||
state.conferenceVisible = true;
|
||||
const parts = new URL(ctx.input.page.url()).pathname.split("/"); state.openedCompanyPrefix = parts[1];
|
||||
state.noBrokenRoute = parts.at(-1) === "board-chat";
|
||||
await ctx.input.evidence("runtime-conference-opened.json", { companyId: o.companyId, url: ctx.input.page.url(), composerVisible: true });
|
||||
state.runsAfterOpen = (await ctx.allRuns()).length;
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
|
||||
await ctx.refreshIssue(); state.standingIssueAfter = ctx.issue()?.id;
|
||||
},
|
||||
async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
|
||||
})];
|
||||
@@ -0,0 +1,20 @@
|
||||
import { it, expect } from "vitest";
|
||||
import { gradeModelOptions } from "./runtime-runs-adapters.js";
|
||||
function valid() {
|
||||
const worker = { id: "morgan", companyId: "company", adapterType: "paperclip_runner", adapterConfig: { model: "gpt-5.6-sol" }, permissions: {} };
|
||||
return { scenarioId: "runs-adapters.list-model-options", companyId: "company", leadId: "ceo", state: { worker,
|
||||
admittedModels: [{ id: "gpt-5.6-sol", label: "GPT 5.6 Sol" }], discoveryStatus: 200, agentRuntimeInventoryStatus: 403, secretScanComplete: true, secretLeaked: false },
|
||||
agents: [structuredClone(worker)], tasks: [], links: [], activity: [], runs: [], humanQueries: [],
|
||||
reply: "The current catalog includes gpt-5.6-sol. Available models still depend on the selected configured account binding and environment qualification." };
|
||||
}
|
||||
it("accepts an actual admitted option with account qualification distinguished", () => expect(gradeModelOptions(valid()).passed).toBe(true));
|
||||
it("rejects missing discovery evidence", () => expect(gradeModelOptions(undefined).passed).toBe(false));
|
||||
it("rejects an invented vendor model presented alongside a real option", () => {
|
||||
const o = valid(); o.reply += " You can also use gpt-99-ultra."; expect(gradeModelOptions(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects silently changing Morgan's selected model", () => {
|
||||
const o = valid(); o.agents[0]!.adapterConfig.model = "other"; expect(gradeModelOptions(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects leaked private account credentials", () => {
|
||||
const o = valid(); o.state.secretLeaked = true; expect(gradeModelOptions(o).passed).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { checks } from "./internal-support.js";
|
||||
import { readAgentResource } from "./identity-support.js";
|
||||
import { runtimeDefinition, runtimeWorker, runtimeObservation, unchangedAgent, type RuntimeAgent } from "./runtime-support.js";
|
||||
interface Model { id: string; label: string }
|
||||
export function gradeModelOptions(value: unknown) {
|
||||
const c = checks(); const o = runtimeObservation(value);
|
||||
c.add("authorized-model-discovery-evidence", !!o, "Actual agent-authorized discovery and saved account/runtime scope are independently read."); if (!o) return c.result();
|
||||
const models = o.state.admittedModels as Model[] | undefined; const worker = o.state.worker as RuntimeAgent | undefined;
|
||||
const current = o.agents.find(row => row.id === worker?.id);
|
||||
c.add("at-least-one-current-discovered-option", Array.isArray(models) && models.length > 0 && o.state.discoveryStatus === 200
|
||||
&& models.some(model => o.reply.includes(model.id)), "At least one returned coding option comes from the current admitted model catalog.");
|
||||
const mentionedIds = [...o.reply.matchAll(/\b(?:gpt-\d[\w.-]*|o[134](?:-[\w.-]+)?|claude-(?:sonnet|opus|haiku)-[\w.-]+)/gi)].map(match => match[0].replace(/[.,;:!?]+$/, ""));
|
||||
c.add("no-unqualified-model-inventory", !!models && mentionedIds.every(id => models.some(model => model.id.toLowerCase() === id.toLowerCase()))
|
||||
&& !/(?:all|every) (?:public |vendor )?models? (?:are |is )?(?:installed|ready|available)|all models.*(?:ready|work)/i.test(o.reply),
|
||||
"The answer does not add unsupported vendor models or equate a public catalog with universal runtime readiness.");
|
||||
c.add("availability-not-account-qualification", /account|binding|connection|connected|configured/i.test(o.reply)
|
||||
&& /available|option|catalog|model/i.test(o.reply) && /valid|qualif|restrict|depends|selected|configured|bound/i.test(o.reply),
|
||||
"The response distinguishes model choices from selected account and environment qualification.");
|
||||
c.add("restricted-runtime-read-denied", o.state.agentRuntimeInventoryStatus === 403,
|
||||
"The actual agent bearer cannot inspect board-only runtime inventory.");
|
||||
c.add("configuration-and-permissions-preserved", !!worker && !!current && unchangedAgent(worker, current) && o.tasks.length === 0,
|
||||
"Discovery does not mutate Morgan's model, account, authority or task assignment.");
|
||||
c.add("private-credential-scan-complete", o.state.secretScanComplete === true && o.state.secretLeaked === false,
|
||||
"The private redaction sink is scanned without recording its values.");
|
||||
return c.result();
|
||||
}
|
||||
export const runtimeRunsAdaptersCases = [runtimeDefinition("runs-adapters.list-model-options", async ctx => {
|
||||
const worker = await runtimeWorker(ctx);
|
||||
if (worker.adapterConfig?.provider !== "codex" && worker.adapterType !== "codex_local")
|
||||
throw new Error("Current runtime model-discovery fixture requires a genuinely qualified Codex account");
|
||||
const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/adapters/codex_local/models`;
|
||||
const discovered = await readAgentResource(ctx, endpoint);
|
||||
const restricted = await readAgentResource(ctx, `/api/agents/${worker.id}/runtime-state`);
|
||||
if (discovered.status !== 200 || !Array.isArray(discovered.data)) throw new Error("Scenario requires admitted real coding model discovery");
|
||||
return { worker, admittedModels: discovered.data, discoveryStatus: discovered.status, agentRuntimeInventoryStatus: restricted.status };
|
||||
}, gradeModelOptions, {
|
||||
async observe(ctx, state) {
|
||||
const { observeRuntime } = await import("./runtime-support.js");
|
||||
const o = await observeRuntime(ctx, state);
|
||||
// Credentials remain only in the private attempt closure. Never put them in
|
||||
// state, messages, diagnostic artifacts or calibration fixtures.
|
||||
state.secretScanComplete = Array.isArray(ctx.input.privateSecrets);
|
||||
state.secretLeaked = ctx.input.privateSecrets?.some(secret => secret.length > 8 && o.reply.includes(secret)) ?? true;
|
||||
return o;
|
||||
},
|
||||
async verifyResult(ctx, state, value) {
|
||||
const o = runtimeObservation(value); if (!o) throw new Error("Model options response is missing");
|
||||
const match = (state.admittedModels as Model[]).find(model => o.reply.includes(model.id));
|
||||
if (!match) throw new Error("No actual discovered option was rendered");
|
||||
await expect(ctx.input.page.getByTestId("task-chat-agent-bubble").filter({ hasText: match.id }).last()).toBeVisible();
|
||||
},
|
||||
})];
|
||||
@@ -0,0 +1,22 @@
|
||||
import { it, expect } from "vitest";
|
||||
import { gradeLocateFiles } from "./runtime-workspaces.js";
|
||||
function valid() {
|
||||
const task = { id: "website", title: "Website task", companyId: "company", projectId: null, status: "done", assigneeAgentId: "morgan" };
|
||||
const view = { workspace: { id: "bound-workspace", companyId: "company", projectId: null, name: "Website task workspace", cwd: "/owned/task-root" } };
|
||||
return { scenarioId: "workspaces.locate-files", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo",
|
||||
state: { task, worker: { id: "morgan" }, workspaceBefore: view, workspaceAfter: structuredClone(view), marker: "saved prerequisite",
|
||||
fileBefore: "saved prerequisite", fileAfter: "saved prerequisite", fixtureRunId: "setup" },
|
||||
agents: [], tasks: [task], links: ["/ACORN/execution-workspaces/bound-workspace"], reply: "The actual files are in /owned/task-root.",
|
||||
activity: [], runs: [{ id: "setup", agentId: "morgan", status: "succeeded" }], humanQueries: [] };
|
||||
}
|
||||
it("accepts the actual bound projectless directory and files", () => expect(gradeLocateFiles(valid()).passed).toBe(true));
|
||||
it("rejects missing task workspace evidence", () => expect(gradeLocateFiles(undefined).passed).toBe(false));
|
||||
it("rejects a plausible agent default directory", () => {
|
||||
const o = valid(); o.reply = "The files are probably in Morgan's /default/work directory."; expect(gradeLocateFiles(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects linking another workspace", () => {
|
||||
const o = valid(); o.links = ["/ACORN/execution-workspaces/other-workspace"]; expect(gradeLocateFiles(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects fabricated saved-file setup without its actual worker run", () => {
|
||||
const o = valid(); o.runs = []; expect(gradeLocateFiles(o).passed).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { checks, sameTask, type Entity } from "./internal-support.js";
|
||||
import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
interface WorkspaceView { workspace: { id: string; companyId: string; projectId?: string | null; name: string; cwd: string }; repositories: unknown[] }
|
||||
async function savedTaskWorkspace(ctx: ChatCatalogContext) {
|
||||
const { execution, fixtures, workspacePath, nonce, api } = ctx.input;
|
||||
const base = execution.profile.buildAgent({ environmentId: fixtures.environment.id, environmentFixtureId: execution.environment.id,
|
||||
workspacePath, secretRefs: fixtures.secretRefs, executionId: nonce });
|
||||
const { cwd: _cwd, ...taskConfig } = base.adapterConfig as Record<string, unknown>;
|
||||
const worker = await runtimeWorker(ctx, "Morgan", { adapterConfig: taskConfig });
|
||||
const company = `/api/companies/${fixtures.company.id}`;
|
||||
for (const scope of [{ scopeType: "company", scopeId: fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
|
||||
await api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
|
||||
const marker = `Saved website prerequisite ${nonce}`;
|
||||
const task = await api.post<Entity>(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id,
|
||||
description: `Create a local file website-fixture.txt containing exactly ${JSON.stringify(marker)} in your admitted task workspace. Save a brief task document as evidence, then complete this fixture task normally. Do not create a project, repository, another task, or remote side effect.` });
|
||||
await pollUntil({ label: "Real projectless website fixture completion", deadlineAt: Date.now() + 180_000,
|
||||
load: async () => ({ task: await api.get<Entity>(`/api/issues/${task.id}`), runs: await ctx.allRuns() }),
|
||||
accept: value => value.task.status === "done" && value.runs.length === 1 && value.runs.every(run => run.status === "succeeded"),
|
||||
reject: value => value.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Projectless prerequisite provider run failed" : undefined });
|
||||
const view = await api.get<WorkspaceView>(`/api/issues/${task.id}/workspace`);
|
||||
const file = await api.get<{ resource?: { workspaceKind?: string }; content?: { data?: string } }>(`/api/issues/${task.id}/file-resources/content?workspace=execution&path=website-fixture.txt`);
|
||||
if (view.workspace.projectId || file.resource?.workspaceKind !== "execution_workspace" || file.content?.data?.trim() !== marker)
|
||||
throw new Error("Projectless fixture has no actual task-bound saved file");
|
||||
return { worker, task: await api.get<Entity>(`/api/issues/${task.id}`), workspaceBefore: view, marker,
|
||||
fileBefore: file.content.data, fixtureRunId: (await ctx.allRuns())[0]!.id };
|
||||
}
|
||||
export function gradeLocateFiles(value: unknown) {
|
||||
const c = checks(); const o = runtimeObservation(value);
|
||||
c.add("actual-task-workspace-evidence", !!o, "The actual admitted binding, file bytes and setup run identity are independently required."); if (!o) return c.result();
|
||||
const before = o.state.workspaceBefore as WorkspaceView | undefined; const after = o.state.workspaceAfter as WorkspaceView | undefined;
|
||||
const task = o.state.task as Entity | undefined;
|
||||
c.add("same-projectless-task-binding", !!before?.workspace?.id && !!after && before.workspace.id === after.workspace.id
|
||||
&& before.workspace.cwd === after.workspace.cwd && after.workspace.companyId === o.companyId && after.workspace.projectId == null
|
||||
&& !!task && sameTask(task, o.tasks.find(row => row.id === task.id)), "The correct projectless binding persists without reassigning or changing the task.");
|
||||
c.add("concrete-authorized-location", !!after && (o.reply.includes(after.workspace.cwd) || o.reply.includes(after.workspace.id)),
|
||||
"The answer identifies the actual bound workspace rather than inferring an agent default or project directory.");
|
||||
c.add("actual-saved-files-retained", o.state.fileAfter === o.state.fileBefore && typeof o.state.fileBefore === "string"
|
||||
&& o.state.fileBefore.trim() === o.state.marker && o.runs.some(run => run.id === o.state.fixtureRunId && run.agentId === (o.state.worker as Entity).id
|
||||
&& run.status === "succeeded"), "A genuine worker created the prerequisite file and it remains unchanged.");
|
||||
c.add("actual-workspace-result-link", !!after && o.links.some(href => workspaceLink(o.origin, href, after.workspace.id)),
|
||||
"The response renders the real workspace view link.");
|
||||
return c.result();
|
||||
}
|
||||
function workspaceLink(origin: string, href: string, id: string) {
|
||||
try { const url = new URL(href, origin); const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf("execution-workspaces");
|
||||
return url.origin === new URL(origin).origin && at >= 0 && parts[at + 1] === id; } catch { return false; }
|
||||
}
|
||||
export const runtimeWorkspacesCases = [runtimeDefinition("workspaces.locate-files", savedTaskWorkspace, gradeLocateFiles, {
|
||||
expectedRunCount: 2, attemptTimeoutMs: 540_000,
|
||||
async observe(ctx, state) {
|
||||
const id = (state.task as Entity).id;
|
||||
state.workspaceAfter = await ctx.input.api.get<WorkspaceView>(`/api/issues/${id}/workspace`);
|
||||
const file = await ctx.input.api.get<{ content: { data: string } }>(`/api/issues/${id}/file-resources/content?workspace=execution&path=website-fixture.txt`);
|
||||
state.fileAfter = file.content.data;
|
||||
return observeRuntime(ctx, state);
|
||||
},
|
||||
async verifyResult(ctx, state, value) {
|
||||
const o = runtimeObservation(value); if (!o) throw new Error("Workspace result missing");
|
||||
const view = state.workspaceAfter as WorkspaceView;
|
||||
const href = o.links.find(link => workspaceLink(o.origin, link, view.workspace.id)); if (!href) throw new Error("No actual workspace result link");
|
||||
const source = ctx.input.page.url();
|
||||
await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();
|
||||
await expect(ctx.input.page.getByRole("heading", { name: view.workspace.name, exact: true }).first()).toBeVisible();
|
||||
await ctx.input.evidence("runtime-task-workspace-opened.json", { taskId: (state.task as Entity).id, workspaceId: view.workspace.id, url: ctx.input.page.url() });
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
},
|
||||
})];
|
||||
Reference in new issue
Block a user