mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
Correct task-bound runtime gates and reviewed boundary evidence
This commit is contained in:
1 parent
beca0672fb
commit
ff31d399e8
11 files changed
+211
-27
No files matched your search
@@ -6,7 +6,7 @@ import type {McpConnection} from "../../../packages/shared/src/public-mcp.js";
|
||||
import {collectRunEvents} from "../run-observations.js";
|
||||
import {pollUntil} from "../api.js";
|
||||
import {beginConnection,oauthPost,mcp,resource,type Tokens} from "../public-mcp-client.js";
|
||||
import {prepareRuntimeWorkerGate,releaseRuntimeWorker} from "./runtime-busy-worker.js";
|
||||
import {prepareRuntimeWorkerGate,readRuntimeTaskGate,releaseRuntimeWorker} from "./runtime-busy-worker.js";
|
||||
import {prepareAssistantSetup} from "./external-assistant-connections.js";
|
||||
import {openStandingChat,probeAgentPermission,waitForReply} from "./identity-support.js";
|
||||
import {definition,externalObservation,observeExternal,checks,root,verifyRenderedReply} from "./external-support.js";
|
||||
@@ -43,10 +43,10 @@ export const personalRevokeExternalCases=[definition("assistant-connections.revo
|
||||
await prepareAssistantSetup(ctx);const fixture:PrivateFixture={};fixtures.set(ctx,fixture);
|
||||
try{
|
||||
const existing=await ctx.input.api.get<McpConnection[]>("/api/mcp/connections");if(existing.length)throw new Error("Personal revoke requires a disposable signed-in person with no preexisting grants");
|
||||
fixture.old=await grant(ctx,"Old test assistant",true);fixture.other=await grant(ctx,"Current test assistant",false);fixture.gate=await prepareRuntimeWorkerGate(ctx);
|
||||
const {worker,gate}=fixture.gate,delegationRequestId=randomUUID(),created=await fixture.old.client.call("paperclip_create_task",{companyId:ctx.input.fixtures.company.id,requestId:delegationRequestId,title:"Website task",assigneeAgentId:worker.id,description:`Collect the pending website brief before implementation. Run node ${JSON.stringify(gate.scriptPath)} once and wait for the actual human-supplied brief. After it returns, save the brief as a task document and complete this brief-collection phase normally. Do not fabricate a brief or finish before the pending tool returns.`});
|
||||
fixture.old=await grant(ctx,"Old test assistant",true);fixture.other=await grant(ctx,"Current test assistant",false);fixture.gate=await prepareRuntimeWorkerGate(ctx,{taskBound:true});
|
||||
const {worker,gate}=fixture.gate,delegationRequestId=randomUUID(),created=await fixture.old.client.call("paperclip_create_task",{companyId:ctx.input.fixtures.company.id,requestId:delegationRequestId,title:"Website task",assigneeAgentId:worker.id,description:`Collect the pending website brief before implementation. Run ${gate.command} once and wait for the actual human-supplied brief. After it returns, save the brief as a task document and complete this brief-collection phase normally. Do not fabricate a brief or finish before the pending tool returns.`});
|
||||
const task=(created.structuredContent?.task as Entity|undefined);if(created.isError||!task?.id||task.assigneeAgentId!==worker.id)throw new Error("Old assistant did not create its real delegated Website task");
|
||||
const running=await pollUntil({label:"old-client delegated native actually waiting",deadlineAt:Date.now()+150000,load:async()=>({runs:await ctx.allRuns(),waiting:await readFile(gate.ready,"utf8").catch(()=>"")}),accept:v=>v.waiting==="waiting"&&v.runs.length===1&&v.runs[0]?.agentId===worker.id&&v.runs[0]?.status==="running"&&v.runs[0]?.contextSnapshot?.issueId===task.id,reject:v=>v.runs.some(run=>["failed","cancelled","timed_out","succeeded"].includes(run.status))?"Delegated worker did not remain active at the fixture boundary":undefined});
|
||||
const running=await pollUntil({label:"old-client delegated native actually waiting",deadlineAt:Date.now()+150000,load:async()=>({runs:await ctx.allRuns(),waiting:await readRuntimeTaskGate(ctx,gate,task.id,worker.id)}),accept:v=>v.waiting==="waiting"&&v.runs.length===1&&v.runs[0]?.agentId===worker.id&&v.runs[0]?.status==="running"&&v.runs[0]?.contextSnapshot?.issueId===task.id,reject:v=>v.runs.some(run=>["failed","cancelled","timed_out","succeeded"].includes(run.status))?"Delegated worker did not remain active at the fixture boundary":undefined});
|
||||
const before=await ctx.input.api.get<McpConnection[]>("/api/mcp/connections"),agentManagementStatus=await probeAgentPermission(ctx,`/api/mcp/connections/${fixture.old.id}`,"DELETE"),source=await openStandingChat(ctx);
|
||||
await ctx.input.api.patch(`/api/issues/${source.id}`,{description:`The person's old client is Old test assistant; Current test assistant is unrelated. Old test assistant already delegated Website task ${task.identifier??task.id} to ${worker.name}; that work is actively collecting its pending brief.`});
|
||||
return {before,task,delegationRequestId,oldGrantId:fixture.old.id,otherGrantId:fixture.other.id,personId:fixture.old.userId,sameHuman:fixture.old.userId===fixture.other.userId,oldCreatedTaskId:task.id,workerRunId:running.runs[0]!.id,workerWasReallyWaiting:true,agentManagementStatus};
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { expect, it, afterEach, vi } from "vitest";
|
||||
import { mkdtemp, mkdir, writeFile, readFile, rm, symlink, realpath } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { spawn } from "node:child_process";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import { readRuntimeTaskGate, releaseRuntimeWorker, taskBoundRuntimeGate, prepareRuntimeWorkerGate } from "./runtime-busy-worker.js";
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => { vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); });
|
||||
async function fixture() {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "runtime-task-gate-")); roots.push(root); vi.stubEnv("PAPERCLIP_RUNNER_E2E_TEMP_ROOT", root);
|
||||
const cwd = path.join(root, "isolated-workspaces", "company", "task"); await mkdir(cwd, { recursive: true });
|
||||
const task = { id: "task", companyId: "company", assigneeAgentId: "worker", executionWorkspaceId: "workspace" };
|
||||
const workspace = { id: "workspace", companyId: "company", cwd };
|
||||
const ctx = { input: { fixtures: { company: { id: "company" } }, api: { get: async (url: string) => url.endsWith("/workspace") ? { workspace } : task } } } as unknown as ChatCatalogContext;
|
||||
return { root, cwd, task, workspace, ctx };
|
||||
}
|
||||
it("a real bounded Node utility creates readiness in actual task cwd and emits only the genuine released brief", async () => {
|
||||
const f = await fixture(), gate = taskBoundRuntimeGate("owned-test", 3000);
|
||||
expect(await readRuntimeTaskGate(f.ctx, gate, "task", "worker")).toBe("");
|
||||
const child = spawn("/bin/sh", ["-c", `exec ${gate.command}`], { cwd: f.cwd, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdout = ""; child.stdout.on("data", bytes => { stdout += bytes; });
|
||||
const exited = new Promise<number | null>(resolve => child.once("exit", resolve));
|
||||
try {
|
||||
for (let i = 0; i < 50 && await readRuntimeTaskGate(f.ctx, gate, "task", "worker") !== "waiting"; i++) await new Promise(resolve => setTimeout(resolve, 20));
|
||||
expect(await readFile(gate.ready, "utf8")).toBe("waiting"); expect(child.exitCode).toBeNull(); expect(stdout).toBe("");
|
||||
await writeFile(gate.gate, "Actual human brief: preserve existing website.");
|
||||
expect(await exited).toBe(0); expect(stdout.trim()).toBe("Actual human brief: preserve existing website.");
|
||||
expect(gate.gate.startsWith(await realpath(f.cwd) + path.sep)).toBe(true);
|
||||
} finally { if (child.exitCode === null) { child.kill("SIGTERM"); await exited; } }
|
||||
});
|
||||
it("rejects another company, assignee, active binding or changed workspace before reading/releasing", async () => {
|
||||
const f = await fixture(), gate = taskBoundRuntimeGate("scope-test");
|
||||
f.task.assigneeAgentId = "other"; await expect(readRuntimeTaskGate(f.ctx, gate, "task", "worker")).rejects.toThrow("assignee");
|
||||
f.task.assigneeAgentId = "worker"; f.workspace.companyId = "foreign"; await expect(readRuntimeTaskGate(f.ctx, gate, "task", "worker")).rejects.toThrow("company");
|
||||
f.workspace.companyId = "company"; f.task.executionWorkspaceId = "other"; await expect(readRuntimeTaskGate(f.ctx, gate, "task", "worker")).rejects.toThrow("binding");
|
||||
f.task.executionWorkspaceId = "workspace"; await readRuntimeTaskGate(f.ctx, gate, "task", "worker");
|
||||
f.workspace.id = "new"; f.task.executionWorkspaceId = "new";
|
||||
await expect(readRuntimeTaskGate(f.ctx, gate, "task", "worker")).rejects.toThrow("changed after admission");
|
||||
});
|
||||
it("rejects escaping workspace and readiness symlinks instead of treating foreign bytes as busy evidence", async () => {
|
||||
const f = await fixture(), gate = taskBoundRuntimeGate("symlink-test");
|
||||
await readRuntimeTaskGate(f.ctx, gate, "task", "worker");
|
||||
await writeFile(path.join(f.root, "outside-ready"), "waiting"); await symlink(path.join(f.root, "outside-ready"), gate.ready);
|
||||
await expect(readRuntimeTaskGate(f.ctx, gate, "task", "worker")).rejects.toThrow("readiness path escapes");
|
||||
f.workspace.cwd = tmpdir();
|
||||
await expect(readRuntimeTaskGate(f.ctx, taskBoundRuntimeGate("outside"), "task", "worker")).rejects.toThrow("owned attempt root");
|
||||
});
|
||||
it("retains neutral cleanup release and refuses timeout widening", async () => {
|
||||
const f = await fixture(), gate = taskBoundRuntimeGate("cleanup"); await readRuntimeTaskGate(f.ctx, gate, "task", "worker");
|
||||
await releaseRuntimeWorker({ gate });
|
||||
expect(await readFile(gate.gate, "utf8")).toBe("The human ended this disposable fixture. Preserve any saved work and finish normally.");
|
||||
expect(() => taskBoundRuntimeGate("too-long", 300001)).toThrow("bounded");
|
||||
});
|
||||
it("funds source and worker before task-bound history, without exposing an agent-home script or admitting work", async () => {
|
||||
const f = await fixture(); vi.stubEnv("PAPERCLIP_HOME", path.join(f.root, "paperclip-home")); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "owned-test");
|
||||
const funded: string[] = [];
|
||||
const ctx = { input: { nonce: "history", execution: { environment: { id: "local" } }, fixtures: { company: { id: "company" }, agent: { id: "source-agent" } },
|
||||
api: { post: async (_url: string, body: { scopeId: string; unpricedUsagePolicy: string }) => { expect(body.unpricedUsagePolicy).toBe("allow"); funded.push(body.scopeId); } } }, allRuns: async () => [] } as unknown as ChatCatalogContext;
|
||||
const prepared = await prepareRuntimeWorkerGate(ctx, { worker: { id: "worker" }, taskBound: true });
|
||||
expect(funded).toEqual(["company", "worker", "source-agent"]);
|
||||
expect(prepared.gate.scriptPath).toBe(""); expect(prepared.gate.command).not.toContain(f.root);
|
||||
});
|
||||
@@ -1,4 +1,5 @@
|
||||
import path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, readFile, realpath, writeFile } from "node:fs/promises";
|
||||
import { resolveDefaultAgentWorkspaceDir } from "../../../server/src/home-paths.js";
|
||||
import { prepareChatBrief } from "../chat-stories.js";
|
||||
@@ -6,28 +7,71 @@ import { pollUntil } from "../api.js";
|
||||
import { runtimeWorker, type RuntimeAgent } from "./runtime-support.js";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import type { Entity } from "./internal-support.js";
|
||||
export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent } = {}) {
|
||||
export interface RuntimeBriefGate { gate: string; ready: string; scriptPath: string; command: string; taskBound: boolean;
|
||||
gateName?: string; script?: string; boundTaskId?: string; boundWorkspaceId?: string }
|
||||
export function taskBoundRuntimeGate(nonce: string, timeoutMs = 300_000): RuntimeBriefGate {
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 300_000) throw new Error("Task brief wait must remain bounded");
|
||||
const gateName = `runtime-brief-${createHash("sha256").update(nonce).digest("hex").slice(0, 24)}.txt`;
|
||||
const script = `const fs=require("node:fs");const gate=${JSON.stringify(gateName)};fs.writeFileSync(gate+".waiting","waiting",{flag:"wx"});const end=Date.now()+${timeoutMs};const timer=setInterval(()=>{if(fs.existsSync(gate)){console.log(fs.readFileSync(gate,"utf8"));clearInterval(timer)}else if(Date.now()>end){clearInterval(timer);process.exitCode=1}},100);`;
|
||||
const quote = (value: string) => "'" + value.replace(/'/g, "'\\''") + "'";
|
||||
return { gate: "", ready: "", scriptPath: "", command: `node -e ${quote(script)}`, taskBound: true, gateName, script };
|
||||
}
|
||||
/** Bind only the actual assigned task's persisted workspace, never AGENT_HOME. */
|
||||
export async function readRuntimeTaskGate(ctx: ChatCatalogContext, gate: RuntimeBriefGate, taskId: string, workerId: string) {
|
||||
if (!gate.taskBound || !gate.gateName) throw new Error("Expected task-bound brief listener");
|
||||
const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT;
|
||||
if (!root) throw new Error("Task gate requires the owned attempt root");
|
||||
const [task, view] = await Promise.all([
|
||||
ctx.input.api.get<{ id: string; companyId: string; assigneeAgentId: string | null; executionWorkspaceId: string | null }>(`/api/issues/${taskId}`),
|
||||
ctx.input.api.get<{ workspace: { id: string; companyId: string; cwd: string } }>(`/api/issues/${taskId}/workspace`),
|
||||
]);
|
||||
if (task.id !== taskId || task.companyId !== ctx.input.fixtures.company.id || task.assigneeAgentId !== workerId
|
||||
|| view.workspace.companyId !== task.companyId || view.workspace.id !== task.executionWorkspaceId)
|
||||
throw new Error("Task gate does not match the actual company, assignee and workspace binding");
|
||||
const cwd = await realpath(view.workspace.cwd), ownedRoot = await realpath(root);
|
||||
if (!cwd.startsWith(`${ownedRoot}${path.sep}`)) throw new Error("Task gate workspace escapes the owned attempt root");
|
||||
const filename = path.join(cwd, gate.gateName);
|
||||
if (gate.boundTaskId && (gate.boundTaskId !== taskId || gate.boundWorkspaceId !== view.workspace.id || gate.gate !== filename))
|
||||
throw new Error("Task gate binding changed after admission");
|
||||
gate.boundTaskId = taskId; gate.boundWorkspaceId = view.workspace.id; gate.gate = filename; gate.ready = `${filename}.waiting`;
|
||||
try {
|
||||
const ready = await realpath(gate.ready);
|
||||
if (path.dirname(ready) !== cwd) throw new Error("Task gate readiness path escapes its binding");
|
||||
return await readFile(ready, "utf8");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return "";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
export async function prepareRuntimeWorkerGate(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent; taskBound?: boolean } = {}) {
|
||||
if (ctx.input.execution.environment.id !== "local") throw new Error("Owned file-gate fixture requires a real local environment");
|
||||
const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root) throw new Error("Busy fixture requires its isolated attempt root");
|
||||
const worker = options.worker ?? await runtimeWorker(ctx); const candidate = path.resolve(resolveDefaultAgentWorkspaceDir(worker.id));
|
||||
const worker = options.worker ?? await runtimeWorker(ctx);
|
||||
const isolated = await realpath(root);
|
||||
if (!candidate.startsWith(`${isolated}${path.sep}`)) throw new Error("Busy fixture escaped its isolated company home");
|
||||
await mkdir(candidate, { recursive: true });
|
||||
const home = await realpath(candidate);
|
||||
if (!home.startsWith(`${isolated}${path.sep}`)) throw new Error("Busy fixture resolved outside its isolated company home");
|
||||
const previousRuns = await ctx.allRuns();
|
||||
const gate = await prepareChatBrief(home, `runtime-${ctx.input.nonce}-${previousRuns.length}`, 300_000);
|
||||
let gate: RuntimeBriefGate;
|
||||
if (options.taskBound) gate = taskBoundRuntimeGate(`${ctx.input.nonce}-${previousRuns.length}`);
|
||||
else {
|
||||
const candidate = path.resolve(resolveDefaultAgentWorkspaceDir(worker.id));
|
||||
if (!candidate.startsWith(`${isolated}${path.sep}`)) throw new Error("Busy fixture escaped its isolated company home");
|
||||
await mkdir(candidate, { recursive: true });
|
||||
const home = await realpath(candidate);
|
||||
if (!home.startsWith(`${isolated}${path.sep}`)) throw new Error("Busy fixture resolved outside its isolated company home");
|
||||
const original = await prepareChatBrief(home, `runtime-${ctx.input.nonce}-${previousRuns.length}`, 300_000);
|
||||
gate = { ...original, command: `node ${JSON.stringify(original.scriptPath)}`, taskBound: false };
|
||||
}
|
||||
const company = `/api/companies/${ctx.input.fixtures.company.id}`;
|
||||
for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }])
|
||||
const fundedAgents = [...new Set(options.taskBound ? [worker.id, ctx.input.fixtures.agent.id] : [worker.id])];
|
||||
for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...fundedAgents.map(scopeId => ({ scopeType: "agent", scopeId }))])
|
||||
await ctx.input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
|
||||
return { worker, gate, previousRuns, company };
|
||||
}
|
||||
export async function busyRuntimeWorker(ctx: ChatCatalogContext, options: { worker?: RuntimeAgent } = {}) {
|
||||
const { worker, gate, previousRuns, company } = await prepareRuntimeWorkerGate(ctx, options);
|
||||
const { worker, gate, previousRuns, company } = await prepareRuntimeWorkerGate(ctx, { ...options, taskBound: true });
|
||||
const task = await ctx.input.api.post<Entity>(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id,
|
||||
description: `The website brief is being supplied by the human. Run node ${JSON.stringify(gate.scriptPath)} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` });
|
||||
description: `The website brief is being supplied by the human. Run ${gate.command} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` });
|
||||
const running = await pollUntil({ label: "Real Morgan process waiting for its human brief", deadlineAt: Date.now() + 150_000,
|
||||
load: async () => ({ runs: (await ctx.allRuns()).filter(run => !previousRuns.some(prior => prior.id === run.id)), waiting: await readFile(gate.ready, "utf8").catch(() => "") }),
|
||||
load: async () => ({ runs: (await ctx.allRuns()).filter(run => !previousRuns.some(prior => prior.id === run.id)), waiting: await readRuntimeTaskGate(ctx, gate, task.id, worker.id) }),
|
||||
accept: value => value.waiting === "waiting" && value.runs.length === 1 && value.runs[0]?.agentId === worker.id && value.runs[0].status === "running",
|
||||
reject: value => value.runs.some(run => ["failed", "cancelled", "timed_out", "succeeded"].includes(run.status)) ? "Busy fixture ended before its required active boundary" : undefined });
|
||||
return { worker, task, gate, workerRunId: running.runs[0]!.id, workerWasReallyWaiting: true };
|
||||
|
||||
@@ -49,12 +49,14 @@ export const runtimeDisabledChatCases: readonly IdentityBoundaryCase[] = [{
|
||||
// actual page with the same genuine human session rather than inspecting
|
||||
// a second page while retaining an unrelated screenshot.
|
||||
await person.installSessionOn(ctx.page.context());
|
||||
person.bindApiSession(ctx.api); // Capture verification uses the same person through private Node fetch.
|
||||
const page = ctx.page;
|
||||
await page.goto(`/${fixtures.company.issuePrefix}/chats/${fixtures.agent.id}`, { waitUntil: "domcontentloaded" });
|
||||
await expect(page.getByText(/Agent Chat is disabled\. Enable it in Experimental settings/)).toBeVisible();
|
||||
await expect(page.getByTestId("task-chat-composer-input")).toHaveCount(0);
|
||||
const unavailableVisible = await page.getByText(/Agent Chat is disabled/).isVisible();
|
||||
const supportedOperatorStepVisible = await page.getByText(/Enable it in Experimental settings/).isVisible();
|
||||
ctx.observeResources({ companyId: fixtures.company.id, agentId: fixtures.agent.id, runIds: [] });
|
||||
await ctx.capture("disabled-agent-chat", "Morgan Chat is genuinely unavailable; canonical intent remains unsent", "runtime-disabled-agent-chat.png");
|
||||
const key = await api.postSensitive<{ id: string; token: string }>(`/api/agents/${fixtures.agent.id}/keys`, { name: "disabled-chat-permission-probe" });
|
||||
ctx.privateSecrets.push(key.token); resource.keyId = key.id;
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { sendChatMessage } from "../chat-flow.js";
|
||||
import { observeCompletionUpdate } from "../completion-update-flow.js";
|
||||
import { completionDelivery } from "../completion-updates.js";
|
||||
import { checks, hasLink, type Entity } from "./internal-support.js";
|
||||
import { openStandingChat } from "./identity-support.js";
|
||||
import { prepareRuntimeWorkerGate, releaseRuntimeWorker } from "./runtime-busy-worker.js";
|
||||
import { prepareRuntimeWorkerGate, readRuntimeTaskGate, releaseRuntimeWorker } from "./runtime-busy-worker.js";
|
||||
import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js";
|
||||
|
||||
export function gradeTrackDelegatedWork(value: unknown) {
|
||||
@@ -35,16 +35,20 @@ export function gradeTrackDelegatedWork(value: unknown) {
|
||||
return c.result();
|
||||
}
|
||||
export const runtimeHandoffCases = [runtimeDefinition("agent-chat.track-delegated-work", async ctx => {
|
||||
const prepared = await prepareRuntimeWorkerGate(ctx); const source = await openStandingChat(ctx);
|
||||
const prepared = await prepareRuntimeWorkerGate(ctx, { taskBound: true }); const source = await openStandingChat(ctx);
|
||||
const marker = `Tracked website delivery ${ctx.input.nonce}`;
|
||||
const instructions = `For the website assignment, first run node ${JSON.stringify(prepared.gate.scriptPath)} once and wait for the real human brief. Do not invent the brief or change the script. Once it returns, save a short website deliverable document with key website-delivery on the assigned task containing the supplied reference and complete that same task normally. Do not create other tasks or perform external actions.`;
|
||||
const instructions = `For the website assignment, first run ${prepared.gate.command} once and wait for the real human brief. Do not invent the brief or change the script. Once it returns, save a short website deliverable document with key website-delivery on the assigned task containing the supplied reference and complete that same task normally. Do not create other tasks or perform external actions.`;
|
||||
await ctx.input.api.saveAgentInstructions(prepared.worker.id, instructions);
|
||||
const prompt = `Create one task titled Website, assigned to Morgan, to draft our launch page. Morgan is waiting for the human brief, which will include the reference ${marker}. Let Morgan implement it on that separate work task. Track the result here and tell me here when the work is complete, with access to the actual result. You may begin that one handoff now; no additional task or project is needed.`;
|
||||
await sendChatMessage(ctx.input.page, prompt);
|
||||
const phase = await pollUntil({ label: "Actual conversation-created Morgan handoff", deadlineAt: Date.now() + 180_000,
|
||||
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(),
|
||||
tasks: await ctx.input.api.get<Entity[]>(`${prepared.company}/issues?includeAgentCreatedTasks=true&limit=1000`), runs: await ctx.allRuns(),
|
||||
waiting: await readFile(prepared.gate.ready, "utf8").catch(() => "") }; },
|
||||
load: async () => {
|
||||
await ctx.refreshIssue();
|
||||
const tasks = await ctx.input.api.get<Entity[]>(`${prepared.company}/issues?includeAgentCreatedTasks=true&limit=1000`);
|
||||
const candidates = tasks.filter(task => task.id !== source.id && task.assigneeAgentId === prepared.worker.id);
|
||||
return { source: ctx.issue(), tasks, runs: await ctx.allRuns(),
|
||||
waiting: candidates.length === 1 ? await readRuntimeTaskGate(ctx, prepared.gate, candidates[0]!.id, prepared.worker.id) : "" };
|
||||
},
|
||||
accept: value => value.source?.conversationState === "waiting" && value.waiting === "waiting" && value.runs.length === 2
|
||||
&& value.runs.some(run => run.agentId === prepared.worker.id && run.status === "running")
|
||||
&& value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded")
|
||||
|
||||
@@ -22,3 +22,13 @@ it("rejects accepting a replacement identity instead of the proposed hire", () =
|
||||
it("rejects readiness language before human approval", () => { const o = valid(); o.state.firstReply = "Morgan is now ready and working."; expect(gradeGovernedHire(o).passed).toBe(false); });
|
||||
it("rejects a hidden nudge classified as a decision notification", () => { const o = valid(); o.runs.push({ id: "extra", agentId: "ceo", status: "succeeded", contextSnapshot: { source: "approval.approved", approvalId: "foreign" } }); expect(gradeGovernedHire(o).passed).toBe(false); });
|
||||
it("accepts a separately counted exact approval notification", () => { const o = valid(); o.runs.push({ id: "extra", agentId: "ceo", status: "succeeded", contextSnapshot: { source: "approval.approved", approvalId: "second" } }); expect(gradeGovernedHire(o).passed).toBe(true); });
|
||||
|
||||
it("accepts the retained request-ready answer with an actual pending proposal and explicit activation gate", () => {
|
||||
const o = valid(); o.state.firstReply = "Morgan’s QA Engineer hire request is ready. Morgan will report to the CEO. Approve Morgan’s hire to activate the agent.";
|
||||
expect(gradeGovernedHire(o).passed).toBe(true);
|
||||
});
|
||||
it("rejects pending language mixed with a named employee readiness claim", () => {
|
||||
const o = valid(); o.state.firstReply = "The proposal is pending, but Morgan is now ready and working.";
|
||||
expect(gradeGovernedHire(o).passed).toBe(false);
|
||||
o.state.firstReply = "Approve the proposal; the QA engineer is active."; expect(gradeGovernedHire(o).passed).toBe(false);
|
||||
});
|
||||
@@ -16,6 +16,13 @@ async function pendingHire(ctx: ChatCatalogContext, name: string) {
|
||||
&& value.runs.every(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded"),
|
||||
reject: value => value.runs.length > 4 || value.runs.some(run => run.agentId !== ctx.input.fixtures.agent.id || ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Governed hire admitted unexpected work or failed" : undefined });
|
||||
}
|
||||
export function pendingHireGuidance(reply: string, name: "Morgan" | "Jules") {
|
||||
if (!/pending|approv(?:al|e)|propos|await|board/i.test(reply) || /successfully hired/i.test(reply)) return false;
|
||||
// A hire request can be ready for the board while its intended employee
|
||||
// remains pending. Do not turn that into a general exemption for ready.
|
||||
const employeeClaim = new RegExp(`\\b(?:${name}|(?:the )?(?:QA engineer|employee|agent|hire)|they|he|she)\\s+(?:is|are|has|have)\\s+(?:now |already )?(?:been )?(?:ready(?! for (?:board )?(?:approval|review))|working|started|active|running)\\b`, "i");
|
||||
return !employeeClaim.test(reply);
|
||||
}
|
||||
export function gradeGovernedHire(value: unknown) {
|
||||
const c = checks(); const o = runtimeObservation(value);
|
||||
c.add("independent-governed-hire-evidence", !!o, "Actual pending proposals, human decisions, preserved intended identity and no premature worker execution are required."); if (!o) return c.result();
|
||||
@@ -44,9 +51,9 @@ export function gradeGovernedHire(value: unknown) {
|
||||
c.add("bounded-authoritative-decision-notifications", extra.every(run => ["approval.approved", "approval.rejected"].includes(String(run.contextSnapshot?.source))
|
||||
&& [rejected?.id, accepted?.id].includes(String(run.contextSnapshot?.approvalId))),
|
||||
"Any extra provider turn is independently tied to one of these exact real human approval decisions.");
|
||||
c.add("honest-pending-phase-guidance", [o.state.firstReply, o.state.secondReply].every(reply => /pending|approval|propos|await|board/i.test(String(reply))
|
||||
&& !/(?:is|has|have) (?:now )?(?:ready|working|started)|successfully hired/i.test(String(reply))),
|
||||
"Each actual request reply describes its own pending approval without claiming a working employee before the human acts.");
|
||||
c.add("honest-pending-phase-guidance", pendingHireGuidance(String(o.state.firstReply), "Morgan")
|
||||
&& pendingHireGuidance(String(o.state.secondReply), "Jules"),
|
||||
"Each actual reply describes an approvable proposal, without claiming that its employee is ready or working before approval.");
|
||||
return c.result();
|
||||
}
|
||||
export const runtimeHiringCases = [runtimeDefinition("agents.hire-needs-approval", async ctx => {
|
||||
@@ -93,7 +100,7 @@ export const runtimeHiringCases = [runtimeDefinition("agents.hire-needs-approval
|
||||
accept: row => row.lifecycleState === "ready" && ["idle", "active"].includes(row.status ?? ""),
|
||||
reject: row => row.lifecycleState === "error" ? "Approved runtime did not qualify; no fake ready state is permitted" : undefined });
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
|
||||
await expect(ctx.input.page.getByText("Approved", { exact: true }).first()).toBeVisible();
|
||||
await expect(ctx.input.page.getByText("approved", { exact: true }).first()).toBeVisible();
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
|
||||
@@ -103,6 +103,9 @@ export function gradeBrowserReconnect(value: unknown) {
|
||||
}
|
||||
|
||||
export const runtimeHistoryCases = [runtimeDefinition("agent-chat.remember-and-follow-up", async ctx => {
|
||||
for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: ctx.input.fixtures.agent.id }])
|
||||
await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500,
|
||||
hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
|
||||
const source = await openStandingChat(ctx); const designMarker = `Selected blue preview ${ctx.input.nonce}`; const filename = `blue-preview-${ctx.input.nonce}.html`;
|
||||
await sendChatMessage(ctx.input.page, `We have selected the blue design. Save an actual small HTML preview named ${filename} with blue (#0000ff) styling and the exact visible marker ${JSON.stringify(designMarker)}. Register this real file as a downloadable artifact work product titled ${filename}. Keep that blue design and this exact preview as our selected version for the next turn. No delegated task or external action is needed.`);
|
||||
const phase = await sourceSettled(ctx, 1);
|
||||
|
||||
@@ -94,6 +94,7 @@ import { resolveRunnerE2ESource } from "./source.js";
|
||||
import { isValidNativePrpEnvelope } from "./native-event-envelope.js";
|
||||
import {
|
||||
isPublicRunnerScreenshotRoute,
|
||||
isDisabledChatBoundaryScreenshotRoute,
|
||||
PUBLIC_RUNNER_SCREENSHOT_MARKER,
|
||||
} from "./screenshot-policy.js";
|
||||
import {
|
||||
@@ -656,6 +657,7 @@ for (const execution of executions) {
|
||||
let nativeInitial: { issueIds: string[]; agentIds: string[]; workspaceDigest: string | null } | undefined;
|
||||
const nativeWorkspaceDigest = () => execution.task.id === "native-blocked-report"
|
||||
? nativeCompletionWorkspaceDigest(workspacePath!) : Promise.resolve(null);
|
||||
let boundaryScreenshotResources: { companyId?: string; agentId?: string } = {};
|
||||
let fixtures: LiveFixtureValues | undefined;
|
||||
let reviewProvider: Awaited<ReturnType<typeof setupConnectionReview>> | undefined;
|
||||
let issue: IssueRecord | undefined;
|
||||
@@ -734,7 +736,22 @@ for (const execution of executions) {
|
||||
label: string,
|
||||
file: string,
|
||||
) => {
|
||||
if (!isReviewedFixtureScreenshotRoute()) {
|
||||
let reviewedBoundary = false;
|
||||
if (execution.suite.id === "chat-features" && execution.task.id === "agent-chat.chat-disabled"
|
||||
&& boundaryScreenshotResources.companyId && boundaryScreenshotResources.agentId
|
||||
&& findChatCatalogCase(execution.task.id)?.journeyKind === "browser-boundary") {
|
||||
const [company, agent, session, settings] = await Promise.all([
|
||||
api.get<{ id: string; issuePrefix: string }>(`/api/companies/${boundaryScreenshotResources.companyId}`),
|
||||
api.get<{ id: string; companyId: string }>(`/api/agents/${boundaryScreenshotResources.agentId}`),
|
||||
api.get<{ user: { id: string } }>("/api/auth/get-session"),
|
||||
api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental"),
|
||||
]);
|
||||
reviewedBoundary = company.id === boundaryScreenshotResources.companyId && agent.id === boundaryScreenshotResources.agentId
|
||||
&& isDisabledChatBoundaryScreenshotRoute(page.url(), { scenarioId: execution.task.id, origin: api.baseURL,
|
||||
companyId: company.id, agentCompanyId: agent.companyId, issuePrefix: company.issuePrefix,
|
||||
agentId: agent.id, authenticatedPersonId: session.user.id, agentChatEnabled: settings.enableAgentChat });
|
||||
}
|
||||
if (!isReviewedFixtureScreenshotRoute() && !reviewedBoundary) {
|
||||
throw new Error(
|
||||
`Public runner screenshot blocked on non-task route ${page.url()}`,
|
||||
);
|
||||
@@ -957,6 +974,7 @@ for (const execution of executions) {
|
||||
if (name === "chat-boundary-cleanup.json") cleanup = (value as { cleanup?: string }).cleanup === "complete" ? "passed" : "failed";
|
||||
},
|
||||
observeResources(value) {
|
||||
if (execution.task.id === "agent-chat.chat-disabled") boundaryScreenshotResources = { companyId: value.companyId, agentId: value.agentId };
|
||||
// Boundaries independently collect their real resources. They do not
|
||||
// manufacture LiveFixtureValues or a source issue for result fields.
|
||||
if (value.runIds.length) matcherResults.push({ matcher: { kind: "json_path", path: "boundary.actual-agent-dispatch", expected: true },
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { isDisabledChatBoundaryScreenshotRoute, isPublicRunnerScreenshotRoute } from "./screenshot-policy.js";
|
||||
const target = { scenarioId: "agent-chat.chat-disabled", origin: "http://127.0.0.1:3100", companyId: "owned-company", agentCompanyId: "owned-company",
|
||||
issuePrefix: "RUN", agentId: "morgan", authenticatedPersonId: "real-fixture-person", agentChatEnabled: false };
|
||||
it("permits only the exact independently owned disabled-chat boundary route without manufacturing a task", () => {
|
||||
expect(isDisabledChatBoundaryScreenshotRoute(`${target.origin}/RUN/chats/morgan`, target)).toBe(true);
|
||||
expect(isPublicRunnerScreenshotRoute(`${target.origin}/RUN/chats/morgan`, { chatAgentId: "morgan", issuePrefix: "RUN", issueId: undefined, issueIdentifier: undefined })).toBe(false);
|
||||
});
|
||||
it("rejects wrong boundary, person, company, agent, prefix, port, host, query and foreign paths", () => {
|
||||
for (const wrong of [{ agentChatEnabled: true }, { scenarioId: "agent-chat.reopen-conversation" }, { authenticatedPersonId: "local-board" }, { authenticatedPersonId: "" },
|
||||
{ agentCompanyId: "foreign" }, { companyId: "" }, { agentId: "other" }, { issuePrefix: "FOREIGN" }])
|
||||
expect(isDisabledChatBoundaryScreenshotRoute(`${target.origin}/RUN/chats/morgan`, { ...target, ...wrong })).toBe(false);
|
||||
for (const wrong of ["http://127.0.0.1:3200/RUN/chats/morgan", "https://example.test/RUN/chats/morgan", `${target.origin}/RUN/chats/other`,
|
||||
`${target.origin}/RUN/chats/morgan?private=1`, `${target.origin}/RUN/chats/morgan#private`, `${target.origin}/RUN/agents/morgan`, `${target.origin}/RUN/dashboard`])
|
||||
expect(isDisabledChatBoundaryScreenshotRoute(wrong, target)).toBe(false);
|
||||
});
|
||||
@@ -32,3 +32,20 @@ export function isPublicRunnerScreenshotRoute(
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** The one zero-task chat boundary has no source issue. Its resource identity
|
||||
* must be independently read through the actual signed-in page before capture.
|
||||
*/
|
||||
export function isDisabledChatBoundaryScreenshotRoute(url: string, target: {
|
||||
scenarioId: string; origin: string; companyId: string; agentCompanyId: string;
|
||||
issuePrefix: string; agentId: string; authenticatedPersonId: string; agentChatEnabled: boolean;
|
||||
}) {
|
||||
try {
|
||||
const candidate = new URL(url), origin = new URL(target.origin);
|
||||
return target.scenarioId === "agent-chat.chat-disabled" && target.agentChatEnabled === false && !!target.companyId && target.agentCompanyId === target.companyId
|
||||
&& !!target.authenticatedPersonId && target.authenticatedPersonId !== "local-board" && !!target.agentId && !!target.issuePrefix
|
||||
&& origin.protocol === "http:" && origin.hostname === "127.0.0.1" && candidate.origin === origin.origin
|
||||
&& !candidate.username && !candidate.password && !candidate.search && !candidate.hash
|
||||
&& candidate.pathname === `/${encodeURIComponent(target.issuePrefix)}/chats/${encodeURIComponent(target.agentId)}`;
|
||||
} catch { return false; }
|
||||
}
|
||||
Reference in new issue
Block a user