mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-20 15:09:57 +02:00
Add Pi remote native evidence and browser denial fixtures
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
fce6e109ad
commit
d1bb5729f9
8 files changed
+376
-24
No files matched your search
@@ -5,28 +5,35 @@ import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
|
||||
import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
|
||||
|
||||
describe("Pi native Product qualification", () => {
|
||||
it("selects three explicit local Pi cases and preserves the basic extended matrix", () => {
|
||||
it("selects four local and three remote Pi cases without changing the basic extended matrix", () => {
|
||||
const suite = runnerSuites.find(row => row.id === "pi-native")!;
|
||||
expect(suite.manualOnly).toBe(true); expect(suite.expectedMatrixSize).toBe(3);
|
||||
expect(suite.manualOnly).toBe(true); expect(suite.expectedMatrixSize).toBe(7);
|
||||
const cells = runnerMatrix.filter(row => row.suite.id === suite.id);
|
||||
expect(cells).toHaveLength(3);
|
||||
expect(cells.every(row => row.environment.id === "local" && row.profile.qualificationCandidate === "pi")).toBe(true);
|
||||
expect(cells.map(row => row.task.expectedRunCount)).toEqual([1, 2, 1]);
|
||||
expect(cells).toHaveLength(7);
|
||||
expect(cells.every(row => row.profile.qualificationCandidate === "pi")).toBe(true);
|
||||
expect(cells.filter(row => row.environment.id === "local").map(row => [row.task.id, row.task.expectedRunCount])).toEqual([
|
||||
["native-questions", 1], ["agent-files-fresh-run", 2], ["restrictive-denial", 1], ["human-permission-denial", 1],
|
||||
]);
|
||||
expect(cells.filter(row => row.environment.id === "daytona").map(row => [row.task.id, row.task.expectedRunCount])).toEqual([
|
||||
["native-questions", 1], ["agent-files-fresh-run", 2], ["human-permission-denial", 1],
|
||||
]);
|
||||
expect(cells.reduce((sum, row) => sum + row.task.expectedRunCount, 0)).toBe(9);
|
||||
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(row => row.suite.id === suite.id)).toBe(false);
|
||||
expect(runnerMatrix.filter(row => row.suite.id === "extended-harnesses")).toHaveLength(30);
|
||||
expect(piNativeTasks[0]!.buildPrompt("fixture")).toContain("paperclip_native_question");
|
||||
expect(piNativeTasks[1]!.buildPrompt("fixture")).toContain("AGENT_HOME");
|
||||
});
|
||||
|
||||
it("admits only the explicit local Pi native candidate and discards ambient admission", () => {
|
||||
it("admits explicit local and remote Pi native candidates and discards ambient admission", () => {
|
||||
const cell = runnerMatrix.find(row => row.suite.id === "pi-native")!;
|
||||
const source = { PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" };
|
||||
expect(JSON.parse(buildRunnerE2EProcessEnvironment(source, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "pi", model: cell.profile.model }]);
|
||||
const remote = runnerMatrix.find(row => row.suite.id === "pi-native" && row.environment.id === "daytona")!;
|
||||
expect(JSON.parse(buildRunnerE2EProcessEnvironment(source, [remote]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "pi", model: remote.profile.model }]);
|
||||
expect(buildRunnerE2EProcessEnvironment(source, []).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
|
||||
for (const changed of [
|
||||
{ ...cell, suite: { ...cell.suite, manualOnly: false } },
|
||||
{ ...cell, suite: { ...cell.suite, id: "implicit" } },
|
||||
{ ...cell, environment: { ...cell.environment, id: "daytona" as const } },
|
||||
{ ...cell, profile: { ...cell.profile, qualificationCandidate: "copilot" as const } },
|
||||
]) expect(() => buildRunnerE2EProcessEnvironment(source, [changed])).toThrow("explicit provider qualification suite");
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ export const piNativeTasks: readonly RunnerTaskFixture[] = [
|
||||
["native-questions", "Four native questions survive browser reconnect", 1],
|
||||
["agent-files-fresh-run", "Agent files save and survive a fresh task", 2],
|
||||
["restrictive-denial", "Restrictive native write denial prevents a file effect", 1],
|
||||
["human-permission-denial", "Browser denial prevents a native write through process retirement", 1],
|
||||
].map(([id, label, turns]) => ({
|
||||
id: String(id), label: String(label), groups: [], workMode: "standard", flow: "pi_native",
|
||||
expectedRunCount: Number(turns), attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 },
|
||||
@@ -35,6 +36,11 @@ export function piNativePrompt(id: string, nonce: string): string {
|
||||
"Do not claim that the file is already saved to managed storage; Paperclip collects it after the provider stops.",
|
||||
piNativeFinish("PI-NATIVE-MEMORY-STAGED"),
|
||||
].join("\n");
|
||||
if (id === "human-permission-denial") return [
|
||||
"Attempt native write exactly once with relative path pi-human-denied.txt and content forbidden. Wait for the actual browser permission decision. Do not retry, use bash or an API, write another path, or work around denial.",
|
||||
"After the native write returns its denial, finish this negative test. Native denial is the expected result; never claim that the file was written.",
|
||||
piNativeFinish(`PI-NATIVE-${id}-${nonce}`),
|
||||
].join("\n");
|
||||
if (id === "restrictive-denial") return [
|
||||
"This fixture deliberately denies native writes. Attempt the native write tool exactly once, with relative path pi-denied.txt and content forbidden. Do not use bash, an API, another path, or a retry to work around the denial.",
|
||||
"After the actual tool failure, report completion of this negative test through the assigned semantic tool. A failed native write is the expected outcome, not a reason to repeat it.",
|
||||
|
||||
@@ -3,7 +3,7 @@ import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { runnerMatrix } from "./catalog.js";
|
||||
import { piNativeTasks } from "./pi-native-cases.js";
|
||||
import { packageEvidence } from "./evidence.js";
|
||||
import { runPiNativeFlow } from "./pi-native-flow.js";
|
||||
import { sanitizeJson } from "./redaction.js";
|
||||
@@ -22,16 +22,16 @@ const roots: string[] = [];
|
||||
afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
|
||||
|
||||
type FlowInput = Parameters<typeof runPiNativeFlow>[0];
|
||||
async function fixture(options: { failComments?: boolean } = {}) {
|
||||
async function fixture(options: { failComments?: boolean; remote?: boolean; incompleteRemote?: boolean } = {}) {
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-native-evidence-")); roots.push(root);
|
||||
const privateDir = join(root, "private"); const uploadDir = join(root, "upload");
|
||||
const workspacePath = join(root, "workspace"); const snapshots = join(privateDir, "snapshots");
|
||||
for (const directory of [workspacePath, snapshots, join(privateDir, "html-report"), join(privateDir, "blob-report")]) await mkdir(directory, { recursive: true });
|
||||
const execution = runnerMatrix.find(row => row.id === "pi-native.runner-acpx-pi.local.native-questions")!;
|
||||
const execution = { id: "pi-native.runner-acpx-pi.local.native-questions", environment: { id: options.remote ? "daytona" : "local" }, profile: { qualificationCandidate: "pi" }, task: piNativeTasks.find(row => row.id === "native-questions")! } as FlowInput["execution"];
|
||||
const issue = { id: "issue-fixture", identifier: "PI-1", title: execution.task.buildTitle("fixture"), status: "done" };
|
||||
const run = { id: "run-fixture", status: "succeeded", runtimeMode: "native", createdAt: "2026-09-29T00:00:00Z" };
|
||||
const headers = ["Pi native color", "Pi native confirmation", "Pi native name", "Pi native draft"];
|
||||
let answered = 0; const typed: string[] = [];
|
||||
let answered = 0; const typed: string[] = []; let answerProof = ""; let remoteFinished = false; const cleanupAssertions: Array<() => Promise<unknown>> = [];
|
||||
const interactions = () => headers.slice(0, Math.min(4, answered + 1)).map((header, index) => ({
|
||||
id: `question-${index}`, kind: "ask_user_questions", status: index < answered ? "answered" : "pending",
|
||||
sourceRunId: run.id, continuationPolicy: "none",
|
||||
@@ -70,24 +70,38 @@ async function fixture(options: { failComments?: boolean } = {}) {
|
||||
getByRole: (role: string) => ({ last: () => ({ click: async () => {
|
||||
if (role !== "button") return;
|
||||
answered += 1;
|
||||
if (answered === 4) await writeFile(join(workspacePath, "pi-native-answers.json"), JSON.stringify([
|
||||
if (answered === 4) { answerProof = JSON.stringify([
|
||||
{ status: "answered", optionId: "blue" }, { status: "negative_or_cancelled", confirmed: false },
|
||||
{ status: "answered", value: typed[0] }, { status: "answered", value: typed[1] },
|
||||
]));
|
||||
]); await writeFile(join(workspacePath, "pi-native-answers.json"), options.remote ? "WRONG HOST COPYBACK" : answerProof); }
|
||||
} }) }),
|
||||
};
|
||||
const evidence = async (name: string, data: unknown) => { await writeFile(join(snapshots, name), JSON.stringify(sanitizeJson(data, [secret]))); };
|
||||
const capture = vi.fn(async (_id: string, _label: string, file: string) => { await writeFile(join(privateDir, file), "fixture screenshot boundary"); });
|
||||
const remoteRoot = { pid: 10, ppid: 1, startTicks: "100", bootId: "boot" };
|
||||
const remoteSnapshot = { observedAtMs: 2, complete: !options.incompleteRemote, workspace: {}, targets: {}, watcher: { complete: true, targetMutationCount: 0, workspaceMutationCount: 0 }, processes: { captured: true, root: remoteRoot, journal: [remoteRoot], live: [] } };
|
||||
const remoteFixture = { binding: { runId: run.id }, remoteCwd: "/owned/remote", outsideTarget: null,
|
||||
snapshot: async () => remoteSnapshot,
|
||||
finish: async () => { remoteFinished = true; return remoteSnapshot; },
|
||||
readFile: async (path: string) => { expect(remoteFinished).toBe(true); expect(path).toBe("pi-native-answers.json"); return Buffer.from(answerProof); },
|
||||
close: vi.fn(async () => {}),
|
||||
};
|
||||
const input = {
|
||||
page: page as unknown as FlowInput["page"], api: api as unknown as FlowInput["api"],
|
||||
fixtures: { company: { id: "company-fixture", issuePrefix: "PI" }, agent: { name: "Pi fixture" }, environment: { id: "environment-fixture" } } as FlowInput["fixtures"],
|
||||
execution, nonce: "fixture", workspacePath, deadlineAt: Date.now() + 5000,
|
||||
restart: async () => { throw new Error("Unexpected restart"); }, observe: () => {}, capture, evidence,
|
||||
...(options.remote ? { registerCleanupAssertion: (fn: () => Promise<unknown>) => cleanupAssertions.push(fn), remoteBootstrap: {
|
||||
prompt: () => "Read the bootstrap file only",
|
||||
bindAndRelease: async (value: { targets: string[]; actionPrompt(fixture: any): Promise<string> | string }) => {
|
||||
expect(value.targets).toEqual(["pi-native-answers.json"]); const prompt = await value.actionPrompt(remoteFixture); expect(prompt).toContain("paperclip_native_question"); return remoteFixture;
|
||||
},
|
||||
} } : {}),
|
||||
};
|
||||
for (const [file, content] of [["result.json", "{}"], ["snapshots/fixtures.json", "{}"], ["junit.xml", "<testsuites/>"], ["html-report/index.html", "fixture report"]]) await writeFile(join(privateDir, file!), content!);
|
||||
const blob = join(privateDir, "blob-report"); await writeFile(join(blob, "fixture.txt"), "unit fixture");
|
||||
execFileSync("zip", ["-q", "report.zip", "fixture.txt"], { cwd: blob });
|
||||
return { input, privateDir, uploadDir, snapshots, paths, capture, issue, run, secret };
|
||||
return { input: input as FlowInput, privateDir, uploadDir, snapshots, paths, capture, issue, run, secret, cleanupAssertions };
|
||||
}
|
||||
|
||||
describe("Pi native terminal evidence", () => {
|
||||
@@ -127,3 +141,15 @@ describe("Pi native terminal evidence", () => {
|
||||
expect(packaged.missing).toEqual(["final-state.png", "snapshots/api-state.json"]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
it("uses sealed remote answer bytes rather than host copyback and requires independent retirement", async () => {
|
||||
const f = await fixture({ remote: true }); const result = await runPiNativeFlow(f.input);
|
||||
expect(result.checks.every(check => check.passed)).toBe(true); expect(f.cleanupAssertions).toHaveLength(1); await f.cleanupAssertions[0]!();
|
||||
const retained = JSON.parse(await readFile(join(f.snapshots, "pi-remote-1-questions-final.json"), "utf8")); expect(retained.snapshot.processes.live).toEqual([]);
|
||||
});
|
||||
it("cannot replace missing remote retirement proof with a succeeded Product run", async () => {
|
||||
const f = await fixture({ remote: true, incompleteRemote: true });
|
||||
await expect(runPiNativeFlow(f.input)).rejects.toThrow("retirement is unproven");
|
||||
await expect(f.cleanupAssertions[0]!()).rejects.toThrow("retirement proof incomplete");
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
/** Public durable permission/tool evidence. Independent filesystem and process
|
||||
* proofs remain required; assistant text can never satisfy this oracle. */
|
||||
type Row = Record<string, any>;
|
||||
const record = (value: unknown): Row => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {};
|
||||
export interface PiDeniedPermission { runId: string; turnId: string; requestId: string; toolCallId: string; target: string }
|
||||
export function piPermissionRequests(rows: readonly unknown[], runId: string): Array<{ row: Row; event: Row; request: Row }> {
|
||||
return rows.map(record).flatMap(row => {
|
||||
const event = record(record(row.payload).prpEvent), request = record(record(event.payload).request), origin = record(request.origin);
|
||||
if (row.eventType !== "runtime_request.created" || event.eventType !== row.eventType || row.runId !== runId || event.runId !== runId
|
||||
|| row.protocolSchemaVersion !== 1 || event.schemaVersion !== 1 || event.schema !== "paperclip.prp.event.v1" || event.sourceKind !== "runner"
|
||||
|| (!Number.isSafeInteger(row.seq) || row.seq < 1) || typeof event.turnId !== "string" || !event.turnId || request.turnId !== event.turnId
|
||||
|| request.type !== "permission" || request.status !== "pending" || origin.adapter !== "acpx-runtime" || origin.provider !== "pi" || origin.method !== "session/request_permission"
|
||||
|| typeof request.requestId !== "string" || !request.requestId || typeof request.details?.toolCallId !== "string" || !request.details.toolCallId
|
||||
|| !Array.isArray(request.choices) || !request.choices.some((choice: unknown) => record(choice).key === "decline")) return [];
|
||||
return [{ row, event, request }];
|
||||
});
|
||||
}
|
||||
export function hasDeliveredPiDenial(rows: readonly unknown[], expected: PiDeniedPermission): boolean {
|
||||
const requests = piPermissionRequests(rows, expected.runId).filter(value => value.request.requestId === expected.requestId);
|
||||
if (requests.length !== 1) return false;
|
||||
const created = requests[0]!;
|
||||
if (created.event.turnId !== expected.turnId || created.request.details.toolCallId !== expected.toolCallId) return false;
|
||||
const events = rows.map(record).map(row => ({ row, event: record(record(row.payload).prpEvent) }))
|
||||
.filter(({ row, event }) => row.runId === expected.runId && event.runId === expected.runId && event.turnId === expected.turnId
|
||||
&& row.eventType === event.eventType && row.protocolSchemaVersion === 1 && event.schemaVersion === 1
|
||||
&& event.schema === "paperclip.prp.event.v1" && event.sourceKind === "runner" && Number.isSafeInteger(row.seq) && row.seq >= 1);
|
||||
const resolutions = events.filter(({ event }) => ["runtime_request.resolved", "runtime_request.expired", "runtime_request.cancelled"].includes(event.eventType) && event.payload?.requestId === expected.requestId);
|
||||
if (resolutions.length !== 1 || resolutions[0]!.event.eventType !== "runtime_request.resolved" || resolutions[0]!.event.payload.action !== "decline"
|
||||
|| resolutions[0]!.event.payload.turnId !== expected.turnId || resolutions[0]!.row.seq <= created.row.seq) return false;
|
||||
const completed = events.filter(({ event }) => event.eventType === "tool.execution.completed" && event.payload?.transport === "builtin" && event.payload?.operation === "edit");
|
||||
return completed.length === 1 && completed.every(({ row, event }) => row.seq > created.row.seq && event.payload.schema === "paperclip.tool.execution.v1"
|
||||
&& event.payload.executionId === expected.toolCallId && event.payload.name === "write" && event.payload.status === "failed"
|
||||
&& event.payload.target === expected.target && typeof event.payload.output === "string" && event.payload.output.includes("Pi operation was denied or cancelled"));
|
||||
}
|
||||
|
||||
/** The shared remote observer supplies these facts directly from the bound
|
||||
* sandbox. Missing watcher/process coverage is never treated as an absent file. */
|
||||
export function hasPiRemoteRetirement(value: unknown): boolean {
|
||||
const snapshot = record(value), processes = record(snapshot.processes), root = record(processes.root);
|
||||
return snapshot.complete === true && record(snapshot.watcher).complete === true && Number.isFinite(snapshot.observedAtMs)
|
||||
&& processes.captured === true && Number.isSafeInteger(root.pid) && root.pid > 1
|
||||
&& typeof root.startTicks === "string" && root.startTicks.length > 0 && typeof root.bootId === "string" && root.bootId.length > 0
|
||||
&& Array.isArray(processes.journal) && processes.journal.length > 0
|
||||
&& processes.journal.some((entry: Row) => entry.pid === root.pid && entry.startTicks === root.startTicks && entry.bootId === root.bootId)
|
||||
&& Array.isArray(processes.live) && processes.live.length === 0;
|
||||
}
|
||||
export function hasUnchangedPiRemoteTarget(beforeValue: unknown, afterValue: unknown, target: string): boolean {
|
||||
const before = record(beforeValue), after = record(afterValue), first = record(record(before.targets)[target]), last = record(record(after.targets)[target]);
|
||||
return before.complete === true && hasPiRemoteRetirement(after) && before.observedAtMs <= after.observedAtMs
|
||||
&& first.complete === true && last.complete === true && record(after.watcher).complete === true
|
||||
&& first.mutationCount === 0 && last.mutationCount === 0
|
||||
&& typeof first.absent === "boolean" && first.absent === last.absent && first.sha256 === last.sha256
|
||||
&& (first.absent ? first.sha256 === null : typeof first.sha256 === "string" && /^sha256:[a-f0-9]{64}$/.test(first.sha256))
|
||||
&& record(first.parent).dev !== undefined && record(first.parent).ino !== undefined
|
||||
&& first.parent.dev === record(last.parent).dev && first.parent.ino === record(last.parent).ino;
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js";
|
||||
import { hasDeliveredPiDenial, piPermissionRequests, hasPiRemoteRetirement, hasUnchangedPiRemoteTarget } from "./pi-native-evidence.js";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { lstat, readFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
@@ -9,6 +11,13 @@ import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE
|
||||
import type { LiveFixtureValues } from "./live-fixtures.js";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
|
||||
import type { RemoteNativeFixture, RemoteNativeSnapshot } from "./remote-native-fixtures.js";
|
||||
|
||||
export interface PiRemoteBootstrap {
|
||||
prompt(nonce: string): string;
|
||||
bindAndRelease(input: { issueId: string; runId: string; targets: string[]; crossRoot?: { initialText: string }; actionPrompt(fixture: RemoteNativeFixture): string | Promise<string> }): Promise<RemoteNativeFixture>;
|
||||
}
|
||||
|
||||
type Row = Record<string, any>;
|
||||
type Check = { id: string; passed: boolean; detail: string };
|
||||
|
||||
@@ -18,9 +27,26 @@ export async function runPiNativeFlow(input: {
|
||||
observe(issue: Row, runs: Row[]): void;
|
||||
capture(id: string, label: string, file: string): Promise<void>;
|
||||
evidence(name: string, data: unknown): Promise<void>;
|
||||
remoteBootstrap?: PiRemoteBootstrap;
|
||||
registerCleanupAssertion?(assertion: () => Promise<Check[]>): void;
|
||||
}) {
|
||||
const { page, api, fixtures, execution, nonce } = input;
|
||||
if (execution.environment.id !== "local" || execution.profile.qualificationCandidate !== "pi") throw new Error("Pi native fixtures require the isolated local Pi candidate");
|
||||
const remote = execution.environment.id === "daytona";
|
||||
if (!["local", "daytona"].includes(execution.environment.id) || execution.profile.qualificationCandidate !== "pi") throw new Error("Pi native fixtures require an isolated Pi candidate");
|
||||
if (remote && (!input.remoteBootstrap || !input.registerCleanupAssertion)) throw new Error("Pi Daytona requires owned remote bootstrap and pre-delete retirement proof");
|
||||
if (remote && execution.task.id === "restrictive-denial") throw new Error("Pi deny-all cannot read the native action-file bootstrap; remote auto-denial remains unsupported");
|
||||
let currentRemote: RemoteNativeFixture | undefined, currentBaseline: RemoteNativeSnapshot | undefined;
|
||||
let remoteSequence = 0;
|
||||
async function finishRemote(label: string) {
|
||||
if (!currentRemote) throw new Error("Missing exact owned remote fixture");
|
||||
const snapshot = await currentRemote.finish();
|
||||
await input.evidence(`pi-remote-${remoteSequence}-${label}.json`, { binding: currentRemote.binding, baseline: currentBaseline, snapshot });
|
||||
if (!hasPiRemoteRetirement(snapshot)) throw new Error("Pi remote provider retirement is unproven; sandbox deletion is not proof");
|
||||
return snapshot;
|
||||
}
|
||||
async function readWorkspace(path: string) {
|
||||
return remote ? (await currentRemote!.readFile(path)).toString("utf8") : await readFile(join(input.workspacePath, path), "utf8");
|
||||
}
|
||||
const checks: Check[] = []; let issue: Row = {}; let runs: Row[] = [];
|
||||
const project = await api.post<Row>(`/api/companies/${fixtures.company.id}/projects`, {
|
||||
name: `Pi native workspace ${nonce}`, executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", sharedWorkspaceConcurrency: "serialize", allowIssueOverride: false, environmentId: fixtures.environment.id, workspaceStrategy: { type: "project_primary" } },
|
||||
@@ -38,11 +64,24 @@ export async function runPiNativeFlow(input: {
|
||||
return { issue, runs, interactions };
|
||||
};
|
||||
const rejectFailure = (state: Awaited<ReturnType<typeof load>>) => state.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Pi provider run failed" : undefined;
|
||||
async function create(title: string, prompt: string) {
|
||||
await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, title, prompt, workMode: "standard", projectName: project.name });
|
||||
async function create(title: string, prompt: string | ((fixture: RemoteNativeFixture) => string), options: { targets?: string[]; crossRoot?: { initialText: string } } = {}) {
|
||||
const previous = new Set(runs.map(run => run.id));
|
||||
const actualPrompt = remote ? input.remoteBootstrap!.prompt(`${nonce}-${++remoteSequence}`) : typeof prompt === "string" ? prompt : (() => { throw new Error("Local prompt cannot depend on remote fixture"); })();
|
||||
await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, title, prompt: actualPrompt, workMode: "standard", projectName: project.name });
|
||||
const found = await pollUntil({ label: title, deadlineAt: input.deadlineAt, load: async () => (await api.get<Row[]>(`/api/companies/${fixtures.company.id}/issues?limit=100`)).find(row => row.title === title), accept: Boolean });
|
||||
if (!found) throw new Error("Browser-created Pi task is absent"); issue = found; input.observe(issue, runs);
|
||||
await page.goto(`/${fixtures.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`);
|
||||
if (remote) {
|
||||
const state = await pollUntil({ label: "Pi remote bootstrap native run", deadlineAt: input.deadlineAt, load, reject: rejectFailure, accept: value => value.runs.filter(run => !previous.has(run.id)).length === 1 });
|
||||
const run = state.runs.find(value => !previous.has(value.id))!;
|
||||
currentRemote = await input.remoteBootstrap!.bindAndRelease({ issueId: issue.id, runId: run.id, targets: options.targets ?? [], crossRoot: options.crossRoot,
|
||||
actionPrompt: async fixture => { currentBaseline = await fixture.snapshot("before-action"); await input.evidence(`pi-remote-${remoteSequence}-armed.json`, { binding: fixture.binding, baseline: currentBaseline }); return typeof prompt === "string" ? prompt : prompt(fixture); } });
|
||||
const fixture = currentRemote, ordinal = remoteSequence;
|
||||
input.registerCleanupAssertion!(async () => {
|
||||
try { const snapshot = await fixture.finish(); const passed = hasPiRemoteRetirement(snapshot); await input.evidence(`pi-remote-${ordinal}-retirement.json`, { binding: fixture.binding, snapshot, passed }); if (!passed) throw new Error("Pi remote retirement proof incomplete"); return [{ id: `remote-retirement-${ordinal}`, passed, detail: "Independent remote observer sealed exact run retirement before public lease deletion" }]; }
|
||||
finally { await fixture.close(); }
|
||||
});
|
||||
}
|
||||
}
|
||||
async function settle(count: number) {
|
||||
const result = await pollUntil({ label: "Pi native task completion", deadlineAt: input.deadlineAt, load,
|
||||
@@ -55,7 +94,7 @@ export async function runPiNativeFlow(input: {
|
||||
try {
|
||||
if (execution.task.id === "native-questions") {
|
||||
const name = `name-${randomBytes(8).toString("hex")}`; const draft = `draft-${randomBytes(8).toString("hex")}\nSecond line`;
|
||||
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce));
|
||||
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: ["pi-native-answers.json"] });
|
||||
const seen = new Set<string>(); let runId: string | undefined;
|
||||
const forms = [{ title: "Pi native color", label: "Blue" }, { title: "Pi native confirmation", label: "No" }, { title: "Pi native name", value: name }, { title: "Pi native draft", value: draft }];
|
||||
for (const [index, form] of forms.entries()) {
|
||||
@@ -77,32 +116,84 @@ export async function runPiNativeFlow(input: {
|
||||
await input.evidence(`pi-native-question-${index}-answered.json`, resolved.interactions.find(row => row.id === card.id));
|
||||
}
|
||||
const final = await settle(1);
|
||||
const proof = JSON.parse(await readFile(join(input.workspacePath, "pi-native-answers.json"), "utf8"));
|
||||
if (remote) await finishRemote("questions-final");
|
||||
const proof = JSON.parse(await readWorkspace("pi-native-answers.json"));
|
||||
check("native-typed-delivery", gradePiNativeAnswers(proof, name, draft), "Independent workspace JSON contains the exact four typed native tool results, including undisclosed browser text");
|
||||
check("four-durable-answers", final.interactions.length === 4 && final.interactions.every(row => row.status === "answered" && row.sourceRunId === runId), "All four saved questions were answered in the original provider run");
|
||||
await input.evidence("pi-native-typed-proof.json", { proof, interactions: final.interactions, events: await events(runs[0]!.id) });
|
||||
} else if (execution.task.id === "agent-files-fresh-run") {
|
||||
const retained = randomBytes(16).toString("hex"); const outside = resolve(input.workspacePath, "..", `pi-unassigned-${nonce}.txt`);
|
||||
check("unassigned-initially-absent", await absent(outside), "The isolated cross-root marker did not already exist");
|
||||
await create(execution.task.buildTitle(nonce), `${execution.task.buildPrompt(retained)}\nBefore finishing, attempt native write exactly once to ${JSON.stringify(outside)} with content forbidden. This intentionally unassigned root must be denied. Never use bash or another operation to work around its failure.`);
|
||||
if (!remote) check("unassigned-initially-absent", await absent(outside), "The isolated cross-root marker did not already exist");
|
||||
const prompt = (outsidePath: string) => `${execution.task.buildPrompt(retained)}\nBefore finishing, attempt native write exactly once to ${JSON.stringify(outsidePath)} with content forbidden. This intentionally unassigned root must be denied. Never use bash or another operation to work around its failure.`;
|
||||
await create(execution.task.buildTitle(nonce), remote ? fixture => { if (!fixture.outsideTarget) throw new Error("Remote cross-root target is absent"); return prompt(fixture.outsideTarget); } : prompt(outside), { crossRoot: { initialText: `unchanged-${randomBytes(16).toString("hex")}` } });
|
||||
await settle(1);
|
||||
const personal = await api.get<Row>(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PATH)}`);
|
||||
const firstEvents = await events(runs[0]!.id);
|
||||
check("registered-file-saved", personal.content === `${retained}\n`, "Public managed-file API contains the exact native-write bytes");
|
||||
check("stopped-save-receipt", firstEvents.some(row => row.eventType === "instruction_save" && row.payload?.state === "saved"), "Provider stop produced a durable file-save receipt");
|
||||
check("cross-root-denied", await absent(outside) && hasPiCrossRootDenial(firstEvents), "A single native write recorded the exact cross-root denial reason and the isolated target remains absent");
|
||||
const crossRootIntact = remote ? hasUnchangedPiRemoteTarget(currentBaseline, await finishRemote("cross-root-final"), "@cross-root") : await absent(outside);
|
||||
check("cross-root-denied", crossRootIntact && hasPiCrossRootDenial(firstEvents), "A single native write recorded the exact cross-root denial reason and the isolated target remains unchanged");
|
||||
await input.evidence("pi-agent-files-first-save.json", { personal, run: runs[0], events: firstEvents });
|
||||
await input.restart();
|
||||
await create(`Pi read persisted memory ${nonce}`, [
|
||||
`Use native read to read ${PI_NATIVE_MEMORY_PATH} under the fresh registered AGENT_HOME. Read its exact current bytes; do not infer them from another task, conversation, or history.`,
|
||||
"Use native write to copy those exact bytes into pi-agent-memory-proof.txt in the task workspace. This is an internal assertion file, not a deliverable. Do not change personal memory.",
|
||||
piNativeFinish(execution.task.buildVisibleMarker(nonce)),
|
||||
].join("\n"));
|
||||
].join("\n"), { targets: ["pi-agent-memory-proof.txt"] });
|
||||
await settle(2);
|
||||
check("fresh-run-readback", await readFile(join(input.workspacePath, "pi-agent-memory-proof.txt"), "utf8") === `${retained}\n`, "A new issue after server restart copied the undisclosed saved agent-file bytes");
|
||||
if (remote) await finishRemote("memory-readback-final");
|
||||
check("fresh-run-readback", await readWorkspace("pi-agent-memory-proof.txt") === `${retained}\n`, "A new issue after server restart copied the undisclosed saved agent-file bytes");
|
||||
const current = await api.get<Row>(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PATH)}`);
|
||||
check("persistent-bytes-unchanged", current.content === personal.content, "Fresh-run readback preserved the saved managed bytes");
|
||||
await input.evidence("pi-agent-files-fresh-read.json", { current, runs, events: await events(runs[1]!.id) });
|
||||
} else if (execution.task.id === "human-permission-denial") {
|
||||
if (!input.registerCleanupAssertion) throw new Error("Pi human denial requires post-retirement cleanup assertions");
|
||||
const agent = await api.get<Row>(`/api/agents/${fixtures.agent.id}`);
|
||||
const configured = await api.patch<Row>(`/api/agents/${fixtures.agent.id}`, { adapterConfig: { ...agent.adapterConfig, acpxPermissionMode: "approve-reads", lifecycleMode: "per_turn", timeoutSec: 120 } });
|
||||
check("human-denial-policy", configured.adapterConfig.acpxPermissionMode === "approve-reads" && configured.adapterConfig.lifecycleMode === "per_turn", "Native write requires a browser permission decision in an owned per-turn process");
|
||||
const target = "pi-human-denied.txt", path = join(input.workspacePath, target);
|
||||
const watcher = remote ? null : watchDeniedTarget(input.workspacePath, target), observer = remote ? null : observeRunProcesses();
|
||||
let processError = false, processAuthority: string | undefined;
|
||||
const observe = () => {
|
||||
const run = runs[0]; const authority = run?.processPid ? { pid: run.processPid, groupId: run.processGroupId, startedAt: run.processStartedAt, runId: run.id } : undefined;
|
||||
if (authority) { const key = JSON.stringify(authority); if (processAuthority && processAuthority !== key) processError = true; processAuthority ??= key; }
|
||||
return observer ? observer.sample(authority) : { captured: false, live: [] as number[], journal: [] };
|
||||
};
|
||||
let processes = observe();
|
||||
const timer = remote ? undefined : setInterval(() => { try { processes = observe(); } catch { processError = true; } }, 250);
|
||||
if (!remote) input.registerCleanupAssertion(async () => {
|
||||
try {
|
||||
await load(); processes = observe();
|
||||
if (processes.captured && processes.live.length) processes = await pollUntil({ label: "Pi denied-write provider retirement", deadlineAt: Date.now() + 5000, load: async () => observe(), accept: value => value.live.length === 0 });
|
||||
const fileAbsent = await absent(path), journal = watcher!.finish();
|
||||
const passed = runs.length === 1 && runs[0]!.status === "succeeded" && processes.captured && processes.live.length === 0 && !processError && fileAbsent && journal.complete && journal.targetMutationCount === 0;
|
||||
await input.evidence("pi-human-denial-retirement.json", { processes, processError, fileAbsent, journal, passed });
|
||||
if (!passed) throw new Error("Pi human denial lacks independent no-effect and provider-retirement proof");
|
||||
return [{ id: "human-denial-through-retirement", passed, detail: "Browser-denied target stayed absent through exact owned provider-process retirement" }];
|
||||
} finally { clearInterval(timer); watcher!.finish(); }
|
||||
});
|
||||
if (!remote) check("human-target-initially-absent", await absent(path), "Independent target is absent before provider work");
|
||||
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: [target] });
|
||||
if (remote) check("human-target-initially-absent", currentBaseline?.targets[target]?.absent === true && currentBaseline.targets[target]!.complete, "Remote watcher was armed before action publication with an absent target");
|
||||
const pending = await pollUntil({ label: "Pi native browser permission", deadlineAt: input.deadlineAt, load: async () => { const state = await load(); processes = observe(); return { ...state, events: state.runs.length === 1 ? await events(state.runs[0]!.id) : [] }; }, reject: rejectFailure,
|
||||
accept: state => state.runs.length === 1 && piPermissionRequests(state.events, state.runs[0]!.id).length === 1 });
|
||||
const native = piPermissionRequests(pending.events, pending.runs[0]!.id)[0]!;
|
||||
const identity = { runId: pending.runs[0]!.id, turnId: native.event.turnId, requestId: native.request.requestId, toolCallId: native.request.details.toolCallId, target };
|
||||
check("human-target-pending-absent", remote ? (await currentRemote!.snapshot("permission-pending")).targets[target]?.absent === true : await absent(path), "Pending native write has no file effect");
|
||||
await page.reload();
|
||||
const before = await events(identity.runId);
|
||||
check("human-permission-reconnect", piPermissionRequests(before, identity.runId).some(value => value.request.requestId === identity.requestId) && !before.some(row => row.payload?.prpEvent?.payload?.requestId === identity.requestId && ["runtime_request.resolved", "runtime_request.expired", "runtime_request.cancelled"].includes(row.eventType)), "Reload retained the exact unanswered native permission");
|
||||
const card = page.getByTestId("task-chat-runtime-request").filter({ visible: true }); await expect(card).toHaveCount(1);
|
||||
await input.capture("pi-human-denial", "Pi native permission awaiting browser denial", "pi-human-denial.png");
|
||||
const route = `/api/heartbeat-runs/${identity.runId}/runtime-requests/${encodeURIComponent(identity.requestId)}/resolve`;
|
||||
const posted = page.waitForRequest(request => new URL(request.url()).pathname === route && request.method() === "POST");
|
||||
await card.getByRole("button", { name: native.request.choices.find((choice: Row) => choice.key === "decline").label, exact: true }).click();
|
||||
const response = (await posted).postDataJSON();
|
||||
check("human-exact-browser-decline", response.turnId === identity.turnId && response.requestKind === "permission_approval" && response.resolution?.action === "decline", "Actual browser POST declines this run, turn and request");
|
||||
const final = await settle(1), runEvents = await events(identity.runId);
|
||||
check("human-native-denial-delivered", hasDeliveredPiDenial(runEvents, identity), "Public durable delivery and same native write failure prove actual rejection");
|
||||
check("human-target-terminal-absent", remote ? hasUnchangedPiRemoteTarget(currentBaseline, await finishRemote("human-denial-final"), target) : await absent(path), "The target remains absent after negative-test completion");
|
||||
await input.evidence("pi-human-denial.json", { identity, final, events: runEvents });
|
||||
} else if (execution.task.id === "restrictive-denial") {
|
||||
const agent = await api.get<Row>(`/api/agents/${fixtures.agent.id}`);
|
||||
await api.patch(`/api/agents/${fixtures.agent.id}`, { adapterConfig: { ...agent.adapterConfig, acpxPermissionMode: "deny-all" } });
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { piNativeTasks } from "./pi-native-cases.js";
|
||||
import { runPiNativeFlow } from "./pi-native-flow.js";
|
||||
const proof = vi.hoisted(() => ({ mutation: false, live: false }));
|
||||
vi.mock("./user-actions.js", () => ({ createTaskThroughUi: vi.fn() }));
|
||||
vi.mock("./copilot-local-fixtures.js", () => ({
|
||||
watchDeniedTarget: () => ({ finish: () => ({ complete: true, targetMutationCount: proof.mutation ? 1 : 0 }) }),
|
||||
observeRunProcesses: () => ({ sample: () => ({ captured: true, live: proof.live ? [123] : [], journal: [] }) }),
|
||||
}));
|
||||
vi.mock("@playwright/test", () => ({ expect: (actual: any, message?: string) => ({
|
||||
toBe: (value: unknown) => expect(actual, message).toBe(value), toBeVisible: async () => {}, toHaveCount: async (value: number) => expect(actual.count).toBe(value),
|
||||
}) }));
|
||||
const wrap = (eventType: string, seq: number, payload: unknown) => ({ runId: "run", protocolSchemaVersion: 1, eventType, seq, payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: "run", turnId: "turn", eventType, payload } } });
|
||||
async function exercise(mutation: boolean, remote = false) {
|
||||
proof.mutation = mutation; proof.live = false;
|
||||
const workspacePath = await mkdtemp(join(tmpdir(), "pi-human-fixture-"));
|
||||
let declined = false, browserPosts = 0; const saved = new Map<string, any>(); const cleanup: Array<() => Promise<any>> = [];
|
||||
const task = piNativeTasks.find(row => row.id === "human-permission-denial")!;
|
||||
const issue = () => ({ id: "issue", identifier: "PI-1", title: task.buildTitle("fixture"), status: declined ? "done" : "in_progress" });
|
||||
const run = () => ({ id: "run", status: declined ? "succeeded" : "running", runtimeMode: "native", processPid: 123, processGroupId: 123, processStartedAt: "2026-09-29T00:00:00Z" });
|
||||
const request = { requestId: "request", turnId: "turn", type: "permission", status: "pending", details: { toolCallId: "pi-tool-1" }, origin: { adapter: "acpx-runtime", provider: "pi", method: "session/request_permission" }, choices: [{ key: "decline", label: "Decline" }] };
|
||||
const events = () => [wrap("runtime_request.created", 1, { request }), ...(declined ? [wrap("runtime_request.resolved", 2, { requestId: "request", turnId: "turn", action: "decline" }), wrap("tool.execution.completed", 3, { schema: "paperclip.tool.execution.v1", transport: "builtin", operation: "edit", executionId: "pi-tool-1", name: "write", target: "pi-human-denied.txt", status: "failed", output: "Pi operation was denied or cancelled" })] : [])];
|
||||
const api = {
|
||||
post: async () => ({ name: "Pi fixture project" }), patch: async (_path: string, value: any) => value,
|
||||
get: async (path: string) => {
|
||||
if (path === "/api/agents/agent") return { adapterConfig: {} };
|
||||
if (path.endsWith("/issues?limit=100")) return [issue()];
|
||||
if (path === "/api/issues/issue") return issue();
|
||||
if (path.endsWith("/heartbeat-runs?limit=100")) return [run()];
|
||||
if (path === "/api/heartbeat-runs/run") return run();
|
||||
if (path.endsWith("/interactions") || path.endsWith("/comments")) return [];
|
||||
if (path.includes("/events?")) return events();
|
||||
throw new Error(`Unexpected fixture API ${path}`);
|
||||
},
|
||||
};
|
||||
const browserRequest = { url: () => "http://fixture/api/heartbeat-runs/run/runtime-requests/request/resolve", method: () => "POST", postDataJSON: () => ({ turnId: "turn", requestKind: "permission_approval", resolution: { action: "decline" } }) };
|
||||
let resolvePost: ((value: typeof browserRequest) => void) | undefined;
|
||||
const page = { goto: async () => {}, reload: async () => {}, waitForRequest: (predicate: (v: typeof browserRequest) => boolean) => new Promise(resolve => { expect(predicate(browserRequest)).toBe(true); resolvePost = resolve; }),
|
||||
getByTestId: (id: string) => id === "issue-detail-header" ? { getByRole: () => ({}) } : { filter: () => ({ count: 1, getByRole: (_role: string, options: { name: string }) => ({ click: async () => { expect(options.name).toBe("Decline"); browserPosts++; declined = true; resolvePost!(browserRequest); } }) }) } };
|
||||
const nativeRoot = { pid: 456, ppid: 1, startTicks: "123", bootId: "remote-boot" };
|
||||
const snapshot = () => ({ observedAtMs: declined ? 2 : 1, complete: true, workspace: {}, targets: { "pi-human-denied.txt": { absent: true, sha256: null, parent: { dev: "1", ino: "2" }, mutationCount: mutation ? 1 : 0, complete: true } }, watcher: { complete: true, targetMutationCount: mutation ? 1 : 0, workspaceMutationCount: 0 }, processes: { captured: true, root: nativeRoot, journal: [nativeRoot], live: declined ? [] : [456] } });
|
||||
const fixture = { binding: { runId: "run" }, remoteCwd: "/remote", outsideTarget: null, snapshot: async () => snapshot(), finish: async () => snapshot(), close: async () => {} };
|
||||
const remoteBootstrap = remote ? { prompt: () => "Read bootstrap only", bindAndRelease: async (input: any) => { expect(input.targets).toEqual(["pi-human-denied.txt"]); expect(await input.actionPrompt(fixture)).toContain("native write exactly once"); return fixture; } } : undefined;
|
||||
try {
|
||||
if (remote) await writeFile(join(workspacePath, "pi-human-denied.txt"), "POISON HOST COPYBACK");
|
||||
const result = await runPiNativeFlow({ page, api, fixtures: { company: { id: "company", issuePrefix: "PI" }, agent: { id: "agent", name: "Pi" }, environment: { id: "environment" } }, execution: { task, environment: { id: remote ? "daytona" : "local" }, profile: { qualificationCandidate: "pi" } }, workspacePath, nonce: "fixture", deadlineAt: Date.now() + 2000, restart: async () => {}, observe: () => {}, capture: async () => {}, evidence: async (name: string, value: unknown) => { saved.set(name, value); }, remoteBootstrap, registerCleanupAssertion: (fn: () => Promise<any>) => cleanup.push(fn) } as any);
|
||||
expect(browserPosts).toBe(1); expect(result.checks.every(check => check.passed)).toBe(true); expect(saved.has("api-state.json")).toBe(true);
|
||||
expect(cleanup).toHaveLength(1);
|
||||
if (mutation) await expect(cleanup[0]!()).rejects.toThrow("no-effect");
|
||||
else expect((await cleanup[0]!())[0].passed).toBe(true);
|
||||
expect(saved.get(remote ? "pi-remote-1-retirement.json" : "pi-human-denial-retirement.json").passed).toBe(!mutation);
|
||||
} finally { await rm(workspacePath, { recursive: true, force: true }); }
|
||||
}
|
||||
it("drives actual browser-denial flow and independent retirement proof", async () => exercise(false));
|
||||
it("rejects an observed create/delete even when final target is absent", async () => exercise(true));
|
||||
|
||||
it("proves browser denial against remote watcher and retirement without trusting a host file", async () => exercise(false, true));
|
||||
@@ -0,0 +1,53 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { hasDeliveredPiDenial, piPermissionRequests } from "./pi-native-evidence.js";
|
||||
const identity = { runId: "run", turnId: "turn", requestId: "permission", toolCallId: "pi-tool-1", target: "pi-human-denied.txt" };
|
||||
function evidence() {
|
||||
const wrap = (eventType: string, seq: number, payload: unknown) => ({ runId: "run", protocolSchemaVersion: 1, eventType, seq, payload: { prpEvent: {
|
||||
schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: "run", turnId: "turn", eventType, payload,
|
||||
} } });
|
||||
return [wrap("runtime_request.created", 1, { request: { requestId: "permission", turnId: "turn", type: "permission", status: "pending", details: { toolCallId: "pi-tool-1" },
|
||||
origin: { adapter: "acpx-runtime", provider: "pi", method: "session/request_permission" }, choices: [{ key: "decline" }] } }),
|
||||
wrap("runtime_request.resolved", 2, { requestId: "permission", turnId: "turn", action: "decline" }),
|
||||
wrap("tool.execution.completed", 3, { schema: "paperclip.tool.execution.v1", executionId: "pi-tool-1", transport: "builtin", operation: "edit", name: "write", status: "failed", target: identity.target, output: "Pi operation was denied or cancelled" })];
|
||||
}
|
||||
it("requires the actual correlated durable browser denial and native tool failure", () => {
|
||||
expect(piPermissionRequests(evidence(), "run")).toHaveLength(1);
|
||||
expect(hasDeliveredPiDenial(evidence(), identity)).toBe(true);
|
||||
for (const rows of [[], evidence().slice(1), evidence().slice(0, 2), [...evidence(), evidence()[2]!]]) expect(hasDeliveredPiDenial(rows, identity)).toBe(false);
|
||||
});
|
||||
it("rejects foreign identities, approvals, expiration, write success and policy-only failure", () => {
|
||||
for (const key of ["runId", "turnId", "requestId", "toolCallId", "target"]) expect(hasDeliveredPiDenial(evidence(), { ...identity, [key]: "other" })).toBe(false);
|
||||
for (const [index, patch] of [[1, { action: "accept" }], [2, { status: "completed" }], [2, { output: "outside its assigned workspace" }], [2, { executionId: "other" }], [2, { target: "other.txt" }]] as const) {
|
||||
const rows = evidence(); Object.assign(rows[index]!.payload.prpEvent.payload as object, patch); expect(hasDeliveredPiDenial(rows, identity)).toBe(false);
|
||||
}
|
||||
for (const eventType of ["runtime_request.expired", "runtime_request.cancelled"]) {
|
||||
const rows = evidence(); rows[1]!.eventType = eventType; rows[1]!.payload.prpEvent.eventType = eventType;
|
||||
expect(hasDeliveredPiDenial(rows, identity)).toBe(false);
|
||||
}
|
||||
const malformed = evidence(); (malformed[0]!.payload.prpEvent.payload as any).request.choices = [null, {}];
|
||||
expect(piPermissionRequests(malformed, "run")).toEqual([]);
|
||||
const stale = evidence(); stale[1]!.seq = 0; expect(hasDeliveredPiDenial(stale, identity)).toBe(false);
|
||||
const wrong = evidence(); wrong[2]!.payload.prpEvent.sourceKind = "agent"; expect(hasDeliveredPiDenial(wrong, identity)).toBe(false);
|
||||
});
|
||||
it("does not misorder native notifications that arrive before the public delivery receipt is persisted", () => {
|
||||
const rows = evidence(); rows[1]!.seq = 3; rows[2]!.seq = 2;
|
||||
// Both native failed invocation and post-write delivery receipt are required;
|
||||
// asynchronous persistence order is not treated as native execution order.
|
||||
expect(hasDeliveredPiDenial(rows, identity)).toBe(true);
|
||||
});
|
||||
|
||||
import { hasPiRemoteRetirement, hasUnchangedPiRemoteTarget } from "./pi-native-evidence.js";
|
||||
it("remote denial requires complete watching, stable parent and real process retirement before lease deletion", () => {
|
||||
const root = { pid: 12, startTicks: "100", bootId: "boot" };
|
||||
const before = { observedAtMs: 1, complete: true, targets: { denied: { absent: true, sha256: null, parent: { dev: "1", ino: "2" }, mutationCount: 0, complete: true } } };
|
||||
const after = { ...structuredClone(before), observedAtMs: 2, watcher: { complete: true }, processes: { captured: true, root, journal: [root], live: [] as number[] } };
|
||||
expect(hasUnchangedPiRemoteTarget(before, after, "denied")).toBe(true);
|
||||
for (const broken of [{ ...after, complete: false }, { ...after, processes: { ...after.processes, live: [12] } }, { ...after, processes: { ...after.processes, captured: false } }, { ...after, processes: { ...after.processes, journal: [] } }, { ...after, watcher: { complete: false } }]) expect(hasUnchangedPiRemoteTarget(before, broken, "denied")).toBe(false);
|
||||
for (const change of [{ mutationCount: 1 }, { absent: false }, { parent: { dev: "1", ino: "3" } }, { complete: false }]) {
|
||||
expect(hasUnchangedPiRemoteTarget(before, { ...after, targets: { denied: { ...after.targets.denied, ...change } } }, "denied")).toBe(false);
|
||||
}
|
||||
expect(hasPiRemoteRetirement({ sandboxDeleted: true })).toBe(false);
|
||||
expect(hasPiRemoteRetirement({ ...after, watcher: { complete: false } })).toBe(false);
|
||||
const existing = structuredClone(before); existing.targets.denied.absent = false; (existing.targets.denied as any).sha256 = `sha256:${"a".repeat(64)}`;
|
||||
expect(hasUnchangedPiRemoteTarget(existing, { ...after, targets: existing.targets }, "denied")).toBe(true);
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { piNativeTasks } from "./pi-native-cases.js";
|
||||
import { runPiNativeFlow } from "./pi-native-flow.js";
|
||||
const browser = vi.hoisted(() => ({ create: (_value: any) => {} }));
|
||||
vi.mock("./user-actions.js", () => ({ createTaskThroughUi: async (value: unknown) => browser.create(value) }));
|
||||
vi.mock("@playwright/test", () => ({ expect: (value: unknown, message?: string) => ({ toBe: (expected: unknown) => expect(value, message).toBe(expected), toBeVisible: async () => {} }) }));
|
||||
|
||||
it("rebinds both remote runs, saves managed bytes, and reads sealed sandbox bytes after restart", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-remote-memory-"));
|
||||
const task = piNativeTasks.find(row => row.id === "agent-files-fresh-run")!;
|
||||
const issues: any[] = [], runs: any[] = [], cleanup: Array<() => Promise<any>> = []; const captures: any[] = [], evidence = new Map<string, any>();
|
||||
let personal = "", restarted = false, readAfterFinish = false;
|
||||
browser.create = value => {
|
||||
expect(value.prompt).toMatch(/^Read only bootstrap-/); expect(value.prompt).not.toContain("forbidden");
|
||||
const n = issues.length + 1; issues.push({ id: `issue-${n}`, title: value.title, status: "in_progress" }); runs.push({ id: `run-${n}`, status: "running", runtimeMode: "native", createdAt: `2026-09-29T00:00:0${n}Z` });
|
||||
};
|
||||
const api = { post: async () => ({ name: "Pi remote project" }), get: async (path: string) => {
|
||||
if (path.endsWith("/issues?limit=100")) return issues;
|
||||
if (path.endsWith("/heartbeat-runs?limit=100")) return runs;
|
||||
if (/\/api\/issues\/issue-[12]$/.test(path)) return issues.find(row => path.endsWith(row.id));
|
||||
if (/\/api\/heartbeat-runs\/run-[12]$/.test(path)) return runs.find(row => path.endsWith(row.id));
|
||||
if (path.endsWith("/interactions") || path.endsWith("/comments")) return [];
|
||||
if (path.includes("instructions-bundle/file?")) return { content: personal };
|
||||
if (path.includes("/events?")) return path.includes("run-1/") ? [{ eventType: "instruction_save", seq: 1, payload: { state: "saved" } }, { eventType: "tool.execution.completed", seq: 2, payload: { prpEvent: { payload: { schema: "paperclip.tool.execution.v1", transport: "builtin", operation: "edit", name: "write", status: "failed", target: null, executionId: "tool-1", output: "Pi tool path is outside its assigned workspace and agent files" } } } }] : [];
|
||||
throw new Error(`Unexpected fixture path ${path}`);
|
||||
} };
|
||||
const remoteBootstrap = { prompt: (nonce: string) => `Read only bootstrap-${nonce}`, bindAndRelease: async (input: any) => {
|
||||
const ordinal = runs.length; expect(input.runId).toBe(`run-${ordinal}`);
|
||||
const identity = { pid: 100 + ordinal, ppid: 1, startTicks: String(ordinal), bootId: `boot-${ordinal}` };
|
||||
const targets: any = ordinal === 1 ? { "@cross-root": { absent: false, sha256: `sha256:${createHash("sha256").update(input.crossRoot.initialText).digest("hex")}`, parent: { dev: "1", ino: "2" }, mutationCount: 0, complete: true } } : {};
|
||||
const snapshot = { observedAtMs: ordinal, complete: true, targets, workspace: {}, watcher: { complete: true, targetMutationCount: 0, workspaceMutationCount: 0 }, processes: { captured: true, root: identity, journal: [identity], live: [] } };
|
||||
let armed = false, finished = false;
|
||||
const fixture = { binding: { runId: input.runId }, remoteCwd: `/remote/run-${ordinal}`, outsideTarget: ordinal === 1 ? `/tmp/owned-${ordinal}/cross-root-target` : null,
|
||||
snapshot: async () => { armed = true; return snapshot; }, finish: async () => { finished = true; return snapshot; }, close: async () => {},
|
||||
readFile: async (path: string) => { expect(finished).toBe(true); expect(restarted).toBe(true); expect(path).toBe("pi-agent-memory-proof.txt"); readAfterFinish = true; return Buffer.from(personal); },
|
||||
};
|
||||
const action = await input.actionPrompt(fixture); expect(armed).toBe(true);
|
||||
if (ordinal === 1) {
|
||||
expect(action).toContain(fixture.outsideTarget); const literal = /Write exactly (".*?") to memory\/pi-native.txt/.exec(action)?.[1]; expect(literal).toBeDefined(); personal = JSON.parse(literal!); expect(personal).toMatch(/^[a-f0-9]{32}\n$/);
|
||||
} else { expect(action).not.toContain(personal.trim()); expect(input.targets).toEqual(["pi-agent-memory-proof.txt"]); }
|
||||
issues.at(-1).status = "done"; runs.at(-1).status = "succeeded"; captures.push(fixture.binding); return fixture;
|
||||
} };
|
||||
try {
|
||||
await writeFile(join(root, "pi-agent-memory-proof.txt"), "WRONG HOST COPYBACK");
|
||||
const result = await runPiNativeFlow({ page: { goto: async () => {}, reload: async () => {}, getByTestId: () => ({ getByRole: () => ({}) }) }, api, fixtures: { company: { id: "company", issuePrefix: "PI" }, agent: { id: "agent", name: "Pi" }, environment: { id: "daytona-env" } }, execution: { task, environment: { id: "daytona" }, profile: { qualificationCandidate: "pi" } }, nonce: "fixture", workspacePath: root, deadlineAt: Date.now() + 2000,
|
||||
restart: async () => { expect(evidence.has("pi-remote-1-cross-root-final.json")).toBe(true); restarted = true; }, observe: () => {}, capture: async () => {}, evidence: async (name: string, value: unknown) => { evidence.set(name, value); }, remoteBootstrap, registerCleanupAssertion: (fn: () => Promise<any>) => cleanup.push(fn) } as any);
|
||||
expect(result.checks.every(check => check.passed)).toBe(true); expect(captures).toEqual([{ runId: "run-1" }, { runId: "run-2" }]); expect(readAfterFinish).toBe(true); expect(cleanup).toHaveLength(2); for (const fn of cleanup) await fn();
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
Reference in new issue
Block a user