fix(evals): recognize native ACPX sidecar requests

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-29 21:40:17 -05:00
1 parent e154e85394
commit 4376dbe0be
7 files changed
+113 -19

No files matched your search

+9
View File
@@ -0,0 +1,9 @@
/** The embedded driver and runtime sidecar are the two production ACPX bridges.
* Keep provider and native method exact; semantic tools are not native proof.
*/
export function hasAcpxNativeOrigin(origin: unknown, provider: string, method: string): boolean {
if (origin === null || typeof origin !== "object" || Array.isArray(origin)) return false;
const value = origin as Record<string, unknown>;
return (value.adapter === "acpx-runtime" || value.adapter === "acpx-runtime-sidecar")
&& value.provider === provider && value.method === method;
}
+54 -5
View File
@@ -1,15 +1,15 @@
import { parsePaperclipQuestionResponse } from "../../packages/paperclip-runner/src/contracts/question-set.js";
import { normalizeCursorPlanRequest } from "../../packages/paperclip-runner/src/drivers/acpx/cursor-extensions.js";
import { cursorNativePlanResponse } from "./cursor-native-flow.js";
import { cursorNativePlanResponse, findCursorNativeRequest, hasCursorNativeCardBinding } from "./cursor-native-flow.js";
import { cursorDeniedCommand, type CursorToolNotice } from "./cursor-native-evidence.js";
import { describe, expect, it } from "vitest";
import { cursorNativeCaseDesigns, cursorNativePrompt, hasDeliveredCursorNativeRequest, hasCursorPlanDecision, hasCursorDenialBoundary, CURSOR_DENIAL_SAMPLE_PHASES, hasExactCursorNativeResponse } from "./cursor-native-cases.js";
const proof = () => {
const proof = (adapter = "acpx-runtime") => {
const wrap = (eventType: string, sourceSeq: number, payload: unknown) => ({ runId: "run", protocolSchemaVersion: 1, payload: { prpEvent: {
schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: "run", turnId: "turn", eventType, sourceSeq, payload,
} } });
return [wrap("runtime_request.created", 1, { request: { requestId: "request", turnId: "turn", type: "input", status: "pending", origin: { adapter: "acpx-runtime", provider: "cursor", method: "cursor/ask_question" } } }),
return [wrap("runtime_request.created", 1, { request: { requestId: "request", turnId: "turn", type: "input", status: "pending", origin: { adapter, provider: "cursor", method: "cursor/ask_question" } } }),
wrap("runtime_request.resolved", 2, { requestId: "request", turnId: "turn", action: "submit" })];
};
const grade = (events: unknown[]) => hasDeliveredCursorNativeRequest({ events, runId: "run", turnId: "turn", requestId: "request", method: "cursor/ask_question", action: "submit" });
@@ -44,13 +44,16 @@ it("binds plan decisions to the full revision and rejects stale answers", () =>
expect(hasCursorPlanDecision({ questions: [{ id: `plan-${"b".repeat(64)}` }] }, answer, "accept")).toBe(false);
expect(hasCursorPlanDecision(questionSet, answer, "reject")).toBe(false);
});
it("requires supported denial choices, native IDs and complete no-effect boundaries", () => {
it.each(["acpx-runtime", "acpx-runtime-sidecar"])("requires supported denial choices, native IDs and complete no-effect boundaries via %s", adapter => {
const input = { request: { requestId: "request", turnId: "turn", type: "permission", status: "pending", details: { toolCallId: "tool" },
origin: { adapter: "acpx-runtime", provider: "cursor", method: "session/request_permission" },
origin: { adapter, provider: "cursor", method: "session/request_permission" },
choices: [{ key: "accept" }, { key: "decline" }, { key: "cancel" }] }, expectedRequestId: "request", expectedToolCallId: "tool", path: "/fixture/denied.txt", runId: "run", turnId: "turn",
notices: ([{ stage: "tool", status: "pending" }, { stage: "permission_requested", requestId: "request", declineOffered: true }, { stage: "permission_delivered", requestId: "request", outcome: "reject_once" }, { stage: "tool", status: "failed" }].map((row, index) => ({ ...row, seq: index + 1, runId: "run", sessionId: "session", turnId: "turn", toolCallId: "tool", operation: "execute", commandSha256: cursorDeniedCommand("/fixture/denied.txt").commandSha256 })) as CursorToolNotice[]),
samples: CURSOR_DENIAL_SAMPLE_PHASES.map((phase, observedAt) => ({ phase, observedAt, absent: true, path: "/fixture/denied.txt" })) };
expect(hasCursorDenialBoundary(input)).toBe(true);
for (const origin of [{ ...input.request.origin, adapter: "semantic" }, { ...input.request.origin, provider: "pi" }, { ...input.request.origin, method: "request_human_input" }]) {
expect(hasCursorDenialBoundary({ ...input, request: { ...input.request, origin } })).toBe(false);
}
expect(hasCursorDenialBoundary({ ...input, samples: input.samples.slice(1) })).toBe(false);
expect(hasCursorDenialBoundary({ ...input, expectedToolCallId: "foreign" })).toBe(false);
input.samples[2]!.absent = false; expect(hasCursorDenialBoundary(input)).toBe(false); input.samples[2]!.absent = true;
@@ -85,3 +88,49 @@ it.each(["accept", "cancel", "reject"] as const)("recognizes canonical delivered
expect(hasExactCursorNativeResponse({ ...input, response: { ...response, answers: { ...response.answers, reason: {} } } })).toBe(false);
}
});
// Retained paid sidecar plan callback exposed both origin and JSONB key-order drift.
it.each(["acpx-runtime", "acpx-runtime-sidecar"])("recognizes a native plan and its durable card via %s", adapter => {
const input = normalizeCursorPlanRequest({ toolCallId: "native-plan", name: "Fixture plan", plan: "Verify complete Markdown.", todos: [{ id: "first", content: "Read", status: "pending" }, { id: "second", content: "Check", status: "pending" }] }).questionSet;
const rows = proof(adapter);
const request = (rows[0]!.payload.prpEvent.payload as any).request;
request.origin.method = "cursor/create_plan";
request.input = input;
const event = findCursorNativeRequest(rows, "cursor/create_plan", "request")!;
expect(event).toBe(rows[0]!.payload.prpEvent);
expect(hasDeliveredCursorNativeRequest({ events: rows, runId: "run", turnId: "turn", requestId: "request", method: "cursor/create_plan", action: "submit" })).toBe(true);
const reorder = (value: any): any => Array.isArray(value) ? value.map(reorder) : value !== null && typeof value === "object"
? Object.fromEntries(Object.entries(value).reverse().map(([key, nested]) => [key, reorder(nested)])) : value;
const card = { sourceRunId: "run", continuationPolicy: "none", payload: { runtimeRequestId: "request", questionSet: reorder(input) } };
expect(JSON.stringify(card.payload.questionSet)).not.toBe(JSON.stringify(input));
expect(hasCursorNativeCardBinding(card, event, "run")).toBe(true);
expect(hasCursorNativeCardBinding(card, event, "other-run")).toBe(false);
expect(hasCursorNativeCardBinding({ ...card, continuationPolicy: "resume" }, event, "run")).toBe(false);
const changed = structuredClone(card);
changed.payload.questionSet.questions[0].options[0].label = "Changed choice";
expect(hasCursorNativeCardBinding(changed, event, "run")).toBe(false);
const reordered = structuredClone(card);
reordered.payload.questionSet.questions[0].options.reverse();
expect(hasCursorNativeCardBinding(reordered, event, "run")).toBe(false);
expect(findCursorNativeRequest(rows, "cursor/create_plan", "other-request")).toBeUndefined();
expect(findCursorNativeRequest(rows, "cursor/ask_question", "request")).toBeUndefined();
});
it.each(["acpx-runtime", "acpx-runtime-sidecar"])("rejects foreign native request identity or origin via %s", adapter => {
for (const patch of [{ adapter: "unknown" }, { adapter: "acpx-runtime-sidecar-unknown" }, { adapter: "semantic" }, { provider: "pi" }, { method: "request_human_input" }]) {
const rows = proof(adapter);
Object.assign((rows[0]!.payload.prpEvent.payload as any).request.origin, patch);
expect(grade(rows)).toBe(false);
expect(findCursorNativeRequest(rows, "cursor/ask_question", "request")).toBeUndefined();
}
for (const patch of [{ requestId: "foreign" }, { turnId: "stale" }]) {
const rows = proof(adapter);
Object.assign((rows[0]!.payload.prpEvent.payload as any).request, patch);
expect(grade(rows)).toBe(false);
}
const rows = proof(adapter);
expect(grade([...rows, rows[0]!])).toBe(false);
expect(grade([...rows, rows[1]!])).toBe(false);
rows[1]!.payload.prpEvent.turnId = "stale";
expect(grade(rows)).toBe(false);
});
+3 -2
View File
@@ -1,3 +1,4 @@
import { hasAcpxNativeOrigin } from "./acpx-native-origin.js";
import type { BootstrapReadProof } from "./native-bootstrap-read-proof.js";
import { cursorDeniedCommand, hasCursorDeniedCommand, type CursorToolNotice } from "./cursor-native-evidence.js";
import type { RunnerTaskFixture } from "./types.js";
@@ -53,7 +54,7 @@ export function hasDeliveredCursorNativeRequest(input: {
if (created.length !== 1) return false;
const request = record(record(created[0]!.event.payload).request);
const origin = record(request.origin);
if (origin.adapter !== "acpx-runtime" || origin.provider !== "cursor" || origin.method !== input.method
if (!hasAcpxNativeOrigin(origin, "cursor", input.method)
|| request.turnId !== input.turnId || request.status !== "pending") return false;
if (input.method === "session/request_permission" ? request.type !== "permission" : request.type !== "input") return false;
const outcomes = rows.filter(({ event }) => ["runtime_request.resolved", "runtime_request.expired", "runtime_request.cancelled"].includes(String(event.eventType))
@@ -88,7 +89,7 @@ export function hasCursorDenialBoundary(input: {
if (!input.expectedRequestId.trim() || !input.expectedToolCallId.trim() || !input.path.trim()
|| request.requestId !== input.expectedRequestId || details.toolCallId !== input.expectedToolCallId
|| request.type !== "permission" || request.status !== "pending"
|| origin.provider !== "cursor" || origin.adapter !== "acpx-runtime" || origin.method !== "session/request_permission") return false;
|| !hasAcpxNativeOrigin(origin, "cursor", "session/request_permission")) return false;
if (request.turnId !== input.turnId || !hasCursorDeniedCommand({ notices: input.notices, runId: input.runId, turnId: input.turnId, requestId: input.expectedRequestId, toolCallId: input.expectedToolCallId, commandSha256: cursorDeniedCommand(input.path).commandSha256, bootstrapReadProof: input.bootstrapReadProof })) return false;
const choices = request.choices;
if (!Array.isArray(choices)) return false;
+21 -7
View File
@@ -1,3 +1,5 @@
import { isDeepStrictEqual } from "node:util";
import { hasAcpxNativeOrigin } from "./acpx-native-origin.js";
import { createHash, randomBytes } from "node:crypto";
import { lstat, readFile, readdir } from "node:fs/promises";
import { join } from "node:path";
@@ -14,6 +16,19 @@ import type { MatrixExecution } from "./types.js";
type Row = Record<string, any>;
type Check = { id: string; passed: boolean; detail: string };
/** Select only the exact native callback, including either production ACPX bridge. */
export function findCursorNativeRequest(rows: Row[], method: CursorNativeMethod, requestId?: string): Row | undefined {
return rows.map(row => row.payload?.prpEvent).find(event => event?.eventType === "runtime_request.created"
&& hasAcpxNativeOrigin(event.payload?.request?.origin, "cursor", method)
&& (!requestId || event.payload.request.requestId === requestId));
}
/** JSON object key order may change during persistence; array order and values may not. */
export function hasCursorNativeCardBinding(card: Row, event: Row, runId: string): boolean {
return card.sourceRunId === runId && card.continuationPolicy === "none"
&& isDeepStrictEqual(card.payload.questionSet, event.payload.request.input);
}
export interface CursorRemoteNativeFixture {
binding: CursorRemoteBinding; remoteCwd: string; actionFile: string;
snapshot(label: string): Promise<CursorRemoteSnapshot>;
@@ -201,14 +216,13 @@ export async function runCursorNativeFlow(input: {
return { issue, runs, interactions, runEvents };
};
const reject = (state: Awaited<ReturnType<typeof load>>) => state.runs.length > 1 ? "Unexpected extra Cursor provider run" : state.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Cursor provider run failed" : undefined;
const createdRequest = (rows: Row[], method: CursorNativeMethod, requestId?: string) => rows.map(row => row.payload?.prpEvent).find(event => event?.eventType === "runtime_request.created" && event.payload?.request?.origin?.adapter === "acpx-runtime" && event.payload.request.origin.provider === "cursor" && event.payload.request.origin.method === method && (!requestId || event.payload.request.requestId === requestId));
async function pending(seen: Set<string>) {
const state = await pollUntil({ label: "exact native Cursor callback", deadlineAt: input.deadlineAt, load,
reject: state => reject(state) ?? (state.runs.some(run => run.status === "succeeded") ? "Native Cursor callback was not observed before completion; qualification remains pending" : undefined),
accept: state => state.interactions.some(card => card.status === "pending" && !seen.has(card.id) && card.payload?.runtimeRequestId && createdRequest(state.runEvents, design!.method, card.payload.runtimeRequestId)) });
accept: state => state.interactions.some(card => card.status === "pending" && !seen.has(card.id) && card.payload?.runtimeRequestId && findCursorNativeRequest(state.runEvents, design!.method, card.payload.runtimeRequestId)) });
const cards = state.interactions.filter(card => card.status === "pending"); check("single-native-request", cards.length === 1 && state.runs.length === 1, "Exactly one native request belongs to one original provider run");
const card = cards[0]!; const event = createdRequest(state.runEvents, design!.method, card.payload.runtimeRequestId)!;
check("native-card-binding", card.sourceRunId === state.runs[0]!.id && card.continuationPolicy === "none" && JSON.stringify(card.payload.questionSet) === JSON.stringify(event.payload.request.input), "Durable card retains complete native input and exact source run");
const card = cards[0]!; const event = findCursorNativeRequest(state.runEvents, design!.method, card.payload.runtimeRequestId)!;
check("native-card-binding", hasCursorNativeCardBinding(card, event, state.runs[0]!.id), "Durable card retains complete native input and exact source run");
await input.evidence(`cursor-native-${seen.size}-pending.json`, { card, event });
await page.reload(); const reloaded = (await load()).interactions.find(row => row.id === card.id);
check("browser-reconnect-identity", reloaded?.status === "pending" && reloaded?.payload.runtimeRequestId === card.payload.runtimeRequestId, "Browser reconnect preserves exact outstanding native request");
@@ -290,12 +304,12 @@ export async function runCursorNativeFlow(input: {
} else {
const state = await pollUntil({ label: "native Cursor permission with exact command provenance", deadlineAt: input.deadlineAt, load, reject,
accept: state => denialNotices.some(notice => notice.stage === "permission_requested" && notice.commandSha256 === deniedCommand!.commandSha256
&& Boolean(createdRequest(state.runEvents, "session/request_permission", notice.requestId))) });
&& Boolean(findCursorNativeRequest(state.runEvents, "session/request_permission", notice.requestId))) });
const native = denialNotices.find(notice => notice.stage === "permission_requested" && notice.commandSha256 === deniedCommand!.commandSha256)!;
const event = createdRequest(state.runEvents, "session/request_permission", native.requestId)!; const request = event.payload.request; deniedRequest = request; denialTurnId = event.turnId;
const event = findCursorNativeRequest(state.runEvents, "session/request_permission", native.requestId)!; const request = event.payload.request; deniedRequest = request; denialTurnId = event.turnId;
check("native-permission-identity", state.runs.length === 1 && request.details?.toolCallId === native.toolCallId && native.turnId === event.turnId && native.declineOffered && request.choices.some((choice: Row) => choice.key === "decline"), "Presented native permission is bound to the exact absolute-target command and supported denial choice");
await sampleDenied("pending"); await page.reload();
const reloaded = await load(); check("permission-reconnect", Boolean(createdRequest(reloaded.runEvents, "session/request_permission", request.requestId)) && !reloaded.runEvents.some(row => row.payload?.prpEvent?.eventType === "runtime_request.resolved" && row.payload.prpEvent.payload?.requestId === request.requestId), "Reconnect preserves the unresolved native permission");
const reloaded = await load(); check("permission-reconnect", Boolean(findCursorNativeRequest(reloaded.runEvents, "session/request_permission", request.requestId)) && !reloaded.runEvents.some(row => row.payload?.prpEvent?.eventType === "runtime_request.resolved" && row.payload.prpEvent.payload?.requestId === request.requestId), "Reconnect preserves the unresolved native permission");
await sampleDenied("browser-reconnected"); await input.capture("cursor-permission", "Native write permission awaiting denial", "cursor-permission.png");
const card = page.getByTestId("task-chat-runtime-request").filter({ visible: true }); await expect(card).toHaveCount(1);
const label = request.choices.find((choice: Row) => choice.key === "decline").label;
+3 -1
View File
@@ -1,3 +1,5 @@
import { hasAcpxNativeOrigin } from "./acpx-native-origin.js";
/** Public durable permission/tool evidence. Independent filesystem and process
* proofs remain required; assistant text can never satisfy this oracle. */
type Row = Record<string, any>;
@@ -9,7 +11,7 @@ export function piPermissionRequests(rows: readonly unknown[], runId: string): A
if (row.eventType !== "runtime_request.created" || event.eventType !== row.eventType || row.runId !== runId || event.runId !== runId
|| row.protocolSchemaVersion !== 1 || event.schemaVersion !== 1 || event.schema !== "paperclip.prp.event.v1" || event.sourceKind !== "runner"
|| (!Number.isSafeInteger(row.seq) || row.seq < 1) || typeof event.turnId !== "string" || !event.turnId || request.turnId !== event.turnId
|| request.type !== "permission" || request.status !== "pending" || origin.adapter !== "acpx-runtime" || origin.provider !== "pi" || origin.method !== "session/request_permission"
|| request.type !== "permission" || request.status !== "pending" || !hasAcpxNativeOrigin(origin, "pi", "session/request_permission")
|| typeof request.requestId !== "string" || !request.requestId || typeof request.details?.toolCallId !== "string" || !request.details.toolCallId
|| !Array.isArray(request.choices) || !request.choices.some((choice: unknown) => record(choice).key === "decline")) return [];
return [{ row, event, request }];
@@ -14,14 +14,14 @@ vi.mock("@playwright/test", () => ({ expect: (actual: any, message?: string) =>
toBe: (value: unknown) => expect(actual, message).toBe(value), toBeVisible: async () => {}, toHaveCount: async (value: number) => expect(actual.count).toBe(value),
}) }));
const wrap = (eventType: string, seq: number, payload: unknown) => ({ runId: "run", protocolSchemaVersion: 1, eventType, seq, payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: "run", turnId: "turn", eventType, payload } } });
async function exercise(mutation: boolean, remote = false) {
async function exercise(mutation: boolean, remote = false, adapter = "acpx-runtime") {
proof.mutation = mutation; proof.live = false;
const workspacePath = await mkdtemp(join(tmpdir(), "pi-human-fixture-"));
let declined = false, browserPosts = 0; const saved = new Map<string, any>(); const cleanup: Array<() => Promise<any>> = [];
const task = piNativeTasks.find(row => row.id === "human-permission-denial")!;
const issue = () => ({ id: "issue", identifier: "PI-1", title: task.buildTitle("fixture"), status: declined ? "done" : "in_progress" });
const run = () => ({ id: "run", status: declined ? "succeeded" : "running", runtimeMode: "native", processPid: 123, processGroupId: 123, processStartedAt: "2026-09-29T00:00:00Z" });
const request = { requestId: "request", turnId: "turn", type: "permission", status: "pending", details: { toolCallId: "pi-tool-1" }, origin: { adapter: "acpx-runtime", provider: "pi", method: "session/request_permission" }, choices: [{ key: "decline", label: "Decline" }] };
const request = { requestId: "request", turnId: "turn", type: "permission", status: "pending", details: { toolCallId: "pi-tool-1" }, origin: { adapter, provider: "pi", method: "session/request_permission" }, choices: [{ key: "decline", label: "Decline" }] };
const events = () => [wrap("runtime_request.created", 1, { request }), ...(declined ? [wrap("runtime_request.resolved", 2, { requestId: "request", turnId: "turn", action: "decline" }), wrap("tool.execution.completed", 3, { schema: "paperclip.tool.execution.v1", transport: "builtin", operation: "edit", executionId: "pi-tool-1", name: "write", target: "pi-human-denied.txt", status: "failed", output: "Pi operation was denied or cancelled" })] : [])];
const api = {
post: async () => ({ name: "Pi fixture project" }), patch: async (_path: string, value: any) => value,
@@ -58,3 +58,5 @@ it("drives actual browser-denial flow and independent retirement proof", async (
it("rejects an observed create/delete even when final target is absent", async () => exercise(true));
it("proves browser denial against remote watcher and retirement without trusting a host file", async () => exercise(false, true));
it.each([false, true])("drives sidecar permission denial with remote=%s", async remote => exercise(false, remote, "acpx-runtime-sidecar"));
+19 -2
View File
@@ -1,12 +1,12 @@
import { expect, it } from "vitest";
import { hasDeliveredPiDenial, piPermissionRequests } from "./pi-native-evidence.js";
const identity = { runId: "run", turnId: "turn", requestId: "permission", toolCallId: "pi-tool-1", target: "pi-human-denied.txt" };
function evidence() {
function evidence(adapter = "acpx-runtime") {
const wrap = (eventType: string, seq: number, payload: unknown) => ({ runId: "run", protocolSchemaVersion: 1, eventType, seq, payload: { prpEvent: {
schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: "run", turnId: "turn", eventType, payload,
} } });
return [wrap("runtime_request.created", 1, { request: { requestId: "permission", turnId: "turn", type: "permission", status: "pending", details: { toolCallId: "pi-tool-1" },
origin: { adapter: "acpx-runtime", provider: "pi", method: "session/request_permission" }, choices: [{ key: "decline" }] } }),
origin: { adapter, provider: "pi", method: "session/request_permission" }, choices: [{ key: "decline" }] } }),
wrap("runtime_request.resolved", 2, { requestId: "permission", turnId: "turn", action: "decline" }),
wrap("tool.execution.completed", 3, { schema: "paperclip.tool.execution.v1", executionId: "pi-tool-1", transport: "builtin", operation: "edit", name: "write", status: "failed", target: identity.target, output: "Pi operation was denied or cancelled" })];
}
@@ -51,3 +51,20 @@ it("remote denial requires complete watching, stable parent and real process ret
const existing = structuredClone(before); existing.targets.denied.absent = false; (existing.targets.denied as any).sha256 = `sha256:${"a".repeat(64)}`;
expect(hasUnchangedPiRemoteTarget(existing, { ...after, targets: existing.targets }, "denied")).toBe(true);
});
it.each(["acpx-runtime", "acpx-runtime-sidecar"])("requires exact Pi permission origin and identity via %s", adapter => {
expect(piPermissionRequests(evidence(adapter), "run")).toHaveLength(1);
expect(hasDeliveredPiDenial(evidence(adapter), identity)).toBe(true);
for (const patch of [{ adapter: "unknown" }, { adapter: "acpx-runtime-sidecar-unknown" }, { adapter: "semantic" }, { provider: "cursor" }, { method: "request_human_input" }]) {
const rows = evidence(adapter);
Object.assign((rows[0]!.payload.prpEvent.payload as any).request.origin, patch);
expect(piPermissionRequests(rows, "run")).toHaveLength(0);
expect(hasDeliveredPiDenial(rows, identity)).toBe(false);
}
for (const key of ["runId", "turnId", "requestId", "toolCallId", "target"]) {
expect(hasDeliveredPiDenial(evidence(adapter), { ...identity, [key]: "foreign" })).toBe(false);
}
const rows = evidence(adapter);
expect(hasDeliveredPiDenial([...rows, rows[0]!], identity)).toBe(false);
expect(hasDeliveredPiDenial([...rows, rows[1]!], identity)).toBe(false);
});