From 3f552d753a78b7b546d23d4f85295b28eb4cc3b2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 17:15:46 -0500 Subject: [PATCH 1/4] test: count all Copilot origins and verify marker after cleanup Co-Authored-By: Paperclip --- .../copilot-protection-evidence.test.ts | 24 +++++++++++++++++++ .../runner-e2e/copilot-protection-evidence.ts | 14 +++++++++++ tests/runner-e2e/copilot-protection-flow.ts | 12 ++++++---- 3 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/copilot-protection-evidence.test.ts create mode 100644 tests/runner-e2e/copilot-protection-evidence.ts diff --git a/tests/runner-e2e/copilot-protection-evidence.test.ts b/tests/runner-e2e/copilot-protection-evidence.test.ts new file mode 100644 index 0000000000..70bef038c0 --- /dev/null +++ b/tests/runner-e2e/copilot-protection-evidence.test.ts @@ -0,0 +1,24 @@ +import { mkdtemp, readFile, rm, unlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; +import type { CopilotToolNotice } from "./copilot-evidence.js"; +const notice: CopilotToolNotice = { runId: "run", sessionId: "session", turnId: "turn", toolCallId: "edit", stage: "tool", operation: "edit", observedAtMs: 1, seq: 1 }; +describe("Copilot protection independent evidence", () => { + it.each(["in_progress", "completed", "failed"] as const)("counts an alternate origin first seen as %s", status => { + const rows = [{ ...notice, status: "pending" as const }, { ...notice, status }, { ...notice, toolCallId: "alternate", status }]; + expect(countCopilotToolOrigins(rows)).toBe(2); + expect(countCopilotToolOrigins([{ ...notice, status }, { ...notice, sessionId: "other", status }])).toBe(2); + }); + it.each(["unchanged", "changed", "deleted"])("independently reads the marker after %s cleanup", async mode => { + const root = await mkdtemp("/tmp/pc-copilot-marker-"); const path = join(root, "marker"); + try { + await writeFile(path, "expected"); expect(await readFile(path, "utf8")).toBe("expected"); + const matches = await readCopilotMarkerAfterCleanup(async () => { + if (mode === "changed") await writeFile(path, "changed"); + if (mode === "deleted") await unlink(path); + }, path, "expected"); + expect(matches).toBe(mode === "unchanged"); + } finally { await rm(root, { recursive: true, force: true }); } + }); +}); diff --git a/tests/runner-e2e/copilot-protection-evidence.ts b/tests/runner-e2e/copilot-protection-evidence.ts new file mode 100644 index 0000000000..01759b0e40 --- /dev/null +++ b/tests/runner-e2e/copilot-protection-evidence.ts @@ -0,0 +1,14 @@ +import { readFile } from "node:fs/promises"; +import type { CopilotToolNotice } from "./copilot-evidence.js"; + +/** A tool origin exists regardless of the first status emitted by the provider. */ +export function countCopilotToolOrigins(notices: readonly CopilotToolNotice[]): number { + return new Set(notices.filter(n => n.stage === "tool").map(n => JSON.stringify([n.runId, n.sessionId, n.turnId, n.toolCallId]))).size; +} + +/** Re-read independently after cleanup; a pre-cleanup value is not evidence. */ +export async function readCopilotMarkerAfterCleanup(close: () => Promise, path: string, expected: string): Promise { + await close(); + try { return await readFile(path, "utf8") === expected; } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; } +} diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index b399d4d1ff..f3f0bac023 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -7,6 +7,7 @@ import { createTaskThroughUi } from "./user-actions.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; import { createAttachedCommandFixture, exists, observeRunProcesses, watchDeniedTarget } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; +import { countCopilotToolOrigins, readCopilotMarkerAfterCleanup } from "./copilot-protection-evidence.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; type Row = Record; @@ -94,16 +95,16 @@ export async function runCopilotProtectionFlow(input: { terminal: { runId: terminalFrame.runId, turnId: terminalFrame.turnId, observedAtMs: terminalAt, status: runs[0]!.status }, cancellation: { requestedAtMs: cancelRequestedAtMs, acknowledged: cancellation?.dispatchState === "acknowledged" && cancellation?.dispatched === true, scope: cancellation?.scope }, cleanup: { observedAtMs: fileObservations.at(-1)!.observedAtMs, ownedProcessesRemaining: processes.live.length }, fileObservations, mutationObservation: watchReceipt, - nativeAttemptsForTarget: new Set(notices.filter(n => n.stage === "tool" && n.operation === "edit" && n.target === target && n.status === "pending").map(n => n.toolCallId)).size, + nativeAttemptsForTarget: countCopilotToolOrigins(notices.filter(n => n.operation === "edit" && n.target === target)), }; await input.evidence("copilot-denial-proof.json", { evidence, processes, notices }); const grade = gradeCopilotDeniedWrite(evidence); check("denial-without-side-effects", grade.passed, grade.failures.join(", ") || "Exact browser denial, explicit cancellation and absence through process cleanup"); check("negative-task-unfinished", issue.status === "in_progress", "The negative test does not claim the task is done"); - check("no-extra-native-operation", new Set(notices.filter(n => n.stage === "tool" && n.status === "pending").map(n => n.toolCallId)).size === 1, "No alternate native edit, command or delegated operation is permitted"); + check("no-extra-native-operation", countCopilotToolOrigins(notices) === 1, "No alternate native edit, command or delegated operation is permitted"); } else { await wait("attached command and task settlement", s => s.issue.status === "done" && s.runs[0]?.status === "succeeded" && s.processes.captured && s.processes.live.length === 0); const call = notices.find(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === command!.commandSha256); - check("single-exact-command", Boolean(call) && new Set(notices.filter(n => n.stage === "tool" && n.status === "pending" && n.commandSha256 === command!.commandSha256).map(n => n.toolCallId)).size === 1, "Exactly one native execution contains the supplied command digest"); + check("single-exact-command", Boolean(call) && countCopilotToolOrigins(notices.filter(n => n.commandSha256 === command!.commandSha256)) === 1, "Exactly one native execution contains the supplied command digest"); const started = notices.find(n => n.toolCallId === call!.toolCallId && n.shellState === "started"); const result = notices.find(n => n.commandToolCallId === call!.toolCallId && n.shellState === "completed"); const terminal = runEvents.find(r => r.eventType === "turn.completed" && r.payload?.prpEvent?.turnId === call!.turnId)?.payload.prpEvent; @@ -112,12 +113,13 @@ export async function runCopilotProtectionFlow(input: { const markerMatches = await readFile(markerPath, "utf8") === command!.marker; check("native-client-before-terminal", Boolean(terminal) && external.clientExitedAtMs !== null && external.clientExitedAtMs < Date.parse(terminal.emittedAt), "Independent PID/start observation confirms native client retirement before turn completion"); check("marker-before-terminal", Boolean(terminal) && external.markerWrittenAtMs !== null && external.markerWrittenAtMs < Date.parse(terminal.emittedAt), "The independent fixture wrote its undisclosed marker before turn completion"); + const afterCleanupMarkerMatches = await readCopilotMarkerAfterCleanup(() => command!.close(), markerPath, command!.marker); const grade = gradeCopilotAttachedSettlement({ expected: copilotOrigin(call!), nativeCall: call ? { ...call, operation: call.operation!, mode: call.mode!, detach: call.detach!, commandSha256: call.commandSha256! } : null, expectedCommandSha256: command!.commandSha256, commandExit: external.commandExit, expectedShellId: started?.shellId ?? "", nativeShellResult: result ? { ...result, shellId: result.shellId!, commandToolCallId: result.commandToolCallId!, status: result.status!, exitCode: result.exitCode! } : null, terminal: terminal ? { observedAtMs: Date.parse(terminal.emittedAt), runId: terminal.runId, turnId: terminal.turnId, status: "succeeded" } : null, - cleanup: { observedAtMs: Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches: markerMatches }); - await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: command!.commandSha256, markerMatches }); + cleanup: { observedAtMs: Date.now(), ownedProcessesRemaining: processes.live.length }, terminalMarkerMatches: markerMatches, afterCleanupMarkerMatches }); + await input.evidence("copilot-attached-proof.json", { external, processes, notices, grade, commandSha256: command!.commandSha256, markerMatches, afterCleanupMarkerMatches }); check("attached-settlement-before-terminal", grade.passed, grade.failures.join(", ") || "Owned finite process and native shell settled before the actual turn terminal"); } await load(); check("one-native-run", runs.length === 1 && runs[0]!.runtimeMode === "native", "Exactly one native run was accounted"); From 0ccdd4f03f938234ec917dbfb224264b7e1d14a9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 17:15:52 -0500 Subject: [PATCH 2/4] Fix canonical Cursor plan delivery expectations Co-Authored-By: Paperclip --- tests/runner-e2e/cursor-native-cases.test.ts | 23 ++++++++++++++++++++ tests/runner-e2e/cursor-native-flow.ts | 7 +++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/cursor-native-cases.test.ts b/tests/runner-e2e/cursor-native-cases.test.ts index d0bdb73f75..63100304b1 100644 --- a/tests/runner-e2e/cursor-native-cases.test.ts +++ b/tests/runner-e2e/cursor-native-cases.test.ts @@ -1,3 +1,6 @@ +import { parsePaperclipQuestionResponse } from "../../packages/paperclip-runner/src/contracts/question-set.js"; +import { normalizeCursorPlanRequest } from "../../packages/paperclip-runner/src/drivers/acpx/cursor-extensions.js"; +import { cursorNativePlanResponse } from "./cursor-native-flow.js"; import { cursorDeniedCommand, type CursorToolNotice } from "./cursor-native-evidence.js"; import { describe, expect, it } from "vitest"; import { cursorNativeCaseDesigns, cursorNativePrompt, hasDeliveredCursorNativeRequest, hasCursorPlanDecision, hasCursorDenialBoundary, CURSOR_DENIAL_SAMPLE_PHASES, hasExactCursorNativeResponse } from "./cursor-native-cases.js"; @@ -62,3 +65,23 @@ it("requires exact delivered answer bytes rather than a saved or different answe expect(hasExactCursorNativeResponse({ ...input, response: { ...response, answers: { q: { selectedOptionIds: ["other"] } } } })).toBe(false); expect(hasExactCursorNativeResponse({ ...input, events: rows.slice(0, 1) })).toBe(false); }); + + +it.each(["accept", "cancel", "reject"] as const)("recognizes canonical delivered native plan %s without an empty optional answer", decision => { + const native = normalizeCursorPlanRequest({ toolCallId: "native-plan", plan: "Verify the revised plan.", todos: [] }); + const planId = native.questionSet.questions[0]!.id; + const response = cursorNativePlanResponse(planId, decision, " Keep exact feedback 漢字\nSecond line "); + const delivered = parsePaperclipQuestionResponse(native.questionSet, response); + const rows = proof(); + (rows[0]!.payload.prpEvent.payload as any).request.origin.method = "cursor/create_plan"; + (rows[1]!.payload.prpEvent.payload as any).response = delivered; + const input = { events: rows, runId: "run", turnId: "turn", requestId: "request", method: "cursor/create_plan" as const, action: "submit" as const, response }; + expect(hasCursorPlanDecision(native.questionSet, response, decision)).toBe(true); + expect(hasExactCursorNativeResponse(input)).toBe(true); + expect(native.resolve(delivered).outcome.outcome).toBe({ accept: "accepted", cancel: "cancelled", reject: "rejected" }[decision]); + if (decision === "reject") expect(delivered.answers.reason?.text).toBe(" Keep exact feedback 漢字\nSecond line "); + else { + expect(delivered.answers).not.toHaveProperty("reason"); + expect(hasExactCursorNativeResponse({ ...input, response: { ...response, answers: { ...response.answers, reason: {} } } })).toBe(false); + } +}); diff --git a/tests/runner-e2e/cursor-native-flow.ts b/tests/runner-e2e/cursor-native-flow.ts index 32e84dd349..c55013cc6d 100644 --- a/tests/runner-e2e/cursor-native-flow.ts +++ b/tests/runner-e2e/cursor-native-flow.ts @@ -14,6 +14,11 @@ import type { MatrixExecution } from "./types.js"; type Row = Record; type Check = { id: string; passed: boolean; detail: string }; +/** Match canonical omission of an unanswered optional feedback field. */ +export function cursorNativePlanResponse(planId: string, decision: "accept" | "reject" | "cancel", feedback: string) { + return { schema: "paperclip.question_response.v1", answers: { [planId]: { selectedOptionIds: [decision] }, ...(decision === "reject" ? { reason: { text: feedback } } : {}) } }; +} + /** Independent bounded snapshot; symlinks are rejected without following them. */ export async function cursorNativeWorkspaceSnapshot(root: string): Promise> { const files: Record = {}; let bytes = 0; @@ -183,7 +188,7 @@ export async function runCursorNativeFlow(input: { await page.getByRole("radio", { name: decision === "accept" ? "Accept plan" : decision === "reject" ? "Reject plan" : "Cancel plan request", exact: true }).last().click(); await page.getByRole("button", { name: "Next", exact: true }).last().click(); if (decision === "reject") await page.getByTestId("question-text-answer-composer").last().locator('[contenteditable="true"],textarea').first().fill(feedback); - const response = { schema: "paperclip.question_response.v1", answers: { [plan.id]: { selectedOptionIds: [decision] }, reason: decision === "reject" ? { text: feedback } : {} } }; + const response = cursorNativePlanResponse(plan.id, decision, feedback); check("revision-bound-decision", hasCursorPlanDecision(set, response, decision), "Decision addresses precisely the displayed native plan revision"); await page.getByRole("button", { name: set.submitLabel ?? "Submit answers", exact: true }).last().click(); await delivery(card, event, response); seen.add(card.id); previousRevision = plan.id; From 11b2842d6919cab60d413f52960e0080c1e4a66d Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 17:16:10 -0500 Subject: [PATCH 3/4] Refresh Pi backend test fixture to current candidate identity Co-Authored-By: Paperclip --- .../runner/crates/runner-core/tests/native_provider_backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index aa380c50cd..e46894cc2f 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -17,7 +17,7 @@ use sha2::{Digest, Sha256}; const CODEX_ACPX_DIGEST: &str = "sha256:c4538599d1ab767db5dff50934f13bb5ba313a59d9c4a83e993fac4617ea63d3"; const PI_ACPX_DIGEST: &str = - "sha256:fec5b1448f4135710887133a9192747c476a825a56c255b52b17b1780ccb3761"; + "sha256:843d30e419914529755c9827d9151a306da1b50b643be7eab1abe797641a37ce"; fn temporary_directory(label: &str) -> PathBuf { let nonce = SystemTime::now() From e7abc92d914501279f0463b555f9fbdf1ce59359 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 17:23:26 -0500 Subject: [PATCH 4/4] Move validation rollup update into qualification follow-up Keep the current validation rollup on the Daytona qualification branch while the prerequisite runtime PR stays below the review file limit. Runtime and tests are unchanged. Co-Authored-By: Paperclip --- ...runner-rich-acp-validation-2026-09-29.json | 46 ++----------------- 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/doc/architecture/runner-rich-acp-validation-2026-09-29.json b/doc/architecture/runner-rich-acp-validation-2026-09-29.json index c7eb43bbf2..792150e8a6 100644 --- a/doc/architecture/runner-rich-acp-validation-2026-09-29.json +++ b/doc/architecture/runner-rich-acp-validation-2026-09-29.json @@ -1,6 +1,6 @@ { "schema": "paperclip.rich-acp-validation-checkpoint.v1", - "updatedAt": "2026-09-29T21:39:23.883903+00:00", + "updatedAt": "2026-09-29T20:58:14.579734+00:00", "qualificationStatus": "pending", "runtimeSource": "bd4cc29c3017ed5e1484423e842fd48e3b2f49f3", "validationSourcePatches": [ @@ -180,47 +180,7 @@ "runnerTypecheck": "passed", "sidecarAndProviderPackBuild": "passed", "daemonBuild": "passed; bytes unchanged", - "paidQualification": "Four cells/seven runs completed: question, plan and controller-restart passed; file case failed exact final response5/7 despite correct file bytes and successful task/run." + "paidQualification": "four-cell batch running; not yet passed" } - ], - "incrementalIntegrationChecks": { - "copilotProjectionSidecarTests": 57, - "mergedPiCopilotProductAndCatalogTests": 77, - "daytonaImageTests": 8, - "runnerTypescriptBuild": "passed", - "productTypecheck": "passed after rebuilding stale runner declaration output", - "typescriptBuildLogSha256": "41a44b38a17d1b6a23ccfb51891372f3da9c9bafc94230771bee103baea9e2f1", - "productTypecheckLogSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "note": "Targeted integration checks, not a new full-suite or final-source qualification." - }, - "latestIncrementalValidation": { - "source": "65b19549e0aac0cc7809142be9f33fb63e0a634c", - "fullRepositoryGates": "pending_on_new_source", - "runnerTypeScript": { - "passed": 389, - "skipped": 1, - "failed": 0, - "filesPassed": 13, - "logSha256": "aca4d7ae2b05c06674dfae5e492434ce36fcefc8f1c5082fd753cd04d107ca11" - }, - "rustAcpxUnits": { - "passed": 30, - "failed": 0 - }, - "serverModeRecovery": { - "passed": 19, - "skipped": 549, - "failed": 0 - }, - "runnerTypeScriptBuild": "passed", - "serverTypecheck": "passed", - "exactLockedAcpxPackageContracts": { - "passed": 22, - "failed": 0, - "acpxPatchSha256": "79aad2d688b03362e8cfcbf7a08f78a8383869f6882a9c9ed66f1d18efb94f2b", - "sdkVersion": "1.4.0", - "logSha256": "a3e60463b4bab0541aadd7f0dda3db91c3013137cf3828f8fb92b817e008d37a" - }, - "scope": "Incremental checks only. Historical broad test failure remains failed; fresh source still requires full repository gates and authenticated qualification." - } + ] }