From 72cbc3503d5871a9a9f064801a175750e11f76ca Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 30 Sep 2026 10:43:26 -0500 Subject: [PATCH] fix(runner-e2e): distinguish Copilot denial settlement from Stop Co-Authored-By: Paperclip --- tests/runner-e2e/FIXTURES.md | 22 ++- tests/runner-e2e/README.md | 14 ++ tests/runner-e2e/catalog.ts | 4 +- .../copilot-protection-cases.test.ts | 60 +++++++- tests/runner-e2e/copilot-protection-cases.ts | 36 ++++- .../runner-e2e/copilot-protection-evidence.ts | 145 +++++++++++++++++- .../copilot-protection-flow.test.ts | 14 +- tests/runner-e2e/copilot-protection-flow.ts | 40 +++-- .../copilot-protection-settlement.test.ts | 121 +++++++++++++++ 9 files changed, 419 insertions(+), 37 deletions(-) create mode 100644 tests/runner-e2e/copilot-protection-settlement.test.ts diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index d252f1fc13f..8c885833743 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -285,7 +285,17 @@ each with one run (120s provider timeout, 300s attempt budget). `native-permission-deny-write` denies one exact native edit through its browser card, waits for the delivered rejection and failed tool, then cancels through the public run API. Its expected outcome is a -cancelled run and an unfinished task, not successful task completion. It rejects +cancelled run and an unfinished task, not successful task completion. +The retained `paperclip.e2e.copilot-denial-settlement.v1` proof separates the exact +provider terminal from the audited controller Stop. It records either +`provider_cancelled_or_interrupted` or `provider_completed_before_stop_settlement`. +The latter requires normal terminal persistence strictly before the server's Stop +acknowledgement, with the same normalized session/source stream and a later source +sequence than the failed denied tool. It cannot establish active-turn cancellation; +a cancelled/interrupted terminal also does not by itself prove Stop reached active +work. Dedicated cancellation coverage must retain its active-operation evidence. +Missing, ambiguous, failed or foreign terminals and incomplete Stop receipts fail. +This distinction does not regrade earlier failed attempts. The case rejects extra native operations/runs and observes the absent target through process retirement. Filesystem event loss or an unexplained parent-directory timestamp change makes no-effect coverage incomplete; stat polling alone cannot pass. @@ -347,3 +357,13 @@ and order; any explicit canonical target must agree. Missing or conflicting attestations fail qualification. PRP retains only the first location and terminal updates may omit it, so canonical targets alone cannot prove bootstrap ownership. Neither tool titles nor output text supplies path evidence. + +Denial ordering uses canonical request, decline-resolution, delivery, failed-edit +and terminal source sequences. Sample checkpoints retain the exact run/turn/source +cursor. File samples and continuous-watch coverage compare only observer-local +times. Provider emission, browser click and server persistence clocks are never +compared to each other. Final remote samples use the sealed, independently verified +process-retirement receipt; their timestamps are not relabeled as host time. +This denial case does not qualify Stop during a definitely pending native request. +That active-turn cancellation boundary needs a separate live case. The attached +async-command oracle is unchanged by this denial-only correction. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 16df5041827..f6bc5071e5a 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1325,6 +1325,10 @@ and the limits of reconnect evidence. The manual `copilot-protection` suite selects two Copilot candidate cases on each of local and Daytona (four cells). Discover them with `pnpm test:e2e:runner -- --list --suite copilot-protection`. +The denial case keeps provider-turn settlement separate from controller run Stop. +A normal provider completion before Stop acknowledgement is not active-turn +cancellation coverage; denial, no-effects, run cancellation and retirement remain +required. Earlier failed attempts retain their original grade. See [Copilot native protection](./FIXTURES.md#copilot-native-protection) for the expected cancelled negative test, finite attached-process oracle, evidence limits, and required rebuilt runtime. Registration is not a qualification claim. @@ -1412,3 +1416,13 @@ lease history. Each turn has a 120-second limit; each three-run cell has a 420-second budget and must be reserved accordingly. Existing Codex warm cells remain separate. All these suites are excluded from `--all`, never automatically retry, and remain pending qualification until their paid evidence passes. + +Denial ordering uses canonical request, decline-resolution, delivery, failed-edit +and terminal source sequences. Sample checkpoints retain the exact run/turn/source +cursor. File samples and continuous-watch coverage compare only observer-local +times. Provider emission, browser click and server persistence clocks are never +compared to each other. Final remote samples use the sealed, independently verified +process-retirement receipt; their timestamps are not relabeled as host time. +This denial case does not qualify Stop during a definitely pending native request. +That active-turn cancellation boundary needs a separate live case. The attached +async-command oracle is unchanged by this denial-only correction. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index d6a34df7b7b..118e12ccb69 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1048,10 +1048,10 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ }, { id: "copilot-protection", label: "Copilot native protection", manualOnly: true, - description: "Exact native denial with explicit operator cancellation, and attached command settlement with independent process evidence.", + description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"), environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "explicit-public-cancellation", settlement: "attached-finite-command-only", remoteEvidence: "owned-lease-sealed-observer" }, + definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v1", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", remoteEvidence: "owned-lease-sealed-observer" }, }, { id: "rich-acp-warm-continuity", label: "Rich ACP warm continuity", manualOnly: true, diff --git a/tests/runner-e2e/copilot-protection-cases.test.ts b/tests/runner-e2e/copilot-protection-cases.test.ts index 5544ae430f6..d41da54c6ff 100644 --- a/tests/runner-e2e/copilot-protection-cases.test.ts +++ b/tests/runner-e2e/copilot-protection-cases.test.ts @@ -7,15 +7,18 @@ const terminal = { observedAtMs: 50, runId: "run", turnId: "turn", status: "succ const cleanup = { observedAtMs: 60, ownedProcessesRemaining: 0 }; function denied(): CopilotDeniedWriteEvidence { return { - expected: identity, terminal: { ...terminal, status: "cancelled" }, cleanup, cancellation: { requestedAtMs: 40, acknowledged: true, scope: "run" }, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt", + expected: identity, terminal: { ...terminal, status: "cancelled" }, cleanup, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v1", ...identity, requestId: "permission-0", + branch: "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: true, + providerTerminal: { eventType: "turn.cancelled", normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolPersistedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, persistedAtMs: 51 }, + runStop: { companyId: "company", issueId: "issue", scope: "run", status: "cancelled", issueStatus: "in_progress", intentId: "intent", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt", request: { ...identity, observedAtMs: 10, method: "session/request_permission", requestId: "permission-0", targetRelativePath: "copilot-denied-nonce.txt", offeredActions: ["accept", "decline"] }, decision: { ...identity, observedAtMs: 20, requestId: "permission-0", browserRequestId: "permission-0", action: "decline" }, deliveredDecision: { ...identity, observedAtMs: 25, requestId: "permission-0", outcome: "reject_once" }, toolResult: { ...identity, observedAtMs: 30, status: "failed" }, nativeAttemptsForTarget: 1, fileObservations: [ - { phase: "before-request", observedAtMs: 0, exists: false }, { phase: "pending", observedAtMs: 15, exists: false }, - { phase: "after-decision", observedAtMs: 25, exists: false }, { phase: "terminal", observedAtMs: 50, exists: false }, - { phase: "after-cleanup", observedAtMs: 60, exists: false }, + { phase: "before-request", observedAtMs: 0, exists: false, providerCursor: null }, { phase: "pending", observedAtMs: 15, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 1 } }, + { phase: "after-decision", observedAtMs: 25, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 4 } }, { phase: "terminal", observedAtMs: 50, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 6 } }, + { phase: "after-cleanup", observedAtMs: 60, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 6 } }, ], mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 }, }; @@ -40,6 +43,49 @@ describe("Copilot protection Product oracles", () => { it("accepts an origin-bound browser denial with an independent continuous absence oracle", () => { expect(gradeCopilotDeniedWrite(denied())).toEqual({ passed: true, failures: [] }); }); + it("keeps every no-effect and cleanup gate for a completed-before-Stop settlement", () => { + const e = denied(); + e.settlement!.branch = "provider_completed_before_stop_settlement"; + e.settlement!.providerCancellationTerminalObserved = false; + e.settlement!.providerTerminal.eventType = "turn.completed"; + expect(gradeCopilotDeniedWrite(e).passed).toBe(true); + for (const mutate of [ + (v: CopilotDeniedWriteEvidence) => { v.cleanup!.ownedProcessesRemaining = 1; }, + (v: CopilotDeniedWriteEvidence) => { v.nativeAttemptsForTarget = 2; }, + (v: CopilotDeniedWriteEvidence) => { v.fileObservations[2]!.exists = true; }, + (v: CopilotDeniedWriteEvidence) => { v.mutationObservation!.targetMutationCount = 1; }, + (v: CopilotDeniedWriteEvidence) => { v.mutationObservation!.complete = false; }, + (v: CopilotDeniedWriteEvidence) => { v.deliveredDecision = null; }, + (v: CopilotDeniedWriteEvidence) => { v.toolResult!.status = "completed"; }, + (v: CopilotDeniedWriteEvidence) => { v.settlement!.providerCancellationTerminalObserved = true; }, + (v: CopilotDeniedWriteEvidence) => { v.settlement!.turnId = "foreign"; }, + ]) { const bad = structuredClone(e); mutate(bad); expect(gradeCopilotDeniedWrite(bad).passed).toBe(false); } + }); + it.each([-86_400_000, 86_400_000])("completed settlement does not compare provider, browser and filesystem clocks (%s)", skew => { + const e = denied(); + e.settlement!.branch = "provider_completed_before_stop_settlement"; + e.settlement!.providerCancellationTerminalObserved = false; + e.settlement!.providerTerminal.eventType = "turn.completed"; + // Distinct positive clock domains, with either provider ahead or behind. + for (const n of [e.request!, e.deliveredDecision!, e.toolResult!, e.terminal!]) n.observedAtMs += 100_000_000 + skew; + e.settlement!.providerTerminal.emittedAtMs = e.terminal!.observedAtMs; + e.decision!.observedAtMs = 200_000_000; + expect(gradeCopilotDeniedWrite(e).passed).toBe(true); + e.fileObservations[1]!.providerCursor!.sourceSeq = e.settlement!.providerTerminal.resolvedSourceSeq; + expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid"); + }); + it.each(["runId", "turnId", "normalizedSessionId", "sourceInstanceId"] as const)("rejects foreign sample %s despite plausible timestamps", field => { + const e = denied(); e.fileObservations[3]!.providerCursor![field] = "foreign"; + expect(gradeCopilotDeniedWrite(e).passed).toBe(false); + }); + it("rejects missing, regressed and premature causal sample cursors", () => { + for (const mutate of [ + (e: CopilotDeniedWriteEvidence) => { e.fileObservations[2]!.providerCursor = null; }, + (e: CopilotDeniedWriteEvidence) => { e.fileObservations[2]!.providerCursor!.sourceSeq = 3; }, + (e: CopilotDeniedWriteEvidence) => { e.fileObservations[3]!.providerCursor!.sourceSeq = 5; }, + (e: CopilotDeniedWriteEvidence) => { e.fileObservations[4]!.providerCursor!.sourceSeq = 5; }, + ]) { const e = denied(); mutate(e); expect(gradeCopilotDeniedWrite(e).passed).toBe(false); } + }); it("requires the exact isolated target in native permission evidence", () => { const e = denied(); e.expectedRelativePath = "pc-denied-ABC123/copilot-denied-nonce.txt"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); @@ -50,11 +96,11 @@ describe("Copilot protection Product oracles", () => { expect(gradeCopilotDeniedWrite(e).passed).toBe(false); } }); - it.each(["request", "decision", "deliveredDecision", "toolResult", "terminal", "cleanup", "mutationObservation"] as const)("rejects missing %s instead of treating no write as denial", field => { + it.each(["request", "decision", "deliveredDecision", "toolResult", "terminal", "cleanup", "mutationObservation", "settlement"] as const)("rejects missing %s instead of treating no write as denial", field => { const e = denied(); e[field] = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); }); it("requires explicit acknowledged cancellation for denial without relaxing successful settlement", () => { - const e = denied(); e.cancellation = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); + const e = denied(); e.settlement = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); const finished = denied(); finished.terminal!.status = "succeeded"; expect(gradeCopilotDeniedWrite(finished).passed).toBe(false); const cancelled = attached(); cancelled.terminal!.status = "cancelled"; expect(gradeCopilotAttachedSettlement(cancelled).passed).toBe(false); }); @@ -72,7 +118,7 @@ describe("Copilot protection Product oracles", () => { const retried = denied(); retried.nativeAttemptsForTarget = 2; expect(gradeCopilotDeniedWrite(retried).passed).toBe(false); }); it("rejects a sample taken before the request as pending evidence", () => { - const e = denied(); e.fileObservations[1]!.observedAtMs = 5; expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid"); + const e = denied(); e.fileObservations[1]!.providerCursor!.sourceSeq = 0; expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid"); }); it("rejects an unrelated request identity even when the chosen decision matches an outer ID", () => { const e = denied(); e.request!.requestId = "foreign-request"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); diff --git a/tests/runner-e2e/copilot-protection-cases.ts b/tests/runner-e2e/copilot-protection-cases.ts index af3cb15000a..ff0f4fa9eb7 100644 --- a/tests/runner-e2e/copilot-protection-cases.ts +++ b/tests/runner-e2e/copilot-protection-cases.ts @@ -1,4 +1,4 @@ -import { matchCopilotFixtureCommand, type CopilotCommandMatch } from "./copilot-protection-evidence.js"; +import { matchCopilotFixtureCommand, validCopilotDenialSettlement, type CopilotDenialSettlement, type CopilotDenialSampleCursor, type CopilotCommandMatch } from "./copilot-protection-evidence.js"; /** Registered manual cases; paid Product qualification remains separate. */ export const copilotProtectionCases = [ { @@ -43,10 +43,12 @@ interface FileObservation { phase: "before-request" | "pending" | "after-decision" | "terminal" | "after-cleanup"; observedAtMs: number; exists: boolean; + /** null only before dispatch/action publication; otherwise read before sampling. */ + providerCursor: CopilotDenialSampleCursor | null; } export interface CopilotDeniedWriteEvidence extends Lifecycle { deliveredDecision: (TimedIdentity & { requestId: string; outcome: string }) | null; - cancellation: { requestedAtMs: number; acknowledged: boolean; scope: string } | null; + settlement: CopilotDenialSettlement | null; request: (TimedIdentity & { method: "session/request_permission"; requestId: string; targetRelativePath: string; offeredActions: string[] }) | null; decision: (TimedIdentity & { requestId: string; action: string; browserRequestId: string }) | null; requestId: string | null; @@ -78,17 +80,35 @@ function lifecycle(e: Lifecycle, failures: string[], expectedStatus: "succeeded" if (!e.cleanup || e.cleanup.ownedProcessesRemaining !== 0 || !time(e.cleanup.observedAtMs) || !e.terminal || e.cleanup.observedAtMs < e.terminal.observedAtMs) failures.push("unsettled-cleanup"); } export function gradeCopilotDeniedWrite(e: CopilotDeniedWriteEvidence): CopilotProtectionGrade { - const failures: string[] = []; lifecycle(e, failures, "cancelled"); - if (!e.cancellation || !e.cancellation.acknowledged || !time(e.cancellation.requestedAtMs) || e.cancellation.scope !== "run" || !e.toolResult || e.cancellation.requestedAtMs < e.toolResult.observedAtMs || !e.terminal || e.cancellation.requestedAtMs > e.terminal.observedAtMs) failures.push("missing-explicit-settled-cancellation"); + const failures: string[] = []; + // The provider, browser and observer clocks are independent. Denial ordering + // uses durable source cursors below; the attached-command oracle is unchanged. + if (!Object.values(e.expected).every(v => typeof v === "string" && v.length > 0)) failures.push("missing-origin-identity"); + if (!e.terminal || e.terminal.runId !== e.expected.runId || e.terminal.turnId !== e.expected.turnId || e.terminal.status !== "cancelled" || !time(e.terminal.observedAtMs)) failures.push("missing-expected-origin-terminal"); + if (!e.cleanup || e.cleanup.ownedProcessesRemaining !== 0 || !time(e.cleanup.observedAtMs)) failures.push("unsettled-cleanup"); + const settlement = e.settlement; + if (!settlement || !validCopilotDenialSettlement(settlement) || !same(settlement, e.expected) || settlement.requestId !== e.requestId + || !e.terminal || e.terminal.observedAtMs !== settlement.providerTerminal.emittedAtMs) failures.push("missing-explicit-settled-run-stop"); if (!/^(?:pc-denied-[a-zA-Z0-9]+\/)?copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request"); - if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs) || !e.request || e.decision.observedAtMs < e.request.observedAtMs) failures.push("missing-exact-browser-denial"); - if (!e.deliveredDecision || !same(e.deliveredDecision, e.expected) || e.deliveredDecision.requestId !== e.requestId || e.deliveredDecision.outcome !== "reject_once" || !time(e.deliveredDecision.observedAtMs) || !e.decision || e.deliveredDecision.observedAtMs < e.decision.observedAtMs || !e.toolResult || e.deliveredDecision.observedAtMs > e.toolResult.observedAtMs) failures.push("missing-delivered-native-rejection"); - if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs) || !e.decision || e.toolResult.observedAtMs < e.decision.observedAtMs || !e.terminal || e.toolResult.observedAtMs > e.terminal.observedAtMs) failures.push("missing-denied-tool-result-before-terminal"); + if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs)) failures.push("missing-exact-browser-denial"); + if (!e.deliveredDecision || !same(e.deliveredDecision, e.expected) || e.deliveredDecision.requestId !== e.requestId || e.deliveredDecision.outcome !== "reject_once" || !time(e.deliveredDecision.observedAtMs)) failures.push("missing-delivered-native-rejection"); + if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs)) failures.push("missing-denied-tool-result-before-terminal"); if (e.nativeAttemptsForTarget !== 1) failures.push("missing-or-retried-native-write"); const phases = ["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const; if (phases.some(phase => !e.fileObservations.some(s => s.phase === phase)) || e.fileObservations.some(s => s.exists || !time(s.observedAtMs))) failures.push("missing-or-mutated-target-observation"); const samples = phases.map(phase => e.fileObservations.find(s => s.phase === phase)); - if (samples.some((s, i) => !s || (i > 0 && s.observedAtMs < samples[i - 1]!.observedAtMs)) || !e.request || !e.decision || !e.terminal || !e.cleanup || (samples[0]?.observedAtMs ?? Infinity) > e.request.observedAtMs || (samples[1]?.observedAtMs ?? -1) < e.request.observedAtMs || (samples[1]?.observedAtMs ?? Infinity) > e.decision.observedAtMs || (samples[2]?.observedAtMs ?? -1) < e.decision.observedAtMs || (samples[3]?.observedAtMs ?? -1) < e.terminal.observedAtMs || (samples[4]?.observedAtMs ?? -1) < e.cleanup.observedAtMs) failures.push("filesystem-observation-order-invalid"); + const t = settlement?.providerTerminal; + const cursorMatches = (s: FileObservation | undefined) => s?.providerCursor && t && s.providerCursor.runId === e.expected.runId + && s.providerCursor.turnId === e.expected.turnId && s.providerCursor.normalizedSessionId === t.normalizedSessionId + && s.providerCursor.sourceInstanceId === t.sourceInstanceId && Number.isSafeInteger(s.providerCursor.sourceSeq); + if (e.fileObservations.length !== phases.length || samples.some((s, i) => !s || (i > 0 && s.observedAtMs < samples[i - 1]!.observedAtMs)) + || samples[0]?.providerCursor !== null || !t || samples.slice(1).some(s => !cursorMatches(s)) + || (samples[1]?.providerCursor?.sourceSeq ?? -1) < t.requestSourceSeq + || (samples[1]?.providerCursor?.sourceSeq ?? Infinity) >= t.resolvedSourceSeq + || (samples[2]?.providerCursor?.sourceSeq ?? -1) < t.failedNoticeSourceSeq + || (samples[3]?.providerCursor?.sourceSeq ?? -1) < t.sourceSeq || (samples[4]?.providerCursor?.sourceSeq ?? -1) < t.sourceSeq + || samples.slice(2).some((s, i) => (s?.providerCursor?.sourceSeq ?? -1) < (samples[i + 1]?.providerCursor?.sourceSeq ?? Infinity)) + || !e.cleanup || samples[4]?.observedAtMs !== e.cleanup.observedAtMs) failures.push("filesystem-observation-order-invalid"); const m = e.mutationObservation; if (!m || !m.complete || m.targetMutationCount !== 0 || !time(m.startedAtMs) || !time(m.endedAtMs) || !samples[0] || !samples[4] || m.startedAtMs > samples[0].observedAtMs || m.endedAtMs < samples[4].observedAtMs) failures.push("missing-or-mutated-filesystem-watch"); return { passed: failures.length === 0, failures }; diff --git a/tests/runner-e2e/copilot-protection-evidence.ts b/tests/runner-e2e/copilot-protection-evidence.ts index dd953f50e7f..4e9a6a39c88 100644 --- a/tests/runner-e2e/copilot-protection-evidence.ts +++ b/tests/runner-e2e/copilot-protection-evidence.ts @@ -1,7 +1,7 @@ -import { withoutProvenBootstrapReads, type BootstrapReadProof } from "./native-bootstrap-read-proof.js"; +import { withoutProvenBootstrapReads, bootstrapReadExecutionId, type BootstrapReadProof } from "./native-bootstrap-read-proof.js"; import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; -import type { CopilotToolNotice } from "./copilot-evidence.js"; +import { readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; /** A tool origin exists regardless of the first status emitted by the provider. */ export function countCopilotToolOrigins(notices: readonly CopilotToolNotice[]): number { @@ -193,3 +193,144 @@ export async function readCopilotRemoteMarkerAfterRetirement(fixture: CopilotRem const bytes = await fixture.readFile(target); return bytes.toString("utf8") === expected && receipt.targets[target]?.sha256 === `sha256:${createHash("sha256").update(bytes).digest("hex")}`; } + +export interface CopilotDenialSettlement { + schema: "paperclip.e2e.copilot-denial-settlement.v1"; + branch: "provider_cancelled_or_interrupted" | "provider_completed_before_stop_settlement"; + /** A cancelled terminal alone does not prove Stop reached active work. */ + providerCancellationTerminalObserved: boolean; + runId: string; sessionId: string; turnId: string; toolCallId: string; requestId: string; + providerTerminal: { + eventType: "turn.completed" | "turn.cancelled" | "turn.interrupted"; + normalizedSessionId: string; sourceInstanceId: string; + requestSourceSeq: number; resolvedSourceSeq: number; deliveredSourceSeq: number; failedNoticeSourceSeq: number; + failedToolSourceSeq: number; failedToolPersistedAtMs: number; sourceSeq: number; + emittedAtMs: number; persistedAtMs: number; + }; + runStop: { + companyId: string; issueId: string; scope: "run"; status: "cancelled"; issueStatus: "in_progress"; + intentId: string; intentAuditId: string; acknowledgementAuditId: string; + requestedAtMs: number; recordedAtMs: number; acknowledgedAtMs: number; finishedAtMs: number; + }; +} +const settlementRecord = (v: unknown): Record => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Record : {}; +const settlementId = (v: unknown): v is string => typeof v === "string" && v.length > 0 && v.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(v) && !v.includes("[REDACTED]"); +const settlementTime = (v: unknown): v is number => Number.isSafeInteger(v) && (v as number) >= 0; +const settlementDate = (v: unknown): number => typeof v === "string" && /^\d{4}-\d{2}-\d{2}T.*Z$/u.test(v) ? Date.parse(v) : NaN; + +export function validCopilotDenialSettlement(s: CopilotDenialSettlement): boolean { + if (!s || s.schema !== "paperclip.e2e.copilot-denial-settlement.v1") return false; + const t = s.providerTerminal, c = s.runStop; + if (!t || !c || ![s.runId, s.sessionId, s.turnId, s.toolCallId, s.requestId, t.normalizedSessionId, t.sourceInstanceId, + c.companyId, c.issueId, c.intentId, c.intentAuditId, c.acknowledgementAuditId].every(settlementId) + || c.scope !== "run" || c.status !== "cancelled" || c.issueStatus !== "in_progress" || c.intentAuditId === c.acknowledgementAuditId + || ![t.failedToolPersistedAtMs, t.emittedAtMs, t.persistedAtMs, c.requestedAtMs, c.recordedAtMs, c.acknowledgedAtMs, c.finishedAtMs].every(settlementTime) + || ![t.requestSourceSeq, t.resolvedSourceSeq, t.deliveredSourceSeq, t.failedNoticeSourceSeq].every(n => Number.isSafeInteger(n) && n > 0) + || !(t.requestSourceSeq < t.resolvedSourceSeq && t.resolvedSourceSeq < t.deliveredSourceSeq && t.deliveredSourceSeq < t.failedNoticeSourceSeq && t.failedNoticeSourceSeq < t.failedToolSourceSeq) + || !Number.isSafeInteger(t.failedToolSourceSeq) || t.failedToolSourceSeq <= 0 || !Number.isSafeInteger(t.sourceSeq) || t.sourceSeq <= t.failedToolSourceSeq + || t.failedToolPersistedAtMs > c.requestedAtMs || t.failedToolPersistedAtMs > t.persistedAtMs + || c.requestedAtMs > c.recordedAtMs || c.recordedAtMs > c.acknowledgedAtMs || c.acknowledgedAtMs > c.finishedAtMs) return false; + if (s.branch === "provider_completed_before_stop_settlement") return t.eventType === "turn.completed" + && s.providerCancellationTerminalObserved === false && t.persistedAtMs < c.acknowledgedAtMs; + return s.branch === "provider_cancelled_or_interrupted" && s.providerCancellationTerminalObserved === true + && ["turn.cancelled", "turn.interrupted"].includes(t.eventType); +} + +/** Denial closes one permission, not necessarily the provider prompt. Match the + * single canonical terminal to the same source stream, then independently bind + * the audited controller Stop. Provider emittedAt never orders server actions. */ +export function readCopilotDenialSettlement(input: { + events: readonly unknown[]; request: CopilotToolNotice; run: unknown; issue: unknown; +}): CopilotDenialSettlement { + const invalid = () => new Error("Copilot denial lacks exact provider settlement and audited run Stop"); + const { request, events } = input, run = settlementRecord(input.run), issue = settlementRecord(input.issue); + const stop = settlementRecord(settlementRecord(run.resultJson).nativeCancellation); + if (events.length > 20_000 || run.id !== request.runId || issue.id !== run.nativeIssueId + || !settlementId(issue.companyId) || run.companyId !== issue.companyId || issue.status !== "in_progress" || run.status !== "cancelled" + || stop.schema !== "paperclip.native-cancellation.v1" || stop.runId !== run.id || stop.companyId !== issue.companyId || stop.issueId !== issue.id + || stop.scope !== "run" || stop.dispatched !== true || stop.dispatchState !== "acknowledged" || stop.reasonCode !== "cancellation_run_only" + || !Array.isArray(stop.effects) || stop.effects.length !== 1 || stop.effects[0] !== "release_run_resources") throw invalid(); + const notices = readCopilotToolEvidence(events, request.runId); + const same = (n: CopilotToolNotice) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => n[k as keyof CopilotToolNotice] === request[k as keyof CopilotToolNotice]); + const requested = notices.filter(n => n.stage === "permission_requested" && same(n)); + const delivered = notices.filter(n => n.stage === "permission_delivered" && same(n)); + const failed = notices.filter(n => n.stage === "tool" && same(n) && ["failed", "completed"].includes(n.status ?? "")); + if (requested.length !== 1 || requested[0]!.seq !== request.seq || requested[0]!.requestId !== request.requestId || request.operation !== "edit" + || delivered.length !== 1 || delivered[0]!.requestId !== request.requestId || delivered[0]!.outcome !== "reject_once" + || failed.length !== 1 || failed[0]!.status !== "failed" || requested[0]!.seq >= delivered[0]!.seq || delivered[0]!.seq >= failed[0]!.seq) throw invalid(); + const rows = events.map(settlementRecord); + const get = (row: Record) => { + const f = settlementRecord(settlementRecord(row.payload).prpEvent); + if (row.runId !== run.id || row.companyId !== issue.companyId || !Number.isSafeInteger(row.seq) || row.seq <= 0 + || f.schema !== "paperclip.prp.event.v1" || f.sourceKind !== "runner" || f.eventType !== row.eventType || f.runId !== run.id || f.turnId !== request.turnId + || !settlementId(f.normalizedSessionId) || !settlementId(f.sourceInstanceId) || !Number.isSafeInteger(f.sourceSeq) || f.sourceSeq <= 0 + || f.sourceEventId !== `${f.sourceInstanceId}:${run.id}:${f.sourceSeq}` || !settlementTime(settlementDate(row.createdAt)) || !settlementTime(settlementDate(f.emittedAt))) throw invalid(); + return f; + }; + const at = (seq: number) => { const found = rows.filter(r => r.seq === seq); if (found.length !== 1) throw invalid(); return get(found[0]!); }; + const origin = at(request.seq), delivery = at(delivered[0]!.seq), failure = at(failed[0]!.seq); + const stream = (f: Record) => f.normalizedSessionId === origin.normalizedSessionId && f.sourceInstanceId === origin.sourceInstanceId; + if (![delivery, failure].every(stream) || origin.sourceSeq >= delivery.sourceSeq || delivery.sourceSeq >= failure.sourceSeq) throw invalid(); + const resolutions = rows.filter(r => ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(r.eventType) + && settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).requestId === request.requestId); + if (resolutions.length !== 1) throw invalid(); + const resolution = get(resolutions[0]!), resolved = settlementRecord(resolution.payload); + if (!stream(resolution) || resolution.eventType !== "runtime_request.resolved" || resolved.requestKind !== "permission_approval" + || resolved.turnId !== request.turnId || resolved.action !== "decline" || resolution.sourceSeq <= origin.sourceSeq + || resolution.sourceSeq >= delivery.sourceSeq || resolutions[0]!.seq <= request.seq || resolutions[0]!.seq >= delivered[0]!.seq) throw invalid(); + // Ignore legacy unwrapped log summaries, never use them as provider evidence. + const terminalRows = rows.filter(r => ["turn.completed", "turn.cancelled", "turn.interrupted", "turn.failed"].includes(r.eventType) + && settlementRecord(r.payload).prpEvent !== undefined); + const toolRows = rows.filter(r => r.eventType === "tool.execution.completed" + && settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).executionId === bootstrapReadExecutionId(request.toolCallId)); + if (terminalRows.length !== 1 || toolRows.length !== 1) throw invalid(); + const tool = get(toolRows[0]!), toolPayload = settlementRecord(tool.payload); + if (!stream(tool) || tool.sourceSeq <= failure.sourceSeq || toolRows[0]!.seq <= failed[0]!.seq + || toolPayload.schema !== "paperclip.tool.execution.v1" || toolPayload.status !== "failed" || toolPayload.operation !== "edit") throw invalid(); + const row = terminalRows[0]!, terminal = get(row), p = settlementRecord(terminal.payload); + if (!stream(terminal) || terminal.sourceSeq <= tool.sourceSeq || row.seq <= failed[0]!.seq + || !["turn.completed", "turn.cancelled", "turn.interrupted"].includes(row.eventType) + || p.status !== row.eventType.slice(5) || p.error != null) throw invalid(); + const result: CopilotDenialSettlement = { + schema: "paperclip.e2e.copilot-denial-settlement.v1", + branch: row.eventType === "turn.completed" ? "provider_completed_before_stop_settlement" : "provider_cancelled_or_interrupted", + providerCancellationTerminalObserved: row.eventType !== "turn.completed", + runId: run.id, sessionId: request.sessionId, turnId: request.turnId, toolCallId: request.toolCallId, requestId: request.requestId!, + providerTerminal: { eventType: row.eventType, normalizedSessionId: terminal.normalizedSessionId, sourceInstanceId: terminal.sourceInstanceId, + requestSourceSeq: origin.sourceSeq, resolvedSourceSeq: resolution.sourceSeq, deliveredSourceSeq: delivery.sourceSeq, failedNoticeSourceSeq: failure.sourceSeq, + failedToolSourceSeq: tool.sourceSeq, failedToolPersistedAtMs: settlementDate(toolRows[0]!.createdAt), sourceSeq: terminal.sourceSeq, emittedAtMs: settlementDate(terminal.emittedAt), persistedAtMs: settlementDate(row.createdAt) }, + runStop: { companyId: issue.companyId, issueId: issue.id, scope: stop.scope, status: run.status, issueStatus: issue.status, + intentId: stop.intentId, intentAuditId: stop.intentAuditId, acknowledgementAuditId: stop.acknowledgementAuditId, + requestedAtMs: settlementDate(settlementRecord(settlementRecord(run.resultJson).startupCancellation).requestedAt), + recordedAtMs: settlementDate(stop.recordedAt), acknowledgedAtMs: settlementDate(stop.acknowledgedAt), finishedAtMs: settlementDate(run.finishedAt) }, + }; + if (!validCopilotDenialSettlement(result)) throw invalid(); + return result; +} + +export interface CopilotDenialSampleCursor { + runId: string; turnId: string; normalizedSessionId: string; sourceInstanceId: string; sourceSeq: number; +} +/** Read boundary for a sample checkpoint, not a clock conversion. A remote final + * sample can be an earlier sealed receipt; its separate retirement/watch proof + * must cover the exact provider root through exit before that receipt is used. */ +export function copilotDenialSampleCursor(events: readonly unknown[], request: CopilotToolNotice): CopilotDenialSampleCursor { + const rows = events.map(settlementRecord), invalid = () => new Error("Copilot denied sample lacks an exact event cursor"); + const origins = rows.filter(r => r.seq === request.seq); + if (origins.length !== 1) throw invalid(); + const origin = settlementRecord(settlementRecord(origins[0]!.payload).prpEvent); + const frames = rows.map(r => ({ row: r, f: settlementRecord(settlementRecord(r.payload).prpEvent) })) + .filter(({ f }) => f.turnId === request.turnId); + if (events.length > 20_000 || frames.length === 0 || origin.runId !== request.runId + || ![origin.normalizedSessionId, origin.sourceInstanceId].every(settlementId)) throw invalid(); + const seen = new Set(); + for (const { row, f } of frames) { + if (row.runId !== request.runId || f.runId !== request.runId || f.schema !== "paperclip.prp.event.v1" || f.sourceKind !== "runner" + || row.eventType !== f.eventType || f.normalizedSessionId !== origin.normalizedSessionId || f.sourceInstanceId !== origin.sourceInstanceId + || !Number.isSafeInteger(f.sourceSeq) || f.sourceSeq <= 0 || seen.has(f.sourceSeq) + || f.sourceEventId !== `${f.sourceInstanceId}:${f.runId}:${f.sourceSeq}`) throw invalid(); + seen.add(f.sourceSeq); + } + return { runId: request.runId, turnId: request.turnId, normalizedSessionId: origin.normalizedSessionId, + sourceInstanceId: origin.sourceInstanceId, sourceSeq: Math.max(...seen) }; +} diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index 19798fcc429..a3c60488146 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -9,7 +9,7 @@ import { validateAcpxRichEvent } from "../../packages/paperclip-runner/src/drive import { copilotOrigin, readCopilotToolEvidence } from "./copilot-evidence.js"; import { copilotProtectionCases, gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite } from "./copilot-protection-cases.js"; import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, watchDeniedTarget, exists, isPerTurnRunProcess } from "./copilot-local-fixtures.js"; -import { runnerMatrix } from "./catalog.js"; +import { runnerMatrix, suiteDefinitionHash } from "./catalog.js"; import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js"; const fixture = JSON.parse(await readFile(new URL("../../packages/paperclip-runner/src/drivers/acpx/fixtures/copilot-tool-evidence.json", import.meta.url), "utf8")); @@ -26,6 +26,9 @@ function projected(name: string, target = "copilot-denied-nonce.txt") { describe("Copilot Product protection integration", () => { it("registers two explicit cases on both environments with honest terminal expectations", () => { const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection"); + expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 3, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", + denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v1", activeTurnCancellation: "not-implied-by-completed-provider-turn" }); + expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } })); expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"])); expect(cells.every(c => c.profile.qualificationCandidate === "copilot" && c.task.expectedRunCount === 1)).toBe(true); expect(cells.find(c => c.task.id === "native-permission-deny-write")!.task.expectedTerminalState).toEqual({ issue: "in_progress", run: "cancelled" }); @@ -38,13 +41,16 @@ describe("Copilot Product protection integration", () => { request: { ...request, requestId: "permission", method: "session/request_permission" as const, targetRelativePath: request.target!, offeredActions: request.declineOffered ? ["decline"] : [] }, decision: { ...delivered, requestId: "permission", browserRequestId: "permission", action: delivered.outcome === "reject_once" ? "decline" : "accept" }, deliveredDecision: { ...delivered, requestId: "permission", outcome: delivered.outcome! }, - toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, cancellation: { requestedAtMs: 40, scope: "run", acknowledged: true }, + toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v1" as const, ...copilotOrigin(request), requestId: "permission", + branch: "provider_cancelled_or_interrupted" as const, providerCancellationTerminalObserved: true, + providerTerminal: { eventType: "turn.cancelled" as const, normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolPersistedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, persistedAtMs: 51 }, + runStop: { companyId: "company", issueId: "issue", scope: "run" as const, status: "cancelled" as const, issueStatus: "in_progress" as const, intentId: "intent", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, cleanup: { observedAtMs: 60, ownedProcessesRemaining: 0 }, nativeAttemptsForTarget: 1, - fileObservations: (["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const).map((phase, index) => ({ phase, observedAtMs: [0, request.observedAtMs, 30, 50, 60][index]!, exists: false })), mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 } }; + fileObservations: (["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const).map((phase, index) => ({ phase, observedAtMs: [0, request.observedAtMs, 30, 50, 60][index]!, exists: false, providerCursor: index === 0 ? null : { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: [0, 1, 4, 6, 6][index]! } })), mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 } }; expect(request.target).toBe(target); expect(gradeCopilotDeniedWrite(e).passed).toBe(true); e.request.targetRelativePath = "foreign.txt"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); - e.request.targetRelativePath = e.expectedRelativePath; e.cancellation.acknowledged = false; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); + e.request.targetRelativePath = e.expectedRelativePath; e.settlement.runStop.acknowledgementAuditId = ""; expect(gradeCopilotDeniedWrite(e).passed).toBe(false); }); it("feeds actual attached wire through canonical notices and rejects early terminal or missing linkage", () => { const { notices } = projected("attached-shell"); const call = notices.find(n => n.commandSha256)!; diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index f1a72d77cba..5bd70634f03 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -7,11 +7,21 @@ import { createTaskThroughUi } from "./user-actions.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; -import { observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js"; +import { copilotDenialSampleCursor, readCopilotDenialSettlement, observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; type Row = Record; type Check = { id: string; passed: boolean; detail: string }; +/** Public run Stop and provider settlement are independent facts. The callback + * drains run events only after independent process retirement has been proved. */ +export async function settleCopilotDeniedRun(input: { + api: Pick; request: CopilotToolNotice; + awaitRetirement(): Promise<{ events: readonly unknown[]; run: unknown; issue: unknown }>; +}) { + await input.api.post(`/api/heartbeat-runs/${input.request.runId}/cancel`); + return readCopilotDenialSettlement({ ...await input.awaitRetirement(), request: input.request }); +} + export async function runCopilotProtectionFlow(input: { page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; workspacePath: string; deadlineAt: number; remoteBootstrap?: CopilotRemoteBootstrap; @@ -42,12 +52,14 @@ export async function runCopilotProtectionFlow(input: { const localTarget = deny && !remote ? await createDeniedTargetFixture(workspacePath, targetName) : undefined; const target = localTarget?.targetRelativePath ?? targetName, targetPath = localTarget?.targetPath ?? join(workspacePath, target); const fileObservations: CopilotDeniedWriteEvidence["fileObservations"] = []; + let deniedRequest: CopilotToolNotice | undefined; const sample = async (phase: CopilotDeniedWriteEvidence["fileObservations"][number]["phase"]) => { + const providerCursor = phase === "before-request" ? null : copilotDenialSampleCursor(runEvents, deniedRequest!); if (remote) { if (!remoteFixture || !baseline) throw new Error("Remote denied target has no baseline"); const snapshot = sealed ?? (phase === "before-request" ? baseline : await remoteFixture.snapshot(phase)); - remoteSnapshots.push(snapshot); fileObservations.push(copilotRemoteDeniedSample(snapshot, baseline, target, phase)); - } else fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath) }); + remoteSnapshots.push(snapshot); fileObservations.push({ ...copilotRemoteDeniedSample(snapshot, baseline, target, phase), providerCursor }); + } else fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath), providerCursor }); }; const watcher = localTarget?.watcher; const markerPath = join(workspacePath, `copilot-settlement-${nonce}.txt`); @@ -114,6 +126,7 @@ export async function runCopilotProtectionFlow(input: { if (deny) { await wait("exact native write permission", s => s.notices.some(n => n.stage === "permission_requested" && n.operation === "edit" && n.target === target && n.declineOffered && s.runEvents.some(r => r.eventType === "runtime_request.created" && r.payload?.prpEvent?.payload?.request?.requestId === n.requestId))); const request = notices.find(n => n.stage === "permission_requested" && n.operation === "edit" && n.target === target)!; + deniedRequest = request; const pending = runEvents.filter(r => r.eventType === "runtime_request.created" && r.payload?.prpEvent?.payload?.request?.requestId === request.requestId).map(r => r.payload.prpEvent.payload.request); check("one-bound-permission", pending.length === 1 && pending[0].requestKind === "permission_approval" && pending[0].origin?.method === "session/request_permission", "The notice maps to the exact durable native permission card"); const retired = new Set(runEvents.filter(r => ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(r.eventType)).map(r => r.payload?.prpEvent?.payload?.requestId)); @@ -129,24 +142,25 @@ export async function runCopilotProtectionFlow(input: { check("browser-exact-denial", posted.turnId === request.turnId && posted.requestKind === "permission_approval" && posted.resolution?.action === "decline", "Browser submitted denial for the exact run/request/turn"); await wait("delivered rejection and failed native edit", s => s.notices.some(n => n.stage === "permission_delivered" && n.requestId === request.requestId && n.outcome === "reject_once") && s.notices.some(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed")); await sample("after-decision"); - const cancelRequestedAtMs = Date.now(); await api.post(`/api/heartbeat-runs/${request.runId}/cancel`); - await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && (remote || (s.processes.captured && s.processes.live.length === 0))); - if (remote) await sealRemote(); - await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup"); + const settlement = await settleCopilotDeniedRun({ api, request, awaitRetirement: async () => { + await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && (remote || (s.processes.captured && s.processes.live.length === 0))); + if (remote) await sealRemote(); + await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup"); + return { events: runEvents, run: runs[0], issue }; + } }); watchReceipt = remote ? { startedAtMs: baseline!.observedAtMs, endedAtMs: sealed!.observedAtMs, complete: sealed!.watcher.complete, targetMutationCount: sealed!.watcher.targetMutationCount } : watcher!.finish(); - const cancellation = runs[0]!.resultJson?.nativeCancellation; const toolResult = notices.find(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed")!; - const terminalFrame = runEvents.find(r => ["turn.cancelled", "turn.interrupted"].includes(r.eventType) && r.payload?.prpEvent?.turnId === request.turnId)?.payload.prpEvent; - check("correlated-cancel-terminal", Boolean(terminalFrame) && terminalFrame.runId === request.runId, "An actual durable cancellation terminal belongs to the denied turn"); - const terminalAt = Date.parse(terminalFrame.emittedAt); + await input.evidence("copilot-denial-settlement.json", settlement); + check("correlated-provider-settlement", true, `Exact provider settlement (${settlement.branch}) and audited run Stop; completed does not cover active-turn cancellation`); + const terminalAt = settlement.providerTerminal.emittedAtMs; const evidence: CopilotDeniedWriteEvidence = { expected: copilotOrigin(request), requestId: request.requestId!, expectedRelativePath: target, request: { ...request, requestId: request.requestId!, targetRelativePath: request.target!, method: "session/request_permission", offeredActions: request.declineOffered ? ["decline"] : [] }, decision: { ...request, observedAtMs: clickedAtMs, requestId: request.requestId!, browserRequestId: request.requestId!, action: "decline" }, deliveredDecision: (() => { const n = notices.find(n => n.stage === "permission_delivered" && n.requestId === request.requestId && n.outcome === "reject_once"); return n ? { ...n, requestId: n.requestId!, outcome: n.outcome! } : null; })(), toolResult: { ...toolResult, status: "failed" }, - terminal: { runId: terminalFrame.runId, turnId: terminalFrame.turnId, observedAtMs: terminalAt, status: runs[0]!.status }, - cancellation: { requestedAtMs: cancelRequestedAtMs, acknowledged: cancellation?.dispatchState === "acknowledged" && cancellation?.dispatched === true, scope: cancellation?.scope }, + terminal: { runId: settlement.runId, turnId: settlement.turnId, observedAtMs: terminalAt, status: runs[0]!.status }, + settlement, cleanup: { observedAtMs: fileObservations.at(-1)!.observedAtMs, ownedProcessesRemaining: processes.live.length }, fileObservations, mutationObservation: watchReceipt, nativeAttemptsForTarget: countCopilotEditOriginsForTarget(notices, target), }; diff --git a/tests/runner-e2e/copilot-protection-settlement.test.ts b/tests/runner-e2e/copilot-protection-settlement.test.ts new file mode 100644 index 00000000000..a25cb49f57a --- /dev/null +++ b/tests/runner-e2e/copilot-protection-settlement.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it, vi } from "vitest"; +import { readCopilotToolEvidence } from "./copilot-evidence.js"; +import { copilotDenialSampleCursor, readCopilotDenialSettlement } from "./copilot-protection-evidence.js"; +import { settleCopilotDeniedRun } from "./copilot-protection-flow.js"; + +// Sanitized durable order from the failed Daytona attempt: completed was emitted +// at .887, persisted at .900; server Stop was requested at .892 and acked at .909. +const date = (ms: number) => new Date(Date.UTC(2026, 8, 30, 15, 13, 7, ms)).toISOString(); +function fixture(eventType = "turn.completed") { + const frame = (seq: number, type: string, payload: unknown, persisted = 305, emitted = 129): any => ({ + seq: seq + 40, companyId: "company", runId: "run", eventType: type, createdAt: date(persisted), + payload: { prpEvent: { schema: "paperclip.prp.event.v1", sourceKind: "runner", runId: "run", turnId: "turn", + normalizedSessionId: "normalized-session", sourceInstanceId: "runner-instance", sourceSeq: seq, + sourceEventId: `runner-instance:run:${seq}`, eventType: type, emittedAt: date(emitted), payload } }, + }); + const notice = (seq: number, stage: string, details: Record) => frame(seq, "provider.notice.recorded", { + schema: "paperclip.provider.notice.v1", category: "copilot_tool_evidence_v1", scope: "turn", + provenance: { sessionId: "native-session", turnId: "turn", eventType: stage, method: stage === "tool" ? "session/update" : "session/request_permission" }, + details: Object.entries({ stage, toolCallId: "denied-edit", operation: "edit", ...details }).map(([name, value]) => ({ name, value })), + }); + const events = [notice(111, "permission_requested", { requestId: "request", target: "copilot-denied-nonce.txt", declineOffered: "true" }), + notice(113, "permission_delivered", { requestId: "request", outcome: "reject_once" }), notice(114, "tool", { status: "failed" }), + frame(115, "tool.execution.completed", { schema: "paperclip.tool.execution.v1", executionId: "denied-edit", operation: "edit", status: "failed" }, 317), + frame(121, eventType, { status: eventType.slice(5), error: null }, 900, 887), + frame(112, "runtime_request.resolved", { requestId: "request", turnId: "turn", requestKind: "permission_approval", action: "decline" })]; + const run: any = { id: "run", companyId: "company", nativeIssueId: "issue", status: "cancelled", finishedAt: date(910), + resultJson: { startupCancellation: { requestedAt: date(892) }, nativeCancellation: { + schema: "paperclip.native-cancellation.v1", companyId: "company", runId: "run", issueId: "issue", scope: "run", dispatched: true, + dispatchState: "acknowledged", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], intentId: "intent", + intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", recordedAt: date(901), acknowledgedAt: date(909), + } } }; + return { events, request: readCopilotToolEvidence(events, "run")[0]!, run, issue: { id: "issue", companyId: "company", status: "in_progress" } }; +} +const terminal = (f: ReturnType) => f.events[4]!.payload.prpEvent; +describe("Copilot denial provider settlement and separate audited run Stop", () => { + it("accepts the retained normal-terminal race without claiming provider cancellation", () => { + const f = fixture(); + expect(readCopilotDenialSettlement(f)).toMatchObject({ branch: "provider_completed_before_stop_settlement", providerCancellationTerminalObserved: false, + providerTerminal: { sourceSeq: 121, failedToolSourceSeq: 115, persistedAtMs: Date.parse(date(900)) }, runStop: { status: "cancelled" } }); + // Native/provider clock skew is irrelevant to server persistence vs ack. + terminal(f).emittedAt = date(999); + expect(readCopilotDenialSettlement(f).branch).toBe("provider_completed_before_stop_settlement"); + }); + it.each(["turn.cancelled", "turn.interrupted"])("retains the distinct observed %s branch", type => { + expect(readCopilotDenialSettlement(fixture(type))).toMatchObject({ branch: "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: true }); + }); + it("cannot use a legacy summary or cancelled run as provider evidence", () => { + const f = fixture(); f.events.splice(4, 1); f.events.push({ eventType: "turn.completed", payload: {} }); + expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/); + }); + it.each(["equal", "later", "invalid"])("rejects %s normal terminal persistence against the server acknowledgement", kind => { + const f = fixture(); f.events[4]!.createdAt = kind === "invalid" ? "not-a-date" : date(kind === "equal" ? 909 : 910); + expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/); + }); + it.each(["runId", "turnId", "normalizedSessionId", "sourceInstanceId", "eventType", "sourceEventId"])("rejects terminal %s mismatch", field => { + const f = fixture(); terminal(f)[field] = "foreign"; + expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/); + }); + it.each(["missing", "duplicate", "failed", "before-tool", "wrong-status", "error"])("rejects %s terminal", kind => { + const f = fixture(); + if (kind === "missing") f.events.splice(4, 1); + if (kind === "duplicate") f.events.push(structuredClone(f.events[4])); + if (kind === "failed") { f.events[4]!.eventType = terminal(f).eventType = "turn.failed"; terminal(f).payload.status = "failed"; } + if (kind === "before-tool") { terminal(f).sourceSeq = 114; terminal(f).sourceEventId = "runner-instance:run:114"; } + if (kind === "wrong-status") terminal(f).payload.status = "cancelled"; + if (kind === "error") terminal(f).payload.error = { code: "failed" }; + expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/); + }); + it.each(["scope", "runId", "companyId", "issueId", "intentId", "intentAuditId", "acknowledgementAuditId", "acknowledgedAt", "recordedAt", "schema", "dispatchState", "dispatched", "reasonCode", "effects"])("rejects malformed or unbound Stop %s", field => { + const f = fixture(); f.run.resultJson.nativeCancellation[field] = null; + expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/); + }); + it.each(["missing-stop", "pending", "wrong-scope", "same-audit", "wrong-run-status", "issue-done", "ack-before-intent"])("rejects %s controller state", kind => { + const f = fixture(), c = f.run.resultJson.nativeCancellation; + if (kind === "missing-stop") delete f.run.resultJson.nativeCancellation; + if (kind === "pending") c.dispatchState = "pending"; + if (kind === "wrong-scope") c.scope = "turn"; + if (kind === "same-audit") c.acknowledgementAuditId = c.intentAuditId; + if (kind === "wrong-run-status") f.run.status = "succeeded"; + if (kind === "issue-done") f.issue.status = "done"; + if (kind === "ack-before-intent") c.recordedAt = date(911); + expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/); + }); + it.each(["native-session", "normalized-session", "failed-tool", "delivery", "duplicate-tool", "tool-order"])("rejects %s corruption", kind => { + const f = fixture(); + if (kind === "native-session") f.events[2]!.payload.prpEvent.payload.provenance.sessionId = "foreign"; + if (kind === "normalized-session") f.events[2]!.payload.prpEvent.normalizedSessionId = "foreign"; + if (kind === "failed-tool") f.events[3]!.payload.prpEvent.payload.status = "completed"; + if (kind === "delivery") f.events[1]!.payload.prpEvent.payload.details.find((d: any) => d.name === "outcome").value = "allow_once"; + if (kind === "duplicate-tool") f.events.push(structuredClone(f.events[3])); + if (kind === "tool-order") f.events[3]!.payload.prpEvent.sourceSeq = 122; + expect(() => readCopilotDenialSettlement(f)).toThrow(); + }); + it.each(["missing", "duplicate", "foreign-turn", "accept", "expired", "after-delivery"])("rejects %s durable resolution", kind => { + const f = fixture(), r = f.events[5]!, p = r.payload.prpEvent; + if (kind === "missing") f.events.pop(); + if (kind === "duplicate") f.events.push(structuredClone(r)); + if (kind === "foreign-turn") p.payload.turnId = "foreign"; + if (kind === "accept") p.payload.action = "accept"; + if (kind === "expired") r.eventType = p.eventType = "runtime_request.expired"; + if (kind === "after-delivery") { p.sourceSeq = 114; p.sourceEventId = "runner-instance:run:114"; } + expect(() => readCopilotDenialSettlement(f)).toThrow(); + }); + it("captures the exact durable stream at the sampling boundary without clock conversion", () => { + const f = fixture(); + expect(copilotDenialSampleCursor([f.events[0]], f.request)).toEqual({ runId: "run", turnId: "turn", normalizedSessionId: "normalized-session", sourceInstanceId: "runner-instance", sourceSeq: 111 }); + expect(copilotDenialSampleCursor(f.events, f.request).sourceSeq).toBe(121); + f.events[1]!.payload.prpEvent.normalizedSessionId = "foreign"; + expect(() => copilotDenialSampleCursor(f.events, f.request)).toThrow(/cursor/); + }); + it("flow sends public Stop, awaits retirement, then reads the raced canonical terminal", async () => { + const f = fixture(), order: string[] = []; const held = f.events.splice(4, 1)[0]; + const post = vi.fn(async (url: string) => { order.push(url); f.events.push(held); }); + const settlement = await settleCopilotDeniedRun({ api: { post } as any, request: f.request, + awaitRetirement: async () => { order.push("retired-and-final-events-read"); return f; } }); + expect(order).toEqual(["/api/heartbeat-runs/run/cancel", "retired-and-final-events-read"]); + expect(post).toHaveBeenCalledTimes(1); expect(settlement.branch).toBe("provider_completed_before_stop_settlement"); + await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, + awaitRetirement: async () => { throw new Error("remote descendants live"); } })).rejects.toThrow("remote descendants live"); + }); +});