mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(runner-e2e): distinguish Copilot denial settlement from Stop
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
0499513435
commit
72cbc3503d
9 files changed
+419
-37
No files matched your search
@@ -285,7 +285,17 @@ each with one run (120s
|
||||
provider timeout, 300s attempt budget). `native-permission-deny-write` denies one
|
||||
exact native edit through its browser card, waits for the delivered rejection and
|
||||
failed tool, then cancels through the public run API. Its expected outcome is a
|
||||
cancelled run and an unfinished task, not successful task completion. It rejects
|
||||
cancelled run and an unfinished task, not successful task completion.
|
||||
The retained `paperclip.e2e.copilot-denial-settlement.v1` proof separates the exact
|
||||
provider terminal from the audited controller Stop. It records either
|
||||
`provider_cancelled_or_interrupted` or `provider_completed_before_stop_settlement`.
|
||||
The latter requires normal terminal persistence strictly before the server's Stop
|
||||
acknowledgement, with the same normalized session/source stream and a later source
|
||||
sequence than the failed denied tool. It cannot establish active-turn cancellation;
|
||||
a cancelled/interrupted terminal also does not by itself prove Stop reached active
|
||||
work. Dedicated cancellation coverage must retain its active-operation evidence.
|
||||
Missing, ambiguous, failed or foreign terminals and incomplete Stop receipts fail.
|
||||
This distinction does not regrade earlier failed attempts. The case rejects
|
||||
extra native operations/runs and observes the absent target through process
|
||||
retirement. Filesystem event loss or an unexplained parent-directory timestamp
|
||||
change makes no-effect coverage incomplete; stat polling alone cannot pass.
|
||||
@@ -347,3 +357,13 @@ and order; any explicit canonical target must agree. Missing or conflicting
|
||||
attestations fail qualification. PRP retains only the first location and terminal
|
||||
updates may omit it, so canonical targets alone cannot prove bootstrap ownership.
|
||||
Neither tool titles nor output text supplies path evidence.
|
||||
|
||||
Denial ordering uses canonical request, decline-resolution, delivery, failed-edit
|
||||
and terminal source sequences. Sample checkpoints retain the exact run/turn/source
|
||||
cursor. File samples and continuous-watch coverage compare only observer-local
|
||||
times. Provider emission, browser click and server persistence clocks are never
|
||||
compared to each other. Final remote samples use the sealed, independently verified
|
||||
process-retirement receipt; their timestamps are not relabeled as host time.
|
||||
This denial case does not qualify Stop during a definitely pending native request.
|
||||
That active-turn cancellation boundary needs a separate live case. The attached
|
||||
async-command oracle is unchanged by this denial-only correction.
|
||||
@@ -1325,6 +1325,10 @@ and the limits of reconnect evidence.
|
||||
|
||||
The manual `copilot-protection` suite selects two Copilot candidate cases on
|
||||
each of local and Daytona (four cells). Discover them with `pnpm test:e2e:runner -- --list --suite copilot-protection`.
|
||||
The denial case keeps provider-turn settlement separate from controller run Stop.
|
||||
A normal provider completion before Stop acknowledgement is not active-turn
|
||||
cancellation coverage; denial, no-effects, run cancellation and retirement remain
|
||||
required. Earlier failed attempts retain their original grade.
|
||||
See [Copilot native protection](./FIXTURES.md#copilot-native-protection) for the
|
||||
expected cancelled negative test, finite attached-process oracle, evidence limits,
|
||||
and required rebuilt runtime. Registration is not a qualification claim.
|
||||
@@ -1412,3 +1416,13 @@ lease history. Each turn has a 120-second limit; each three-run cell has a
|
||||
420-second budget and must be reserved accordingly. Existing Codex warm cells
|
||||
remain separate. All these suites are excluded from `--all`, never automatically
|
||||
retry, and remain pending qualification until their paid evidence passes.
|
||||
|
||||
Denial ordering uses canonical request, decline-resolution, delivery, failed-edit
|
||||
and terminal source sequences. Sample checkpoints retain the exact run/turn/source
|
||||
cursor. File samples and continuous-watch coverage compare only observer-local
|
||||
times. Provider emission, browser click and server persistence clocks are never
|
||||
compared to each other. Final remote samples use the sealed, independently verified
|
||||
process-retirement receipt; their timestamps are not relabeled as host time.
|
||||
This denial case does not qualify Stop during a definitely pending native request.
|
||||
That active-turn cancellation boundary needs a separate live case. The attached
|
||||
async-command oracle is unchanged by this denial-only correction.
|
||||
@@ -1048,10 +1048,10 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
},
|
||||
{
|
||||
id: "copilot-protection", label: "Copilot native protection", manualOnly: true,
|
||||
description: "Exact native denial with explicit operator cancellation, and attached command settlement with independent process evidence.",
|
||||
description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.",
|
||||
groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"),
|
||||
environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4,
|
||||
definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "explicit-public-cancellation", settlement: "attached-finite-command-only", remoteEvidence: "owned-lease-sealed-observer" },
|
||||
definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v1", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", remoteEvidence: "owned-lease-sealed-observer" },
|
||||
},
|
||||
{
|
||||
id: "rich-acp-warm-continuity", label: "Rich ACP warm continuity", manualOnly: true,
|
||||
|
||||
@@ -7,15 +7,18 @@ const terminal = { observedAtMs: 50, runId: "run", turnId: "turn", status: "succ
|
||||
const cleanup = { observedAtMs: 60, ownedProcessesRemaining: 0 };
|
||||
function denied(): CopilotDeniedWriteEvidence {
|
||||
return {
|
||||
expected: identity, terminal: { ...terminal, status: "cancelled" }, cleanup, cancellation: { requestedAtMs: 40, acknowledged: true, scope: "run" }, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt",
|
||||
expected: identity, terminal: { ...terminal, status: "cancelled" }, cleanup, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v1", ...identity, requestId: "permission-0",
|
||||
branch: "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: true,
|
||||
providerTerminal: { eventType: "turn.cancelled", normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolPersistedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, persistedAtMs: 51 },
|
||||
runStop: { companyId: "company", issueId: "issue", scope: "run", status: "cancelled", issueStatus: "in_progress", intentId: "intent", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt",
|
||||
request: { ...identity, observedAtMs: 10, method: "session/request_permission", requestId: "permission-0", targetRelativePath: "copilot-denied-nonce.txt", offeredActions: ["accept", "decline"] },
|
||||
decision: { ...identity, observedAtMs: 20, requestId: "permission-0", browserRequestId: "permission-0", action: "decline" },
|
||||
deliveredDecision: { ...identity, observedAtMs: 25, requestId: "permission-0", outcome: "reject_once" },
|
||||
toolResult: { ...identity, observedAtMs: 30, status: "failed" }, nativeAttemptsForTarget: 1,
|
||||
fileObservations: [
|
||||
{ phase: "before-request", observedAtMs: 0, exists: false }, { phase: "pending", observedAtMs: 15, exists: false },
|
||||
{ phase: "after-decision", observedAtMs: 25, exists: false }, { phase: "terminal", observedAtMs: 50, exists: false },
|
||||
{ phase: "after-cleanup", observedAtMs: 60, exists: false },
|
||||
{ phase: "before-request", observedAtMs: 0, exists: false, providerCursor: null }, { phase: "pending", observedAtMs: 15, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 1 } },
|
||||
{ phase: "after-decision", observedAtMs: 25, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 4 } }, { phase: "terminal", observedAtMs: 50, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 6 } },
|
||||
{ phase: "after-cleanup", observedAtMs: 60, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 6 } },
|
||||
],
|
||||
mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 },
|
||||
};
|
||||
@@ -40,6 +43,49 @@ describe("Copilot protection Product oracles", () => {
|
||||
it("accepts an origin-bound browser denial with an independent continuous absence oracle", () => {
|
||||
expect(gradeCopilotDeniedWrite(denied())).toEqual({ passed: true, failures: [] });
|
||||
});
|
||||
it("keeps every no-effect and cleanup gate for a completed-before-Stop settlement", () => {
|
||||
const e = denied();
|
||||
e.settlement!.branch = "provider_completed_before_stop_settlement";
|
||||
e.settlement!.providerCancellationTerminalObserved = false;
|
||||
e.settlement!.providerTerminal.eventType = "turn.completed";
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(true);
|
||||
for (const mutate of [
|
||||
(v: CopilotDeniedWriteEvidence) => { v.cleanup!.ownedProcessesRemaining = 1; },
|
||||
(v: CopilotDeniedWriteEvidence) => { v.nativeAttemptsForTarget = 2; },
|
||||
(v: CopilotDeniedWriteEvidence) => { v.fileObservations[2]!.exists = true; },
|
||||
(v: CopilotDeniedWriteEvidence) => { v.mutationObservation!.targetMutationCount = 1; },
|
||||
(v: CopilotDeniedWriteEvidence) => { v.mutationObservation!.complete = false; },
|
||||
(v: CopilotDeniedWriteEvidence) => { v.deliveredDecision = null; },
|
||||
(v: CopilotDeniedWriteEvidence) => { v.toolResult!.status = "completed"; },
|
||||
(v: CopilotDeniedWriteEvidence) => { v.settlement!.providerCancellationTerminalObserved = true; },
|
||||
(v: CopilotDeniedWriteEvidence) => { v.settlement!.turnId = "foreign"; },
|
||||
]) { const bad = structuredClone(e); mutate(bad); expect(gradeCopilotDeniedWrite(bad).passed).toBe(false); }
|
||||
});
|
||||
it.each([-86_400_000, 86_400_000])("completed settlement does not compare provider, browser and filesystem clocks (%s)", skew => {
|
||||
const e = denied();
|
||||
e.settlement!.branch = "provider_completed_before_stop_settlement";
|
||||
e.settlement!.providerCancellationTerminalObserved = false;
|
||||
e.settlement!.providerTerminal.eventType = "turn.completed";
|
||||
// Distinct positive clock domains, with either provider ahead or behind.
|
||||
for (const n of [e.request!, e.deliveredDecision!, e.toolResult!, e.terminal!]) n.observedAtMs += 100_000_000 + skew;
|
||||
e.settlement!.providerTerminal.emittedAtMs = e.terminal!.observedAtMs;
|
||||
e.decision!.observedAtMs = 200_000_000;
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(true);
|
||||
e.fileObservations[1]!.providerCursor!.sourceSeq = e.settlement!.providerTerminal.resolvedSourceSeq;
|
||||
expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid");
|
||||
});
|
||||
it.each(["runId", "turnId", "normalizedSessionId", "sourceInstanceId"] as const)("rejects foreign sample %s despite plausible timestamps", field => {
|
||||
const e = denied(); e.fileObservations[3]!.providerCursor![field] = "foreign";
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
});
|
||||
it("rejects missing, regressed and premature causal sample cursors", () => {
|
||||
for (const mutate of [
|
||||
(e: CopilotDeniedWriteEvidence) => { e.fileObservations[2]!.providerCursor = null; },
|
||||
(e: CopilotDeniedWriteEvidence) => { e.fileObservations[2]!.providerCursor!.sourceSeq = 3; },
|
||||
(e: CopilotDeniedWriteEvidence) => { e.fileObservations[3]!.providerCursor!.sourceSeq = 5; },
|
||||
(e: CopilotDeniedWriteEvidence) => { e.fileObservations[4]!.providerCursor!.sourceSeq = 5; },
|
||||
]) { const e = denied(); mutate(e); expect(gradeCopilotDeniedWrite(e).passed).toBe(false); }
|
||||
});
|
||||
it("requires the exact isolated target in native permission evidence", () => {
|
||||
const e = denied(); e.expectedRelativePath = "pc-denied-ABC123/copilot-denied-nonce.txt";
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
@@ -50,11 +96,11 @@ describe("Copilot protection Product oracles", () => {
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
}
|
||||
});
|
||||
it.each(["request", "decision", "deliveredDecision", "toolResult", "terminal", "cleanup", "mutationObservation"] as const)("rejects missing %s instead of treating no write as denial", field => {
|
||||
it.each(["request", "decision", "deliveredDecision", "toolResult", "terminal", "cleanup", "mutationObservation", "settlement"] as const)("rejects missing %s instead of treating no write as denial", field => {
|
||||
const e = denied(); e[field] = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
});
|
||||
it("requires explicit acknowledged cancellation for denial without relaxing successful settlement", () => {
|
||||
const e = denied(); e.cancellation = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
const e = denied(); e.settlement = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
const finished = denied(); finished.terminal!.status = "succeeded"; expect(gradeCopilotDeniedWrite(finished).passed).toBe(false);
|
||||
const cancelled = attached(); cancelled.terminal!.status = "cancelled"; expect(gradeCopilotAttachedSettlement(cancelled).passed).toBe(false);
|
||||
});
|
||||
@@ -72,7 +118,7 @@ describe("Copilot protection Product oracles", () => {
|
||||
const retried = denied(); retried.nativeAttemptsForTarget = 2; expect(gradeCopilotDeniedWrite(retried).passed).toBe(false);
|
||||
});
|
||||
it("rejects a sample taken before the request as pending evidence", () => {
|
||||
const e = denied(); e.fileObservations[1]!.observedAtMs = 5; expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid");
|
||||
const e = denied(); e.fileObservations[1]!.providerCursor!.sourceSeq = 0; expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid");
|
||||
});
|
||||
it("rejects an unrelated request identity even when the chosen decision matches an outer ID", () => {
|
||||
const e = denied(); e.request!.requestId = "foreign-request"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { matchCopilotFixtureCommand, type CopilotCommandMatch } from "./copilot-protection-evidence.js";
|
||||
import { matchCopilotFixtureCommand, validCopilotDenialSettlement, type CopilotDenialSettlement, type CopilotDenialSampleCursor, type CopilotCommandMatch } from "./copilot-protection-evidence.js";
|
||||
/** Registered manual cases; paid Product qualification remains separate. */
|
||||
export const copilotProtectionCases = [
|
||||
{
|
||||
@@ -43,10 +43,12 @@ interface FileObservation {
|
||||
phase: "before-request" | "pending" | "after-decision" | "terminal" | "after-cleanup";
|
||||
observedAtMs: number;
|
||||
exists: boolean;
|
||||
/** null only before dispatch/action publication; otherwise read before sampling. */
|
||||
providerCursor: CopilotDenialSampleCursor | null;
|
||||
}
|
||||
export interface CopilotDeniedWriteEvidence extends Lifecycle {
|
||||
deliveredDecision: (TimedIdentity & { requestId: string; outcome: string }) | null;
|
||||
cancellation: { requestedAtMs: number; acknowledged: boolean; scope: string } | null;
|
||||
settlement: CopilotDenialSettlement | null;
|
||||
request: (TimedIdentity & { method: "session/request_permission"; requestId: string; targetRelativePath: string; offeredActions: string[] }) | null;
|
||||
decision: (TimedIdentity & { requestId: string; action: string; browserRequestId: string }) | null;
|
||||
requestId: string | null;
|
||||
@@ -78,17 +80,35 @@ function lifecycle(e: Lifecycle, failures: string[], expectedStatus: "succeeded"
|
||||
if (!e.cleanup || e.cleanup.ownedProcessesRemaining !== 0 || !time(e.cleanup.observedAtMs) || !e.terminal || e.cleanup.observedAtMs < e.terminal.observedAtMs) failures.push("unsettled-cleanup");
|
||||
}
|
||||
export function gradeCopilotDeniedWrite(e: CopilotDeniedWriteEvidence): CopilotProtectionGrade {
|
||||
const failures: string[] = []; lifecycle(e, failures, "cancelled");
|
||||
if (!e.cancellation || !e.cancellation.acknowledged || !time(e.cancellation.requestedAtMs) || e.cancellation.scope !== "run" || !e.toolResult || e.cancellation.requestedAtMs < e.toolResult.observedAtMs || !e.terminal || e.cancellation.requestedAtMs > e.terminal.observedAtMs) failures.push("missing-explicit-settled-cancellation");
|
||||
const failures: string[] = [];
|
||||
// The provider, browser and observer clocks are independent. Denial ordering
|
||||
// uses durable source cursors below; the attached-command oracle is unchanged.
|
||||
if (!Object.values(e.expected).every(v => typeof v === "string" && v.length > 0)) failures.push("missing-origin-identity");
|
||||
if (!e.terminal || e.terminal.runId !== e.expected.runId || e.terminal.turnId !== e.expected.turnId || e.terminal.status !== "cancelled" || !time(e.terminal.observedAtMs)) failures.push("missing-expected-origin-terminal");
|
||||
if (!e.cleanup || e.cleanup.ownedProcessesRemaining !== 0 || !time(e.cleanup.observedAtMs)) failures.push("unsettled-cleanup");
|
||||
const settlement = e.settlement;
|
||||
if (!settlement || !validCopilotDenialSettlement(settlement) || !same(settlement, e.expected) || settlement.requestId !== e.requestId
|
||||
|| !e.terminal || e.terminal.observedAtMs !== settlement.providerTerminal.emittedAtMs) failures.push("missing-explicit-settled-run-stop");
|
||||
if (!/^(?:pc-denied-[a-zA-Z0-9]+\/)?copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request");
|
||||
if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs) || !e.request || e.decision.observedAtMs < e.request.observedAtMs) failures.push("missing-exact-browser-denial");
|
||||
if (!e.deliveredDecision || !same(e.deliveredDecision, e.expected) || e.deliveredDecision.requestId !== e.requestId || e.deliveredDecision.outcome !== "reject_once" || !time(e.deliveredDecision.observedAtMs) || !e.decision || e.deliveredDecision.observedAtMs < e.decision.observedAtMs || !e.toolResult || e.deliveredDecision.observedAtMs > e.toolResult.observedAtMs) failures.push("missing-delivered-native-rejection");
|
||||
if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs) || !e.decision || e.toolResult.observedAtMs < e.decision.observedAtMs || !e.terminal || e.toolResult.observedAtMs > e.terminal.observedAtMs) failures.push("missing-denied-tool-result-before-terminal");
|
||||
if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs)) failures.push("missing-exact-browser-denial");
|
||||
if (!e.deliveredDecision || !same(e.deliveredDecision, e.expected) || e.deliveredDecision.requestId !== e.requestId || e.deliveredDecision.outcome !== "reject_once" || !time(e.deliveredDecision.observedAtMs)) failures.push("missing-delivered-native-rejection");
|
||||
if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs)) failures.push("missing-denied-tool-result-before-terminal");
|
||||
if (e.nativeAttemptsForTarget !== 1) failures.push("missing-or-retried-native-write");
|
||||
const phases = ["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const;
|
||||
if (phases.some(phase => !e.fileObservations.some(s => s.phase === phase)) || e.fileObservations.some(s => s.exists || !time(s.observedAtMs))) failures.push("missing-or-mutated-target-observation");
|
||||
const samples = phases.map(phase => e.fileObservations.find(s => s.phase === phase));
|
||||
if (samples.some((s, i) => !s || (i > 0 && s.observedAtMs < samples[i - 1]!.observedAtMs)) || !e.request || !e.decision || !e.terminal || !e.cleanup || (samples[0]?.observedAtMs ?? Infinity) > e.request.observedAtMs || (samples[1]?.observedAtMs ?? -1) < e.request.observedAtMs || (samples[1]?.observedAtMs ?? Infinity) > e.decision.observedAtMs || (samples[2]?.observedAtMs ?? -1) < e.decision.observedAtMs || (samples[3]?.observedAtMs ?? -1) < e.terminal.observedAtMs || (samples[4]?.observedAtMs ?? -1) < e.cleanup.observedAtMs) failures.push("filesystem-observation-order-invalid");
|
||||
const t = settlement?.providerTerminal;
|
||||
const cursorMatches = (s: FileObservation | undefined) => s?.providerCursor && t && s.providerCursor.runId === e.expected.runId
|
||||
&& s.providerCursor.turnId === e.expected.turnId && s.providerCursor.normalizedSessionId === t.normalizedSessionId
|
||||
&& s.providerCursor.sourceInstanceId === t.sourceInstanceId && Number.isSafeInteger(s.providerCursor.sourceSeq);
|
||||
if (e.fileObservations.length !== phases.length || samples.some((s, i) => !s || (i > 0 && s.observedAtMs < samples[i - 1]!.observedAtMs))
|
||||
|| samples[0]?.providerCursor !== null || !t || samples.slice(1).some(s => !cursorMatches(s))
|
||||
|| (samples[1]?.providerCursor?.sourceSeq ?? -1) < t.requestSourceSeq
|
||||
|| (samples[1]?.providerCursor?.sourceSeq ?? Infinity) >= t.resolvedSourceSeq
|
||||
|| (samples[2]?.providerCursor?.sourceSeq ?? -1) < t.failedNoticeSourceSeq
|
||||
|| (samples[3]?.providerCursor?.sourceSeq ?? -1) < t.sourceSeq || (samples[4]?.providerCursor?.sourceSeq ?? -1) < t.sourceSeq
|
||||
|| samples.slice(2).some((s, i) => (s?.providerCursor?.sourceSeq ?? -1) < (samples[i + 1]?.providerCursor?.sourceSeq ?? Infinity))
|
||||
|| !e.cleanup || samples[4]?.observedAtMs !== e.cleanup.observedAtMs) failures.push("filesystem-observation-order-invalid");
|
||||
const m = e.mutationObservation;
|
||||
if (!m || !m.complete || m.targetMutationCount !== 0 || !time(m.startedAtMs) || !time(m.endedAtMs) || !samples[0] || !samples[4] || m.startedAtMs > samples[0].observedAtMs || m.endedAtMs < samples[4].observedAtMs) failures.push("missing-or-mutated-filesystem-watch");
|
||||
return { passed: failures.length === 0, failures };
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { withoutProvenBootstrapReads, type BootstrapReadProof } from "./native-bootstrap-read-proof.js";
|
||||
import { withoutProvenBootstrapReads, bootstrapReadExecutionId, type BootstrapReadProof } from "./native-bootstrap-read-proof.js";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { CopilotToolNotice } from "./copilot-evidence.js";
|
||||
import { readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js";
|
||||
|
||||
/** A tool origin exists regardless of the first status emitted by the provider. */
|
||||
export function countCopilotToolOrigins(notices: readonly CopilotToolNotice[]): number {
|
||||
@@ -193,3 +193,144 @@ export async function readCopilotRemoteMarkerAfterRetirement(fixture: CopilotRem
|
||||
const bytes = await fixture.readFile(target);
|
||||
return bytes.toString("utf8") === expected && receipt.targets[target]?.sha256 === `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
||||
}
|
||||
|
||||
export interface CopilotDenialSettlement {
|
||||
schema: "paperclip.e2e.copilot-denial-settlement.v1";
|
||||
branch: "provider_cancelled_or_interrupted" | "provider_completed_before_stop_settlement";
|
||||
/** A cancelled terminal alone does not prove Stop reached active work. */
|
||||
providerCancellationTerminalObserved: boolean;
|
||||
runId: string; sessionId: string; turnId: string; toolCallId: string; requestId: string;
|
||||
providerTerminal: {
|
||||
eventType: "turn.completed" | "turn.cancelled" | "turn.interrupted";
|
||||
normalizedSessionId: string; sourceInstanceId: string;
|
||||
requestSourceSeq: number; resolvedSourceSeq: number; deliveredSourceSeq: number; failedNoticeSourceSeq: number;
|
||||
failedToolSourceSeq: number; failedToolPersistedAtMs: number; sourceSeq: number;
|
||||
emittedAtMs: number; persistedAtMs: number;
|
||||
};
|
||||
runStop: {
|
||||
companyId: string; issueId: string; scope: "run"; status: "cancelled"; issueStatus: "in_progress";
|
||||
intentId: string; intentAuditId: string; acknowledgementAuditId: string;
|
||||
requestedAtMs: number; recordedAtMs: number; acknowledgedAtMs: number; finishedAtMs: number;
|
||||
};
|
||||
}
|
||||
const settlementRecord = (v: unknown): Record<string, any> => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Record<string, any> : {};
|
||||
const settlementId = (v: unknown): v is string => typeof v === "string" && v.length > 0 && v.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(v) && !v.includes("[REDACTED]");
|
||||
const settlementTime = (v: unknown): v is number => Number.isSafeInteger(v) && (v as number) >= 0;
|
||||
const settlementDate = (v: unknown): number => typeof v === "string" && /^\d{4}-\d{2}-\d{2}T.*Z$/u.test(v) ? Date.parse(v) : NaN;
|
||||
|
||||
export function validCopilotDenialSettlement(s: CopilotDenialSettlement): boolean {
|
||||
if (!s || s.schema !== "paperclip.e2e.copilot-denial-settlement.v1") return false;
|
||||
const t = s.providerTerminal, c = s.runStop;
|
||||
if (!t || !c || ![s.runId, s.sessionId, s.turnId, s.toolCallId, s.requestId, t.normalizedSessionId, t.sourceInstanceId,
|
||||
c.companyId, c.issueId, c.intentId, c.intentAuditId, c.acknowledgementAuditId].every(settlementId)
|
||||
|| c.scope !== "run" || c.status !== "cancelled" || c.issueStatus !== "in_progress" || c.intentAuditId === c.acknowledgementAuditId
|
||||
|| ![t.failedToolPersistedAtMs, t.emittedAtMs, t.persistedAtMs, c.requestedAtMs, c.recordedAtMs, c.acknowledgedAtMs, c.finishedAtMs].every(settlementTime)
|
||||
|| ![t.requestSourceSeq, t.resolvedSourceSeq, t.deliveredSourceSeq, t.failedNoticeSourceSeq].every(n => Number.isSafeInteger(n) && n > 0)
|
||||
|| !(t.requestSourceSeq < t.resolvedSourceSeq && t.resolvedSourceSeq < t.deliveredSourceSeq && t.deliveredSourceSeq < t.failedNoticeSourceSeq && t.failedNoticeSourceSeq < t.failedToolSourceSeq)
|
||||
|| !Number.isSafeInteger(t.failedToolSourceSeq) || t.failedToolSourceSeq <= 0 || !Number.isSafeInteger(t.sourceSeq) || t.sourceSeq <= t.failedToolSourceSeq
|
||||
|| t.failedToolPersistedAtMs > c.requestedAtMs || t.failedToolPersistedAtMs > t.persistedAtMs
|
||||
|| c.requestedAtMs > c.recordedAtMs || c.recordedAtMs > c.acknowledgedAtMs || c.acknowledgedAtMs > c.finishedAtMs) return false;
|
||||
if (s.branch === "provider_completed_before_stop_settlement") return t.eventType === "turn.completed"
|
||||
&& s.providerCancellationTerminalObserved === false && t.persistedAtMs < c.acknowledgedAtMs;
|
||||
return s.branch === "provider_cancelled_or_interrupted" && s.providerCancellationTerminalObserved === true
|
||||
&& ["turn.cancelled", "turn.interrupted"].includes(t.eventType);
|
||||
}
|
||||
|
||||
/** Denial closes one permission, not necessarily the provider prompt. Match the
|
||||
* single canonical terminal to the same source stream, then independently bind
|
||||
* the audited controller Stop. Provider emittedAt never orders server actions. */
|
||||
export function readCopilotDenialSettlement(input: {
|
||||
events: readonly unknown[]; request: CopilotToolNotice; run: unknown; issue: unknown;
|
||||
}): CopilotDenialSettlement {
|
||||
const invalid = () => new Error("Copilot denial lacks exact provider settlement and audited run Stop");
|
||||
const { request, events } = input, run = settlementRecord(input.run), issue = settlementRecord(input.issue);
|
||||
const stop = settlementRecord(settlementRecord(run.resultJson).nativeCancellation);
|
||||
if (events.length > 20_000 || run.id !== request.runId || issue.id !== run.nativeIssueId
|
||||
|| !settlementId(issue.companyId) || run.companyId !== issue.companyId || issue.status !== "in_progress" || run.status !== "cancelled"
|
||||
|| stop.schema !== "paperclip.native-cancellation.v1" || stop.runId !== run.id || stop.companyId !== issue.companyId || stop.issueId !== issue.id
|
||||
|| stop.scope !== "run" || stop.dispatched !== true || stop.dispatchState !== "acknowledged" || stop.reasonCode !== "cancellation_run_only"
|
||||
|| !Array.isArray(stop.effects) || stop.effects.length !== 1 || stop.effects[0] !== "release_run_resources") throw invalid();
|
||||
const notices = readCopilotToolEvidence(events, request.runId);
|
||||
const same = (n: CopilotToolNotice) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => n[k as keyof CopilotToolNotice] === request[k as keyof CopilotToolNotice]);
|
||||
const requested = notices.filter(n => n.stage === "permission_requested" && same(n));
|
||||
const delivered = notices.filter(n => n.stage === "permission_delivered" && same(n));
|
||||
const failed = notices.filter(n => n.stage === "tool" && same(n) && ["failed", "completed"].includes(n.status ?? ""));
|
||||
if (requested.length !== 1 || requested[0]!.seq !== request.seq || requested[0]!.requestId !== request.requestId || request.operation !== "edit"
|
||||
|| delivered.length !== 1 || delivered[0]!.requestId !== request.requestId || delivered[0]!.outcome !== "reject_once"
|
||||
|| failed.length !== 1 || failed[0]!.status !== "failed" || requested[0]!.seq >= delivered[0]!.seq || delivered[0]!.seq >= failed[0]!.seq) throw invalid();
|
||||
const rows = events.map(settlementRecord);
|
||||
const get = (row: Record<string, any>) => {
|
||||
const f = settlementRecord(settlementRecord(row.payload).prpEvent);
|
||||
if (row.runId !== run.id || row.companyId !== issue.companyId || !Number.isSafeInteger(row.seq) || row.seq <= 0
|
||||
|| f.schema !== "paperclip.prp.event.v1" || f.sourceKind !== "runner" || f.eventType !== row.eventType || f.runId !== run.id || f.turnId !== request.turnId
|
||||
|| !settlementId(f.normalizedSessionId) || !settlementId(f.sourceInstanceId) || !Number.isSafeInteger(f.sourceSeq) || f.sourceSeq <= 0
|
||||
|| f.sourceEventId !== `${f.sourceInstanceId}:${run.id}:${f.sourceSeq}` || !settlementTime(settlementDate(row.createdAt)) || !settlementTime(settlementDate(f.emittedAt))) throw invalid();
|
||||
return f;
|
||||
};
|
||||
const at = (seq: number) => { const found = rows.filter(r => r.seq === seq); if (found.length !== 1) throw invalid(); return get(found[0]!); };
|
||||
const origin = at(request.seq), delivery = at(delivered[0]!.seq), failure = at(failed[0]!.seq);
|
||||
const stream = (f: Record<string, any>) => f.normalizedSessionId === origin.normalizedSessionId && f.sourceInstanceId === origin.sourceInstanceId;
|
||||
if (![delivery, failure].every(stream) || origin.sourceSeq >= delivery.sourceSeq || delivery.sourceSeq >= failure.sourceSeq) throw invalid();
|
||||
const resolutions = rows.filter(r => ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(r.eventType)
|
||||
&& settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).requestId === request.requestId);
|
||||
if (resolutions.length !== 1) throw invalid();
|
||||
const resolution = get(resolutions[0]!), resolved = settlementRecord(resolution.payload);
|
||||
if (!stream(resolution) || resolution.eventType !== "runtime_request.resolved" || resolved.requestKind !== "permission_approval"
|
||||
|| resolved.turnId !== request.turnId || resolved.action !== "decline" || resolution.sourceSeq <= origin.sourceSeq
|
||||
|| resolution.sourceSeq >= delivery.sourceSeq || resolutions[0]!.seq <= request.seq || resolutions[0]!.seq >= delivered[0]!.seq) throw invalid();
|
||||
// Ignore legacy unwrapped log summaries, never use them as provider evidence.
|
||||
const terminalRows = rows.filter(r => ["turn.completed", "turn.cancelled", "turn.interrupted", "turn.failed"].includes(r.eventType)
|
||||
&& settlementRecord(r.payload).prpEvent !== undefined);
|
||||
const toolRows = rows.filter(r => r.eventType === "tool.execution.completed"
|
||||
&& settlementRecord(settlementRecord(settlementRecord(r.payload).prpEvent).payload).executionId === bootstrapReadExecutionId(request.toolCallId));
|
||||
if (terminalRows.length !== 1 || toolRows.length !== 1) throw invalid();
|
||||
const tool = get(toolRows[0]!), toolPayload = settlementRecord(tool.payload);
|
||||
if (!stream(tool) || tool.sourceSeq <= failure.sourceSeq || toolRows[0]!.seq <= failed[0]!.seq
|
||||
|| toolPayload.schema !== "paperclip.tool.execution.v1" || toolPayload.status !== "failed" || toolPayload.operation !== "edit") throw invalid();
|
||||
const row = terminalRows[0]!, terminal = get(row), p = settlementRecord(terminal.payload);
|
||||
if (!stream(terminal) || terminal.sourceSeq <= tool.sourceSeq || row.seq <= failed[0]!.seq
|
||||
|| !["turn.completed", "turn.cancelled", "turn.interrupted"].includes(row.eventType)
|
||||
|| p.status !== row.eventType.slice(5) || p.error != null) throw invalid();
|
||||
const result: CopilotDenialSettlement = {
|
||||
schema: "paperclip.e2e.copilot-denial-settlement.v1",
|
||||
branch: row.eventType === "turn.completed" ? "provider_completed_before_stop_settlement" : "provider_cancelled_or_interrupted",
|
||||
providerCancellationTerminalObserved: row.eventType !== "turn.completed",
|
||||
runId: run.id, sessionId: request.sessionId, turnId: request.turnId, toolCallId: request.toolCallId, requestId: request.requestId!,
|
||||
providerTerminal: { eventType: row.eventType, normalizedSessionId: terminal.normalizedSessionId, sourceInstanceId: terminal.sourceInstanceId,
|
||||
requestSourceSeq: origin.sourceSeq, resolvedSourceSeq: resolution.sourceSeq, deliveredSourceSeq: delivery.sourceSeq, failedNoticeSourceSeq: failure.sourceSeq,
|
||||
failedToolSourceSeq: tool.sourceSeq, failedToolPersistedAtMs: settlementDate(toolRows[0]!.createdAt), sourceSeq: terminal.sourceSeq, emittedAtMs: settlementDate(terminal.emittedAt), persistedAtMs: settlementDate(row.createdAt) },
|
||||
runStop: { companyId: issue.companyId, issueId: issue.id, scope: stop.scope, status: run.status, issueStatus: issue.status,
|
||||
intentId: stop.intentId, intentAuditId: stop.intentAuditId, acknowledgementAuditId: stop.acknowledgementAuditId,
|
||||
requestedAtMs: settlementDate(settlementRecord(settlementRecord(run.resultJson).startupCancellation).requestedAt),
|
||||
recordedAtMs: settlementDate(stop.recordedAt), acknowledgedAtMs: settlementDate(stop.acknowledgedAt), finishedAtMs: settlementDate(run.finishedAt) },
|
||||
};
|
||||
if (!validCopilotDenialSettlement(result)) throw invalid();
|
||||
return result;
|
||||
}
|
||||
|
||||
export interface CopilotDenialSampleCursor {
|
||||
runId: string; turnId: string; normalizedSessionId: string; sourceInstanceId: string; sourceSeq: number;
|
||||
}
|
||||
/** Read boundary for a sample checkpoint, not a clock conversion. A remote final
|
||||
* sample can be an earlier sealed receipt; its separate retirement/watch proof
|
||||
* must cover the exact provider root through exit before that receipt is used. */
|
||||
export function copilotDenialSampleCursor(events: readonly unknown[], request: CopilotToolNotice): CopilotDenialSampleCursor {
|
||||
const rows = events.map(settlementRecord), invalid = () => new Error("Copilot denied sample lacks an exact event cursor");
|
||||
const origins = rows.filter(r => r.seq === request.seq);
|
||||
if (origins.length !== 1) throw invalid();
|
||||
const origin = settlementRecord(settlementRecord(origins[0]!.payload).prpEvent);
|
||||
const frames = rows.map(r => ({ row: r, f: settlementRecord(settlementRecord(r.payload).prpEvent) }))
|
||||
.filter(({ f }) => f.turnId === request.turnId);
|
||||
if (events.length > 20_000 || frames.length === 0 || origin.runId !== request.runId
|
||||
|| ![origin.normalizedSessionId, origin.sourceInstanceId].every(settlementId)) throw invalid();
|
||||
const seen = new Set<number>();
|
||||
for (const { row, f } of frames) {
|
||||
if (row.runId !== request.runId || f.runId !== request.runId || f.schema !== "paperclip.prp.event.v1" || f.sourceKind !== "runner"
|
||||
|| row.eventType !== f.eventType || f.normalizedSessionId !== origin.normalizedSessionId || f.sourceInstanceId !== origin.sourceInstanceId
|
||||
|| !Number.isSafeInteger(f.sourceSeq) || f.sourceSeq <= 0 || seen.has(f.sourceSeq)
|
||||
|| f.sourceEventId !== `${f.sourceInstanceId}:${f.runId}:${f.sourceSeq}`) throw invalid();
|
||||
seen.add(f.sourceSeq);
|
||||
}
|
||||
return { runId: request.runId, turnId: request.turnId, normalizedSessionId: origin.normalizedSessionId,
|
||||
sourceInstanceId: origin.sourceInstanceId, sourceSeq: Math.max(...seen) };
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import { validateAcpxRichEvent } from "../../packages/paperclip-runner/src/drive
|
||||
import { copilotOrigin, readCopilotToolEvidence } from "./copilot-evidence.js";
|
||||
import { copilotProtectionCases, gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite } from "./copilot-protection-cases.js";
|
||||
import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, watchDeniedTarget, exists, isPerTurnRunProcess } from "./copilot-local-fixtures.js";
|
||||
import { runnerMatrix } from "./catalog.js";
|
||||
import { runnerMatrix, suiteDefinitionHash } from "./catalog.js";
|
||||
import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js";
|
||||
|
||||
const fixture = JSON.parse(await readFile(new URL("../../packages/paperclip-runner/src/drivers/acpx/fixtures/copilot-tool-evidence.json", import.meta.url), "utf8"));
|
||||
@@ -26,6 +26,9 @@ function projected(name: string, target = "copilot-denied-nonce.txt") {
|
||||
describe("Copilot Product protection integration", () => {
|
||||
it("registers two explicit cases on both environments with honest terminal expectations", () => {
|
||||
const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection");
|
||||
expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 3, denialTerminal: "correlated-provider-settlement-and-audited-run-stop",
|
||||
denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v1", activeTurnCancellation: "not-implied-by-completed-provider-turn" });
|
||||
expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } }));
|
||||
expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"]));
|
||||
expect(cells.every(c => c.profile.qualificationCandidate === "copilot" && c.task.expectedRunCount === 1)).toBe(true);
|
||||
expect(cells.find(c => c.task.id === "native-permission-deny-write")!.task.expectedTerminalState).toEqual({ issue: "in_progress", run: "cancelled" });
|
||||
@@ -38,13 +41,16 @@ describe("Copilot Product protection integration", () => {
|
||||
request: { ...request, requestId: "permission", method: "session/request_permission" as const, targetRelativePath: request.target!, offeredActions: request.declineOffered ? ["decline"] : [] },
|
||||
decision: { ...delivered, requestId: "permission", browserRequestId: "permission", action: delivered.outcome === "reject_once" ? "decline" : "accept" },
|
||||
deliveredDecision: { ...delivered, requestId: "permission", outcome: delivered.outcome! },
|
||||
toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, cancellation: { requestedAtMs: 40, scope: "run", acknowledged: true },
|
||||
toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v1" as const, ...copilotOrigin(request), requestId: "permission",
|
||||
branch: "provider_cancelled_or_interrupted" as const, providerCancellationTerminalObserved: true,
|
||||
providerTerminal: { eventType: "turn.cancelled" as const, normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolPersistedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, persistedAtMs: 51 },
|
||||
runStop: { companyId: "company", issueId: "issue", scope: "run" as const, status: "cancelled" as const, issueStatus: "in_progress" as const, intentId: "intent", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } },
|
||||
cleanup: { observedAtMs: 60, ownedProcessesRemaining: 0 }, nativeAttemptsForTarget: 1,
|
||||
fileObservations: (["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const).map((phase, index) => ({ phase, observedAtMs: [0, request.observedAtMs, 30, 50, 60][index]!, exists: false })), mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 } };
|
||||
fileObservations: (["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const).map((phase, index) => ({ phase, observedAtMs: [0, request.observedAtMs, 30, 50, 60][index]!, exists: false, providerCursor: index === 0 ? null : { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: [0, 1, 4, 6, 6][index]! } })), mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 } };
|
||||
expect(request.target).toBe(target);
|
||||
expect(gradeCopilotDeniedWrite(e).passed).toBe(true);
|
||||
e.request.targetRelativePath = "foreign.txt"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
e.request.targetRelativePath = e.expectedRelativePath; e.cancellation.acknowledged = false; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
e.request.targetRelativePath = e.expectedRelativePath; e.settlement.runStop.acknowledgementAuditId = ""; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
|
||||
});
|
||||
it("feeds actual attached wire through canonical notices and rejects early terminal or missing linkage", () => {
|
||||
const { notices } = projected("attached-shell"); const call = notices.find(n => n.commandSha256)!;
|
||||
|
||||
@@ -7,11 +7,21 @@ import { createTaskThroughUi } from "./user-actions.js";
|
||||
import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js";
|
||||
import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js";
|
||||
import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js";
|
||||
import { observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js";
|
||||
import { copilotDenialSampleCursor, readCopilotDenialSettlement, observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js";
|
||||
import type { LiveFixtureValues } from "./live-fixtures.js";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
type Row = Record<string, any>;
|
||||
type Check = { id: string; passed: boolean; detail: string };
|
||||
/** Public run Stop and provider settlement are independent facts. The callback
|
||||
* drains run events only after independent process retirement has been proved. */
|
||||
export async function settleCopilotDeniedRun(input: {
|
||||
api: Pick<RunnerApi, "post">; request: CopilotToolNotice;
|
||||
awaitRetirement(): Promise<{ events: readonly unknown[]; run: unknown; issue: unknown }>;
|
||||
}) {
|
||||
await input.api.post(`/api/heartbeat-runs/${input.request.runId}/cancel`);
|
||||
return readCopilotDenialSettlement({ ...await input.awaitRetirement(), request: input.request });
|
||||
}
|
||||
|
||||
export async function runCopilotProtectionFlow(input: {
|
||||
page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; workspacePath: string; deadlineAt: number;
|
||||
remoteBootstrap?: CopilotRemoteBootstrap;
|
||||
@@ -42,12 +52,14 @@ export async function runCopilotProtectionFlow(input: {
|
||||
const localTarget = deny && !remote ? await createDeniedTargetFixture(workspacePath, targetName) : undefined;
|
||||
const target = localTarget?.targetRelativePath ?? targetName, targetPath = localTarget?.targetPath ?? join(workspacePath, target);
|
||||
const fileObservations: CopilotDeniedWriteEvidence["fileObservations"] = [];
|
||||
let deniedRequest: CopilotToolNotice | undefined;
|
||||
const sample = async (phase: CopilotDeniedWriteEvidence["fileObservations"][number]["phase"]) => {
|
||||
const providerCursor = phase === "before-request" ? null : copilotDenialSampleCursor(runEvents, deniedRequest!);
|
||||
if (remote) {
|
||||
if (!remoteFixture || !baseline) throw new Error("Remote denied target has no baseline");
|
||||
const snapshot = sealed ?? (phase === "before-request" ? baseline : await remoteFixture.snapshot(phase));
|
||||
remoteSnapshots.push(snapshot); fileObservations.push(copilotRemoteDeniedSample(snapshot, baseline, target, phase));
|
||||
} else fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath) });
|
||||
remoteSnapshots.push(snapshot); fileObservations.push({ ...copilotRemoteDeniedSample(snapshot, baseline, target, phase), providerCursor });
|
||||
} else fileObservations.push({ phase, observedAtMs: Date.now(), exists: await exists(targetPath), providerCursor });
|
||||
};
|
||||
const watcher = localTarget?.watcher;
|
||||
const markerPath = join(workspacePath, `copilot-settlement-${nonce}.txt`);
|
||||
@@ -114,6 +126,7 @@ export async function runCopilotProtectionFlow(input: {
|
||||
if (deny) {
|
||||
await wait("exact native write permission", s => s.notices.some(n => n.stage === "permission_requested" && n.operation === "edit" && n.target === target && n.declineOffered && s.runEvents.some(r => r.eventType === "runtime_request.created" && r.payload?.prpEvent?.payload?.request?.requestId === n.requestId)));
|
||||
const request = notices.find(n => n.stage === "permission_requested" && n.operation === "edit" && n.target === target)!;
|
||||
deniedRequest = request;
|
||||
const pending = runEvents.filter(r => r.eventType === "runtime_request.created" && r.payload?.prpEvent?.payload?.request?.requestId === request.requestId).map(r => r.payload.prpEvent.payload.request);
|
||||
check("one-bound-permission", pending.length === 1 && pending[0].requestKind === "permission_approval" && pending[0].origin?.method === "session/request_permission", "The notice maps to the exact durable native permission card");
|
||||
const retired = new Set(runEvents.filter(r => ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(r.eventType)).map(r => r.payload?.prpEvent?.payload?.requestId));
|
||||
@@ -129,24 +142,25 @@ export async function runCopilotProtectionFlow(input: {
|
||||
check("browser-exact-denial", posted.turnId === request.turnId && posted.requestKind === "permission_approval" && posted.resolution?.action === "decline", "Browser submitted denial for the exact run/request/turn");
|
||||
await wait("delivered rejection and failed native edit", s => s.notices.some(n => n.stage === "permission_delivered" && n.requestId === request.requestId && n.outcome === "reject_once") && s.notices.some(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed"));
|
||||
await sample("after-decision");
|
||||
const cancelRequestedAtMs = Date.now(); await api.post(`/api/heartbeat-runs/${request.runId}/cancel`);
|
||||
await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && (remote || (s.processes.captured && s.processes.live.length === 0)));
|
||||
if (remote) await sealRemote();
|
||||
await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup");
|
||||
const settlement = await settleCopilotDeniedRun({ api, request, awaitRetirement: async () => {
|
||||
await wait("explicitly cancelled native run and retired processes", s => s.runs[0]?.status === "cancelled" && (remote || (s.processes.captured && s.processes.live.length === 0)));
|
||||
if (remote) await sealRemote();
|
||||
await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup");
|
||||
return { events: runEvents, run: runs[0], issue };
|
||||
} });
|
||||
watchReceipt = remote ? { startedAtMs: baseline!.observedAtMs, endedAtMs: sealed!.observedAtMs, complete: sealed!.watcher.complete, targetMutationCount: sealed!.watcher.targetMutationCount } : watcher!.finish();
|
||||
const cancellation = runs[0]!.resultJson?.nativeCancellation;
|
||||
const toolResult = notices.find(n => n.stage === "tool" && n.toolCallId === request.toolCallId && n.status === "failed")!;
|
||||
const terminalFrame = runEvents.find(r => ["turn.cancelled", "turn.interrupted"].includes(r.eventType) && r.payload?.prpEvent?.turnId === request.turnId)?.payload.prpEvent;
|
||||
check("correlated-cancel-terminal", Boolean(terminalFrame) && terminalFrame.runId === request.runId, "An actual durable cancellation terminal belongs to the denied turn");
|
||||
const terminalAt = Date.parse(terminalFrame.emittedAt);
|
||||
await input.evidence("copilot-denial-settlement.json", settlement);
|
||||
check("correlated-provider-settlement", true, `Exact provider settlement (${settlement.branch}) and audited run Stop; completed does not cover active-turn cancellation`);
|
||||
const terminalAt = settlement.providerTerminal.emittedAtMs;
|
||||
const evidence: CopilotDeniedWriteEvidence = {
|
||||
expected: copilotOrigin(request), requestId: request.requestId!, expectedRelativePath: target,
|
||||
request: { ...request, requestId: request.requestId!, targetRelativePath: request.target!, method: "session/request_permission", offeredActions: request.declineOffered ? ["decline"] : [] },
|
||||
decision: { ...request, observedAtMs: clickedAtMs, requestId: request.requestId!, browserRequestId: request.requestId!, action: "decline" },
|
||||
deliveredDecision: (() => { const n = notices.find(n => n.stage === "permission_delivered" && n.requestId === request.requestId && n.outcome === "reject_once"); return n ? { ...n, requestId: n.requestId!, outcome: n.outcome! } : null; })(),
|
||||
toolResult: { ...toolResult, status: "failed" },
|
||||
terminal: { runId: terminalFrame.runId, turnId: terminalFrame.turnId, observedAtMs: terminalAt, status: runs[0]!.status },
|
||||
cancellation: { requestedAtMs: cancelRequestedAtMs, acknowledged: cancellation?.dispatchState === "acknowledged" && cancellation?.dispatched === true, scope: cancellation?.scope },
|
||||
terminal: { runId: settlement.runId, turnId: settlement.turnId, observedAtMs: terminalAt, status: runs[0]!.status },
|
||||
settlement,
|
||||
cleanup: { observedAtMs: fileObservations.at(-1)!.observedAtMs, ownedProcessesRemaining: processes.live.length }, fileObservations, mutationObservation: watchReceipt,
|
||||
nativeAttemptsForTarget: countCopilotEditOriginsForTarget(notices, target),
|
||||
};
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { readCopilotToolEvidence } from "./copilot-evidence.js";
|
||||
import { copilotDenialSampleCursor, readCopilotDenialSettlement } from "./copilot-protection-evidence.js";
|
||||
import { settleCopilotDeniedRun } from "./copilot-protection-flow.js";
|
||||
|
||||
// Sanitized durable order from the failed Daytona attempt: completed was emitted
|
||||
// at .887, persisted at .900; server Stop was requested at .892 and acked at .909.
|
||||
const date = (ms: number) => new Date(Date.UTC(2026, 8, 30, 15, 13, 7, ms)).toISOString();
|
||||
function fixture(eventType = "turn.completed") {
|
||||
const frame = (seq: number, type: string, payload: unknown, persisted = 305, emitted = 129): any => ({
|
||||
seq: seq + 40, companyId: "company", runId: "run", eventType: type, createdAt: date(persisted),
|
||||
payload: { prpEvent: { schema: "paperclip.prp.event.v1", sourceKind: "runner", runId: "run", turnId: "turn",
|
||||
normalizedSessionId: "normalized-session", sourceInstanceId: "runner-instance", sourceSeq: seq,
|
||||
sourceEventId: `runner-instance:run:${seq}`, eventType: type, emittedAt: date(emitted), payload } },
|
||||
});
|
||||
const notice = (seq: number, stage: string, details: Record<string, string>) => frame(seq, "provider.notice.recorded", {
|
||||
schema: "paperclip.provider.notice.v1", category: "copilot_tool_evidence_v1", scope: "turn",
|
||||
provenance: { sessionId: "native-session", turnId: "turn", eventType: stage, method: stage === "tool" ? "session/update" : "session/request_permission" },
|
||||
details: Object.entries({ stage, toolCallId: "denied-edit", operation: "edit", ...details }).map(([name, value]) => ({ name, value })),
|
||||
});
|
||||
const events = [notice(111, "permission_requested", { requestId: "request", target: "copilot-denied-nonce.txt", declineOffered: "true" }),
|
||||
notice(113, "permission_delivered", { requestId: "request", outcome: "reject_once" }), notice(114, "tool", { status: "failed" }),
|
||||
frame(115, "tool.execution.completed", { schema: "paperclip.tool.execution.v1", executionId: "denied-edit", operation: "edit", status: "failed" }, 317),
|
||||
frame(121, eventType, { status: eventType.slice(5), error: null }, 900, 887),
|
||||
frame(112, "runtime_request.resolved", { requestId: "request", turnId: "turn", requestKind: "permission_approval", action: "decline" })];
|
||||
const run: any = { id: "run", companyId: "company", nativeIssueId: "issue", status: "cancelled", finishedAt: date(910),
|
||||
resultJson: { startupCancellation: { requestedAt: date(892) }, nativeCancellation: {
|
||||
schema: "paperclip.native-cancellation.v1", companyId: "company", runId: "run", issueId: "issue", scope: "run", dispatched: true,
|
||||
dispatchState: "acknowledged", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], intentId: "intent",
|
||||
intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", recordedAt: date(901), acknowledgedAt: date(909),
|
||||
} } };
|
||||
return { events, request: readCopilotToolEvidence(events, "run")[0]!, run, issue: { id: "issue", companyId: "company", status: "in_progress" } };
|
||||
}
|
||||
const terminal = (f: ReturnType<typeof fixture>) => f.events[4]!.payload.prpEvent;
|
||||
describe("Copilot denial provider settlement and separate audited run Stop", () => {
|
||||
it("accepts the retained normal-terminal race without claiming provider cancellation", () => {
|
||||
const f = fixture();
|
||||
expect(readCopilotDenialSettlement(f)).toMatchObject({ branch: "provider_completed_before_stop_settlement", providerCancellationTerminalObserved: false,
|
||||
providerTerminal: { sourceSeq: 121, failedToolSourceSeq: 115, persistedAtMs: Date.parse(date(900)) }, runStop: { status: "cancelled" } });
|
||||
// Native/provider clock skew is irrelevant to server persistence vs ack.
|
||||
terminal(f).emittedAt = date(999);
|
||||
expect(readCopilotDenialSettlement(f).branch).toBe("provider_completed_before_stop_settlement");
|
||||
});
|
||||
it.each(["turn.cancelled", "turn.interrupted"])("retains the distinct observed %s branch", type => {
|
||||
expect(readCopilotDenialSettlement(fixture(type))).toMatchObject({ branch: "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: true });
|
||||
});
|
||||
it("cannot use a legacy summary or cancelled run as provider evidence", () => {
|
||||
const f = fixture(); f.events.splice(4, 1); f.events.push({ eventType: "turn.completed", payload: {} });
|
||||
expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/);
|
||||
});
|
||||
it.each(["equal", "later", "invalid"])("rejects %s normal terminal persistence against the server acknowledgement", kind => {
|
||||
const f = fixture(); f.events[4]!.createdAt = kind === "invalid" ? "not-a-date" : date(kind === "equal" ? 909 : 910);
|
||||
expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/);
|
||||
});
|
||||
it.each(["runId", "turnId", "normalizedSessionId", "sourceInstanceId", "eventType", "sourceEventId"])("rejects terminal %s mismatch", field => {
|
||||
const f = fixture(); terminal(f)[field] = "foreign";
|
||||
expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/);
|
||||
});
|
||||
it.each(["missing", "duplicate", "failed", "before-tool", "wrong-status", "error"])("rejects %s terminal", kind => {
|
||||
const f = fixture();
|
||||
if (kind === "missing") f.events.splice(4, 1);
|
||||
if (kind === "duplicate") f.events.push(structuredClone(f.events[4]));
|
||||
if (kind === "failed") { f.events[4]!.eventType = terminal(f).eventType = "turn.failed"; terminal(f).payload.status = "failed"; }
|
||||
if (kind === "before-tool") { terminal(f).sourceSeq = 114; terminal(f).sourceEventId = "runner-instance:run:114"; }
|
||||
if (kind === "wrong-status") terminal(f).payload.status = "cancelled";
|
||||
if (kind === "error") terminal(f).payload.error = { code: "failed" };
|
||||
expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/);
|
||||
});
|
||||
it.each(["scope", "runId", "companyId", "issueId", "intentId", "intentAuditId", "acknowledgementAuditId", "acknowledgedAt", "recordedAt", "schema", "dispatchState", "dispatched", "reasonCode", "effects"])("rejects malformed or unbound Stop %s", field => {
|
||||
const f = fixture(); f.run.resultJson.nativeCancellation[field] = null;
|
||||
expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/);
|
||||
});
|
||||
it.each(["missing-stop", "pending", "wrong-scope", "same-audit", "wrong-run-status", "issue-done", "ack-before-intent"])("rejects %s controller state", kind => {
|
||||
const f = fixture(), c = f.run.resultJson.nativeCancellation;
|
||||
if (kind === "missing-stop") delete f.run.resultJson.nativeCancellation;
|
||||
if (kind === "pending") c.dispatchState = "pending";
|
||||
if (kind === "wrong-scope") c.scope = "turn";
|
||||
if (kind === "same-audit") c.acknowledgementAuditId = c.intentAuditId;
|
||||
if (kind === "wrong-run-status") f.run.status = "succeeded";
|
||||
if (kind === "issue-done") f.issue.status = "done";
|
||||
if (kind === "ack-before-intent") c.recordedAt = date(911);
|
||||
expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/);
|
||||
});
|
||||
it.each(["native-session", "normalized-session", "failed-tool", "delivery", "duplicate-tool", "tool-order"])("rejects %s corruption", kind => {
|
||||
const f = fixture();
|
||||
if (kind === "native-session") f.events[2]!.payload.prpEvent.payload.provenance.sessionId = "foreign";
|
||||
if (kind === "normalized-session") f.events[2]!.payload.prpEvent.normalizedSessionId = "foreign";
|
||||
if (kind === "failed-tool") f.events[3]!.payload.prpEvent.payload.status = "completed";
|
||||
if (kind === "delivery") f.events[1]!.payload.prpEvent.payload.details.find((d: any) => d.name === "outcome").value = "allow_once";
|
||||
if (kind === "duplicate-tool") f.events.push(structuredClone(f.events[3]));
|
||||
if (kind === "tool-order") f.events[3]!.payload.prpEvent.sourceSeq = 122;
|
||||
expect(() => readCopilotDenialSettlement(f)).toThrow();
|
||||
});
|
||||
it.each(["missing", "duplicate", "foreign-turn", "accept", "expired", "after-delivery"])("rejects %s durable resolution", kind => {
|
||||
const f = fixture(), r = f.events[5]!, p = r.payload.prpEvent;
|
||||
if (kind === "missing") f.events.pop();
|
||||
if (kind === "duplicate") f.events.push(structuredClone(r));
|
||||
if (kind === "foreign-turn") p.payload.turnId = "foreign";
|
||||
if (kind === "accept") p.payload.action = "accept";
|
||||
if (kind === "expired") r.eventType = p.eventType = "runtime_request.expired";
|
||||
if (kind === "after-delivery") { p.sourceSeq = 114; p.sourceEventId = "runner-instance:run:114"; }
|
||||
expect(() => readCopilotDenialSettlement(f)).toThrow();
|
||||
});
|
||||
it("captures the exact durable stream at the sampling boundary without clock conversion", () => {
|
||||
const f = fixture();
|
||||
expect(copilotDenialSampleCursor([f.events[0]], f.request)).toEqual({ runId: "run", turnId: "turn", normalizedSessionId: "normalized-session", sourceInstanceId: "runner-instance", sourceSeq: 111 });
|
||||
expect(copilotDenialSampleCursor(f.events, f.request).sourceSeq).toBe(121);
|
||||
f.events[1]!.payload.prpEvent.normalizedSessionId = "foreign";
|
||||
expect(() => copilotDenialSampleCursor(f.events, f.request)).toThrow(/cursor/);
|
||||
});
|
||||
it("flow sends public Stop, awaits retirement, then reads the raced canonical terminal", async () => {
|
||||
const f = fixture(), order: string[] = []; const held = f.events.splice(4, 1)[0];
|
||||
const post = vi.fn(async (url: string) => { order.push(url); f.events.push(held); });
|
||||
const settlement = await settleCopilotDeniedRun({ api: { post } as any, request: f.request,
|
||||
awaitRetirement: async () => { order.push("retired-and-final-events-read"); return f; } });
|
||||
expect(order).toEqual(["/api/heartbeat-runs/run/cancel", "retired-and-final-events-read"]);
|
||||
expect(post).toHaveBeenCalledTimes(1); expect(settlement.branch).toBe("provider_completed_before_stop_settlement");
|
||||
await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request,
|
||||
awaitRetirement: async () => { throw new Error("remote descendants live"); } })).rejects.toThrow("remote descendants live");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user