From eecd64509ab4c8c50343ff5ccea481c952727fa1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 30 Sep 2026 11:31:24 -0500 Subject: [PATCH] fix(runner-e2e): require causal pre-Stop denial observations Co-Authored-By: Paperclip --- tests/runner-e2e/FIXTURES.md | 15 +- tests/runner-e2e/README.md | 12 +- tests/runner-e2e/catalog.ts | 2 +- .../copilot-protection-cases.test.ts | 11 +- .../runner-e2e/copilot-protection-evidence.ts | 88 ++++++++--- .../copilot-protection-flow.test.ts | 9 +- tests/runner-e2e/copilot-protection-flow.ts | 33 ++++- .../copilot-protection-settlement.test.ts | 138 ++++++++++++------ 8 files changed, 221 insertions(+), 87 deletions(-) diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 8c885833743..46dadd4049d 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -286,12 +286,17 @@ provider timeout, 300s attempt budget). `native-permission-deny-write` denies on exact native edit through its browser card, waits for the delivered rejection and failed tool, then cancels through the public run API. Its expected outcome is a cancelled run and an unfinished task, not successful task completion. -The retained `paperclip.e2e.copilot-denial-settlement.v1` proof separates the exact +The retained `paperclip.e2e.copilot-denial-settlement.v2` proof separates the exact provider terminal from the audited controller Stop. It records either -`provider_cancelled_or_interrupted` or `provider_completed_before_stop_settlement`. -The latter requires normal terminal persistence strictly before the server's Stop -acknowledgement, with the same normalized session/source stream and a later source -sequence than the failed denied tool. It cannot establish active-turn cancellation; +`provider_cancelled_or_interrupted` or `provider_completed_observed_before_stop`. +The latter requires the exact terminal row to be returned by the operator API +before Stop dispatch. The fixture awaits retention of its scoped row-hash receipt +before sending Stop, then matches that receipt against the final durable rows. +The same normalized session/source stream and a later source sequence than the +failed edit are required. A fixed 2s observation window, inside the existing case +deadline, allows natural completion. A normal terminal first seen after Stop is +insufficient evidence. Database createdAt is transaction-start metadata; it cannot +prove that an event committed before Stop acknowledgement. It cannot establish active-turn cancellation; a cancelled/interrupted terminal also does not by itself prove Stop reached active work. Dedicated cancellation coverage must retain its active-operation evidence. Missing, ambiguous, failed or foreign terminals and incomplete Stop receipts fail. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index f6bc5071e5a..50762e55972 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1326,7 +1326,7 @@ and the limits of reconnect evidence. The manual `copilot-protection` suite selects two Copilot candidate cases on each of local and Daytona (four cells). Discover them with `pnpm test:e2e:runner -- --list --suite copilot-protection`. The denial case keeps provider-turn settlement separate from controller run Stop. -A normal provider completion before Stop acknowledgement is not active-turn +A normal provider completion observed by API before Stop dispatch is not active-turn cancellation coverage; denial, no-effects, run cancellation and retirement remain required. Earlier failed attempts retain their original grade. See [Copilot native protection](./FIXTURES.md#copilot-native-protection) for the @@ -1420,9 +1420,17 @@ retry, and remain pending qualification until their paid evidence passes. Denial ordering uses canonical request, decline-resolution, delivery, failed-edit and terminal source sequences. Sample checkpoints retain the exact run/turn/source cursor. File samples and continuous-watch coverage compare only observer-local -times. Provider emission, browser click and server persistence clocks are never +times. Provider emission, browser click and database transaction clocks are never compared to each other. Final remote samples use the sealed, independently verified process-retirement receipt; their timestamps are not relabeled as host time. This denial case does not qualify Stop during a definitely pending native request. That active-turn cancellation boundary needs a separate live case. The attached async-command oracle is unchanged by this denial-only correction. + +Copilot denial settlement v2 (suite definition v4) requires a retained pre-Stop API +observation. The fixture awaits that artifact write before dispatching Stop and +matches exact source identities and row hashes against final evidence. It waits +at most 2s for natural settlement within the existing deadline. Normal completion +first observed afterward cannot pass. A row's createdAt is transaction-start time, +not proof of commit order. Older failed artifacts lack this observation and cannot +establish a completed-before-Stop causal boundary; they remain failed. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 118e12ccb69..9f42eb8c767 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1051,7 +1051,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"), environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v1", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", remoteEvidence: "owned-lease-sealed-observer" }, + definitionMetadata: { version: 4, qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v2", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", remoteEvidence: "owned-lease-sealed-observer" }, }, { id: "rich-acp-warm-continuity", label: "Rich ACP warm continuity", manualOnly: true, diff --git a/tests/runner-e2e/copilot-protection-cases.test.ts b/tests/runner-e2e/copilot-protection-cases.test.ts index d41da54c6ff..3f8c647ffde 100644 --- a/tests/runner-e2e/copilot-protection-cases.test.ts +++ b/tests/runner-e2e/copilot-protection-cases.test.ts @@ -7,9 +7,10 @@ const terminal = { observedAtMs: 50, runId: "run", turnId: "turn", status: "succ const cleanup = { observedAtMs: 60, ownedProcessesRemaining: 0 }; function denied(): CopilotDeniedWriteEvidence { return { - expected: identity, terminal: { ...terminal, status: "cancelled" }, cleanup, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v1", ...identity, requestId: "permission-0", + expected: identity, terminal: { ...terminal, status: "cancelled" }, cleanup, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v2", ...identity, requestId: "permission-0", branch: "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: true, - providerTerminal: { eventType: "turn.cancelled", normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolPersistedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, persistedAtMs: 51 }, + preStop: { schema: "paperclip.e2e.copilot-pre-stop-observation.v1" as const, ...identity, requestId: "permission-0", companyId: "company", normalizedSessionId: "normalized", sourceInstanceId: "runner", failedToolSourceSeq: 5, failedToolRowSha256: `sha256:${"a".repeat(64)}`, terminal: null, apiReadCompletedMonotonicNs: "10" }, stopDispatchMonotonicNs: "20", + providerTerminal: { rowSha256: `sha256:${"b".repeat(64)}`, failedToolRowSha256: `sha256:${"a".repeat(64)}`, eventType: "turn.cancelled", normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolRowCreatedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, rowCreatedAtMs: 51 }, runStop: { companyId: "company", issueId: "issue", scope: "run", status: "cancelled", issueStatus: "in_progress", intentId: "intent", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt", request: { ...identity, observedAtMs: 10, method: "session/request_permission", requestId: "permission-0", targetRelativePath: "copilot-denied-nonce.txt", offeredActions: ["accept", "decline"] }, decision: { ...identity, observedAtMs: 20, requestId: "permission-0", browserRequestId: "permission-0", action: "decline" }, @@ -45,9 +46,10 @@ describe("Copilot protection Product oracles", () => { }); it("keeps every no-effect and cleanup gate for a completed-before-Stop settlement", () => { const e = denied(); - e.settlement!.branch = "provider_completed_before_stop_settlement"; + e.settlement!.branch = "provider_completed_observed_before_stop"; e.settlement!.providerCancellationTerminalObserved = false; e.settlement!.providerTerminal.eventType = "turn.completed"; + e.settlement!.preStop.terminal = { eventType: "turn.completed", sourceSeq: 6, rowSha256: e.settlement!.providerTerminal.rowSha256 }; expect(gradeCopilotDeniedWrite(e).passed).toBe(true); for (const mutate of [ (v: CopilotDeniedWriteEvidence) => { v.cleanup!.ownedProcessesRemaining = 1; }, @@ -63,9 +65,10 @@ describe("Copilot protection Product oracles", () => { }); it.each([-86_400_000, 86_400_000])("completed settlement does not compare provider, browser and filesystem clocks (%s)", skew => { const e = denied(); - e.settlement!.branch = "provider_completed_before_stop_settlement"; + e.settlement!.branch = "provider_completed_observed_before_stop"; e.settlement!.providerCancellationTerminalObserved = false; e.settlement!.providerTerminal.eventType = "turn.completed"; + e.settlement!.preStop.terminal = { eventType: "turn.completed", sourceSeq: 6, rowSha256: e.settlement!.providerTerminal.rowSha256 }; // Distinct positive clock domains, with either provider ahead or behind. for (const n of [e.request!, e.deliveredDecision!, e.toolResult!, e.terminal!]) n.observedAtMs += 100_000_000 + skew; e.settlement!.providerTerminal.emittedAtMs = e.terminal!.observedAtMs; diff --git a/tests/runner-e2e/copilot-protection-evidence.ts b/tests/runner-e2e/copilot-protection-evidence.ts index 65dc9c64016..88911c2be1f 100644 --- a/tests/runner-e2e/copilot-protection-evidence.ts +++ b/tests/runner-e2e/copilot-protection-evidence.ts @@ -1,4 +1,5 @@ import { withoutProvenBootstrapReads, bootstrapReadExecutionId, type BootstrapReadProof } from "./native-bootstrap-read-proof.js"; +import { canonicalJson } from "../../packages/shared/src/portability-hash.js"; import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; import { readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; @@ -195,17 +196,19 @@ export async function readCopilotRemoteMarkerAfterRetirement(fixture: CopilotRem } export interface CopilotDenialSettlement { - schema: "paperclip.e2e.copilot-denial-settlement.v1"; - branch: "provider_cancelled_or_interrupted" | "provider_completed_before_stop_settlement"; + schema: "paperclip.e2e.copilot-denial-settlement.v2"; + branch: "provider_cancelled_or_interrupted" | "provider_completed_observed_before_stop"; /** A cancelled terminal alone does not prove Stop reached active work. */ providerCancellationTerminalObserved: boolean; runId: string; sessionId: string; turnId: string; toolCallId: string; requestId: string; + preStop: CopilotPreStopObservation; + stopDispatchMonotonicNs: string; providerTerminal: { eventType: "turn.completed" | "turn.cancelled" | "turn.interrupted"; normalizedSessionId: string; sourceInstanceId: string; requestSourceSeq: number; resolvedSourceSeq: number; deliveredSourceSeq: number; failedNoticeSourceSeq: number; - failedToolSourceSeq: number; failedToolPersistedAtMs: number; sourceSeq: number; - emittedAtMs: number; persistedAtMs: number; + failedToolSourceSeq: number; failedToolRowCreatedAtMs: number; sourceSeq: number; + emittedAtMs: number; rowCreatedAtMs: number; rowSha256: string; failedToolRowSha256: string; }; runStop: { companyId: string; issueId: string; scope: "run"; status: "cancelled"; issueStatus: "in_progress"; @@ -219,19 +222,27 @@ const settlementTime = (v: unknown): v is number => Number.isSafeInteger(v) && ( const settlementDate = (v: unknown): number => typeof v === "string" && /^\d{4}-\d{2}-\d{2}T.*Z$/u.test(v) ? Date.parse(v) : NaN; export function validCopilotDenialSettlement(s: CopilotDenialSettlement): boolean { - if (!s || s.schema !== "paperclip.e2e.copilot-denial-settlement.v1") return false; + if (!s || s.schema !== "paperclip.e2e.copilot-denial-settlement.v2") return false; const t = s.providerTerminal, c = s.runStop; if (!t || !c || ![s.runId, s.sessionId, s.turnId, s.toolCallId, s.requestId, t.normalizedSessionId, t.sourceInstanceId, c.companyId, c.issueId, c.intentId, c.intentAuditId, c.acknowledgementAuditId].every(settlementId) || c.scope !== "run" || c.status !== "cancelled" || c.issueStatus !== "in_progress" || c.intentAuditId === c.acknowledgementAuditId - || ![t.failedToolPersistedAtMs, t.emittedAtMs, t.persistedAtMs, c.requestedAtMs, c.recordedAtMs, c.acknowledgedAtMs, c.finishedAtMs].every(settlementTime) + || ![t.failedToolRowCreatedAtMs, t.emittedAtMs, t.rowCreatedAtMs, c.requestedAtMs, c.recordedAtMs, c.acknowledgedAtMs, c.finishedAtMs].every(settlementTime) || ![t.requestSourceSeq, t.resolvedSourceSeq, t.deliveredSourceSeq, t.failedNoticeSourceSeq].every(n => Number.isSafeInteger(n) && n > 0) || !(t.requestSourceSeq < t.resolvedSourceSeq && t.resolvedSourceSeq < t.deliveredSourceSeq && t.deliveredSourceSeq < t.failedNoticeSourceSeq && t.failedNoticeSourceSeq < t.failedToolSourceSeq) || !Number.isSafeInteger(t.failedToolSourceSeq) || t.failedToolSourceSeq <= 0 || !Number.isSafeInteger(t.sourceSeq) || t.sourceSeq <= t.failedToolSourceSeq - || t.failedToolPersistedAtMs > c.requestedAtMs || t.failedToolPersistedAtMs > t.persistedAtMs || c.requestedAtMs > c.recordedAtMs || c.recordedAtMs > c.acknowledgedAtMs || c.acknowledgedAtMs > c.finishedAtMs) return false; - if (s.branch === "provider_completed_before_stop_settlement") return t.eventType === "turn.completed" - && s.providerCancellationTerminalObserved === false && t.persistedAtMs < c.acknowledgedAtMs; + const before = s.preStop; + if (!before || before.schema !== "paperclip.e2e.copilot-pre-stop-observation.v1" + || !["runId", "sessionId", "turnId", "toolCallId", "requestId"].every(k => before[k as keyof CopilotPreStopObservation] === s[k as keyof CopilotDenialSettlement]) + || before.companyId !== c.companyId || before.normalizedSessionId !== t.normalizedSessionId || before.sourceInstanceId !== t.sourceInstanceId + || before.failedToolSourceSeq !== t.failedToolSourceSeq || before.failedToolRowSha256 !== t.failedToolRowSha256 + || ![t.rowSha256, t.failedToolRowSha256].every(v => /^sha256:[a-f0-9]{64}$/u.test(v)) + || !/^[1-9][0-9]{0,29}$/u.test(before.apiReadCompletedMonotonicNs) || !/^[1-9][0-9]{0,29}$/u.test(s.stopDispatchMonotonicNs) + || BigInt(before.apiReadCompletedMonotonicNs) >= BigInt(s.stopDispatchMonotonicNs)) return false; + if (before.terminal !== null && (!before.terminal || before.terminal.eventType !== t.eventType || before.terminal.sourceSeq !== t.sourceSeq || before.terminal.rowSha256 !== t.rowSha256)) return false; + if (s.branch === "provider_completed_observed_before_stop") return t.eventType === "turn.completed" + && s.providerCancellationTerminalObserved === false && before.terminal?.eventType === "turn.completed"; return s.branch === "provider_cancelled_or_interrupted" && s.providerCancellationTerminalObserved === true && ["turn.cancelled", "turn.interrupted"].includes(t.eventType); } @@ -279,11 +290,48 @@ export function readCopilotDeniedEdit(input: { events: readonly unknown[]; reque return { rows, get, stream, origin, delivery, failure, resolution, tool, toolRows, failed }; } +export interface CopilotPreStopObservation { + schema: "paperclip.e2e.copilot-pre-stop-observation.v1"; + companyId: string; runId: string; sessionId: string; turnId: string; toolCallId: string; requestId: string; + normalizedSessionId: string; sourceInstanceId: string; + failedToolSourceSeq: number; failedToolRowSha256: string; + terminal: { eventType: "turn.completed" | "turn.cancelled" | "turn.interrupted"; sourceSeq: number; rowSha256: string } | null; + /** Fixture-process monotonic clock, never provider or database wall time. */ + apiReadCompletedMonotonicNs: string; +} +const denialRowSha = (row: unknown) => `sha256:${createHash("sha256").update(canonicalJson(row)).digest("hex")}`; +function readDeniedTerminal(proof: ReturnType) { + const { rows, get, stream, tool, failed } = proof; + const invalid = () => new Error("Copilot denial lacks exact provider settlement"); + // Ignore legacy unwrapped log summaries, never use them as provider evidence. + const terminalRows = rows.filter(r => ["turn.completed", "turn.cancelled", "turn.interrupted", "turn.failed"].includes(r.eventType) + && settlementRecord(r.payload).prpEvent !== undefined); + if (terminalRows.length === 0) return null; + if (terminalRows.length !== 1) throw invalid(); + const row = terminalRows[0]!, terminal = get(row), p = settlementRecord(terminal.payload); + if (!stream(terminal) || terminal.sourceSeq <= tool.sourceSeq || row.seq <= failed[0]!.seq + || !["turn.completed", "turn.cancelled", "turn.interrupted"].includes(row.eventType) + || p.status !== row.eventType.slice(5) || p.error != null) throw invalid(); + return { row, terminal }; +} +/** Call only on rows returned by the current operator API read. Retain this + * receipt before dispatching Stop; historical rows cannot recreate that fact. */ +export function observeCopilotPreStop(input: { events: readonly unknown[]; request: CopilotToolNotice; companyId: string }): CopilotPreStopObservation { + const proof = readCopilotDeniedEdit(input), observed = readDeniedTerminal(proof), r = input.request; + return { schema: "paperclip.e2e.copilot-pre-stop-observation.v1", companyId: input.companyId, + runId: r.runId, sessionId: r.sessionId, turnId: r.turnId, toolCallId: r.toolCallId, requestId: r.requestId!, + normalizedSessionId: proof.origin.normalizedSessionId, sourceInstanceId: proof.origin.sourceInstanceId, + failedToolSourceSeq: proof.tool.sourceSeq, failedToolRowSha256: denialRowSha(proof.toolRows[0]), + terminal: observed ? { eventType: observed.terminal.eventType, sourceSeq: observed.terminal.sourceSeq, rowSha256: denialRowSha(observed.row) } : null, + apiReadCompletedMonotonicNs: process.hrtime.bigint().toString() }; +} + /** Denial closes one permission, not necessarily the provider prompt. Match the * single canonical terminal to the same source stream, then independently bind - * the audited controller Stop. Provider emittedAt never orders server actions. */ + * the audited controller Stop. Database createdAt is transaction-start metadata, not a commit boundary. */ export function readCopilotDenialSettlement(input: { events: readonly unknown[]; request: CopilotToolNotice; run: unknown; issue: unknown; + preStop: CopilotPreStopObservation; stopDispatchMonotonicNs: string; }): CopilotDenialSettlement { const invalid = () => new Error("Copilot denial lacks exact provider settlement and audited run Stop"); const { request, events } = input, run = settlementRecord(input.run), issue = settlementRecord(input.issue); @@ -294,22 +342,18 @@ export function readCopilotDenialSettlement(input: { || stop.scope !== "run" || stop.dispatched !== true || stop.dispatchState !== "acknowledged" || stop.reasonCode !== "cancellation_run_only" || !Array.isArray(stop.effects) || stop.effects.length !== 1 || stop.effects[0] !== "release_run_resources") throw invalid(); const { rows, get, stream, origin, delivery, failure, resolution, tool, toolRows, failed } = readCopilotDeniedEdit({ events, request, companyId: issue.companyId }); - // Ignore legacy unwrapped log summaries, never use them as provider evidence. - const terminalRows = rows.filter(r => ["turn.completed", "turn.cancelled", "turn.interrupted", "turn.failed"].includes(r.eventType) - && settlementRecord(r.payload).prpEvent !== undefined); - if (terminalRows.length !== 1) throw invalid(); - const row = terminalRows[0]!, terminal = get(row), p = settlementRecord(terminal.payload); - if (!stream(terminal) || terminal.sourceSeq <= tool.sourceSeq || row.seq <= failed[0]!.seq - || !["turn.completed", "turn.cancelled", "turn.interrupted"].includes(row.eventType) - || p.status !== row.eventType.slice(5) || p.error != null) throw invalid(); + const observed = readDeniedTerminal({ rows, get, stream, origin, delivery, failure, resolution, tool, toolRows, failed }); + if (!observed) throw invalid(); + const { row, terminal } = observed; const result: CopilotDenialSettlement = { - schema: "paperclip.e2e.copilot-denial-settlement.v1", - branch: row.eventType === "turn.completed" ? "provider_completed_before_stop_settlement" : "provider_cancelled_or_interrupted", + schema: "paperclip.e2e.copilot-denial-settlement.v2", + branch: row.eventType === "turn.completed" ? "provider_completed_observed_before_stop" : "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: row.eventType !== "turn.completed", runId: run.id, sessionId: request.sessionId, turnId: request.turnId, toolCallId: request.toolCallId, requestId: request.requestId!, - providerTerminal: { eventType: row.eventType, normalizedSessionId: terminal.normalizedSessionId, sourceInstanceId: terminal.sourceInstanceId, + preStop: input.preStop, stopDispatchMonotonicNs: input.stopDispatchMonotonicNs, + providerTerminal: { eventType: row.eventType, rowSha256: denialRowSha(row), failedToolRowSha256: denialRowSha(toolRows[0]), normalizedSessionId: terminal.normalizedSessionId, sourceInstanceId: terminal.sourceInstanceId, requestSourceSeq: origin.sourceSeq, resolvedSourceSeq: resolution.sourceSeq, deliveredSourceSeq: delivery.sourceSeq, failedNoticeSourceSeq: failure.sourceSeq, - failedToolSourceSeq: tool.sourceSeq, failedToolPersistedAtMs: settlementDate(toolRows[0]!.createdAt), sourceSeq: terminal.sourceSeq, emittedAtMs: settlementDate(terminal.emittedAt), persistedAtMs: settlementDate(row.createdAt) }, + failedToolSourceSeq: tool.sourceSeq, failedToolRowCreatedAtMs: settlementDate(toolRows[0]!.createdAt), sourceSeq: terminal.sourceSeq, emittedAtMs: settlementDate(terminal.emittedAt), rowCreatedAtMs: settlementDate(row.createdAt) }, runStop: { companyId: issue.companyId, issueId: issue.id, scope: stop.scope, status: run.status, issueStatus: issue.status, intentId: stop.intentId, intentAuditId: stop.intentAuditId, acknowledgementAuditId: stop.acknowledgementAuditId, requestedAtMs: settlementDate(settlementRecord(settlementRecord(run.resultJson).startupCancellation).requestedAt), diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index a3c60488146..b13b2647399 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -26,8 +26,8 @@ function projected(name: string, target = "copilot-denied-nonce.txt") { describe("Copilot Product protection integration", () => { it("registers two explicit cases on both environments with honest terminal expectations", () => { const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection"); - expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 3, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", - denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v1", activeTurnCancellation: "not-implied-by-completed-provider-turn" }); + expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 4, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", + denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v2", activeTurnCancellation: "not-implied-by-completed-provider-turn" }); expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } })); expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"])); expect(cells.every(c => c.profile.qualificationCandidate === "copilot" && c.task.expectedRunCount === 1)).toBe(true); @@ -41,9 +41,10 @@ describe("Copilot Product protection integration", () => { request: { ...request, requestId: "permission", method: "session/request_permission" as const, targetRelativePath: request.target!, offeredActions: request.declineOffered ? ["decline"] : [] }, decision: { ...delivered, requestId: "permission", browserRequestId: "permission", action: delivered.outcome === "reject_once" ? "decline" : "accept" }, deliveredDecision: { ...delivered, requestId: "permission", outcome: delivered.outcome! }, - toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v1" as const, ...copilotOrigin(request), requestId: "permission", + toolResult: { ...failed, status: "failed" as const }, terminal: { runId: "run", turnId: "turn", observedAtMs: 50, status: "cancelled" as const }, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v2" as const, ...copilotOrigin(request), requestId: "permission", branch: "provider_cancelled_or_interrupted" as const, providerCancellationTerminalObserved: true, - providerTerminal: { eventType: "turn.cancelled" as const, normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolPersistedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, persistedAtMs: 51 }, + preStop: { schema: "paperclip.e2e.copilot-pre-stop-observation.v1" as const, ...copilotOrigin(request), requestId: "permission", companyId: "company", normalizedSessionId: "normalized", sourceInstanceId: "runner", failedToolSourceSeq: 5, failedToolRowSha256: `sha256:${"a".repeat(64)}`, terminal: null, apiReadCompletedMonotonicNs: "10" }, stopDispatchMonotonicNs: "20", + providerTerminal: { rowSha256: `sha256:${"b".repeat(64)}`, failedToolRowSha256: `sha256:${"a".repeat(64)}`, eventType: "turn.cancelled" as const, normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolRowCreatedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, rowCreatedAtMs: 51 }, runStop: { companyId: "company", issueId: "issue", scope: "run" as const, status: "cancelled" as const, issueStatus: "in_progress" as const, intentId: "intent", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, cleanup: { observedAtMs: 60, ownedProcessesRemaining: 0 }, nativeAttemptsForTarget: 1, fileObservations: (["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const).map((phase, index) => ({ phase, observedAtMs: [0, request.observedAtMs, 30, 50, 60][index]!, exists: false, providerCursor: index === 0 ? null : { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: [0, 1, 4, 6, 6][index]! } })), mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 } }; diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index 904821a3f31..91fc0633e1e 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -7,7 +7,7 @@ import { createTaskThroughUi } from "./user-actions.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; -import { readCopilotDeniedEdit, copilotDenialSampleCursor, readCopilotDenialSettlement, observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js"; +import { observeCopilotPreStop, type CopilotPreStopObservation, readCopilotDeniedEdit, copilotDenialSampleCursor, readCopilotDenialSettlement, observeCopilotFixtureCommand, readCopilotRemoteMarkerAfterRetirement, prepareCopilotRemoteAction, assertCopilotRemoteRetirement, assertCopilotRemoteAttached, copilotRemoteDeniedSample, copilotActionNotices, type CopilotRemoteBootstrap, type CopilotRemoteFixture, type CopilotRemoteSnapshot, countCopilotToolOrigins, countCopilotEditOriginsForTarget } from "./copilot-protection-evidence.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; type Row = Record; @@ -18,23 +18,45 @@ export async function settleCopilotDeniedRun(input: { api: Pick; request: CopilotToolNotice; deadlineAt: number; load(): Promise<{ events: readonly unknown[]; run: Row; issue: Row; retired: boolean }>; afterDeniedEdit(): Promise; + retainPreStop(receipt: CopilotPreStopObservation): Promise; afterSettlement(): Promise; }) { const poll = (label: string, accept: (state: Awaited>) => boolean) => pollUntil({ - label, deadlineAt: input.deadlineAt, load: input.load, accept, intervalMs: 200, - reject: state => ["failed", "timed_out"].includes(state.run.status) ? "Copilot provider run failed" : undefined, + label, deadlineAt: input.deadlineAt, load: async () => { + const state = await input.load(); + // pollUntil recognizes this definitive rejection before invoking readers. + if (["failed", "timed_out"].includes(state.run.status)) throw new Error(`Stopped waiting for ${label}: Copilot provider run failed`); + return state; + }, accept, intervalMs: 200, }); await poll("persisted correlated failed native edit", state => { readCopilotDeniedEdit({ events: state.events, request: input.request, companyId: state.issue.companyId }); return true; }); await input.afterDeniedEdit(); + // This negative case does not qualify active-turn cancellation. Give natural + // settlement a fixed observation window inside the unchanged case deadline. + const observeUntil = Math.min(input.deadlineAt, Date.now() + 2000); + let preStop: CopilotPreStopObservation; + while (true) { + const state = await input.load(); + if (["failed", "timed_out"].includes(state.run.status)) throw new Error("Copilot provider run failed before Stop"); + preStop = observeCopilotPreStop({ events: state.events, request: input.request, companyId: state.issue.companyId }); + if (preStop.terminal || Date.now() >= observeUntil) break; + await new Promise(resolve => setTimeout(resolve, Math.min(200, observeUntil - Date.now()))); + } + if (Date.now() >= input.deadlineAt) throw new Error("Copilot denial deadline reached before Stop"); + // Await the artifact write before dispatch. Database createdAt cannot supply + // this causal boundary, and a later replay must never manufacture it. + await input.retainPreStop(preStop); + if (Date.now() >= input.deadlineAt) throw new Error("Copilot denial deadline reached before Stop"); + const stopDispatchMonotonicNs = process.hrtime.bigint().toString(); await input.api.post(`/api/heartbeat-runs/${input.request.runId}/cancel`); await poll("correlated provider settlement and retired run", state => { if (!state.retired) return false; - readCopilotDenialSettlement({ ...state, request: input.request }); return true; + readCopilotDenialSettlement({ ...state, request: input.request, preStop, stopDispatchMonotonicNs }); return true; }); await input.afterSettlement(); - return readCopilotDenialSettlement({ ...await input.load(), request: input.request }); + return readCopilotDenialSettlement({ ...await input.load(), request: input.request, preStop, stopDispatchMonotonicNs }); } export async function runCopilotProtectionFlow(input: { @@ -162,6 +184,7 @@ export async function runCopilotProtectionFlow(input: { retired: remote || (state.processes.captured && state.processes.live.length === 0) }; }, afterDeniedEdit: () => sample("after-decision"), + retainPreStop: receipt => input.evidence("copilot-pre-stop-observation.json", receipt), afterSettlement: async () => { if (remote) await sealRemote(); await sample("terminal"); await new Promise(resolve => setTimeout(resolve, 100)); await load(); await sample("after-cleanup"); diff --git a/tests/runner-e2e/copilot-protection-settlement.test.ts b/tests/runner-e2e/copilot-protection-settlement.test.ts index 276e371e937..a330870f1d5 100644 --- a/tests/runner-e2e/copilot-protection-settlement.test.ts +++ b/tests/runner-e2e/copilot-protection-settlement.test.ts @@ -1,10 +1,11 @@ import { describe, expect, it, vi } from "vitest"; import { readCopilotToolEvidence } from "./copilot-evidence.js"; -import { copilotDenialSampleCursor, readCopilotDenialSettlement } from "./copilot-protection-evidence.js"; +import { observeCopilotPreStop, copilotDenialSampleCursor, readCopilotDenialSettlement } from "./copilot-protection-evidence.js"; import { settleCopilotDeniedRun } from "./copilot-protection-flow.js"; -// Sanitized durable order from the failed Daytona attempt: completed was emitted -// at .887, persisted at .900; server Stop was requested at .892 and acked at .909. +// Sanitized source order from the failed Daytona attempt. Its .900 createdAt +// is transaction time, not commit evidence. Pre-Stop receipts below are synthetic +// test observations; the historical paid attempt did not capture one. const date = (ms: number) => new Date(Date.UTC(2026, 8, 30, 15, 13, 7, ms)).toISOString(); function fixture(eventType = "turn.completed") { const frame = (seq: number, type: string, payload: unknown, persisted = 305, emitted = 129): any => ({ @@ -29,17 +30,21 @@ function fixture(eventType = "turn.completed") { dispatchState: "acknowledged", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], intentId: "intent", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", recordedAt: date(901), acknowledgedAt: date(909), } } }; - return { events, request: readCopilotToolEvidence(events, "run")[0]!, run, issue: { id: "issue", companyId: "company", status: "in_progress" } }; + const request = readCopilotToolEvidence(events, "run")[0]!; + const preStop = observeCopilotPreStop({ events, request, companyId: "company" }); + return { events, request, run, issue: { id: "issue", companyId: "company", status: "in_progress" }, preStop, stopDispatchMonotonicNs: process.hrtime.bigint().toString() }; + } const terminal = (f: ReturnType) => f.events[4]!.payload.prpEvent; describe("Copilot denial provider settlement and separate audited run Stop", () => { - it("accepts the retained normal-terminal race without claiming provider cancellation", () => { + it("accepts a synthetic pre-Stop API observation without claiming provider cancellation", () => { const f = fixture(); - expect(readCopilotDenialSettlement(f)).toMatchObject({ branch: "provider_completed_before_stop_settlement", providerCancellationTerminalObserved: false, - providerTerminal: { sourceSeq: 121, failedToolSourceSeq: 115, persistedAtMs: Date.parse(date(900)) }, runStop: { status: "cancelled" } }); - // Native/provider clock skew is irrelevant to server persistence vs ack. + expect(readCopilotDenialSettlement(f)).toMatchObject({ branch: "provider_completed_observed_before_stop", providerCancellationTerminalObserved: false, + providerTerminal: { sourceSeq: 121, failedToolSourceSeq: 115, rowCreatedAtMs: Date.parse(date(900)) }, runStop: { status: "cancelled" } }); + // Native/provider clock skew is irrelevant to the operator causal boundary. terminal(f).emittedAt = date(999); - expect(readCopilotDenialSettlement(f).branch).toBe("provider_completed_before_stop_settlement"); + f.preStop = observeCopilotPreStop({ ...f, companyId: "company" }); f.stopDispatchMonotonicNs = process.hrtime.bigint().toString(); + expect(readCopilotDenialSettlement(f).branch).toBe("provider_completed_observed_before_stop"); }); it.each(["turn.cancelled", "turn.interrupted"])("retains the distinct observed %s branch", type => { expect(readCopilotDenialSettlement(fixture(type))).toMatchObject({ branch: "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: true }); @@ -48,8 +53,23 @@ describe("Copilot denial provider settlement and separate audited run Stop", () const f = fixture(); f.events.splice(4, 1); f.events.push({ eventType: "turn.completed", payload: {} }); expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/); }); - it.each(["equal", "later", "invalid"])("rejects %s normal terminal persistence against the server acknowledgement", kind => { - const f = fixture(); f.events[4]!.createdAt = kind === "invalid" ? "not-a-date" : date(kind === "equal" ? 909 : 910); + it.each(["equal", "later"])("accepts observed pre-Stop completion with %s transaction time, without ordering it against ack", kind => { + const f = fixture(); f.events[4]!.createdAt = date(kind === "equal" ? 909 : 910); + f.preStop = observeCopilotPreStop({ ...f, companyId: "company" }); f.stopDispatchMonotonicNs = process.hrtime.bigint().toString(); + expect(readCopilotDenialSettlement(f).branch).toBe("provider_completed_observed_before_stop"); + }); + it("rejects late normal completion even when its transaction began before Stop ack", () => { + const f = fixture(); f.preStop.terminal = null; f.events[4]!.createdAt = date(800); + expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/); + }); + it.each(["missing", "foreign", "hash", "sequence", "equal-clock", "reverse-clock"])("rejects %s pre-dispatch observation", kind => { + const f = fixture(); + if (kind === "missing") (f as any).preStop = null; + if (kind === "foreign") f.preStop.runId = "foreign"; + if (kind === "hash") f.preStop.terminal!.rowSha256 = `sha256:${"a".repeat(64)}`; + if (kind === "sequence") f.preStop.terminal!.sourceSeq++; + if (kind === "equal-clock") f.stopDispatchMonotonicNs = f.preStop.apiReadCompletedMonotonicNs; + if (kind === "reverse-clock") f.stopDispatchMonotonicNs = "1"; expect(() => readCopilotDenialSettlement(f)).toThrow(/settlement/); }); it.each(["runId", "turnId", "normalizedSessionId", "sourceInstanceId", "eventType", "sourceEventId"])("rejects terminal %s mismatch", field => { @@ -108,53 +128,83 @@ describe("Copilot denial provider settlement and separate audited run Stop", () f.events[1]!.payload.prpEvent.normalizedSessionId = "foreign"; expect(() => copilotDenialSampleCursor(f.events, f.request)).toThrow(/cursor/); }); - it("waits for failed-edit persistence before Stop and terminal persistence before sampling", async () => { - vi.useFakeTimers(); + it("awaits exact pre-Stop receipt retention and waits for post-Stop cancelled terminal before sampling", async () => { + vi.useFakeTimers(); let mono = 100n; vi.spyOn(process.hrtime, "bigint").mockImplementation(() => ++mono); try { - const f = fixture("turn.cancelled"), order: string[] = []; let loads = 0; - const post = vi.fn(async (url: string) => { order.push(url); }); - const load = async () => { - loads++; - // Native failed notice arrives first. Run cancellation and retirement - // also become visible before the durable canonical provider terminal. - const events = f.events.filter((_, i) => !(loads === 1 && i === 3) && !(loads < 4 && i === 4)); - order.push(`load-${loads}`); - return { ...f, events, run: loads < 3 ? { ...f.run, status: "running" } : f.run, retired: loads >= 3 }; - }; - const afterDeniedEdit = vi.fn(async () => { order.push("sample-after-decision"); }); - const afterSettlement = vi.fn(async () => { order.push("sample-terminal"); }); - const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 2000, - load, afterDeniedEdit, afterSettlement }); - await vi.advanceTimersByTimeAsync(0); - expect(post).not.toHaveBeenCalled(); expect(afterDeniedEdit).not.toHaveBeenCalled(); - await vi.advanceTimersByTimeAsync(200); - expect(post).toHaveBeenCalledTimes(1); expect(afterSettlement).not.toHaveBeenCalled(); - await vi.advanceTimersByTimeAsync(200); - expect((await result).branch).toBe("provider_cancelled_or_interrupted"); - expect(order).toEqual(["load-1", "load-2", "sample-after-decision", "/api/heartbeat-runs/run/cancel", "load-3", "load-4", "sample-terminal", "load-5"]); - } finally { vi.useRealTimers(); } + const f = fixture("turn.cancelled"); let loads = 0, dispatched = false, postLoads = 0; + let release!: () => void; const retention = new Promise(resolve => { release = resolve; }); + const post = vi.fn(async () => { dispatched = true; }); + const retainPreStop = vi.fn(async (receipt) => { expect(receipt.terminal).toBeNull(); await retention; }); + const afterDeniedEdit = vi.fn(async () => {}), afterSettlement = vi.fn(async () => {}); + const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 5000, + load: async () => { + loads++; if (dispatched) postLoads++; + const events = f.events.filter((_, i) => !(loads === 1 && i === 3) && !((!dispatched || postLoads === 1) && i === 4)); + return { ...f, events, run: dispatched ? f.run : { ...f.run, status: "running" }, retired: dispatched }; + }, afterDeniedEdit, retainPreStop, afterSettlement }); + await vi.advanceTimersByTimeAsync(0); expect(post).not.toHaveBeenCalled(); expect(afterDeniedEdit).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(200); expect(afterDeniedEdit).toHaveBeenCalledOnce(); expect(post).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(2000); expect(retainPreStop).toHaveBeenCalledOnce(); expect(post).not.toHaveBeenCalled(); + release(); await vi.advanceTimersByTimeAsync(0); expect(post).toHaveBeenCalledOnce(); expect(afterSettlement).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(200); expect((await result).branch).toBe("provider_cancelled_or_interrupted"); expect(afterSettlement).toHaveBeenCalledOnce(); + } finally { vi.restoreAllMocks(); vi.useRealTimers(); } + }); + it("rejects normal completion first returned after Stop even with an earlier transaction timestamp", async () => { + vi.useFakeTimers(); let mono = 100n; vi.spyOn(process.hrtime, "bigint").mockImplementation(() => ++mono); + try { + const f = fixture(); f.events[4]!.createdAt = date(1); let dispatched = false; + const sample = vi.fn(async () => {}), post = vi.fn(async () => { dispatched = true; }); + const retain = vi.fn(async (receipt) => { expect(receipt.terminal).toBeNull(); }); + const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 3000, + load: async () => ({ ...f, events: dispatched ? f.events : f.events.filter((_, i) => i !== 4), retired: dispatched }), + afterDeniedEdit: async () => {}, retainPreStop: retain, afterSettlement: sample }); + const rejected = expect(result).rejects.toThrow(/Timed out waiting for correlated provider settlement/); + await vi.advanceTimersByTimeAsync(3200); await rejected; + expect(retain).toHaveBeenCalledOnce(); expect(post).toHaveBeenCalledOnce(); expect(sample).not.toHaveBeenCalled(); + } finally { vi.restoreAllMocks(); vi.useRealTimers(); } }); it.each(["missing-edit", "foreign-edit", "missing-terminal", "foreign-terminal", "live-process"])("fails closed on %s without premature side effects or terminal samples", async kind => { - vi.useFakeTimers(); + vi.useFakeTimers(); let mono = 100n; vi.spyOn(process.hrtime, "bigint").mockImplementation(() => ++mono); try { const f = fixture(), post = vi.fn(async () => {}), sample = vi.fn(async () => {}); if (kind === "missing-edit") f.events.splice(3, 1); if (kind === "foreign-edit") f.events[3]!.payload.prpEvent.normalizedSessionId = "foreign"; if (kind === "missing-terminal") f.events.splice(4, 1); if (kind === "foreign-terminal") terminal(f).sourceInstanceId = "foreign"; - const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 500, - load: async () => ({ ...f, retired: kind !== "live-process" }), afterDeniedEdit: async () => {}, afterSettlement: sample }); - const rejected = expect(result).rejects.toThrow(/Timed out waiting/); - await vi.advanceTimersByTimeAsync(600); await rejected; - expect(post).toHaveBeenCalledTimes(kind.endsWith("edit") ? 0 : 1); + const result = settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 3000, + load: async () => ({ ...f, retired: kind !== "live-process" }), afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: sample }); + const rejected = expect(result).rejects.toThrow(); + await vi.advanceTimersByTimeAsync(3200); await rejected; + expect(post).toHaveBeenCalledTimes(kind.endsWith("edit") || kind === "foreign-terminal" ? 0 : 1); expect(sample).not.toHaveBeenCalled(); - } finally { vi.useRealTimers(); } + } finally { vi.restoreAllMocks(); vi.useRealTimers(); } + }); + it.each(["failed", "timed_out"])("reports %s promptly even when required rows are missing", async status => { + const f = fixture(), post = vi.fn(async () => {}), sample = vi.fn(async () => {}); + const load = vi.fn(async () => ({ ...f, events: [], run: { ...f.run, status }, retired: false })); + await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 10000, + load, afterDeniedEdit: sample, retainPreStop: sample, afterSettlement: sample })).rejects.toThrow(`Stopped waiting for persisted correlated failed native edit: Copilot provider run failed`); + expect(load).toHaveBeenCalledOnce(); expect(post).not.toHaveBeenCalled(); expect(sample).not.toHaveBeenCalled(); + }); + it.each(["failed", "timed_out"])("reports post-Stop %s before attempting a missing terminal reader", async status => { + const f = fixture(), post = vi.fn(async () => {}); let loads = 0; + const load = vi.fn(async () => ({ ...f, events: ++loads < 3 ? f.events : [], run: { ...f.run, status: loads < 3 ? "running" : status }, retired: true })); + await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 10000, + load, afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: async () => {} })).rejects.toThrow("Stopped waiting for correlated provider settlement and retired run: Copilot provider run failed"); + expect(load).toHaveBeenCalledTimes(3); expect(post).toHaveBeenCalledOnce(); + }); + it("does not dispatch Stop if the pre-dispatch receipt cannot be retained", async () => { + const f = fixture(), post = vi.fn(async () => {}); + await expect(settleCopilotDeniedRun({ api: { post } as any, request: f.request, deadlineAt: Date.now() + 1000, + load: async () => ({ ...f, retired: true }), afterDeniedEdit: async () => {}, afterSettlement: async () => {}, + retainPreStop: async () => { throw new Error("artifact write failed"); } })).rejects.toThrow("artifact write failed"); + expect(post).not.toHaveBeenCalled(); }); it("retains completed-before-Stop settlement and propagates retirement sampling failure", async () => { const f = fixture(), post = vi.fn(async () => {}); const input = { api: { post } as any, request: f.request, deadlineAt: Date.now() + 1000, - load: async () => ({ ...f, retired: true }), afterDeniedEdit: async () => {}, afterSettlement: async () => {} }; - expect((await settleCopilotDeniedRun(input)).branch).toBe("provider_completed_before_stop_settlement"); + load: async () => ({ ...f, retired: true }), afterDeniedEdit: async () => {}, retainPreStop: async () => {}, afterSettlement: async () => {} }; + expect((await settleCopilotDeniedRun(input)).branch).toBe("provider_completed_observed_before_stop"); await expect(settleCopilotDeniedRun({ ...input, afterSettlement: async () => { throw new Error("remote descendants live"); } })).rejects.toThrow("remote descendants live"); }); });