PaperClipAI/tests/runner-e2e/copilot-protection-cases.test.ts

import { createHash } from "node:crypto";
import { matchCopilotFixtureCommand } from "./copilot-protection-evidence.js";
import { describe, expect, it } from "vitest";
import { copilotProtectionCases, gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotAttachedSettlementEvidence, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js";
const identity = { runId: "run", sessionId: "session", turnId: "turn", toolCallId: "tool" };
const terminal = { observedAtMs: 50, runId: "run", turnId: "turn", status: "succeeded" as const };
const cleanup = { observedAtMs: 60, ownedProcessesRemaining: 0 };
function denied(): CopilotDeniedWriteEvidence {
  return {
    expected: identity, terminal: { ...terminal, status: "cancelled" }, cleanup, settlement: { schema: "paperclip.e2e.copilot-denial-settlement.v4", ...identity, requestId: "permission-0",
      branch: "provider_cancelled_or_interrupted", providerCancellationTerminalObserved: true,
      preStop: { schema: "paperclip.e2e.copilot-pre-stop-observation.v2" as const, ...identity, requestId: "permission-0", companyId: "company", normalizedSessionId: "normalized", sourceInstanceId: "runner", failedToolSourceSeq: 5, failedToolRowSha256: `sha256:${"a".repeat(64)}`, terminal: null, apiReadCompletedMonotonicNs: "10", cancellationRequestId: "11111111-1111-4111-8111-111111111111" }, stopDispatchMonotonicNs: "20",
      stopAcknowledgement: { schema: "paperclip.e2e.copilot-stop-acknowledgement.v1" as const, companyId: "company", runId: "run", issueId: "issue", callerUserId: "local-board", fixtureCorrelationId: "11111111-1111-4111-8111-111111111111", dispatchMonotonicNs: "20", apiResponseObservedMonotonicNs: "21", intentId: "native-cancellation:11111111-1111-4111-8111-111111111111", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", responseMetadataSha256: `sha256:${"c".repeat(64)}` },
      providerTerminal: { rowSha256: `sha256:${"b".repeat(64)}`, failedToolRowSha256: `sha256:${"a".repeat(64)}`, eventType: "turn.cancelled", normalizedSessionId: "normalized", sourceInstanceId: "runner", requestSourceSeq: 1, resolvedSourceSeq: 2, deliveredSourceSeq: 3, failedNoticeSourceSeq: 4, failedToolSourceSeq: 5, failedToolRowCreatedAtMs: 31, sourceSeq: 6, emittedAtMs: 50, rowCreatedAtMs: 51 },
      runStop: { companyId: "company", issueId: "issue", scope: "run", status: "cancelled", issueStatus: "in_progress", intentId: "native-cancellation:11111111-1111-4111-8111-111111111111", intentAuditId: "intent-audit", acknowledgementAuditId: "ack-audit", requestedAtMs: 40, recordedAtMs: 41, acknowledgedAtMs: 52, finishedAtMs: 53 } }, requestId: "permission-0", expectedRelativePath: "copilot-denied-nonce.txt",
    request: { ...identity, observedAtMs: 10, method: "session/request_permission", requestId: "permission-0", targetRelativePath: "copilot-denied-nonce.txt", offeredActions: ["accept", "decline"] },
    decision: { ...identity, observedAtMs: 20, requestId: "permission-0", browserRequestId: "permission-0", action: "decline" },
    deliveredDecision: { ...identity, observedAtMs: 25, requestId: "permission-0", outcome: "reject_once" },
    toolResult: { ...identity, observedAtMs: 30, status: "failed" }, nativeAttemptsForTarget: 1,
    fileObservations: [
      { phase: "before-request", observedAtMs: 0, exists: false, providerCursor: null }, { phase: "pending", observedAtMs: 15, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 1 } },
      { phase: "after-decision", observedAtMs: 25, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 4 } }, { phase: "terminal", observedAtMs: 50, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 6 } },
      { phase: "after-cleanup", observedAtMs: 60, exists: false, providerCursor: { runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "runner", sourceSeq: 6 } },
    ],
    mutationObservation: { startedAtMs: 0, endedAtMs: 60, complete: true, targetMutationCount: 0 },
  };
}
function attached(): CopilotAttachedSettlementEvidence {
  const command = "'/pinned/node' '/fixture/client.cjs' '/fixture/private/socket' 'nonce'";
  const digest = `sha256:${createHash("sha256").update(command).digest("hex")}`;
  return {
    expected: identity, terminal: { ...terminal }, cleanup: { ...cleanup }, expectedCommand: command, expectedCommandSha256: digest, commandMatch: matchCopilotFixtureCommand(command, digest), expectedShellId: "0",
    nativeCall: { ...identity, observedAtMs: 10, operation: "execute", mode: "async", detach: false, commandSha256: digest },
    commandExit: { observedAtMs: 30, code: 0, ownedProcessIdentityVerified: true, commandSha256: digest },
    nativeShellResult: { ...identity, toolCallId: "read-shell-tool", commandToolCallId: "tool", observedAtMs: 40, shellId: "0", status: "completed", exitCode: 0 },
    terminalMarkerMatches: true, afterCleanupMarkerMatches: true,
  };
}
describe("Copilot protection Product oracles", () => {
  it("declares one bounded run and explicit discovery integration", () => {
    expect(copilotProtectionCases.map(c => c.id)).toEqual(["native-permission-deny-write", "attached-async-settlement"]);
    for (const c of copilotProtectionCases) { expect(c.expectedRunCount).toBe(1); expect(c.providerTimeoutSec).toBe(120); expect(c.integration).toBe("registered-copilot-protection-flow"); }
    expect(copilotProtectionCases[1].prompt("nonce")).toContain("detach false");
  });
  it("accepts an origin-bound browser denial with an independent continuous absence oracle", () => {
    expect(gradeCopilotDeniedWrite(denied())).toEqual({ passed: true, failures: [] });
  });
  it("keeps every no-effect and cleanup gate for a completed-before-Stop settlement", () => {
    const e = denied();
    e.settlement!.branch = "provider_completed_observed_before_stop";
    e.settlement!.providerCancellationTerminalObserved = false;
    e.settlement!.providerTerminal.eventType = "turn.completed";
    e.settlement!.preStop.terminal = { eventType: "turn.completed", sourceSeq: 6, rowSha256: e.settlement!.providerTerminal.rowSha256 };
    expect(gradeCopilotDeniedWrite(e).passed).toBe(true);
    for (const mutate of [
      (v: CopilotDeniedWriteEvidence) => { v.cleanup!.ownedProcessesRemaining = 1; },
      (v: CopilotDeniedWriteEvidence) => { v.nativeAttemptsForTarget = 2; },
      (v: CopilotDeniedWriteEvidence) => { v.fileObservations[2]!.exists = true; },
      (v: CopilotDeniedWriteEvidence) => { v.mutationObservation!.targetMutationCount = 1; },
      (v: CopilotDeniedWriteEvidence) => { v.mutationObservation!.complete = false; },
      (v: CopilotDeniedWriteEvidence) => { v.deliveredDecision = null; },
      (v: CopilotDeniedWriteEvidence) => { v.toolResult!.status = "completed"; },
      (v: CopilotDeniedWriteEvidence) => { v.settlement!.providerCancellationTerminalObserved = true; },
      (v: CopilotDeniedWriteEvidence) => { v.settlement!.turnId = "foreign"; },
    ]) { const bad = structuredClone(e); mutate(bad); expect(gradeCopilotDeniedWrite(bad).passed).toBe(false); }
  });
  it.each([-86_400_000, 86_400_000])("completed settlement does not compare provider, browser and filesystem clocks (%s)", skew => {
    const e = denied();
    e.settlement!.branch = "provider_completed_observed_before_stop";
    e.settlement!.providerCancellationTerminalObserved = false;
    e.settlement!.providerTerminal.eventType = "turn.completed";
    e.settlement!.preStop.terminal = { eventType: "turn.completed", sourceSeq: 6, rowSha256: e.settlement!.providerTerminal.rowSha256 };
    // Distinct positive clock domains, with either provider ahead or behind.
    for (const n of [e.request!, e.deliveredDecision!, e.toolResult!, e.terminal!]) n.observedAtMs += 100_000_000 + skew;
    e.settlement!.providerTerminal.emittedAtMs = e.terminal!.observedAtMs;
    e.decision!.observedAtMs = 200_000_000;
    expect(gradeCopilotDeniedWrite(e).passed).toBe(true);
    e.fileObservations[1]!.providerCursor!.sourceSeq = e.settlement!.providerTerminal.resolvedSourceSeq;
    expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid");
  });
  it.each(["runId", "turnId", "normalizedSessionId", "sourceInstanceId"] as const)("rejects foreign sample %s despite plausible timestamps", field => {
    const e = denied(); e.fileObservations[3]!.providerCursor![field] = "foreign";
    expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
  });
  it("rejects missing, regressed and premature causal sample cursors", () => {
    for (const mutate of [
      (e: CopilotDeniedWriteEvidence) => { e.fileObservations[2]!.providerCursor = null; },
      (e: CopilotDeniedWriteEvidence) => { e.fileObservations[2]!.providerCursor!.sourceSeq = 3; },
      (e: CopilotDeniedWriteEvidence) => { e.fileObservations[3]!.providerCursor!.sourceSeq = 5; },
      (e: CopilotDeniedWriteEvidence) => { e.fileObservations[4]!.providerCursor!.sourceSeq = 5; },
    ]) { const e = denied(); mutate(e); expect(gradeCopilotDeniedWrite(e).passed).toBe(false); }
  });
  it("requires the exact isolated target in native permission evidence", () => {
    const e = denied(); e.expectedRelativePath = "pc-denied-ABC123/copilot-denied-nonce.txt";
    expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
    e.request!.targetRelativePath = e.expectedRelativePath;
    expect(gradeCopilotDeniedWrite(e).passed).toBe(true);
    for (const path of ["../copilot-denied-nonce.txt", "other/copilot-denied-nonce.txt", "pc-denied-ABC123/../copilot-denied-nonce.txt"]) {
      e.expectedRelativePath = path; e.request!.targetRelativePath = path;
      expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
    }
  });
  it.each(["request", "decision", "deliveredDecision", "toolResult", "terminal", "cleanup", "mutationObservation", "settlement"] as const)("rejects missing %s instead of treating no write as denial", field => {
    const e = denied(); e[field] = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
  });
  it("requires explicit acknowledged cancellation for denial without relaxing successful settlement", () => {
    const e = denied(); e.settlement = null; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
    const finished = denied(); finished.terminal!.status = "succeeded"; expect(gradeCopilotDeniedWrite(finished).passed).toBe(false);
    const cancelled = attached(); cancelled.terminal!.status = "cancelled"; expect(gradeCopilotAttachedSettlement(cancelled).passed).toBe(false);
  });
  it("rejects a transient create/delete even when all five stat samples are absent", () => {
    const e = denied(); e.mutationObservation!.targetMutationCount = 2; expect(gradeCopilotDeniedWrite(e).failures).toContain("missing-or-mutated-filesystem-watch");
  });
  it("rejects missing watch coverage, observation gaps and late mutation", () => {
    const gap = denied(); gap.mutationObservation!.complete = false; expect(gradeCopilotDeniedWrite(gap).passed).toBe(false);
    const missing = denied(); missing.fileObservations.pop(); expect(gradeCopilotDeniedWrite(missing).passed).toBe(false);
    const late = denied(); late.fileObservations.at(-1)!.exists = true; expect(gradeCopilotDeniedWrite(late).passed).toBe(false);
  });
  it("rejects another request's click, a successful edit, and a retry through a second native tool", () => {
    const foreign = denied(); foreign.decision!.browserRequestId = "other-request"; expect(gradeCopilotDeniedWrite(foreign).passed).toBe(false);
    const success = denied(); success.toolResult!.status = "completed"; expect(gradeCopilotDeniedWrite(success).passed).toBe(false);
    const retried = denied(); retried.nativeAttemptsForTarget = 2; expect(gradeCopilotDeniedWrite(retried).passed).toBe(false);
  });
  it("rejects a sample taken before the request as pending evidence", () => {
    const e = denied(); e.fileObservations[1]!.providerCursor!.sourceSeq = 0; expect(gradeCopilotDeniedWrite(e).failures).toContain("filesystem-observation-order-invalid");
  });
  it("rejects an unrelated request identity even when the chosen decision matches an outer ID", () => {
    const e = denied(); e.request!.requestId = "foreign-request"; expect(gradeCopilotDeniedWrite(e).passed).toBe(false);
  });
  it("accepts attached async completion using a separately correlated read_bash call", () => {
    expect(gradeCopilotAttachedSettlement(attached())).toEqual({ passed: true, failures: [] });
  });
  it("verifies the raw-to-fixture digest relation without weakening independent marker proof", () => {
    const e = attached(); const raw = `sha256:${createHash("sha256").update(" " + e.expectedCommand).digest("hex")}`;
    e.nativeCall!.commandSha256 = raw; e.commandMatch = matchCopilotFixtureCommand(e.expectedCommand, raw);
    expect(e.commandMatch?.leadingWhitespace).toBe(" ");
    expect(gradeCopilotAttachedSettlement(e).passed).toBe(true);
    expect(e.nativeCall!.commandSha256).toBe(raw);
    for (const field of ["algorithm", "canonicalCommandSha256", "nativeCommandSha256", "leadingWhitespace"] as const) {
      const bad = structuredClone(e); (bad.commandMatch as any)[field] = "tampered";
      expect(gradeCopilotAttachedSettlement(bad).passed).toBe(false);
    }
    const missing = structuredClone(e); missing.commandMatch = null; expect(gradeCopilotAttachedSettlement(missing).passed).toBe(false);
    const forged = structuredClone(e); forged.expectedCommand += " changed"; expect(gradeCopilotAttachedSettlement(forged).passed).toBe(false);
    const unbound = structuredClone(e); unbound.expectedCommandSha256 = raw; expect(gradeCopilotAttachedSettlement(unbound).passed).toBe(false);
    const wrongExit = structuredClone(e); wrongExit.commandExit!.commandSha256 = raw; expect(gradeCopilotAttachedSettlement(wrongExit).passed).toBe(false);
    e.afterCleanupMarkerMatches = false; expect(gradeCopilotAttachedSettlement(e).passed).toBe(false);
  });
  it.each(["runId", "sessionId", "turnId", "toolCallId"] as const)("rejects a matched digest from a foreign %s", field => {
    const e = attached(); e.nativeCall![field] = "foreign";
    expect(gradeCopilotAttachedSettlement(e).passed).toBe(false);
  });
  it("rejects detached policy denial as settlement", () => {
    const e = attached(); e.nativeCall!.detach = true; expect(gradeCopilotAttachedSettlement(e).failures).toContain("missing-exact-attached-async-call");
  });
  it("rejects prompt-only mode claims, fabricated marker receipts, and missing native completion", () => {
    const missing = attached(); missing.nativeCall = null; expect(gradeCopilotAttachedSettlement(missing).passed).toBe(false);
    const fake = attached(); fake.commandExit!.ownedProcessIdentityVerified = false; expect(gradeCopilotAttachedSettlement(fake).passed).toBe(false);
    const incomplete = attached(); incomplete.nativeShellResult = null; expect(gradeCopilotAttachedSettlement(incomplete).passed).toBe(false);
  });
  it("rejects work finishing at or after terminal even when the final marker is correct", () => {
    for (const at of [50, 51]) { const e = attached(); e.commandExit!.observedAtMs = at; expect(gradeCopilotAttachedSettlement(e).passed).toBe(false); }
  });
  it("rejects shell identity reuse across turns and foreign command completion", () => {
    const foreign = attached(); foreign.nativeShellResult!.turnId = "previous-turn"; expect(gradeCopilotAttachedSettlement(foreign).passed).toBe(false);
    const other = attached(); other.nativeShellResult!.commandToolCallId = "other-tool"; expect(gradeCopilotAttachedSettlement(other).passed).toBe(false);
  });
  it("rejects nonzero exit, live descendants, and marker loss during cleanup", () => {
    const exit = attached(); exit.commandExit!.code = 1; expect(gradeCopilotAttachedSettlement(exit).passed).toBe(false);
    const live = attached(); live.cleanup = { observedAtMs: 60, ownedProcessesRemaining: 1 }; expect(gradeCopilotAttachedSettlement(live).passed).toBe(false);
    const marker = attached(); marker.afterCleanupMarkerMatches = false; expect(gradeCopilotAttachedSettlement(marker).passed).toBe(false);
  });
});