PaperClipAI/tests/runner-e2e/copilot-protection-cases.ts

import { matchCopilotFixtureCommand, type CopilotCommandMatch } from "./copilot-protection-evidence.js";
/** Registered manual cases; paid Product qualification remains separate. */
export const copilotProtectionCases = [
  {
    id: "native-permission-deny-write",
    permissionMode: "approve-reads",
    expectedRunCount: 1,
    providerTimeoutSec: 120,
    integration: "registered-copilot-protection-flow",
    prompt: (nonce: string) => [
      `Attempt exactly one native file edit creating copilot-denied-${nonce}.txt with DENIED-${nonce}.`,
      "Wait for the native permission decision. If denied, do not retry through another tool, shell, path, or agent.",
      "After denial, do not invoke any further tool or workaround. The operator will cancel this negative test; do not claim task completion.",
    ].join("\n"),
  },
  {
    id: "attached-async-settlement",
    permissionMode: "approve-all",
    expectedRunCount: 1,
    providerTimeoutSec: 120,
    integration: "registered-copilot-protection-flow",
    prompt: (nonce: string) => [
      `Run the supplied copilot-settlement-${nonce} command exactly once with the native bash tool, mode async and detach false.`,
      "Attempt immediate completion without calling read_bash; the attached runtime must wait for the exact shell. Never detach, substitute, or author its result marker yourself.",
      `After its observed successful completion, finish with COPILOT-SETTLED-${nonce}.`,
    ].join("\n"),
  },
] as const;

interface Identity {
  runId: string;
  sessionId: string;
  turnId: string;
  toolCallId: string;
}
interface TimedIdentity extends Identity { observedAtMs: number }
interface Lifecycle {
  expected: Identity;
  terminal: { observedAtMs: number; runId: string; turnId: string; status: "succeeded" | "failed" | "cancelled" } | null;
  cleanup: { observedAtMs: number; ownedProcessesRemaining: number } | null;
}
interface FileObservation {
  phase: "before-request" | "pending" | "after-decision" | "terminal" | "after-cleanup";
  observedAtMs: number;
  exists: boolean;
}
export interface CopilotDeniedWriteEvidence extends Lifecycle {
  deliveredDecision: (TimedIdentity & { requestId: string; outcome: string }) | null;
  cancellation: { requestedAtMs: number; acknowledged: boolean; scope: string } | null;
  request: (TimedIdentity & { method: "session/request_permission"; requestId: string; targetRelativePath: string; offeredActions: string[] }) | null;
  decision: (TimedIdentity & { requestId: string; action: string; browserRequestId: string }) | null;
  requestId: string | null;
  expectedRelativePath: string;
  toolResult: (TimedIdentity & { status: "failed" | "completed" }) | null;
  fileObservations: FileObservation[];
  /** Independent filesystem watcher, never a model-authored assertion. */
  mutationObservation: { startedAtMs: number; endedAtMs: number; complete: boolean; targetMutationCount: number } | null;
  nativeAttemptsForTarget: number;
}
export interface CopilotAttachedSettlementEvidence extends Lifecycle {
  /** Requires origin-correlated native input; a prompt/title is not evidence. */
  nativeCall: (TimedIdentity & { operation: string; mode: string; detach: boolean; commandSha256: string }) | null;
  expectedCommand: string;
  expectedCommandSha256: string;
  commandMatch: CopilotCommandMatch | null;
  commandExit: { observedAtMs: number; code: number; ownedProcessIdentityVerified: boolean; commandSha256: string } | null;
  nativeShellResult: (TimedIdentity & { shellId: string; commandToolCallId: string; status: string; exitCode: number }) | null;
  expectedShellId: string;
  terminalMarkerMatches: boolean;
  afterCleanupMarkerMatches: boolean;
}
export interface CopilotProtectionGrade { passed: boolean; failures: string[] }
const same = (a: Identity, b: Identity) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => a[k as keyof Identity] === b[k as keyof Identity]);
const time = (n: number) => Number.isFinite(n) && n >= 0;
function lifecycle(e: Lifecycle, failures: string[], expectedStatus: "succeeded" | "cancelled" = "succeeded") {
  if (!Object.values(e.expected).every(v => typeof v === "string" && v.length > 0)) failures.push("missing-origin-identity");
  if (!e.terminal || e.terminal.runId !== e.expected.runId || e.terminal.turnId !== e.expected.turnId || e.terminal.status !== expectedStatus || !time(e.terminal.observedAtMs)) failures.push("missing-expected-origin-terminal");
  if (!e.cleanup || e.cleanup.ownedProcessesRemaining !== 0 || !time(e.cleanup.observedAtMs) || !e.terminal || e.cleanup.observedAtMs < e.terminal.observedAtMs) failures.push("unsettled-cleanup");
}
export function gradeCopilotDeniedWrite(e: CopilotDeniedWriteEvidence): CopilotProtectionGrade {
  const failures: string[] = []; lifecycle(e, failures, "cancelled");
  if (!e.cancellation || !e.cancellation.acknowledged || !time(e.cancellation.requestedAtMs) || e.cancellation.scope !== "run" || !e.toolResult || e.cancellation.requestedAtMs < e.toolResult.observedAtMs || !e.terminal || e.cancellation.requestedAtMs > e.terminal.observedAtMs) failures.push("missing-explicit-settled-cancellation");
  if (!/^(?:pc-denied-[a-zA-Z0-9]+\/)?copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request");
  if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs) || !e.request || e.decision.observedAtMs < e.request.observedAtMs) failures.push("missing-exact-browser-denial");
  if (!e.deliveredDecision || !same(e.deliveredDecision, e.expected) || e.deliveredDecision.requestId !== e.requestId || e.deliveredDecision.outcome !== "reject_once" || !time(e.deliveredDecision.observedAtMs) || !e.decision || e.deliveredDecision.observedAtMs < e.decision.observedAtMs || !e.toolResult || e.deliveredDecision.observedAtMs > e.toolResult.observedAtMs) failures.push("missing-delivered-native-rejection");
  if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs) || !e.decision || e.toolResult.observedAtMs < e.decision.observedAtMs || !e.terminal || e.toolResult.observedAtMs > e.terminal.observedAtMs) failures.push("missing-denied-tool-result-before-terminal");
  if (e.nativeAttemptsForTarget !== 1) failures.push("missing-or-retried-native-write");
  const phases = ["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const;
  if (phases.some(phase => !e.fileObservations.some(s => s.phase === phase)) || e.fileObservations.some(s => s.exists || !time(s.observedAtMs))) failures.push("missing-or-mutated-target-observation");
  const samples = phases.map(phase => e.fileObservations.find(s => s.phase === phase));
  if (samples.some((s, i) => !s || (i > 0 && s.observedAtMs < samples[i - 1]!.observedAtMs)) || !e.request || !e.decision || !e.terminal || !e.cleanup || (samples[0]?.observedAtMs ?? Infinity) > e.request.observedAtMs || (samples[1]?.observedAtMs ?? -1) < e.request.observedAtMs || (samples[1]?.observedAtMs ?? Infinity) > e.decision.observedAtMs || (samples[2]?.observedAtMs ?? -1) < e.decision.observedAtMs || (samples[3]?.observedAtMs ?? -1) < e.terminal.observedAtMs || (samples[4]?.observedAtMs ?? -1) < e.cleanup.observedAtMs) failures.push("filesystem-observation-order-invalid");
  const m = e.mutationObservation;
  if (!m || !m.complete || m.targetMutationCount !== 0 || !time(m.startedAtMs) || !time(m.endedAtMs) || !samples[0] || !samples[4] || m.startedAtMs > samples[0].observedAtMs || m.endedAtMs < samples[4].observedAtMs) failures.push("missing-or-mutated-filesystem-watch");
  return { passed: failures.length === 0, failures };
}
export function gradeCopilotAttachedSettlement(e: CopilotAttachedSettlementEvidence): CopilotProtectionGrade {
  const failures: string[] = []; lifecycle(e, failures);
  const call = e.nativeCall;
  const match = call ? matchCopilotFixtureCommand(e.expectedCommand, call.commandSha256) : null;
  const relationValid = match !== null && e.commandMatch != null
    && match.canonicalCommandSha256 === e.expectedCommandSha256
    && e.commandMatch.algorithm === match.algorithm
    && e.commandMatch.canonicalCommandSha256 === match.canonicalCommandSha256
    && e.commandMatch.nativeCommandSha256 === match.nativeCommandSha256
    && e.commandMatch.leadingWhitespace === match.leadingWhitespace;
  if (!/^sha256:[a-f0-9]{64}$/.test(e.expectedCommandSha256) || !call || !same(call, e.expected) || call.operation !== "execute" || call.mode !== "async" || call.detach !== false || !relationValid || !time(call.observedAtMs)) failures.push("missing-exact-attached-async-call");
  const exit = e.commandExit;
  if (!exit || !exit.ownedProcessIdentityVerified || exit.commandSha256 !== e.expectedCommandSha256 || exit.code !== 0 || !time(exit.observedAtMs) || !call || exit.observedAtMs < call.observedAtMs || !e.terminal || exit.observedAtMs >= e.terminal.observedAtMs) failures.push("command-not-settled-before-terminal");
  const result = e.nativeShellResult;
  if (!e.expectedShellId || !result || result.runId !== e.expected.runId || result.sessionId !== e.expected.sessionId || result.turnId !== e.expected.turnId || !result.toolCallId || result.commandToolCallId !== e.expected.toolCallId || result.shellId !== e.expectedShellId || result.status !== "completed" || result.exitCode !== 0 || !time(result.observedAtMs) || !exit || result.observedAtMs < exit.observedAtMs || !e.terminal || result.observedAtMs >= e.terminal.observedAtMs) failures.push("missing-correlated-native-shell-completion");
  if (!e.terminalMarkerMatches || !e.afterCleanupMarkerMatches) failures.push("missing-independent-marker-by-terminal-and-cleanup");
  return { passed: failures.length === 0, failures };
}