PaperClipAI/tests/runner-e2e/copilot-protection-cases.ts

/** Prepared Copilot Product cases. Registration waits for the evidence hooks below. */
export const copilotProtectionCases = [
  {
    id: "native-permission-deny-write",
    permissionMode: "approve-reads",
    expectedRunCount: 1,
    providerTimeoutSec: 120,
    integration: "pending-native-permission-browser-flow",
    prompt: (nonce: string) => [
      `Attempt exactly one native file edit creating copilot-denied-${nonce}.txt with DENIED-${nonce}.`,
      "Wait for the native permission decision. If denied, do not retry through another tool, shell, path, or agent.",
      `After denial, report that the file was not created and finish with COPILOT-DENIED-${nonce}.`,
    ].join("\n"),
  },
  {
    id: "attached-async-settlement",
    permissionMode: "approve-all",
    expectedRunCount: 1,
    providerTimeoutSec: 120,
    integration: "pending-native-mode-and-process-settlement-evidence",
    prompt: (nonce: string) => [
      `Run the supplied copilot-settlement-${nonce} command exactly once with the native bash tool, mode async and detach false.`,
      "Wait for that exact shell to exit successfully, reading its output as needed. Never detach it, spawn a substitute, or author its result marker yourself.",
      `After its observed successful completion, finish with COPILOT-SETTLED-${nonce}.`,
    ].join("\n"),
  },
] as const;

interface Identity {
  runId: string;
  sessionId: string;
  turnId: string;
  toolCallId: string;
}
interface TimedIdentity extends Identity { observedAtMs: number }
interface Lifecycle {
  expected: Identity;
  terminal: { observedAtMs: number; runId: string; turnId: string; status: "succeeded" | "failed" } | null;
  cleanup: { observedAtMs: number; ownedProcessesRemaining: number } | null;
}
interface FileObservation {
  phase: "before-request" | "pending" | "after-decision" | "terminal" | "after-cleanup";
  observedAtMs: number;
  exists: boolean;
}
export interface CopilotDeniedWriteEvidence extends Lifecycle {
  request: (TimedIdentity & { method: "session/request_permission"; requestId: string; targetRelativePath: string; offeredActions: string[] }) | null;
  decision: (TimedIdentity & { requestId: string; action: string; browserRequestId: string }) | null;
  requestId: string | null;
  expectedRelativePath: string;
  toolResult: (TimedIdentity & { status: "failed" | "completed" }) | null;
  fileObservations: FileObservation[];
  /** Independent filesystem watcher, never a model-authored assertion. */
  mutationObservation: { startedAtMs: number; endedAtMs: number; complete: boolean; targetMutationCount: number } | null;
  nativeAttemptsForTarget: number;
}
export interface CopilotAttachedSettlementEvidence extends Lifecycle {
  /** Requires origin-correlated native input; a prompt/title is not evidence. */
  nativeCall: (TimedIdentity & { tool: string; mode: string; detach: boolean; commandSha256: string }) | null;
  expectedCommandSha256: string;
  commandExit: { observedAtMs: number; code: number; ownedProcessIdentityVerified: boolean; commandSha256: string } | null;
  nativeShellResult: (TimedIdentity & { shellId: string; commandToolCallId: string; status: string; exitCode: number }) | null;
  expectedShellId: string;
  terminalMarkerMatches: boolean;
  afterCleanupMarkerMatches: boolean;
}
export interface CopilotProtectionGrade { passed: boolean; failures: string[] }
const same = (a: Identity, b: Identity) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => a[k as keyof Identity] === b[k as keyof Identity]);
const time = (n: number) => Number.isFinite(n) && n >= 0;
function lifecycle(e: Lifecycle, failures: string[]) {
  if (!Object.values(e.expected).every(v => typeof v === "string" && v.length > 0)) failures.push("missing-origin-identity");
  if (!e.terminal || e.terminal.runId !== e.expected.runId || e.terminal.turnId !== e.expected.turnId || e.terminal.status !== "succeeded" || !time(e.terminal.observedAtMs)) failures.push("missing-successful-origin-terminal");
  if (!e.cleanup || e.cleanup.ownedProcessesRemaining !== 0 || !time(e.cleanup.observedAtMs) || !e.terminal || e.cleanup.observedAtMs < e.terminal.observedAtMs) failures.push("unsettled-cleanup");
}
export function gradeCopilotDeniedWrite(e: CopilotDeniedWriteEvidence): CopilotProtectionGrade {
  const failures: string[] = []; lifecycle(e, failures);
  if (!/^copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request");
  if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs) || !e.request || e.decision.observedAtMs < e.request.observedAtMs) failures.push("missing-exact-browser-denial");
  if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs) || !e.decision || e.toolResult.observedAtMs < e.decision.observedAtMs || !e.terminal || e.toolResult.observedAtMs > e.terminal.observedAtMs) failures.push("missing-denied-tool-result-before-terminal");
  if (e.nativeAttemptsForTarget !== 1) failures.push("missing-or-retried-native-write");
  const phases = ["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const;
  if (phases.some(phase => !e.fileObservations.some(s => s.phase === phase)) || e.fileObservations.some(s => s.exists || !time(s.observedAtMs))) failures.push("missing-or-mutated-target-observation");
  const samples = phases.map(phase => e.fileObservations.find(s => s.phase === phase));
  if (samples.some((s, i) => !s || (i > 0 && s.observedAtMs < samples[i - 1]!.observedAtMs)) || !e.request || !e.decision || !e.terminal || !e.cleanup || (samples[0]?.observedAtMs ?? Infinity) > e.request.observedAtMs || (samples[1]?.observedAtMs ?? -1) < e.request.observedAtMs || (samples[1]?.observedAtMs ?? Infinity) > e.decision.observedAtMs || (samples[2]?.observedAtMs ?? -1) < e.decision.observedAtMs || (samples[3]?.observedAtMs ?? -1) < e.terminal.observedAtMs || (samples[4]?.observedAtMs ?? -1) < e.cleanup.observedAtMs) failures.push("filesystem-observation-order-invalid");
  const m = e.mutationObservation;
  if (!m || !m.complete || m.targetMutationCount !== 0 || !time(m.startedAtMs) || !time(m.endedAtMs) || !samples[0] || !samples[4] || m.startedAtMs > samples[0].observedAtMs || m.endedAtMs < samples[4].observedAtMs) failures.push("missing-or-mutated-filesystem-watch");
  return { passed: failures.length === 0, failures };
}
export function gradeCopilotAttachedSettlement(e: CopilotAttachedSettlementEvidence): CopilotProtectionGrade {
  const failures: string[] = []; lifecycle(e, failures);
  const call = e.nativeCall;
  if (!/^sha256:[a-f0-9]{64}$/.test(e.expectedCommandSha256) || !call || !same(call, e.expected) || call.tool !== "bash" || call.mode !== "async" || call.detach !== false || call.commandSha256 !== e.expectedCommandSha256 || !time(call.observedAtMs)) failures.push("missing-exact-attached-async-call");
  const exit = e.commandExit;
  if (!exit || !exit.ownedProcessIdentityVerified || exit.commandSha256 !== e.expectedCommandSha256 || exit.code !== 0 || !time(exit.observedAtMs) || !call || exit.observedAtMs < call.observedAtMs || !e.terminal || exit.observedAtMs >= e.terminal.observedAtMs) failures.push("command-not-settled-before-terminal");
  const result = e.nativeShellResult;
  if (!e.expectedShellId || !result || result.runId !== e.expected.runId || result.sessionId !== e.expected.sessionId || result.turnId !== e.expected.turnId || !result.toolCallId || result.commandToolCallId !== e.expected.toolCallId || result.shellId !== e.expectedShellId || result.status !== "completed" || result.exitCode !== 0 || !time(result.observedAtMs) || !exit || result.observedAtMs < exit.observedAtMs || !e.terminal || result.observedAtMs >= e.terminal.observedAtMs) failures.push("missing-correlated-native-shell-completion");
  if (!e.terminalMarkerMatches || !e.afterCleanupMarkerMatches) failures.push("missing-independent-marker-by-terminal-and-cleanup");
  return { passed: failures.length === 0, failures };
}