mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 00:53:38 +02:00
104 lines
9.3 KiB
TypeScript
104 lines
9.3 KiB
TypeScript
/** Registered manual cases; paid Product qualification remains separate. */
|
|
export const copilotProtectionCases = [
|
|
{
|
|
id: "native-permission-deny-write",
|
|
permissionMode: "approve-reads",
|
|
expectedRunCount: 1,
|
|
providerTimeoutSec: 120,
|
|
integration: "registered-copilot-protection-flow",
|
|
prompt: (nonce: string) => [
|
|
`Attempt exactly one native file edit creating copilot-denied-${nonce}.txt with DENIED-${nonce}.`,
|
|
"Wait for the native permission decision. If denied, do not retry through another tool, shell, path, or agent.",
|
|
"After denial, do not invoke any further tool or workaround. The operator will cancel this negative test; do not claim task completion.",
|
|
].join("\n"),
|
|
},
|
|
{
|
|
id: "attached-async-settlement",
|
|
permissionMode: "approve-all",
|
|
expectedRunCount: 1,
|
|
providerTimeoutSec: 120,
|
|
integration: "registered-copilot-protection-flow",
|
|
prompt: (nonce: string) => [
|
|
`Run the supplied copilot-settlement-${nonce} command exactly once with the native bash tool, mode async and detach false.`,
|
|
"Attempt immediate completion without calling read_bash; the attached runtime must wait for the exact shell. Never detach, substitute, or author its result marker yourself.",
|
|
`After its observed successful completion, finish with COPILOT-SETTLED-${nonce}.`,
|
|
].join("\n"),
|
|
},
|
|
] as const;
|
|
|
|
interface Identity {
|
|
runId: string;
|
|
sessionId: string;
|
|
turnId: string;
|
|
toolCallId: string;
|
|
}
|
|
interface TimedIdentity extends Identity { observedAtMs: number }
|
|
interface Lifecycle {
|
|
expected: Identity;
|
|
terminal: { observedAtMs: number; runId: string; turnId: string; status: "succeeded" | "failed" | "cancelled" } | null;
|
|
cleanup: { observedAtMs: number; ownedProcessesRemaining: number } | null;
|
|
}
|
|
interface FileObservation {
|
|
phase: "before-request" | "pending" | "after-decision" | "terminal" | "after-cleanup";
|
|
observedAtMs: number;
|
|
exists: boolean;
|
|
}
|
|
export interface CopilotDeniedWriteEvidence extends Lifecycle {
|
|
deliveredDecision: (TimedIdentity & { requestId: string; outcome: string }) | null;
|
|
cancellation: { requestedAtMs: number; acknowledged: boolean; scope: string } | null;
|
|
request: (TimedIdentity & { method: "session/request_permission"; requestId: string; targetRelativePath: string; offeredActions: string[] }) | null;
|
|
decision: (TimedIdentity & { requestId: string; action: string; browserRequestId: string }) | null;
|
|
requestId: string | null;
|
|
expectedRelativePath: string;
|
|
toolResult: (TimedIdentity & { status: "failed" | "completed" }) | null;
|
|
fileObservations: FileObservation[];
|
|
/** Independent filesystem watcher, never a model-authored assertion. */
|
|
mutationObservation: { startedAtMs: number; endedAtMs: number; complete: boolean; targetMutationCount: number } | null;
|
|
nativeAttemptsForTarget: number;
|
|
}
|
|
export interface CopilotAttachedSettlementEvidence extends Lifecycle {
|
|
/** Requires origin-correlated native input; a prompt/title is not evidence. */
|
|
nativeCall: (TimedIdentity & { operation: string; mode: string; detach: boolean; commandSha256: string }) | null;
|
|
expectedCommandSha256: string;
|
|
commandExit: { observedAtMs: number; code: number; ownedProcessIdentityVerified: boolean; commandSha256: string } | null;
|
|
nativeShellResult: (TimedIdentity & { shellId: string; commandToolCallId: string; status: string; exitCode: number }) | null;
|
|
expectedShellId: string;
|
|
terminalMarkerMatches: boolean;
|
|
afterCleanupMarkerMatches: boolean;
|
|
}
|
|
export interface CopilotProtectionGrade { passed: boolean; failures: string[] }
|
|
const same = (a: Identity, b: Identity) => ["runId", "sessionId", "turnId", "toolCallId"].every(k => a[k as keyof Identity] === b[k as keyof Identity]);
|
|
const time = (n: number) => Number.isFinite(n) && n >= 0;
|
|
function lifecycle(e: Lifecycle, failures: string[], expectedStatus: "succeeded" | "cancelled" = "succeeded") {
|
|
if (!Object.values(e.expected).every(v => typeof v === "string" && v.length > 0)) failures.push("missing-origin-identity");
|
|
if (!e.terminal || e.terminal.runId !== e.expected.runId || e.terminal.turnId !== e.expected.turnId || e.terminal.status !== expectedStatus || !time(e.terminal.observedAtMs)) failures.push("missing-expected-origin-terminal");
|
|
if (!e.cleanup || e.cleanup.ownedProcessesRemaining !== 0 || !time(e.cleanup.observedAtMs) || !e.terminal || e.cleanup.observedAtMs < e.terminal.observedAtMs) failures.push("unsettled-cleanup");
|
|
}
|
|
export function gradeCopilotDeniedWrite(e: CopilotDeniedWriteEvidence): CopilotProtectionGrade {
|
|
const failures: string[] = []; lifecycle(e, failures, "cancelled");
|
|
if (!e.cancellation || !e.cancellation.acknowledged || !time(e.cancellation.requestedAtMs) || e.cancellation.scope !== "run" || !e.toolResult || e.cancellation.requestedAtMs < e.toolResult.observedAtMs || !e.terminal || e.cancellation.requestedAtMs > e.terminal.observedAtMs) failures.push("missing-explicit-settled-cancellation");
|
|
if (!/^(?:pc-denied-[a-zA-Z0-9]+\/)?copilot-denied-[a-z0-9-]+\.txt$/.test(e.expectedRelativePath) || !e.request || !same(e.request, e.expected) || e.request.method !== "session/request_permission" || e.request.requestId !== e.requestId || e.request.targetRelativePath !== e.expectedRelativePath || !e.request.offeredActions.includes("decline") || !time(e.request.observedAtMs)) failures.push("missing-exact-native-write-request");
|
|
if (!e.requestId || !e.decision || !same(e.decision, e.expected) || e.decision.requestId !== e.requestId || e.decision.browserRequestId !== e.requestId || e.decision.action !== "decline" || !time(e.decision.observedAtMs) || !e.request || e.decision.observedAtMs < e.request.observedAtMs) failures.push("missing-exact-browser-denial");
|
|
if (!e.deliveredDecision || !same(e.deliveredDecision, e.expected) || e.deliveredDecision.requestId !== e.requestId || e.deliveredDecision.outcome !== "reject_once" || !time(e.deliveredDecision.observedAtMs) || !e.decision || e.deliveredDecision.observedAtMs < e.decision.observedAtMs || !e.toolResult || e.deliveredDecision.observedAtMs > e.toolResult.observedAtMs) failures.push("missing-delivered-native-rejection");
|
|
if (!e.toolResult || !same(e.toolResult, e.expected) || e.toolResult.status !== "failed" || !time(e.toolResult.observedAtMs) || !e.decision || e.toolResult.observedAtMs < e.decision.observedAtMs || !e.terminal || e.toolResult.observedAtMs > e.terminal.observedAtMs) failures.push("missing-denied-tool-result-before-terminal");
|
|
if (e.nativeAttemptsForTarget !== 1) failures.push("missing-or-retried-native-write");
|
|
const phases = ["before-request", "pending", "after-decision", "terminal", "after-cleanup"] as const;
|
|
if (phases.some(phase => !e.fileObservations.some(s => s.phase === phase)) || e.fileObservations.some(s => s.exists || !time(s.observedAtMs))) failures.push("missing-or-mutated-target-observation");
|
|
const samples = phases.map(phase => e.fileObservations.find(s => s.phase === phase));
|
|
if (samples.some((s, i) => !s || (i > 0 && s.observedAtMs < samples[i - 1]!.observedAtMs)) || !e.request || !e.decision || !e.terminal || !e.cleanup || (samples[0]?.observedAtMs ?? Infinity) > e.request.observedAtMs || (samples[1]?.observedAtMs ?? -1) < e.request.observedAtMs || (samples[1]?.observedAtMs ?? Infinity) > e.decision.observedAtMs || (samples[2]?.observedAtMs ?? -1) < e.decision.observedAtMs || (samples[3]?.observedAtMs ?? -1) < e.terminal.observedAtMs || (samples[4]?.observedAtMs ?? -1) < e.cleanup.observedAtMs) failures.push("filesystem-observation-order-invalid");
|
|
const m = e.mutationObservation;
|
|
if (!m || !m.complete || m.targetMutationCount !== 0 || !time(m.startedAtMs) || !time(m.endedAtMs) || !samples[0] || !samples[4] || m.startedAtMs > samples[0].observedAtMs || m.endedAtMs < samples[4].observedAtMs) failures.push("missing-or-mutated-filesystem-watch");
|
|
return { passed: failures.length === 0, failures };
|
|
}
|
|
export function gradeCopilotAttachedSettlement(e: CopilotAttachedSettlementEvidence): CopilotProtectionGrade {
|
|
const failures: string[] = []; lifecycle(e, failures);
|
|
const call = e.nativeCall;
|
|
if (!/^sha256:[a-f0-9]{64}$/.test(e.expectedCommandSha256) || !call || !same(call, e.expected) || call.operation !== "execute" || call.mode !== "async" || call.detach !== false || call.commandSha256 !== e.expectedCommandSha256 || !time(call.observedAtMs)) failures.push("missing-exact-attached-async-call");
|
|
const exit = e.commandExit;
|
|
if (!exit || !exit.ownedProcessIdentityVerified || exit.commandSha256 !== e.expectedCommandSha256 || exit.code !== 0 || !time(exit.observedAtMs) || !call || exit.observedAtMs < call.observedAtMs || !e.terminal || exit.observedAtMs >= e.terminal.observedAtMs) failures.push("command-not-settled-before-terminal");
|
|
const result = e.nativeShellResult;
|
|
if (!e.expectedShellId || !result || result.runId !== e.expected.runId || result.sessionId !== e.expected.sessionId || result.turnId !== e.expected.turnId || !result.toolCallId || result.commandToolCallId !== e.expected.toolCallId || result.shellId !== e.expectedShellId || result.status !== "completed" || result.exitCode !== 0 || !time(result.observedAtMs) || !exit || result.observedAtMs < exit.observedAtMs || !e.terminal || result.observedAtMs >= e.terminal.observedAtMs) failures.push("missing-correlated-native-shell-completion");
|
|
if (!e.terminalMarkerMatches || !e.afterCleanupMarkerMatches) failures.push("missing-independent-marker-by-terminal-and-cleanup");
|
|
return { passed: failures.length === 0, failures };
|
|
}
|