mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-22 00:30:19 +02:00
test(runner-e2e): require correlated accepted Copilot completion
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
35fbc123a7
commit
4a876efef5
8 files changed
+226
-6
No files matched your search
@@ -455,3 +455,21 @@ The existing `copilot-protection` denial remains distinct: rejecting a permissio
|
||||
before Stop does not exercise this pending-callback boundary. This new suite has
|
||||
pure calibration and wiring tests, not a paid qualification result. Provider
|
||||
process death is not simulated by substituting the chat runner-worker crash hook.
|
||||
|
||||
### Copilot attached semantic completion evidence (suite 7)
|
||||
|
||||
Attached settlement now requires one native `paperclip_finish` lifecycle joined
|
||||
to an invocation-captured, bounded Paperclip bridge receipt by exact call, input
|
||||
and result digests in the same run/turn/native session and durable source stream.
|
||||
A matching display title is not authority. The exact proposed completion input
|
||||
must match a control-plane `run.result.accepted` body and the requested summary.
|
||||
Transport `returned` alone, including a returned rejection, cannot pass. The
|
||||
visible exact terminal comment remains an independent assertion. Both local and
|
||||
Daytona cases reject extra native operations; existing attested bootstrap reads,
|
||||
command settlement, process retirement and marker checks remain required.
|
||||
|
||||
The receipt reader fails closed when the production native receipt projection is
|
||||
missing, partial, duplicated or outside its bounds (including 256 KiB for the
|
||||
whole native rawOutput, which may repeat text). This oracle depends on the new
|
||||
production semantic-receipt contract and does not regrade earlier failed runs.
|
||||
Denial case version 5 and denial settlement schema v3 are unchanged.
|
||||
@@ -1503,3 +1503,21 @@ older denial or cancellation results. Current live qualification is pending.
|
||||
Provider death during a pending callback remains a separate uncovered case:
|
||||
the existing chat worker-crash hook targets a runner worker and cannot establish
|
||||
safe ownership of the native provider process in both environments.
|
||||
|
||||
### Copilot attached semantic completion evidence (suite 7)
|
||||
|
||||
Attached settlement now requires one native `paperclip_finish` lifecycle joined
|
||||
to an invocation-captured, bounded Paperclip bridge receipt by exact call, input
|
||||
and result digests in the same run/turn/native session and durable source stream.
|
||||
A matching display title is not authority. The exact proposed completion input
|
||||
must match a control-plane `run.result.accepted` body and the requested summary.
|
||||
Transport `returned` alone, including a returned rejection, cannot pass. The
|
||||
visible exact terminal comment remains an independent assertion. Both local and
|
||||
Daytona cases reject extra native operations; existing attested bootstrap reads,
|
||||
command settlement, process retirement and marker checks remain required.
|
||||
|
||||
The receipt reader fails closed when the production native receipt projection is
|
||||
missing, partial, duplicated or outside its bounds (including 256 KiB for the
|
||||
whole native rawOutput, which may repeat text). This oracle depends on the new
|
||||
production semantic-receipt contract and does not regrade earlier failed runs.
|
||||
Denial case version 5 and denial settlement schema v3 are unchanged.
|
||||
@@ -1059,7 +1059,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.",
|
||||
groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"),
|
||||
environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4,
|
||||
definitionMetadata: { version: 6, qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" },
|
||||
definitionMetadata: { version: 7, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v1", qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" },
|
||||
},
|
||||
{
|
||||
id: "rich-acp-warm-continuity", label: "Rich ACP warm continuity", manualOnly: true,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
export interface CopilotToolNotice {
|
||||
runId: string; sessionId: string; turnId: string; toolCallId: string;
|
||||
observedAtMs: number; seq: number;
|
||||
semanticOperationId?: string; semanticCallIdentitySha256?: string; semanticInputSha256?: string; semanticResultSha256?: string; semanticOutcome?: "returned" | "error";
|
||||
stage: "tool" | "permission_requested" | "permission_delivered";
|
||||
status?: "pending" | "in_progress" | "completed" | "failed";
|
||||
operation?: "edit" | "execute" | "read"; target?: string; requestId?: string;
|
||||
@@ -12,7 +13,7 @@ export interface CopilotToolNotice {
|
||||
const rec = (v: unknown): Record<string, any> => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Record<string, any> : {};
|
||||
const id = (v: unknown): v is string => typeof v === "string" && v.length > 0 && v.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(v) && !v.includes("[REDACTED]");
|
||||
const enums = { stage: ["tool", "permission_requested", "permission_delivered"], status: ["pending", "in_progress", "completed", "failed"], operation: ["edit", "execute", "read"], outcome: ["allow_once", "allow_always", "reject_once", "cancel"], mode: ["sync", "async"], shellState: ["started", "completed"] };
|
||||
const names = new Set(["stage", "toolCallId", "status", "operation", "target", "requestId", "declineOffered", "outcome", "commandSha256", "readTargetSha256", "mode", "detach", "shellId", "commandToolCallId", "shellState", "exitCode"]);
|
||||
const names = new Set(["stage", "toolCallId", "status", "operation", "target", "requestId", "declineOffered", "outcome", "commandSha256", "readTargetSha256", "mode", "detach", "shellId", "commandToolCallId", "shellState", "exitCode", "semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256", "semanticOutcome"]);
|
||||
export function readCopilotToolEvidence(rows: readonly unknown[], expectedRunId: string): CopilotToolNotice[] {
|
||||
const result: CopilotToolNotice[] = [];
|
||||
for (const row of rows) {
|
||||
@@ -42,6 +43,13 @@ export function readCopilotToolEvidence(rows: readonly unknown[], expectedRunId:
|
||||
if (fields.commandSha256 !== undefined) { if (!/^sha256:[a-f0-9]{64}$/u.test(fields.commandSha256)) throw new Error("Invalid command digest"); notice.commandSha256 = fields.commandSha256; }
|
||||
if (fields.shellId !== undefined) { if (!/^[A-Za-z0-9_.-]{1,80}$/u.test(fields.shellId)) throw new Error("Invalid shell identity"); notice.shellId = fields.shellId; }
|
||||
if (fields.exitCode !== undefined) { if (!/^-?\d{1,10}$/u.test(fields.exitCode) || !Number.isSafeInteger(Number(fields.exitCode))) throw new Error("Invalid exit code"); notice.exitCode = Number(fields.exitCode); }
|
||||
const semanticKeys = ["semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256", "semanticOutcome"];
|
||||
if (semanticKeys.some(k => fields[k] !== undefined)) {
|
||||
if (!semanticKeys.every(k => fields[k] !== undefined) || fields.stage !== "tool" || !["completed", "failed"].includes(fields.status!)
|
||||
|| !/^[A-Za-z0-9_.:-]{1,256}$/.test(fields.semanticOperationId!) || !["returned", "error"].includes(fields.semanticOutcome!)
|
||||
|| ["semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256"].some(k => !/^[a-f0-9]{64}$/.test(fields[k]!))) throw new Error("Invalid semantic receipt fields");
|
||||
for (const key of semanticKeys) notice[key] = fields[key];
|
||||
}
|
||||
result.push(notice as unknown as CopilotToolNotice);
|
||||
}
|
||||
return result;
|
||||
|
||||
@@ -26,7 +26,7 @@ function projected(name: string, target = "copilot-denied-nonce.txt") {
|
||||
describe("Copilot Product protection integration", () => {
|
||||
it("registers two explicit cases on both environments with honest terminal expectations", () => {
|
||||
const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection");
|
||||
expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 6, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop",
|
||||
expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 7, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop",
|
||||
denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn" });
|
||||
expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } }));
|
||||
expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"]));
|
||||
|
||||
@@ -4,6 +4,7 @@ import { expect, type Page } from "@playwright/test";
|
||||
import { pollUntil, type RunnerApi } from "./api.js";
|
||||
import { collectRunEvents } from "./run-observations.js";
|
||||
import { createTaskThroughUi } from "./user-actions.js";
|
||||
import { onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js";
|
||||
import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js";
|
||||
import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses } from "./copilot-local-fixtures.js";
|
||||
import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js";
|
||||
@@ -239,9 +240,11 @@ export async function runCopilotProtectionFlow(input: {
|
||||
markerMatches: localMarkerMatches, afterCleanupMarkerMatches: localAfterCleanupMarkerMatches });
|
||||
const call = matched?.call;
|
||||
check("single-exact-command", Boolean(matched), "Exactly one native execution matches the fixture command with at most eight leading ASCII SPACE/TAB bytes");
|
||||
if (remote) check("no-extra-native-operation", copilotActionNotices(notices, call!, { actionFile: remoteFixture!.actionFile, events: runEvents }).every(n =>
|
||||
n.runId === call!.runId && n.sessionId === call!.sessionId && n.turnId === call!.turnId
|
||||
&& (n.toolCallId === call!.toolCallId || n.commandToolCallId === call!.toolCallId)), "Only setup reads and the exact native attached command/result are allowed");
|
||||
const semantic = readCopilotSemanticCompletion(runEvents, { companyId: fixtures.company.id, runId: call!.runId, turnId: call!.turnId,
|
||||
nativeSessionId: call!.sessionId, command: call!, summary: execution.task.buildVisibleMarker(nonce) });
|
||||
await input.evidence("copilot-semantic-completion.json", semantic);
|
||||
check("accepted-canonical-completion", true, "One exact native finish receipt matches the proposed and control-plane accepted result; transport success alone is insufficient");
|
||||
check("no-extra-native-operation", onlyCopilotAttachedOperations(copilotActionNotices(notices, call!, remoteFixture ? { actionFile: remoteFixture.actionFile, events: runEvents } : undefined), call!, semantic), "Only attested setup reads, the exact attached command/result, and one authoritatively correlated accepted finish are allowed");
|
||||
const started = notices.find(n => n.toolCallId === call!.toolCallId && n.shellState === "started");
|
||||
const result = notices.find(n => n.commandToolCallId === call!.toolCallId && n.shellState === "completed");
|
||||
const terminal = runEvents.find(r => r.eventType === "turn.completed" && r.payload?.prpEvent?.turnId === call!.turnId)?.payload.prpEvent;
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js";
|
||||
import { readCopilotToolEvidence } from "./copilot-evidence.js";
|
||||
import { runnerSuites, suiteDefinitionHash } from "./catalog.js";
|
||||
const hash = (v: string) => createHash("sha256").update(v).digest("hex");
|
||||
const canonical = (v: any): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v !== null && typeof v === "object" ? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical(v[k])}`).join(",")}}` : JSON.stringify(v);
|
||||
const result = { schema: "paperclip.run_result.v1", summary: "EXACT-MARKER", reportedWorkDisposition: "done", completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, evidence: [], artifacts: [], verification: [], attentionRequests: [] };
|
||||
const receipt = { schema: "paperclip.semantic_tool_receipt.v1", operationId: "paperclip_finish", callIdentitySha256: hash("3"), inputSha256: hash(canonical(result)), resultSha256: hash("opaque original returned encoding"), outcome: "returned" };
|
||||
function row(seq: number, eventType: string, payload: any, overrides: any = {}) {
|
||||
const e = { schema: "paperclip.prp.event.v1", eventType, runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "source", sourceKind: "runner", sourceSeq: seq, emittedAt: "2026-09-30T12:00:00Z", payload, ...overrides };
|
||||
return { companyId: "company", runId: "run", seq, eventType, sourceInstanceId: e.sourceInstanceId, sourceSeq: e.sourceSeq, payload: { prpEvent: e } };
|
||||
}
|
||||
function notice(seq: number, toolCallId: string, status: string, fields: any = {}) {
|
||||
return row(seq, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "copilot_tool_evidence_v1", scope: "turn", provenance: { method: "session/update", eventType: "tool", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "tool", toolCallId, status, ...fields }).map(([name, value]) => ({ name, value: String(value) })) });
|
||||
}
|
||||
function fixture() {
|
||||
const rows = [notice(1, "command", "pending", { operation: "execute" }), notice(2, "finish-native", "pending"), row(3, "run.result.proposed", structuredClone(result), { itemId: "3" }),
|
||||
row(4, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "paperclip_semantic_tool_receipt_v1", scope: "turn", provenance: { method: "paperclip/semantic_tool_result", eventType: "semantic_result", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "semantic_result", ...receipt }).map(([name, value]) => ({ name, value })) }),
|
||||
notice(5, "finish-native", "completed", { semanticOperationId: receipt.operationId, semanticCallIdentitySha256: receipt.callIdentitySha256, semanticInputSha256: receipt.inputSha256, semanticResultSha256: receipt.resultSha256, semanticOutcome: receipt.outcome }),
|
||||
row(6, "turn.completed", {}), row(7, "run.result.accepted", { result: structuredClone(result) }, { sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 1 })];
|
||||
const expected = { companyId: "company", runId: "run", turnId: "turn", nativeSessionId: "native", command: readCopilotToolEvidence(rows, "run")[0]!, summary: "EXACT-MARKER" };
|
||||
return { rows, expected };
|
||||
}
|
||||
const frame = (r: any) => r.payload.prpEvent;
|
||||
function setField(r: any, key: string, value: string) { frame(r).payload.details.find((d: any) => d.name === key).value = value; }
|
||||
describe("Copilot semantic completion public-event oracle", () => {
|
||||
it("versions the actual Copilot suite oracle without changing denial settlement", () => {
|
||||
const suite = runnerSuites.find(s => s.id === "copilot-protection")!;
|
||||
expect(suite.definitionMetadata).toMatchObject({ version: 7, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v1", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3" });
|
||||
expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 6 } }));
|
||||
});
|
||||
it("joins exact native lifecycle, authoritative callback, proposed content and accepted control-plane result", () => {
|
||||
const { rows, expected } = fixture(); const proof = readCopilotSemanticCompletion(rows, expected);
|
||||
expect(proof.nativeToolCallId).toBe("finish-native"); expect(proof.acceptedSeq).toBe(7); expect(proof.summarySha256).toBe(hash("EXACT-MARKER"));
|
||||
expect(JSON.stringify(proof)).not.toContain("EXACT-MARKER");
|
||||
});
|
||||
it.each([0, 1, 2, 3, 4, 5, 6])("rejects missing required row %s", index => { const f = fixture(); f.rows.splice(index, 1); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each([1, 2, 3, 4, 5, 6])("rejects duplicate required row %s", index => { const f = fixture(); f.rows.push(structuredClone(f.rows[index]!)); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each(["companyId", "runId"])("rejects foreign durable %s", key => { const f = fixture(); (f.rows[3] as any)[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each(["turnId", "normalizedSessionId", "sourceInstanceId", "sourceKind", "eventType", "schema"])("rejects foreign or malformed envelope %s", key => {
|
||||
for (const index of [1, 2, 3, 4, 5, 6]) { const f = fixture(); (frame(f.rows[index]!))[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }
|
||||
});
|
||||
it.each([null, undefined])("rejects missing envelope turn %s even with native provenance intact", turn => { const f = fixture(); frame(f.rows[4]!).turnId = turn; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each(["sessionId", "turnId", "method", "eventType"])("rejects authoritative provenance %s mismatch", key => { const f = fixture(); frame(f.rows[3]!).payload.provenance[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each(["callIdentitySha256", "inputSha256", "resultSha256", "operationId", "outcome"])("rejects native %s disagreement", key => { const f = fixture(); setField(f.rows[4], `semantic${key[0]!.toUpperCase()}${key.slice(1)}`, key.endsWith("Sha256") ? "a".repeat(64) : "wrong"); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it("does not infer acceptance from returned receipt or a tool named paperclip_finish", () => {
|
||||
const f = fixture(); frame(f.rows[6]!).payload.result = { accepted: false, summary: "EXACT-MARKER" }; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow();
|
||||
const g = fixture(); frame(g.rows[4]!).payload.title = "paperclip_finish"; frame(g.rows[4]!).payload.details = frame(g.rows[4]!).payload.details.filter((d: any) => !d.name.startsWith("semantic")); expect(() => readCopilotSemanticCompletion(g.rows, g.expected)).toThrow();
|
||||
});
|
||||
it.each(["failed", "cancelled", "interrupted"])("rejects %s terminal", status => { const f = fixture(); f.rows[5]!.eventType = `turn.${status}`; frame(f.rows[5]!).eventType = `turn.${status}`; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it("rejects changed accepted body, forged call ID, wrong summary and rejected result", () => {
|
||||
for (const mutate of [(f: ReturnType<typeof fixture>) => { frame(f.rows[6]!).payload.result.completionClaim.remainingWork.push("unfinished"); }, (f: ReturnType<typeof fixture>) => { frame(f.rows[2]!).itemId = "4"; }, (f: ReturnType<typeof fixture>) => { f.expected.summary = "other"; }, (f: ReturnType<typeof fixture>) => { f.rows.push(row(8, "run.result.rejected", {})); }]) { const f = fixture(); mutate(f); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }
|
||||
});
|
||||
it("rejects duplicate/unknown/oversized receipt details and partial native fields", () => {
|
||||
for (const mutate of [(r: any) => frame(r).payload.details.push({ name: "secret", value: "untrusted" }), (r: any) => frame(r).payload.details.push(frame(r).payload.details[0]), (r: any) => setField(r, "operationId", "x".repeat(300))]) { const f = fixture(); mutate(f.rows[3]); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }
|
||||
const f = fixture(); frame(f.rows[4]!).payload.details.pop(); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow();
|
||||
});
|
||||
it("rejects mutation metadata, extra lifecycle, reordered receipt and source sequence drift", () => {
|
||||
for (const mutate of [(f: ReturnType<typeof fixture>) => frame(f.rows[1]!).payload.details.push({ name: "operation", value: "edit" }), (f: ReturnType<typeof fixture>) => f.rows.push(notice(8, "finish-native", "pending")), (f: ReturnType<typeof fixture>) => { f.rows[3]!.seq = 8; }, (f: ReturnType<typeof fixture>) => { frame(f.rows[3]!).sourceSeq = 20; f.rows[3]!.sourceSeq = 20; }, (f: ReturnType<typeof fixture>) => { frame(f.rows[1]!).sourceSeq = 20; f.rows[1]!.sourceSeq = 20; }]) { const f = fixture(); mutate(f); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); }
|
||||
});
|
||||
it("allows only exact command and correlated finish; unknown reads/edits/delegation remain failures", () => {
|
||||
const f = fixture(), proof = readCopilotSemanticCompletion(f.rows, f.expected), notices = readCopilotToolEvidence(f.rows, "run");
|
||||
expect(onlyCopilotAttachedOperations(notices, f.expected.command, proof)).toBe(true);
|
||||
for (const operation of [undefined, "read", "edit", "execute"] as const) expect(onlyCopilotAttachedOperations([...notices, { ...f.expected.command, toolCallId: "extra", operation }], f.expected.command, proof)).toBe(false);
|
||||
expect(onlyCopilotAttachedOperations(notices, f.expected.command, { ...proof, nativeToolCallId: "command" })).toBe(false);
|
||||
expect(onlyCopilotAttachedOperations([{ ...notices[1]!, sessionId: "foreign" }], f.expected.command, proof)).toBe(false);
|
||||
});
|
||||
it("keeps actual flow correlation before no-extra-operation gate and keeps visible marker independent", async () => {
|
||||
const source = await readFile(new URL("./copilot-protection-flow.ts", import.meta.url), "utf8");
|
||||
expect(source.indexOf("readCopilotSemanticCompletion(runEvents")).toBeLessThan(source.indexOf('check("no-extra-native-operation", onlyCopilotAttachedOperations('));
|
||||
expect(source).toContain("undefined), call!, semantic)");
|
||||
expect(source).toContain('check("exact-completion-marker", comments.filter');
|
||||
expect(source).not.toContain('if (remote) check("no-extra-native-operation"');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js";
|
||||
|
||||
type Row = Record<string, any>;
|
||||
const rec = (v: unknown): Row => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Row : {};
|
||||
const id = (v: unknown): v is string => typeof v === "string" && v.length > 0 && v.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(v) && !v.includes("[REDACTED]");
|
||||
const hash = (v: string) => createHash("sha256").update(v).digest("hex");
|
||||
const canonical = (v: unknown): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v !== null && typeof v === "object"
|
||||
? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical((v as Row)[k])}`).join(",")}}` : JSON.stringify(v) ?? "undefined";
|
||||
const digest = (v: unknown) => hash(canonical(v));
|
||||
const receiptKeys = ["callIdentitySha256", "inputSha256", "operationId", "outcome", "resultSha256", "schema", "stage"].sort().join(",");
|
||||
export interface CopilotSemanticCompletionProof {
|
||||
schema: "paperclip.e2e.copilot-semantic-completion.v1";
|
||||
runId: string; turnId: string; nativeSessionId: string; normalizedSessionId: string; sourceInstanceId: string;
|
||||
nativeToolCallId: string; callIdentitySha256: string; inputSha256: string; resultSha256: string;
|
||||
proposedSeq: number; receiptSeq: number; nativeCompletedSeq: number; turnCompletedSeq: number; acceptedSeq: number;
|
||||
summarySha256: string;
|
||||
}
|
||||
/** Independent public-event oracle. A transport receipt alone never proves completion acceptance. */
|
||||
export function readCopilotSemanticCompletion(rows: readonly unknown[], expected: {
|
||||
companyId: string; runId: string; turnId: string; nativeSessionId: string; command: CopilotToolNotice; summary: string;
|
||||
}): CopilotSemanticCompletionProof {
|
||||
const fail = (message: string): never => { throw new Error(`Copilot semantic completion: ${message}`); };
|
||||
const all = rows.map(rec), notices = readCopilotToolEvidence(rows, expected.runId);
|
||||
const commandRows = all.filter(r => r.seq === expected.command.seq);
|
||||
if (commandRows.length !== 1) fail("missing exact command origin");
|
||||
const base = rec(rec(commandRows[0]!.payload).prpEvent);
|
||||
if (!id(base.normalizedSessionId) || !id(base.sourceInstanceId)) fail("missing command stream");
|
||||
function frame(r: Row, sourceKind: "runner" | "control_plane") {
|
||||
const e = rec(rec(r.payload).prpEvent);
|
||||
if (r.companyId !== expected.companyId || r.runId !== expected.runId || !Number.isSafeInteger(r.seq) || r.seq < 1
|
||||
|| e.schema !== "paperclip.prp.event.v1" || e.eventType !== r.eventType || e.sourceKind !== sourceKind
|
||||
|| e.runId !== expected.runId || e.turnId !== expected.turnId || e.normalizedSessionId !== base.normalizedSessionId
|
||||
|| !id(e.sourceInstanceId) || r.sourceInstanceId !== e.sourceInstanceId || r.sourceSeq !== e.sourceSeq
|
||||
|| !Number.isSafeInteger(e.sourceSeq) || e.sourceSeq < 1
|
||||
|| (sourceKind === "runner" && e.sourceInstanceId !== base.sourceInstanceId)) fail("foreign or malformed durable frame");
|
||||
return e;
|
||||
}
|
||||
frame(commandRows[0]!, "runner");
|
||||
if (expected.command.runId !== expected.runId || expected.command.turnId !== expected.turnId || expected.command.sessionId !== expected.nativeSessionId) fail("foreign command");
|
||||
const authorityRows = all.filter(r => rec(rec(rec(r.payload).prpEvent).payload).category === "paperclip_semantic_tool_receipt_v1");
|
||||
// This authored case asks for one finish, not arbitrary semantic actions.
|
||||
if (authorityRows.length !== 1) fail("expected one authoritative finish receipt");
|
||||
const authorityRow = authorityRows[0]!, authority = frame(authorityRow, "runner"), payload = rec(authority.payload), origin = rec(payload.provenance);
|
||||
if (payload.schema !== "paperclip.provider.notice.v1" || payload.scope !== "turn" || authorityRow.eventType !== "provider.notice.recorded"
|
||||
|| origin.method !== "paperclip/semantic_tool_result" || origin.eventType !== "semantic_result"
|
||||
|| origin.sessionId !== expected.nativeSessionId || origin.turnId !== expected.turnId || !Array.isArray(payload.details) || payload.details.length !== 7) fail("invalid receipt provenance");
|
||||
const fields: Record<string, string> = {};
|
||||
for (const d of payload.details) {
|
||||
if (Object.keys(rec(d)).sort().join(",") !== "name,value" || typeof d.name !== "string" || typeof d.value !== "string" || d.value.length > 256 || Object.hasOwn(fields, d.name)) fail("malformed receipt detail");
|
||||
fields[d.name] = d.value;
|
||||
}
|
||||
if (Object.keys(fields).sort().join(",") !== receiptKeys || fields.stage !== "semantic_result" || fields.schema !== "paperclip.semantic_tool_receipt.v1"
|
||||
|| fields.operationId !== "paperclip_finish" || fields.outcome !== "returned"
|
||||
|| ![fields.callIdentitySha256, fields.inputSha256, fields.resultSha256].every(v => /^[a-f0-9]{64}$/.test(v!))) fail("invalid finish receipt");
|
||||
const matches = notices.filter(n => n.semanticCallIdentitySha256 === fields.callIdentitySha256);
|
||||
if (matches.length !== 1) fail("missing or duplicate native match");
|
||||
const native = matches[0]!;
|
||||
if (native.stage !== "tool" || native.status !== "completed" || native.sessionId !== expected.nativeSessionId || native.turnId !== expected.turnId
|
||||
|| native.semanticOperationId !== fields.operationId || native.semanticInputSha256 !== fields.inputSha256 || native.semanticResultSha256 !== fields.resultSha256 || native.semanticOutcome !== "returned") fail("native receipt mismatch");
|
||||
const group = notices.filter(n => n.toolCallId === native.toolCallId);
|
||||
const pending = group.filter(n => n.stage === "tool" && n.status === "pending");
|
||||
if (pending.length !== 1 || group.filter(n => n.status === "completed" || n.status === "failed").length !== 1
|
||||
|| group.some(n => n.stage !== "tool" || n.sessionId !== expected.nativeSessionId || n.turnId !== expected.turnId
|
||||
|| n.operation !== undefined || n.target !== undefined || n.commandSha256 !== undefined || n.shellId !== undefined || n.commandToolCallId !== undefined
|
||||
|| !["pending", "in_progress", "completed"].includes(n.status ?? "") || n.seq < pending[0]!.seq || n.seq > native.seq)) fail("not one semantic-only native lifecycle");
|
||||
for (const n of group) {
|
||||
const rs = all.filter(r => r.seq === n.seq); if (rs.length !== 1) fail("duplicate native row"); frame(rs[0]!, "runner");
|
||||
}
|
||||
const proposedRows = all.filter(r => r.eventType === "run.result.proposed"), acceptedRows = all.filter(r => r.eventType === "run.result.accepted"), terminals = all.filter(r => ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(r.eventType) && rec(rec(r.payload).prpEvent).turnId === expected.turnId);
|
||||
if (proposedRows.length !== 1 || acceptedRows.length !== 1 || terminals.length !== 1 || terminals[0]!.eventType !== "turn.completed"
|
||||
|| all.some(r => r.eventType === "run.result.rejected")) fail("missing, rejected or ambiguous completion");
|
||||
const proposed = frame(proposedRows[0]!, "runner"), accepted = frame(acceptedRows[0]!, "control_plane"), terminal = frame(terminals[0]!, "runner");
|
||||
const result = rec(proposed.payload), acceptedResult = rec(rec(accepted.payload).result);
|
||||
if (!id(proposed.itemId) || hash(proposed.itemId) !== fields.callIdentitySha256 || digest(result) !== fields.inputSha256
|
||||
|| result.schema !== "paperclip.run_result.v1" || result.reportedWorkDisposition !== "done" || result.summary !== expected.summary
|
||||
|| canonical(result) !== canonical(acceptedResult)) fail("canonical accepted result does not match exact finish");
|
||||
const pendingFrame = frame(all.find(r => r.seq === pending[0]!.seq)!, "runner");
|
||||
const completedFrame = frame(all.find(r => r.seq === native.seq)!, "runner");
|
||||
if (!(expected.command.seq < pending[0]!.seq && pending[0]!.seq < proposedRows[0]!.seq && proposedRows[0]!.seq < authorityRow.seq
|
||||
&& authorityRow.seq < native.seq && native.seq < terminals[0]!.seq && terminals[0]!.seq < acceptedRows[0]!.seq)
|
||||
|| !(base.sourceSeq < pendingFrame.sourceSeq && pendingFrame.sourceSeq < proposed.sourceSeq && proposed.sourceSeq < authority.sourceSeq
|
||||
&& authority.sourceSeq < completedFrame.sourceSeq && completedFrame.sourceSeq < terminal.sourceSeq)) fail("invalid causal order");
|
||||
return { schema: "paperclip.e2e.copilot-semantic-completion.v1", runId: expected.runId, turnId: expected.turnId, nativeSessionId: expected.nativeSessionId,
|
||||
normalizedSessionId: base.normalizedSessionId, sourceInstanceId: base.sourceInstanceId, nativeToolCallId: native.toolCallId,
|
||||
callIdentitySha256: fields.callIdentitySha256!, inputSha256: fields.inputSha256!, resultSha256: fields.resultSha256!,
|
||||
proposedSeq: proposedRows[0]!.seq, receiptSeq: authorityRow.seq, nativeCompletedSeq: native.seq, turnCompletedSeq: terminals[0]!.seq, acceptedSeq: acceptedRows[0]!.seq, summarySha256: hash(expected.summary) };
|
||||
}
|
||||
|
||||
/** Called only after the completion proof and bootstrap-read oracle succeed. */
|
||||
export function onlyCopilotAttachedOperations(notices: readonly CopilotToolNotice[], command: CopilotToolNotice, proof: CopilotSemanticCompletionProof): boolean {
|
||||
return proof.runId === command.runId && proof.turnId === command.turnId && proof.nativeSessionId === command.sessionId
|
||||
&& proof.nativeToolCallId !== command.toolCallId && notices.every(n =>
|
||||
n.runId === command.runId && n.sessionId === command.sessionId && n.turnId === command.turnId
|
||||
&& (n.toolCallId === command.toolCallId || n.commandToolCallId === command.toolCallId || n.toolCallId === proof.nativeToolCallId));
|
||||
}
|
||||
Reference in new issue
Block a user