mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 08:22:20 +02:00
test(runner-e2e): join normalized Copilot completion receipts
Keep raw invocation provenance separate from the validated proposal digest. Advance the semantic evidence contract to v2 and Copilot suite to 8; preserve denial and attached-operation invariants. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
a1fa21c7b3
commit
f2da0ed5fc
7 files changed
+114
-33
No files matched your search
@@ -442,7 +442,7 @@ before Stop does not exercise this pending-callback boundary. This new suite has
|
||||
pure calibration and wiring tests, not a paid qualification result. Provider
|
||||
process death is not simulated by substituting the chat runner-worker crash hook.
|
||||
|
||||
### Copilot attached semantic completion evidence (suite 7)
|
||||
### Copilot attached semantic completion evidence (suite 8)
|
||||
|
||||
Attached settlement now requires one native `paperclip_finish` lifecycle joined
|
||||
to an invocation-captured, bounded Paperclip bridge receipt by exact call, input
|
||||
@@ -460,8 +460,21 @@ whole native rawOutput, which may repeat text). This oracle depends on the new
|
||||
production semantic-receipt contract and does not regrade earlier failed runs.
|
||||
The bridge call hash identifies the invocation and its native receipt. It is not
|
||||
inferred from `run.result.proposed.itemId`, which can identify the run instead.
|
||||
Exactly one proposed body must hash to the receipt input and equal exactly one
|
||||
accepted body; identical duplicate proposals or receipts still fail. The shell
|
||||
The v2 receipt keeps the raw invocation input digest separate from the normalized
|
||||
input digest captured by that same authenticated invocation after production
|
||||
validation. Raw arguments may omit schema, artifacts and attention requests;
|
||||
the production validator supplies those defaults. The oracle never reconstructs
|
||||
or normalizes a body to make it match. The raw call/input/result hashes still join
|
||||
the native lifecycle; its normalized input hash must also match the authority.
|
||||
Exactly one proposed body must hash to that normalized digest and equal exactly
|
||||
one accepted body; identical duplicate proposals or receipts still fail.
|
||||
|
||||
Suite 8 writes `paperclip.e2e.copilot-semantic-completion.v2` and requires
|
||||
`paperclip.semantic_tool_receipt.v2` under `paperclip_semantic_tool_receipt_v2`.
|
||||
The authority has exactly eight bounded details. An explicit null normalized
|
||||
digest is diagnostic only and cannot qualify a successful finish. Legacy v1
|
||||
receipts do not qualify fresh runs. This fixture depends on the corresponding
|
||||
production receipt and durable-redaction fixes; it does not supply them. The shell
|
||||
result is one separate complete read lifecycle: pending and any progress name the
|
||||
started shell, and its successful terminal also names the original command. A
|
||||
completed-only read or a second shell read cannot inherit that exemption.
|
||||
|
||||
@@ -1473,7 +1473,7 @@ Provider death during a pending callback remains a separate uncovered case:
|
||||
the existing chat worker-crash hook targets a runner worker and cannot establish
|
||||
safe ownership of the native provider process in both environments.
|
||||
|
||||
### Copilot attached semantic completion evidence (suite 7)
|
||||
### Copilot attached semantic completion evidence (suite 8)
|
||||
|
||||
Attached settlement now requires one native `paperclip_finish` lifecycle joined
|
||||
to an invocation-captured, bounded Paperclip bridge receipt by exact call, input
|
||||
@@ -1491,8 +1491,21 @@ whole native rawOutput, which may repeat text). This oracle depends on the new
|
||||
production semantic-receipt contract and does not regrade earlier failed runs.
|
||||
The bridge call hash identifies the invocation and its native receipt. It is not
|
||||
inferred from `run.result.proposed.itemId`, which can identify the run instead.
|
||||
Exactly one proposed body must hash to the receipt input and equal exactly one
|
||||
accepted body; identical duplicate proposals or receipts still fail. The shell
|
||||
The v2 receipt keeps the raw invocation input digest separate from the normalized
|
||||
input digest captured by that same authenticated invocation after production
|
||||
validation. Raw arguments may omit schema, artifacts and attention requests;
|
||||
the production validator supplies those defaults. The oracle never reconstructs
|
||||
or normalizes a body to make it match. The raw call/input/result hashes still join
|
||||
the native lifecycle; its normalized input hash must also match the authority.
|
||||
Exactly one proposed body must hash to that normalized digest and equal exactly
|
||||
one accepted body; identical duplicate proposals or receipts still fail.
|
||||
|
||||
Suite 8 writes `paperclip.e2e.copilot-semantic-completion.v2` and requires
|
||||
`paperclip.semantic_tool_receipt.v2` under `paperclip_semantic_tool_receipt_v2`.
|
||||
The authority has exactly eight bounded details. An explicit null normalized
|
||||
digest is diagnostic only and cannot qualify a successful finish. Legacy v1
|
||||
receipts do not qualify fresh runs. This fixture depends on the corresponding
|
||||
production receipt and durable-redaction fixes; it does not supply them. The shell
|
||||
result is one separate complete read lifecycle: pending and any progress name the
|
||||
started shell, and its successful terminal also names the original command. A
|
||||
completed-only read or a second shell read cannot inherit that exemption.
|
||||
|
||||
@@ -1059,7 +1059,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.",
|
||||
groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"),
|
||||
environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4,
|
||||
definitionMetadata: { version: 7, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v1", qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" },
|
||||
definitionMetadata: { version: 8, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v2", qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" },
|
||||
},
|
||||
{
|
||||
id: "rich-acp-warm-continuity", label: "Rich ACP warm continuity", manualOnly: true,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
export interface CopilotToolNotice {
|
||||
runId: string; sessionId: string; turnId: string; toolCallId: string;
|
||||
observedAtMs: number; seq: number;
|
||||
semanticOperationId?: string; semanticCallIdentitySha256?: string; semanticInputSha256?: string; semanticResultSha256?: string; semanticOutcome?: "returned" | "error";
|
||||
semanticOperationId?: string; semanticCallIdentitySha256?: string; semanticInputSha256?: string; semanticNormalizedInputSha256?: string | null; semanticResultSha256?: string; semanticOutcome?: "returned" | "error";
|
||||
stage: "tool" | "permission_requested" | "permission_delivered";
|
||||
status?: "pending" | "in_progress" | "completed" | "failed";
|
||||
operation?: "edit" | "execute" | "read"; target?: string; requestId?: string;
|
||||
@@ -13,7 +13,7 @@ export interface CopilotToolNotice {
|
||||
const rec = (v: unknown): Record<string, any> => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Record<string, any> : {};
|
||||
const id = (v: unknown): v is string => typeof v === "string" && v.length > 0 && v.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(v) && !v.includes("[REDACTED]");
|
||||
const enums = { stage: ["tool", "permission_requested", "permission_delivered"], status: ["pending", "in_progress", "completed", "failed"], operation: ["edit", "execute", "read"], outcome: ["allow_once", "allow_always", "reject_once", "cancel"], mode: ["sync", "async"], shellState: ["started", "completed"] };
|
||||
const names = new Set(["stage", "toolCallId", "status", "operation", "target", "requestId", "declineOffered", "outcome", "commandSha256", "readTargetSha256", "mode", "detach", "shellId", "commandToolCallId", "shellState", "exitCode", "semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256", "semanticOutcome"]);
|
||||
const names = new Set(["stage", "toolCallId", "status", "operation", "target", "requestId", "declineOffered", "outcome", "commandSha256", "readTargetSha256", "mode", "detach", "shellId", "commandToolCallId", "shellState", "exitCode", "semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticNormalizedInputSha256", "semanticResultSha256", "semanticOutcome"]);
|
||||
export function readCopilotToolEvidence(rows: readonly unknown[], expectedRunId: string): CopilotToolNotice[] {
|
||||
const result: CopilotToolNotice[] = [];
|
||||
for (const row of rows) {
|
||||
@@ -43,12 +43,14 @@ export function readCopilotToolEvidence(rows: readonly unknown[], expectedRunId:
|
||||
if (fields.commandSha256 !== undefined) { if (!/^sha256:[a-f0-9]{64}$/u.test(fields.commandSha256)) throw new Error("Invalid command digest"); notice.commandSha256 = fields.commandSha256; }
|
||||
if (fields.shellId !== undefined) { if (!/^[A-Za-z0-9_.-]{1,80}$/u.test(fields.shellId)) throw new Error("Invalid shell identity"); notice.shellId = fields.shellId; }
|
||||
if (fields.exitCode !== undefined) { if (!/^-?\d{1,10}$/u.test(fields.exitCode) || !Number.isSafeInteger(Number(fields.exitCode))) throw new Error("Invalid exit code"); notice.exitCode = Number(fields.exitCode); }
|
||||
const semanticKeys = ["semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256", "semanticOutcome"];
|
||||
const semanticKeys = ["semanticOperationId", "semanticCallIdentitySha256", "semanticInputSha256", "semanticNormalizedInputSha256", "semanticResultSha256", "semanticOutcome"];
|
||||
if (semanticKeys.some(k => fields[k] !== undefined)) {
|
||||
if (!semanticKeys.every(k => fields[k] !== undefined) || fields.stage !== "tool" || !["completed", "failed"].includes(fields.status!)
|
||||
|| !/^[A-Za-z0-9_.:-]{1,256}$/.test(fields.semanticOperationId!) || !["returned", "error"].includes(fields.semanticOutcome!)
|
||||
|| ["semanticCallIdentitySha256", "semanticInputSha256", "semanticResultSha256"].some(k => !/^[a-f0-9]{64}$/.test(fields[k]!))) throw new Error("Invalid semantic receipt fields");
|
||||
if (fields.semanticNormalizedInputSha256 !== "null" && !/^[a-f0-9]{64}$/.test(fields.semanticNormalizedInputSha256!)) throw new Error("Invalid normalized semantic digest");
|
||||
for (const key of semanticKeys) notice[key] = fields[key];
|
||||
notice.semanticNormalizedInputSha256 = fields.semanticNormalizedInputSha256 === "null" ? null : fields.semanticNormalizedInputSha256;
|
||||
}
|
||||
result.push(notice as unknown as CopilotToolNotice);
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ function projected(name: string, target = "copilot-denied-nonce.txt") {
|
||||
describe("Copilot Product protection integration", () => {
|
||||
it("registers two explicit cases on both environments with honest terminal expectations", () => {
|
||||
const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection");
|
||||
expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 7, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop",
|
||||
expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 8, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop",
|
||||
denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn" });
|
||||
expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } }));
|
||||
expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"]));
|
||||
|
||||
@@ -3,11 +3,15 @@ import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js";
|
||||
import { readCopilotToolEvidence } from "./copilot-evidence.js";
|
||||
import { validatePrpStructuredRunResult } from "../../packages/paperclip-runner/src/protocol/replay-contract.js";
|
||||
import { runnerSuites, suiteDefinitionHash } from "./catalog.js";
|
||||
const hash = (v: string) => createHash("sha256").update(v).digest("hex");
|
||||
const canonical = (v: any): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v !== null && typeof v === "object" ? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical(v[k])}`).join(",")}}` : JSON.stringify(v);
|
||||
const result = { schema: "paperclip.run_result.v1", summary: "EXACT-MARKER", reportedWorkDisposition: "done", completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, evidence: [], artifacts: [], verification: [], attentionRequests: [] };
|
||||
const receipt = { schema: "paperclip.semantic_tool_receipt.v1", operationId: "paperclip_finish", callIdentitySha256: hash("3"), inputSha256: hash(canonical(result)), resultSha256: hash("opaque original returned encoding"), outcome: "returned" };
|
||||
const rawInput = { summary: "EXACT-MARKER", reportedWorkDisposition: "done", completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, evidence: [], verification: [] };
|
||||
const validation = validatePrpStructuredRunResult(rawInput);
|
||||
if (!validation.ok) throw new Error("Semantic fixture input must pass the production validator");
|
||||
const result = validation.result;
|
||||
const receipt = { schema: "paperclip.semantic_tool_receipt.v2", operationId: "paperclip_finish", callIdentitySha256: hash("3"), inputSha256: hash(canonical(rawInput)), normalizedInputSha256: hash(canonical(result)), resultSha256: hash("opaque original returned encoding"), outcome: "returned" };
|
||||
function row(seq: number, eventType: string, payload: any, overrides: any = {}) {
|
||||
const e = { schema: "paperclip.prp.event.v1", eventType, runId: "run", turnId: "turn", normalizedSessionId: "normalized", sourceInstanceId: "source", sourceKind: "runner", sourceSeq: seq, emittedAt: "2026-09-30T12:00:00Z", payload, ...overrides };
|
||||
return { companyId: "company", runId: "run", seq, eventType, sourceInstanceId: e.sourceInstanceId, sourceSeq: e.sourceSeq, payload: { prpEvent: e } };
|
||||
@@ -17,8 +21,8 @@ function notice(seq: number, toolCallId: string, status: string, fields: any = {
|
||||
}
|
||||
function fixture() {
|
||||
const rows = [notice(1, "command", "pending", { operation: "execute" }), notice(2, "finish-native", "pending"), row(3, "run.result.proposed", structuredClone(result), { itemId: "item-run" }),
|
||||
row(4, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "paperclip_semantic_tool_receipt_v1", scope: "turn", provenance: { method: "paperclip/semantic_tool_result", eventType: "semantic_result", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "semantic_result", ...receipt }).map(([name, value]) => ({ name, value })) }),
|
||||
notice(5, "finish-native", "completed", { semanticOperationId: receipt.operationId, semanticCallIdentitySha256: receipt.callIdentitySha256, semanticInputSha256: receipt.inputSha256, semanticResultSha256: receipt.resultSha256, semanticOutcome: receipt.outcome }),
|
||||
row(4, "provider.notice.recorded", { schema: "paperclip.provider.notice.v1", category: "paperclip_semantic_tool_receipt_v2", scope: "turn", provenance: { method: "paperclip/semantic_tool_result", eventType: "semantic_result", sessionId: "native", turnId: "turn" }, details: Object.entries({ stage: "semantic_result", ...receipt }).map(([name, value]) => ({ name, value })) }),
|
||||
notice(5, "finish-native", "completed", { semanticOperationId: receipt.operationId, semanticCallIdentitySha256: receipt.callIdentitySha256, semanticInputSha256: receipt.inputSha256, semanticNormalizedInputSha256: receipt.normalizedInputSha256, semanticResultSha256: receipt.resultSha256, semanticOutcome: receipt.outcome }),
|
||||
row(6, "turn.completed", {}), row(7, "run.result.accepted", { result: structuredClone(result) }, { sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 1 })];
|
||||
const expected = { companyId: "company", runId: "run", turnId: "turn", nativeSessionId: "native", command: readCopilotToolEvidence(rows, "run")[0]!, summary: "EXACT-MARKER" };
|
||||
return { rows, expected };
|
||||
@@ -28,14 +32,59 @@ function setField(r: any, key: string, value: string) { frame(r).payload.details
|
||||
describe("Copilot semantic completion public-event oracle", () => {
|
||||
it("versions the actual Copilot suite oracle without changing denial settlement", () => {
|
||||
const suite = runnerSuites.find(s => s.id === "copilot-protection")!;
|
||||
expect(suite.definitionMetadata).toMatchObject({ version: 7, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v1", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3" });
|
||||
expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 6 } }));
|
||||
expect(suite.definitionMetadata).toMatchObject({ version: 8, semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v2", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3" });
|
||||
expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 7 } }));
|
||||
});
|
||||
it("joins exact native lifecycle, authoritative callback, proposed content and accepted control-plane result", () => {
|
||||
const { rows, expected } = fixture(); const proof = readCopilotSemanticCompletion(rows, expected);
|
||||
expect(proof.nativeToolCallId).toBe("finish-native"); expect(proof.acceptedSeq).toBe(7); expect(proof.summarySha256).toBe(hash("EXACT-MARKER"));
|
||||
expect(JSON.stringify(proof)).not.toContain("EXACT-MARKER");
|
||||
});
|
||||
it("binds distinct raw and production-normalized input digests without filling defaults in the oracle", () => {
|
||||
const { rows, expected } = fixture();
|
||||
expect(rawInput).not.toHaveProperty("schema");
|
||||
expect(rawInput).not.toHaveProperty("artifacts");
|
||||
expect(rawInput).not.toHaveProperty("attentionRequests");
|
||||
expect(result).toMatchObject({ schema: "paperclip.run_result.v1", artifacts: [], attentionRequests: [] });
|
||||
expect(receipt.inputSha256).not.toBe(receipt.normalizedInputSha256);
|
||||
expect(readCopilotSemanticCompletion(rows, expected)).toMatchObject({
|
||||
schema: "paperclip.e2e.copilot-semantic-completion.v2",
|
||||
inputSha256: receipt.inputSha256, normalizedInputSha256: receipt.normalizedInputSha256,
|
||||
});
|
||||
// Same authenticated raw/native join is necessary but cannot substitute for
|
||||
// the invocation-captured normalized body digest.
|
||||
setField(rows[3], "normalizedInputSha256", receipt.inputSha256);
|
||||
setField(rows[4], "semanticNormalizedInputSha256", receipt.inputSha256);
|
||||
expect(() => readCopilotSemanticCompletion(rows, expected)).toThrow("canonical accepted result");
|
||||
});
|
||||
it.each(["missing", "null", "wrong", "uppercase", "malformed"])("rejects %s normalized finish digest", value => {
|
||||
const f = fixture();
|
||||
if (value === "missing") {
|
||||
frame(f.rows[3]!).payload.details = frame(f.rows[3]!).payload.details.filter((d: any) => d.name !== "normalizedInputSha256");
|
||||
frame(f.rows[4]!).payload.details = frame(f.rows[4]!).payload.details.filter((d: any) => d.name !== "semanticNormalizedInputSha256");
|
||||
} else {
|
||||
const digest = value === "null" ? "null" : value === "wrong" ? hash("foreign-normalized-input") : value === "uppercase" ? "A".repeat(64) : "not-a-digest";
|
||||
setField(f.rows[3], "normalizedInputSha256", digest); setField(f.rows[4], "semanticNormalizedInputSha256", digest);
|
||||
}
|
||||
expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow();
|
||||
});
|
||||
it("allows an explicit null native digest as diagnostic data, never as successful finish authority", () => {
|
||||
const f = fixture(); setField(f.rows[4], "semanticNormalizedInputSha256", "null");
|
||||
expect(readCopilotToolEvidence(f.rows, "run").at(-1)?.semanticNormalizedInputSha256).toBeNull();
|
||||
expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow("native receipt mismatch");
|
||||
});
|
||||
it.each(["only", "alongside-v2"])("rejects legacy v1 authority %s as fresh qualification", mode => {
|
||||
const f = fixture(), old = structuredClone(f.rows[3]!);
|
||||
frame(old).payload.category = "paperclip_semantic_tool_receipt_v1";
|
||||
setField(old, "schema", "paperclip.semantic_tool_receipt.v1");
|
||||
frame(old).payload.details = frame(old).payload.details.filter((d: any) => d.name !== "normalizedInputSha256");
|
||||
if (mode === "only") f.rows[3] = old; else f.rows.push(old);
|
||||
expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow();
|
||||
});
|
||||
it("does not repair an unnormalized proposed and accepted body by adding defaults", () => {
|
||||
const f = fixture(); frame(f.rows[2]!).payload = structuredClone(rawInput); frame(f.rows[6]!).payload.result = structuredClone(rawInput);
|
||||
expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow("canonical accepted result");
|
||||
});
|
||||
it.each([0, 1, 2, 3, 4, 5, 6])("rejects missing required row %s", index => { const f = fixture(); f.rows.splice(index, 1); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each([1, 2, 3, 4, 5, 6])("rejects duplicate required row %s", index => { const f = fixture(); f.rows.push(structuredClone(f.rows[index]!)); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each(["companyId", "runId"])("rejects foreign durable %s", key => { const f = fixture(); (f.rows[3] as any)[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
@@ -44,7 +93,7 @@ describe("Copilot semantic completion public-event oracle", () => {
|
||||
});
|
||||
it.each([null, undefined])("rejects missing envelope turn %s even with native provenance intact", turn => { const f = fixture(); frame(f.rows[4]!).turnId = turn; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each(["sessionId", "turnId", "method", "eventType"])("rejects authoritative provenance %s mismatch", key => { const f = fixture(); frame(f.rows[3]!).payload.provenance[key] = "foreign"; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each(["callIdentitySha256", "inputSha256", "resultSha256", "operationId", "outcome"])("rejects native %s disagreement", key => { const f = fixture(); setField(f.rows[4], `semantic${key[0]!.toUpperCase()}${key.slice(1)}`, key.endsWith("Sha256") ? "a".repeat(64) : "wrong"); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it.each(["callIdentitySha256", "inputSha256", "normalizedInputSha256", "resultSha256", "operationId", "outcome"])("rejects native %s disagreement", key => { const f = fixture(); setField(f.rows[4], `semantic${key[0]!.toUpperCase()}${key.slice(1)}`, key.endsWith("Sha256") ? "a".repeat(64) : "wrong"); expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow(); });
|
||||
it("does not infer acceptance from returned receipt or a tool named paperclip_finish", () => {
|
||||
const f = fixture(); frame(f.rows[6]!).payload.result = { accepted: false, summary: "EXACT-MARKER" }; expect(() => readCopilotSemanticCompletion(f.rows, f.expected)).toThrow();
|
||||
const g = fixture(); frame(g.rows[4]!).payload.title = "paperclip_finish"; frame(g.rows[4]!).payload.details = frame(g.rows[4]!).payload.details.filter((d: any) => !d.name.startsWith("semantic")); expect(() => readCopilotSemanticCompletion(g.rows, g.expected)).toThrow();
|
||||
@@ -86,7 +135,7 @@ describe("Copilot semantic completion public-event oracle", () => {
|
||||
frame(r).sourceInstanceId = "foreign"; r.sourceInstanceId = "foreign";
|
||||
expect(() => readCopilotSemanticCompletion(f.rows, { companyId: "company", runId: "run", turnId: "turn", nativeSessionId: "native", command: f.command, summary: "EXACT-MARKER" })).toThrow();
|
||||
});
|
||||
it.each(["missing-pending", "missing-terminal", "duplicate-pending", "duplicate-terminal", "extra-completed-only", "extra-complete-read", "wrong-shell", "wrong-command", "failed", "nonzero", "reordered", "late", "wrong-operation", "mutation", "read-path", "foreign-session", "foreign-turn", "foreign-run", "permission", "semantic-fields"])("rejects invalid shell-read lifecycle: %s", failure => {
|
||||
it.each(["missing-pending", "missing-terminal", "duplicate-pending", "duplicate-terminal", "extra-completed-only", "extra-complete-read", "wrong-shell", "wrong-command", "failed", "nonzero", "reordered", "late", "wrong-operation", "mutation", "read-path", "foreign-session", "foreign-turn", "foreign-run", "permission", "semantic-fields", "normalized-semantic-field"])("rejects invalid shell-read lifecycle: %s", failure => {
|
||||
const f = attachedFixture(), start = f.notices.find(n => n.toolCallId === "shell-read" && n.status === "pending")!, end = f.notices.find(n => n.toolCallId === "shell-read" && n.status === "completed")!;
|
||||
if (failure === "missing-pending") f.notices.splice(f.notices.indexOf(start), 1);
|
||||
if (failure === "missing-terminal") f.notices.splice(f.notices.indexOf(end), 1);
|
||||
@@ -108,6 +157,7 @@ describe("Copilot semantic completion public-event oracle", () => {
|
||||
if (failure === "foreign-run") start.runId = "foreign";
|
||||
if (failure === "permission") start.stage = "permission_requested";
|
||||
if (failure === "semantic-fields") end.semanticCallIdentitySha256 = receipt.callIdentitySha256;
|
||||
if (failure === "normalized-semantic-field") end.semanticNormalizedInputSha256 = receipt.normalizedInputSha256;
|
||||
expect(onlyCopilotAttachedOperations(f.notices, f.command, f.proof)).toBe(false);
|
||||
});
|
||||
it("rejects duplicate shell origins and arbitrary extra operations", () => {
|
||||
|
||||
@@ -8,11 +8,11 @@ const hash = (v: string) => createHash("sha256").update(v).digest("hex");
|
||||
const canonical = (v: unknown): string => Array.isArray(v) ? `[${v.map(canonical).join(",")}]` : v !== null && typeof v === "object"
|
||||
? `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical((v as Row)[k])}`).join(",")}}` : JSON.stringify(v) ?? "undefined";
|
||||
const digest = (v: unknown) => hash(canonical(v));
|
||||
const receiptKeys = ["callIdentitySha256", "inputSha256", "operationId", "outcome", "resultSha256", "schema", "stage"].sort().join(",");
|
||||
const receiptKeys = ["callIdentitySha256", "inputSha256", "normalizedInputSha256", "operationId", "outcome", "resultSha256", "schema", "stage"].sort().join(",");
|
||||
export interface CopilotSemanticCompletionProof {
|
||||
schema: "paperclip.e2e.copilot-semantic-completion.v1";
|
||||
schema: "paperclip.e2e.copilot-semantic-completion.v2";
|
||||
runId: string; turnId: string; nativeSessionId: string; normalizedSessionId: string; sourceInstanceId: string;
|
||||
nativeToolCallId: string; callIdentitySha256: string; inputSha256: string; resultSha256: string;
|
||||
nativeToolCallId: string; callIdentitySha256: string; inputSha256: string; normalizedInputSha256: string; resultSha256: string;
|
||||
proposedSeq: number; receiptSeq: number; nativeCompletedSeq: number; turnCompletedSeq: number; acceptedSeq: number;
|
||||
summarySha256: string;
|
||||
}
|
||||
@@ -45,26 +45,26 @@ export function readCopilotSemanticCompletion(rows: readonly unknown[], expected
|
||||
frame(rs[0]!, "runner");
|
||||
}
|
||||
if (expected.command.runId !== expected.runId || expected.command.turnId !== expected.turnId || expected.command.sessionId !== expected.nativeSessionId) fail("foreign command");
|
||||
const authorityRows = all.filter(r => rec(rec(rec(r.payload).prpEvent).payload).category === "paperclip_semantic_tool_receipt_v1");
|
||||
const authorityRows = all.filter(r => rec(rec(rec(r.payload).prpEvent).payload).category === "paperclip_semantic_tool_receipt_v2" || rec(rec(rec(r.payload).prpEvent).payload).category === "paperclip_semantic_tool_receipt_v1");
|
||||
// This authored case asks for one finish, not arbitrary semantic actions.
|
||||
if (authorityRows.length !== 1) fail("expected one authoritative finish receipt");
|
||||
const authorityRow = authorityRows[0]!, authority = frame(authorityRow, "runner"), payload = rec(authority.payload), origin = rec(payload.provenance);
|
||||
if (payload.schema !== "paperclip.provider.notice.v1" || payload.scope !== "turn" || authorityRow.eventType !== "provider.notice.recorded"
|
||||
if (payload.category !== "paperclip_semantic_tool_receipt_v2" || payload.schema !== "paperclip.provider.notice.v1" || payload.scope !== "turn" || authorityRow.eventType !== "provider.notice.recorded"
|
||||
|| origin.method !== "paperclip/semantic_tool_result" || origin.eventType !== "semantic_result"
|
||||
|| origin.sessionId !== expected.nativeSessionId || origin.turnId !== expected.turnId || !Array.isArray(payload.details) || payload.details.length !== 7) fail("invalid receipt provenance");
|
||||
|| origin.sessionId !== expected.nativeSessionId || origin.turnId !== expected.turnId || !Array.isArray(payload.details) || payload.details.length !== 8) fail("invalid receipt provenance");
|
||||
const fields: Record<string, string> = {};
|
||||
for (const d of payload.details) {
|
||||
if (Object.keys(rec(d)).sort().join(",") !== "name,value" || typeof d.name !== "string" || typeof d.value !== "string" || d.value.length > 256 || Object.hasOwn(fields, d.name)) fail("malformed receipt detail");
|
||||
fields[d.name] = d.value;
|
||||
}
|
||||
if (Object.keys(fields).sort().join(",") !== receiptKeys || fields.stage !== "semantic_result" || fields.schema !== "paperclip.semantic_tool_receipt.v1"
|
||||
if (Object.keys(fields).sort().join(",") !== receiptKeys || fields.stage !== "semantic_result" || fields.schema !== "paperclip.semantic_tool_receipt.v2"
|
||||
|| fields.operationId !== "paperclip_finish" || fields.outcome !== "returned"
|
||||
|| ![fields.callIdentitySha256, fields.inputSha256, fields.resultSha256].every(v => /^[a-f0-9]{64}$/.test(v!))) fail("invalid finish receipt");
|
||||
|| ![fields.callIdentitySha256, fields.inputSha256, fields.normalizedInputSha256, fields.resultSha256].every(v => /^[a-f0-9]{64}$/.test(v!))) fail("invalid finish receipt");
|
||||
const matches = notices.filter(n => n.semanticCallIdentitySha256 === fields.callIdentitySha256);
|
||||
if (matches.length !== 1) fail("missing or duplicate native match");
|
||||
const native = matches[0]!;
|
||||
if (native.stage !== "tool" || native.status !== "completed" || native.sessionId !== expected.nativeSessionId || native.turnId !== expected.turnId
|
||||
|| native.semanticOperationId !== fields.operationId || native.semanticInputSha256 !== fields.inputSha256 || native.semanticResultSha256 !== fields.resultSha256 || native.semanticOutcome !== "returned") fail("native receipt mismatch");
|
||||
|| native.semanticOperationId !== fields.operationId || native.semanticInputSha256 !== fields.inputSha256 || native.semanticNormalizedInputSha256 !== fields.normalizedInputSha256 || native.semanticResultSha256 !== fields.resultSha256 || native.semanticOutcome !== "returned") fail("native receipt mismatch");
|
||||
const group = notices.filter(n => n.toolCallId === native.toolCallId);
|
||||
const pending = group.filter(n => n.stage === "tool" && n.status === "pending");
|
||||
if (pending.length !== 1 || group.filter(n => n.status === "completed" || n.status === "failed").length !== 1
|
||||
@@ -82,8 +82,11 @@ export function readCopilotSemanticCompletion(rows: readonly unknown[], expected
|
||||
// Native correlation uses the invocation-captured callback call hash. The
|
||||
// canonical proposal itemId may instead identify the run: never interpret it
|
||||
// as a bridge call ID. Unique same-turn result bodies supply the independent
|
||||
// acceptance proof, joined to that exact invocation's input digest.
|
||||
if (digest(result) !== fields.inputSha256
|
||||
// acceptance proof, joined to the normalized digest captured by the same
|
||||
// invocation after validation. Raw provider arguments may omit defaults;
|
||||
// their separate digest remains joined to the native receipt above. Never
|
||||
// reconstruct or normalize a proposal here to make its digest match.
|
||||
if (digest(result) !== fields.normalizedInputSha256
|
||||
|| result.schema !== "paperclip.run_result.v1" || result.reportedWorkDisposition !== "done" || result.summary !== expected.summary
|
||||
|| canonical(result) !== canonical(acceptedResult)) fail("canonical accepted result does not match exact finish");
|
||||
const pendingFrame = frame(all.find(r => r.seq === pending[0]!.seq)!, "runner");
|
||||
@@ -92,9 +95,9 @@ export function readCopilotSemanticCompletion(rows: readonly unknown[], expected
|
||||
&& authorityRow.seq < native.seq && native.seq < terminals[0]!.seq && terminals[0]!.seq < acceptedRows[0]!.seq)
|
||||
|| !(base.sourceSeq < pendingFrame.sourceSeq && pendingFrame.sourceSeq < proposed.sourceSeq && proposed.sourceSeq < authority.sourceSeq
|
||||
&& authority.sourceSeq < completedFrame.sourceSeq && completedFrame.sourceSeq < terminal.sourceSeq)) fail("invalid causal order");
|
||||
return { schema: "paperclip.e2e.copilot-semantic-completion.v1", runId: expected.runId, turnId: expected.turnId, nativeSessionId: expected.nativeSessionId,
|
||||
return { schema: "paperclip.e2e.copilot-semantic-completion.v2", runId: expected.runId, turnId: expected.turnId, nativeSessionId: expected.nativeSessionId,
|
||||
normalizedSessionId: base.normalizedSessionId, sourceInstanceId: base.sourceInstanceId, nativeToolCallId: native.toolCallId,
|
||||
callIdentitySha256: fields.callIdentitySha256!, inputSha256: fields.inputSha256!, resultSha256: fields.resultSha256!,
|
||||
callIdentitySha256: fields.callIdentitySha256!, inputSha256: fields.inputSha256!, normalizedInputSha256: fields.normalizedInputSha256!, resultSha256: fields.resultSha256!,
|
||||
proposedSeq: proposedRows[0]!.seq, receiptSeq: authorityRow.seq, nativeCompletedSeq: native.seq, turnCompletedSeq: terminals[0]!.seq, acceptedSeq: acceptedRows[0]!.seq, summarySha256: hash(expected.summary) };
|
||||
}
|
||||
|
||||
@@ -117,7 +120,7 @@ export function onlyCopilotAttachedOperations(notices: readonly CopilotToolNotic
|
||||
: i === group.length - 1 ? n.status === "completed" : n.status === "in_progress"));
|
||||
};
|
||||
const semanticFree = (n: CopilotToolNotice) => n.semanticOperationId === undefined && n.semanticCallIdentitySha256 === undefined
|
||||
&& n.semanticInputSha256 === undefined && n.semanticResultSha256 === undefined && n.semanticOutcome === undefined;
|
||||
&& n.semanticInputSha256 === undefined && n.semanticNormalizedInputSha256 === undefined && n.semanticResultSha256 === undefined && n.semanticOutcome === undefined;
|
||||
const commandGroup = groups.get(command.toolCallId) ?? [];
|
||||
if (!complete(commandGroup) || commandGroup[0]!.seq !== command.seq) return false;
|
||||
const started = commandGroup.at(-1)!;
|
||||
|
||||
Reference in new issue
Block a user