fix(runner): preserve Copilot evidence in the direct driver

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-29 16:05:55 -05:00
1 parent bd44f41e9e
commit 54dfc1bc8e
10 files changed
+202 -153

No files matched your search

@@ -754,7 +754,8 @@ model qualification. The profile remains pending.
### Bounded native tool observations and protection evaluations
The sidecar now projects an allowlist of active-turn ACP tool/permission fields into
The sidecar and direct TypeScript driver share one projector under `drivers/acpx`,
which projects an allowlist of active-turn ACP tool/permission fields into
existing `provider.notice.recorded` details and provenance: validated relative target,
request/tool identities, command SHA-256, explicit mode/detach, and linked shell
start/completion with provider-reported exit code. Session passthrough notifications
@@ -61,7 +61,7 @@ import {
type AcpxSidecarResponse,
} from "../drivers/acpx/sidecar-protocol.js";
import { safeAcpxLocations } from "./acpx-sidecar-locations.js";
import { createCopilotToolEvidence, type CopilotToolEvidence } from "./copilot-tool-evidence.js";
import { createCopilotToolEvidence, type CopilotToolEvidence } from "../drivers/acpx/copilot-tool-evidence.js";
import {
persistedAcpxTurnUsage,
acpxUsageEstimateNotice,
@@ -1,139 +1,6 @@
import { lstatSync, realpathSync } from "node:fs";
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
import { classifyGeneratedAcpxToolOperation } from "../drivers/acpx/generated-sidecar-contract.js";
export const ACPX_WORKSPACE_RELATIVE_DISPLAY_BOUNDARY =
"paperclip.workspace_relative_display.v2";
export const ACPX_WORKSPACE_ENTRY_ATTESTATION = "paperclip.workspace_entry.v1";
export const ACPX_WORKSPACE_CREATE_TARGET_ATTESTATION =
"paperclip.workspace_create_target.v1";
const RFC_URI_SCHEME_PREFIX = /^[A-Za-z][A-Za-z0-9+.-]*:/u;
const WINDOWS_DRIVE_PREFIX = /^[A-Za-z]:/u;
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function permitsCreateTarget(toolKind: unknown, toolTitle: unknown): boolean {
return classifyGeneratedAcpxToolOperation(toolKind, toolTitle) === "edit";
}
/**
* Converts provider paths to workspace-relative display targets using the
* sidecar host's path semantics. URI-scheme and Windows drive-shaped values
* are ambiguous on POSIX, so they require a real, in-workspace filesystem
* entry before the sidecar may attest them as filename data. Leading
* backslashes likewise require native-host attestation because they are rooted
* syntax on Windows but valid filename data on POSIX. Consumers must
* treat the result as display data, never as file-access authorization.
*/
export function safeAcpxLocations(
locations: readonly unknown[] | null | undefined,
workingDirectory: string | null | undefined,
toolKind?: unknown,
toolTitle?: unknown,
): Array<Record<string, unknown>> {
if (!workingDirectory) return [];
const cwd = resolve(workingDirectory);
let canonicalCwd: string | null | undefined;
return (locations ?? []).slice(0, 2_000).flatMap((location) => {
const candidate = record(location);
const rawPath = typeof candidate.path === "string" ? candidate.path : "";
if (!rawPath || rawPath.includes("\0")) return [];
const absolute = isAbsolute(rawPath)
? resolve(rawPath)
: resolve(cwd, rawPath);
const local = relative(cwd, absolute);
if (!local || isAbsolute(local)) return [];
const portable = sep === "\\" ? local.replaceAll("\\", "/") : local;
if (
portable.startsWith("/") ||
portable.split("/").some((segment) => segment === "..")
) {
return [];
}
// Classify the value we actually emit as well as the provider's spelling.
// Dot-relative and absolute paths can normalize to an ambiguous display
// target even though their raw forms did not begin with an ambiguous shape.
const requiresEntryAttestation =
rawPath.startsWith("\\") ||
portable.startsWith("\\") ||
WINDOWS_DRIVE_PREFIX.test(rawPath) ||
RFC_URI_SCHEME_PREFIX.test(rawPath) ||
WINDOWS_DRIVE_PREFIX.test(portable) ||
RFC_URI_SCHEME_PREFIX.test(portable);
let pathAttestation: string | undefined;
if (requiresEntryAttestation) {
if (canonicalCwd === undefined) {
try {
canonicalCwd = realpathSync(cwd);
} catch {
canonicalCwd = null;
}
}
if (canonicalCwd === null) return [];
let entryExists = true;
try {
lstatSync(absolute);
} catch (error) {
entryExists = false;
if (
!permitsCreateTarget(toolKind, toolTitle) ||
!(
error !== null &&
typeof error === "object" &&
"code" in error &&
error.code === "ENOENT"
)
) {
return [];
}
}
if (entryExists) {
try {
const canonicalEntry = realpathSync(absolute);
const entryRelative = relative(canonicalCwd, canonicalEntry);
if (
!entryRelative ||
isAbsolute(entryRelative) ||
entryRelative.split(sep).some((segment) => segment === "..")
) {
return [];
}
pathAttestation = ACPX_WORKSPACE_ENTRY_ATTESTATION;
} catch {
// An existing but unresolved entry includes dangling links. It must
// not be downgraded to a create target by an ENOENT from realpath.
return [];
}
} else {
try {
const canonicalParent = realpathSync(dirname(absolute));
const parentRelative = relative(canonicalCwd, canonicalParent);
if (
isAbsolute(parentRelative) ||
parentRelative.split(sep).some((segment) => segment === "..")
) {
return [];
}
pathAttestation = ACPX_WORKSPACE_CREATE_TARGET_ATTESTATION;
} catch {
return [];
}
}
}
return [
{
path: [...portable].slice(0, 4_000).join(""),
line: candidate.line ?? null,
pathBoundary: ACPX_WORKSPACE_RELATIVE_DISPLAY_BOUNDARY,
...(pathAttestation ? { pathAttestation } : {}),
},
];
});
}
export {
ACPX_WORKSPACE_RELATIVE_DISPLAY_BOUNDARY,
ACPX_WORKSPACE_ENTRY_ATTESTATION,
ACPX_WORKSPACE_CREATE_TARGET_ATTESTATION,
safeAcpxLocations,
} from "../drivers/acpx/safe-locations.js";
@@ -1,3 +1,4 @@
import { readFileSync } from "node:fs";
import { describe, expect, it, vi } from "vitest";
import type { AcpRuntimeEvent } from "acpx/runtime";
@@ -1543,10 +1544,11 @@ describe("Codex ACPX harness driver", () => {
const callback = fixture.host.startTurn.mock.calls[0]![0].onPermissionRequest!;
const receipt = deferred<void>();
const providerResponse = callback({ inferredKind: "execute", raw: {
sessionId: "agent-session-1", toolCall: { toolCallId: "receipt-tool", title: "Run validation" },
sessionId: "backend-1", toolCall: { toolCallId: "receipt-tool", title: "Run validation", kind: "execute", rawInput: { command: "printf safe", mode: "async", detach: false } },
options: [{ optionId: "session", kind: "allow_always", name: "Allow for session" }],
} } as Parameters<typeof callback>[0], { signal: new AbortController().signal, responseDelivery: receipt.promise });
await created;
const requested = await created;
expect(requested.filter(event => event.eventType === "provider.notice.recorded" && event.payload.category === "copilot_tool_evidence_v1").map(event => Object.fromEntries((event.payload.details as Array<{ name: string; value: string }>).map(x => [x.name, x.value])))).toEqual([expect.objectContaining({ stage: "permission_requested", toolCallId: "receipt-tool", mode: "async", detach: "false" })]);
const request = session.pendingRuntimeRequests!()[0]!;
expect(request.details).toMatchObject({ choices: [{ key: "accept_for_session" }, { key: "cancel" }] });
let acknowledged = false;
@@ -1561,6 +1563,9 @@ describe("Codex ACPX harness driver", () => {
receipt.reject(new Error("pipe failed")); await rejected;
}
const events = await emitted;
const evidence = events.filter(event => event.eventType === "provider.notice.recorded" && event.payload.category === "copilot_tool_evidence_v1" && (event.payload.details as Array<{ name: string; value: string }>).some(x => x.name === "stage" && x.value === "permission_delivered"));
expect(evidence).toHaveLength(outcome === "written" ? 1 : 0);
if (outcome === "written") expect(evidence[0]!.payload.details).toEqual(expect.arrayContaining([{ name: "stage", value: "permission_delivered" }, { name: "outcome", value: "allow_always" }]));
expect(events.filter(event => event.eventType === "runtime_request.resolved")).toHaveLength(outcome === "written" ? 1 : 0);
if (outcome === "failed") expect(events.at(-1)?.payload).toMatchObject({ replayAllowed: false, reason: "response_delivery_failed" });
expect(session.pendingRuntimeRequests!()).toHaveLength(0);
@@ -1568,6 +1573,26 @@ describe("Codex ACPX harness driver", () => {
await session.close({ reason: "receipt checked" });
});
it.each(["copilot", "codex"] as const)("preserves pinned attached-shell evidence only on the Copilot direct driver: %s", async agent => {
const wire = JSON.parse(readFileSync(new URL("./fixtures/copilot-tool-evidence.json", import.meta.url), "utf8"))["attached-shell"];
const runtimeEvents = wire.filter((frame: any) => frame.method === "session/update").map((frame: any) => ({ ...frame.params.update, type: "tool_call", tag: frame.params.update.sessionUpdate })) as AcpRuntimeEvent[];
const fixture = driverFixture({ agent, providerPolicy: { readOnly: false } }, { runtimeEvents });
const session = await fixture.driver.openSession({ runId: "run-attached", normalizedSessionId: "session-1", workingDirectory: "/fixture/workspace" });
const completed = collectUntil(session.events(), "turn.completed");
const { turnId } = await session.startTurn({ message: { text: "Start attached work." } });
fixture.finishTurn({ status: "completed", stopReason: "end_turn" });
const events = await completed;
const evidence = events.filter(event => event.eventType === "provider.notice.recorded" && event.payload.category === "copilot_tool_evidence_v1");
if (agent === "copilot") {
expect(evidence.length).toBeGreaterThan(2);
expect(evidence.every(event => event.turnId === turnId && event.runId === "run-attached")).toBe(true);
const fields = evidence.map(event => Object.fromEntries((event.payload.details as Array<{ name: string; value: string }>).map(x => [x.name, x.value])));
expect(fields).toEqual(expect.arrayContaining([expect.objectContaining({ operation: "execute", mode: "async", detach: "false", commandSha256: expect.stringMatching(/^sha256:/u) }), expect.objectContaining({ shellState: "started", shellId: "0" }), expect.objectContaining({ shellState: "completed", shellId: "0", exitCode: "0", commandToolCallId: expect.any(String) })]));
expect(JSON.stringify(evidence)).not.toContain("sleep 2");
} else expect(evidence).toEqual([]);
await session.close({ reason: "projection verified" });
});
it("round-trips a provider-neutral ACP form through the runtime request boundary", async () => {
const fixture = driverFixture();
const session = await fixture.driver.openSession({
@@ -1,3 +1,4 @@
import { createCopilotToolEvidence, type CopilotToolEvidence } from "./copilot-tool-evidence.js";
import { requireAcpxResponseDelivery } from "./response-delivery.js";
import { acpxProfileClientCapabilities, bindAcpxExtensionTurn, validateAcpxRichEvent, createAcpxProfileExtensionAdapter, type AcpxExtensionInput } from "./profile-extensions.js";
import { createHash, randomBytes } from "node:crypto";
@@ -119,6 +120,7 @@ const QUARANTINED_HOST_ADMISSION_GRACE_MS =
interface PendingAcpxRuntimeRequest {
request: HarnessRuntimeRequest;
responseDelivery: Promise<void>;
deliveredEvidence?: () => void;
prepareResolution(resolution: HarnessRuntimeRequestResolution): () => void;
cancel(): void;
cleanup(): void;
@@ -896,6 +898,14 @@ class CodexAcpxSession implements HarnessSession {
}
},
});
const toolEvidence = this.#agent === "copilot" ? createCopilotToolEvidence({
sessionId: this.#host.identity().backendSessionId, turnId, workingDirectory: this.#input.workingDirectory,
active: () => this.#activeTurnId === turnId && !this.#closingStarted,
emit: event => {
validateAcpxRichEvent(event);
if (!this.#emit(event.eventType, event.payload, { turnId, itemId: event.itemId })) throw new Error("Copilot activity could not be retained");
},
}) : undefined;
let turn: AcpxRuntimeTurn;
const usageBefore = await readUsageStatus(this.#host);
try {
@@ -907,7 +917,7 @@ class CodexAcpxSession implements HarnessSession {
onExtensionRequest: extensions.onExtensionRequest,
onExtensionNotification: extensions.onExtensionNotification,
onPermissionRequest: (request, context) =>
this.#handlePermission(turnId, request, context),
this.#handlePermission(turnId, request, context, toolEvidence),
onElicitation: (request, context) =>
this.#handleElicitation(turnId, request, context),
});
@@ -920,7 +930,7 @@ class CodexAcpxSession implements HarnessSession {
);
throw error;
}
const pump = this.#pumpTurn(turnId, turn, extensions.drain, usageBefore);
const pump = this.#pumpTurn(turnId, turn, extensions.drain, usageBefore, toolEvidence);
this.#activePump = pump;
void pump
.finally(() => {
@@ -1014,6 +1024,7 @@ class CodexAcpxSession implements HarnessSession {
dispatched = true;
deliver();
await pending.responseDelivery;
pending.deliveredEvidence?.();
this.#emit(
"runtime_request.resolved",
harnessRuntimeRequestOutcome(pending.request, {
@@ -1423,7 +1434,7 @@ class CodexAcpxSession implements HarnessSession {
.catch(() => undefined);
}
async #pumpTurn(turnId: string, turn: AcpxRuntimeTurn, drainExtensions: () => Promise<void>, usageBefore: unknown): Promise<void> {
async #pumpTurn(turnId: string, turn: AcpxRuntimeTurn, drainExtensions: () => Promise<void>, usageBefore: unknown, toolEvidence?: CopilotToolEvidence): Promise<void> {
try {
let index = 0;
const normalizeToolEvent =
@@ -1431,6 +1442,7 @@ class CodexAcpxSession implements HarnessSession {
const normalizeMessage = this.#agent === "grok"
? createGrokMessageNormalizer<AcpRuntimeEvent>() : (event: AcpRuntimeEvent) => event;
for await (const event of turn.events) {
toolEvidence?.tool(event);
this.#mapRuntimeEvent(normalizeMessage(normalizeToolEvent(event)), turnId, ++index);
}
const result = await turn.result;
@@ -1709,6 +1721,7 @@ class CodexAcpxSession implements HarnessSession {
turnId: string,
request: AcpPermissionRequest,
context: { signal: AbortSignal; responseDelivery?: Promise<void> },
toolEvidence?: CopilotToolEvidence,
): Promise<AcpPermissionDecision> {
const { signal } = context;
if (this.#closed || this.#activeTurnId !== turnId || signal.aborted
@@ -1724,6 +1737,8 @@ class CodexAcpxSession implements HarnessSession {
details: { choices: normalized.choices, toolCallId: normalized.toolCallId, kind: normalized.kind },
origin: { adapter: "acpx-runtime", provider: this.#agent, method: "session/request_permission" },
};
const deliveredEvidence = toolEvidence?.permission(request, requestId, normalized.choices.map(choice => choice.key));
let deliveredOutcome: AcpPermissionDecision["outcome"] | undefined;
return await new Promise<AcpPermissionDecision>((settle) => {
const cancel = () => {
const pending = this.#pendingRuntimeRequests.get(requestId);
@@ -1736,8 +1751,10 @@ class CodexAcpxSession implements HarnessSession {
};
this.#pendingRuntimeRequests.set(requestId, {
request: runtimeRequest, responseDelivery,
deliveredEvidence: () => { if (deliveredOutcome) deliveredEvidence?.(deliveredOutcome); },
prepareResolution: (resolution) => {
const response = normalized.resolve(resolution);
deliveredOutcome = response.outcome;
return () => settle(response);
},
cancel: () => settle({ outcome: "cancel" }),
@@ -1,8 +1,8 @@
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { describe, expect, it, vi } from "vitest";
import { validateAcpxRichEvent } from "../drivers/acpx/profile-extensions.js";
import type { CanonicalProviderEvent } from "../provider-events.js";
import { validateAcpxRichEvent } from "./profile-extensions.js";
import type { CanonicalProviderEvent } from "../../provider-events.js";
import { createCopilotToolEvidence } from "./copilot-tool-evidence.js";
const fixture = JSON.parse(readFileSync(new URL("./fixtures/copilot-tool-evidence.json", import.meta.url), "utf8"));
const details = (e: CanonicalProviderEvent) => Object.fromEntries((e.payload.details as Array<{ name: string; value: string }>).map(d => [d.name, d.value]));
@@ -1,7 +1,7 @@
import { createHash } from "node:crypto";
import { redactPaperclipSemanticValue } from "../semantic-tools/redaction.js";
import type { CanonicalProviderEvent } from "../provider-events.js";
import { safeAcpxLocations } from "./acpx-sidecar-locations.js";
import { redactPaperclipSemanticValue } from "../../semantic-tools/redaction.js";
import type { CanonicalProviderEvent } from "../../provider-events.js";
import { safeAcpxLocations } from "./safe-locations.js";
const LIMIT = 256;
const CATEGORY = "copilot_tool_evidence_v1";
@@ -0,0 +1,139 @@
import { lstatSync, realpathSync } from "node:fs";
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
import { classifyGeneratedAcpxToolOperation } from "./generated-sidecar-contract.js";
export const ACPX_WORKSPACE_RELATIVE_DISPLAY_BOUNDARY =
"paperclip.workspace_relative_display.v2";
export const ACPX_WORKSPACE_ENTRY_ATTESTATION = "paperclip.workspace_entry.v1";
export const ACPX_WORKSPACE_CREATE_TARGET_ATTESTATION =
"paperclip.workspace_create_target.v1";
const RFC_URI_SCHEME_PREFIX = /^[A-Za-z][A-Za-z0-9+.-]*:/u;
const WINDOWS_DRIVE_PREFIX = /^[A-Za-z]:/u;
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function permitsCreateTarget(toolKind: unknown, toolTitle: unknown): boolean {
return classifyGeneratedAcpxToolOperation(toolKind, toolTitle) === "edit";
}
/**
* Converts provider paths to workspace-relative display targets using the
* sidecar host's path semantics. URI-scheme and Windows drive-shaped values
* are ambiguous on POSIX, so they require a real, in-workspace filesystem
* entry before the sidecar may attest them as filename data. Leading
* backslashes likewise require native-host attestation because they are rooted
* syntax on Windows but valid filename data on POSIX. Consumers must
* treat the result as display data, never as file-access authorization.
*/
export function safeAcpxLocations(
locations: readonly unknown[] | null | undefined,
workingDirectory: string | null | undefined,
toolKind?: unknown,
toolTitle?: unknown,
): Array<Record<string, unknown>> {
if (!workingDirectory) return [];
const cwd = resolve(workingDirectory);
let canonicalCwd: string | null | undefined;
return (locations ?? []).slice(0, 2_000).flatMap((location) => {
const candidate = record(location);
const rawPath = typeof candidate.path === "string" ? candidate.path : "";
if (!rawPath || rawPath.includes("\0")) return [];
const absolute = isAbsolute(rawPath)
? resolve(rawPath)
: resolve(cwd, rawPath);
const local = relative(cwd, absolute);
if (!local || isAbsolute(local)) return [];
const portable = sep === "\\" ? local.replaceAll("\\", "/") : local;
if (
portable.startsWith("/") ||
portable.split("/").some((segment) => segment === "..")
) {
return [];
}
// Classify the value we actually emit as well as the provider's spelling.
// Dot-relative and absolute paths can normalize to an ambiguous display
// target even though their raw forms did not begin with an ambiguous shape.
const requiresEntryAttestation =
rawPath.startsWith("\\") ||
portable.startsWith("\\") ||
WINDOWS_DRIVE_PREFIX.test(rawPath) ||
RFC_URI_SCHEME_PREFIX.test(rawPath) ||
WINDOWS_DRIVE_PREFIX.test(portable) ||
RFC_URI_SCHEME_PREFIX.test(portable);
let pathAttestation: string | undefined;
if (requiresEntryAttestation) {
if (canonicalCwd === undefined) {
try {
canonicalCwd = realpathSync(cwd);
} catch {
canonicalCwd = null;
}
}
if (canonicalCwd === null) return [];
let entryExists = true;
try {
lstatSync(absolute);
} catch (error) {
entryExists = false;
if (
!permitsCreateTarget(toolKind, toolTitle) ||
!(
error !== null &&
typeof error === "object" &&
"code" in error &&
error.code === "ENOENT"
)
) {
return [];
}
}
if (entryExists) {
try {
const canonicalEntry = realpathSync(absolute);
const entryRelative = relative(canonicalCwd, canonicalEntry);
if (
!entryRelative ||
isAbsolute(entryRelative) ||
entryRelative.split(sep).some((segment) => segment === "..")
) {
return [];
}
pathAttestation = ACPX_WORKSPACE_ENTRY_ATTESTATION;
} catch {
// An existing but unresolved entry includes dangling links. It must
// not be downgraded to a create target by an ENOENT from realpath.
return [];
}
} else {
try {
const canonicalParent = realpathSync(dirname(absolute));
const parentRelative = relative(canonicalCwd, canonicalParent);
if (
isAbsolute(parentRelative) ||
parentRelative.split(sep).some((segment) => segment === "..")
) {
return [];
}
pathAttestation = ACPX_WORKSPACE_CREATE_TARGET_ATTESTATION;
} catch {
return [];
}
}
}
return [
{
path: [...portable].slice(0, 4_000).join(""),
line: candidate.line ?? null,
pathBoundary: ACPX_WORKSPACE_RELATIVE_DISPLAY_BOUNDARY,
...(pathAttestation ? { pathAttestation } : {}),
},
];
});
}
@@ -2,7 +2,7 @@ import { spawn } from "node:child_process";
import { readFile, mkdtemp, rm, writeFile, unlink, rename, mkdir } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { createCopilotToolEvidence } from "../../packages/paperclip-runner/src/cli/copilot-tool-evidence.js";
import { createCopilotToolEvidence } from "../../packages/paperclip-runner/src/drivers/acpx/copilot-tool-evidence.js";
import { validateAcpxRichEvent } from "../../packages/paperclip-runner/src/drivers/acpx/profile-extensions.js";
import { copilotOrigin, readCopilotToolEvidence } from "./copilot-evidence.js";
import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite } from "./copilot-protection-cases.js";
@@ -10,7 +10,7 @@ import { createAttachedCommandFixture, watchDeniedTarget, exists, isPerTurnRunPr
import { runnerMatrix } from "./catalog.js";
import { selectRunnerExecutions, parseRunnerSelectors } from "./selectors.js";
const fixture = JSON.parse(await readFile(new URL("../../packages/paperclip-runner/src/cli/fixtures/copilot-tool-evidence.json", import.meta.url), "utf8"));
const fixture = JSON.parse(await readFile(new URL("../../packages/paperclip-runner/src/drivers/acpx/fixtures/copilot-tool-evidence.json", import.meta.url), "utf8"));
function projected(name: string) {
const frames = fixture[name], rows: any[] = []; let clock = 10;
const projector = createCopilotToolEvidence({ sessionId: frames[0].params.sessionId, turnId: "turn", workingDirectory: "/fixture/workspace", active: () => true,