diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 2014a717cf9..752b86b6afc 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1161,7 +1161,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"), environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 21, titleCreation: "explicit-search-composer", taskBinding: "browser-creation-response-id", remoteTerminalDiagnostics: "bounded-incomplete-reasons-with-strict-failure", remoteSetupOrder: "observed-action-published-before-context-approval", remoteBootstrapContext: "already-exposed-dedicated-tool-no-http-substitution", denialContext: "already-exposed-dedicated-tool-before-write", stopAcknowledgement: "retained-current-mainline-api-response", contextPermission: "context-and-bounded-attested-discovery-sequence-await-origin-rows", permissionCard: "current-operation-and-decision-control-excludes-history", semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v4", qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v4", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" }, + definitionMetadata: { version: 22, titleCreation: "explicit-search-composer", taskBinding: "browser-creation-response-id", remoteTerminalDiagnostics: "bounded-incomplete-reasons-with-strict-failure", remoteSetupOrder: "observed-action-published-before-context-approval", remoteBootstrapContext: "already-exposed-dedicated-tool-no-http-substitution", denialContext: "already-exposed-dedicated-tool-before-write", stopAcknowledgement: "retained-current-mainline-api-response", contextPermission: "context-and-bounded-attested-discovery-sequence-await-origin-rows", permissionCard: "current-operation-and-decision-control-excludes-history", semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v4", qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v4", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-released-after-native-finish-receipt", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" }, }, { id: "native-provider-loss", label: "Lose a runtime with an unanswered native permission", manualOnly: true, diff --git a/tests/runner-e2e/copilot-active-stop.test.ts b/tests/runner-e2e/copilot-active-stop.test.ts index 6e540eded7f..843c73396bb 100644 --- a/tests/runner-e2e/copilot-active-stop.test.ts +++ b/tests/runner-e2e/copilot-active-stop.test.ts @@ -377,7 +377,7 @@ describe("explicit active Stop discovery", () => { expect(cells).toHaveLength(2); expect(suite.manualOnly).toBe(true); expect(cells.map(e => `${e.profile.qualificationCandidate}/${e.environment.id}`).sort()).toEqual(["copilot/daytona", "copilot/local"]); expect(cells.every(e => e.task.expectedRunCount === 1 && e.task.flow === "native_active_stop" && e.task.expectedTerminalState?.run === "cancelled")).toBe(true); - expect(suite.definitionMetadata?.copilotDefinition).toMatchObject({ version: 18, titleCreation: "explicit-search-composer", taskBinding: "browser-creation-response-id", bootstrapPrompt: "already-exposed-dedicated-context-before-native-edit", evidence: "paperclip.e2e.native-active-stop-settlement.v3", normalCompletionAccepted: false, providerDeath: "not-covered" }); + expect(suite.definitionMetadata?.copilotDefinition).toMatchObject({ version: 19, titleCreation: "explicit-search-composer", taskBinding: "browser-creation-response-id", bootstrapPrompt: "already-exposed-dedicated-context-before-native-edit", evidence: "paperclip.e2e.native-active-stop-settlement.v3", normalCompletionAccepted: false, providerDeath: "not-covered" }); expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 3 } })); expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, normalCompletionAccepted: true } })); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(e => e.suite.id === suite.id)).toBe(false); diff --git a/tests/runner-e2e/copilot-local-fixtures.ts b/tests/runner-e2e/copilot-local-fixtures.ts index 9fa10877c69..41f4473e56c 100644 --- a/tests/runner-e2e/copilot-local-fixtures.ts +++ b/tests/runner-e2e/copilot-local-fixtures.ts @@ -183,12 +183,12 @@ export function observeRunProcesses() { * arguments or file path. The test owns/reaps the fixed finite child, records its * OS exit status, and holds the exact provider-launched client until that exit. */ -export async function createAttachedCommandFixture(markerPath: string, delayMs = 4000) { +export async function createAttachedCommandFixture(markerPath: string, delayMs = 4000, waitForFinishAttempt = false) { if (!Number.isInteger(delayMs) || delayMs < 100 || delayMs > 8000) throw new Error("Invalid bounded fixture delay"); const root = await mkdtemp("/tmp/pc-copilot-"); await mkdir(join(root, "private"), { mode: 0o700 }); const socketPath = join(root, "private", "socket"), scriptPath = join(root, "client.cjs"); const nonce = randomBytes(16).toString("hex"), marker = `${randomBytes(24).toString("hex")}\n`; - const script = `const net=require('node:net');const s=net.connect(process.argv[2]);let b='';s.setTimeout(15000,()=>process.exit(3));s.on('error',()=>process.exit(4));s.on('connect',()=>s.write(JSON.stringify({nonce:process.argv[3],pid:process.pid})+'\\n'));s.on('data',x=>{b+=x;if(b.includes('\\n')){const r=JSON.parse(b);s.end();process.exit(r.code===0?0:5);}});`; + const script = `const net=require('node:net');const s=net.connect(process.argv[2]);let b='';s.setTimeout(${waitForFinishAttempt ? 45000 : 15000},()=>process.exit(3));s.on('error',()=>process.exit(4));s.on('connect',()=>s.write(JSON.stringify({nonce:process.argv[3],pid:process.pid})+'\\n'));s.on('data',x=>{b+=x;if(b.includes('\\n')){const r=JSON.parse(b);s.end();process.exit(r.code===0?0:5);}});`; await writeFile(scriptPath, script, { mode: 0o400 }); const command = `${quote(process.execPath)} ${quote(scriptPath)} ${quote(socketPath)} ${quote(nonce)}`; const commandSha256 = sha256(command); @@ -199,6 +199,7 @@ export async function createAttachedCommandFixture(markerPath: string, delayMs = let exit: { observedAtMs: number; code: number; ownedProcessIdentityVerified: boolean; commandSha256: string } | null = null; const sockets = new Set(); let closed = false; + let releasedAtMs: number | null = null; const server = createServer(socket => { sockets.add(socket); socket.on("close", () => sockets.delete(socket)); socket.on("error", () => { failure = "fixture_socket_error"; }); let buffer = ""; @@ -220,7 +221,10 @@ export async function createAttachedCommandFixture(markerPath: string, delayMs = } } catch { failure = "fixture_client_observation_failed"; clearInterval(clientObservation); } }, 25); - child = spawn(process.execPath, ["-e", `setTimeout(()=>process.exit(0),${delayMs})`], { env: { PATH: "/usr/bin:/bin" }, stdio: "ignore" }); + const childScript = waitForFinishAttempt + ? `const deadline=setTimeout(()=>process.exit(3),30000);process.stdin.once('data',b=>{if(b.toString()!=='release\\n')process.exit(2);clearTimeout(deadline);setTimeout(()=>process.exit(0),${delayMs})});` + : `setTimeout(()=>process.exit(0),${delayMs})`; + child = spawn(process.execPath, ["-e", childScript], { env: { PATH: "/usr/bin:/bin" }, stdio: waitForFinishAttempt ? ["pipe", "ignore", "ignore"] : "ignore" }); child.once("error", () => { failure = "fixture_child_start_failed"; socket.destroy(); }); child.once("exit", (code, signal) => { exit = { observedAtMs: Date.now(), code: code ?? -1, ownedProcessIdentityVerified: Number.isInteger(child?.pid), commandSha256 }; @@ -237,6 +241,10 @@ export async function createAttachedCommandFixture(markerPath: string, delayMs = catch (error) { server.close(); await rm(root, { recursive: true, force: true }); throw error; } return { command, commandSha256, marker, + releaseAfterFinish() { + if (!waitForFinishAttempt || closed || releasedAtMs !== null || !child?.stdin || child.exitCode !== null || child.signalCode !== null) throw new Error("Attached fixture release requires one live owned child"); + releasedAtMs = Date.now(); child.stdin.end("release\n"); return { releasedAtMs }; + }, snapshot() { const table = processTable(); return { connections, failure, markerWrittenAtMs, clientExitedAtMs, commandExit: exit, childPid: child?.pid ?? null, clientPid: client?.pid ?? null, clientGone: Boolean(client) && !table.some(p => sameObservedProcess(client, p)), childGone: Boolean(exit), observedAtMs: Date.now() }; }, async close() { if (closed) return; closed = true; clearInterval(clientObservation); diff --git a/tests/runner-e2e/copilot-protection-evidence.test.ts b/tests/runner-e2e/copilot-protection-evidence.test.ts index a47e9a97b68..75486245068 100644 --- a/tests/runner-e2e/copilot-protection-evidence.test.ts +++ b/tests/runner-e2e/copilot-protection-evidence.test.ts @@ -143,7 +143,8 @@ describe("Copilot sealed remote proof", () => { const fixture: CopilotRemoteFixture = { binding: s.binding, remoteCwd: s.binding.remoteCwd, actionFile: s.setup.path, killOwnedCopilot: async () => { throw new Error("Unexpected provider-death dispatch"); }, snapshot: async () => { await Promise.resolve(); order.push("baseline"); return s; }, - setupAttachedCommand: async input => { expect(input.markerText).toBe("private-marker"); order.push("setup"); return { command: "exact-remote-command", commandSha256: `sha256:${"c".repeat(64)}` }; }, + releaseAttachedCommand: async () => { throw new Error("Unexpected fixture release during setup"); }, + setupAttachedCommand: async input => { expect(input.waitForFinishAttempt).toBe(true); expect(input.markerText).toBe("private-marker"); order.push("setup"); return { command: "exact-remote-command", commandSha256: `sha256:${"c".repeat(64)}` }; }, finish: async () => { throw new Error("must not finish during setup"); }, readFile: async () => { throw new Error("must not read host file"); }, close: async () => {} }; const prepared = await prepareCopilotRemoteAction({ fixture, companyId: "company", environmentId: "env", runId: "run", target, prompt: "test", markerText: "private-marker" }); order.push("publish"); expect(order).toEqual(["setup", "baseline", "publish"]); diff --git a/tests/runner-e2e/copilot-protection-evidence.ts b/tests/runner-e2e/copilot-protection-evidence.ts index 413321403ff..5ff5c562c0a 100644 --- a/tests/runner-e2e/copilot-protection-evidence.ts +++ b/tests/runner-e2e/copilot-protection-evidence.ts @@ -123,7 +123,8 @@ export interface CopilotRemoteFixture { binding: CopilotRemoteBinding; remoteCwd: string; actionFile: string; snapshot(label: string): Promise; killOwnedCopilot(): Promise>; - setupAttachedCommand(input: { marker: string; markerText: string; delayMs: number }): Promise<{ command: string; commandSha256: string }>; + setupAttachedCommand(input: { marker: string; markerText: string; delayMs: number; waitForFinishAttempt?: boolean }): Promise<{ command: string; commandSha256: string }>; + releaseAttachedCommand(): Promise<{ releasedAtMs: number }>; finish(): Promise; readFile(relative: string): Promise; close(): Promise; } export interface CopilotRemoteBootstrap { @@ -185,7 +186,7 @@ export async function prepareCopilotRemoteAction(input: { }) { const f = input.fixture; if (f.binding.companyId !== input.companyId || f.binding.environmentId !== input.environmentId || f.binding.runId !== input.runId || f.remoteCwd !== f.binding.remoteCwd) throw new Error("Foreign Copilot remote bootstrap binding"); - const command = input.markerText === undefined ? undefined : await f.setupAttachedCommand({ marker: input.target, markerText: input.markerText, delayMs: 4000 }); + const command = input.markerText === undefined ? undefined : await f.setupAttachedCommand({ marker: input.target, markerText: input.markerText, delayMs: 4000, waitForFinishAttempt: true }); const baseline = await f.snapshot("before-action-publication"); assertCopilotRemoteSnapshot(baseline, f.binding); if (baseline.setup.published || baseline.setup.path !== f.actionFile || !baseline.processes.captured || baseline.processes.live.length === 0) throw new Error("Copilot action was not held behind the remote observer"); const target = baseline.targets[input.target]; diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index 7b0f32aa0ca..49822010fb0 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -26,7 +26,7 @@ function projected(name: string, target = "copilot-denied-nonce.txt") { describe("Copilot Product protection integration", () => { it("registers two explicit cases on both environments with honest terminal expectations", () => { const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection"); - expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 20, titleCreation: "explicit-search-composer", taskBinding: "browser-creation-response-id", naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", + expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 22, titleCreation: "explicit-search-composer", taskBinding: "browser-creation-response-id", naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v4", activeTurnCancellation: "not-implied-by-completed-provider-turn" }); expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } })); expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"])); @@ -155,6 +155,22 @@ describe("Copilot Product protection integration", () => { expect(await readFile(join(root, "marker"), "utf8")).toBe(fixture.marker); } finally { await fixture.close(); await rm(root, { recursive: true, force: true }); } }); + it("holds only the owned child until the controller releases the observed finish boundary", async () => { + const root = await mkdtemp("/tmp/pc-copilot-gated-command-"); const fixture = await createAttachedCommandFixture(join(root, "marker"), 100, true); + const client = spawn("/bin/sh", ["-c", fixture.command], { stdio: "ignore" }); + const exited = new Promise(resolve => client.once("exit", resolve)); + try { + const deadline = Date.now() + 3000; + while (!fixture.snapshot().childPid && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 20)); + await new Promise(resolve => setTimeout(resolve, 200)); + expect(fixture.snapshot()).toMatchObject({ connections: 1, failure: null, commandExit: null, childGone: false, clientGone: false }); + const release = fixture.releaseAfterFinish(); + expect(() => fixture.releaseAfterFinish()).toThrow(); + expect(await exited).toBe(0); + expect(fixture.snapshot().commandExit!.observedAtMs).toBeGreaterThan(release.releasedAtMs); + expect(await readFile(join(root, "marker"), "utf8")).toBe(fixture.marker); + } finally { await fixture.close(); await exited; await rm(root, { recursive: true, force: true }); } + }); }); it("rejects remote protection before any API access without bootstrap and pre-teardown authority", async () => { diff --git a/tests/runner-e2e/copilot-protection-flow.ts b/tests/runner-e2e/copilot-protection-flow.ts index 3cfd85f777d..7a5acf7905e 100644 --- a/tests/runner-e2e/copilot-protection-flow.ts +++ b/tests/runner-e2e/copilot-protection-flow.ts @@ -7,7 +7,7 @@ import { pollUntil, type RunnerApi } from "./api.js"; import { collectRunEvents } from "./run-observations.js"; import { createTaskThroughUi } from "./user-actions.js"; import { approveCopilotContextThroughUi, prepareCopilotContext } from "./copilot-context-permission.js"; -import { copilotFinishAttemptBeforeCommandExit, onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js"; +import { copilotAttachedFinishReady, copilotFinishAttemptBeforeCommandExit, onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js"; import { copilotOrigin, readCopilotToolEvidence, type CopilotToolNotice } from "./copilot-evidence.js"; import { createAttachedCommandFixture, createDeniedTargetFixture, bindDeniedTargetPrompt, exists, observeRunProcesses, retainRunProcessIdentity } from "./copilot-local-fixtures.js"; import { gradeCopilotAttachedSettlement, gradeCopilotDeniedWrite, type CopilotDeniedWriteEvidence } from "./copilot-protection-cases.js"; @@ -125,7 +125,7 @@ export async function runCopilotProtectionFlow(input: { }; const watcher = localTarget?.watcher; const markerPath = join(workspacePath, `copilot-settlement-${nonce}.txt`); - const command = deny || remote ? undefined : await createAttachedCommandFixture(markerPath); + const command = deny || remote ? undefined : await createAttachedCommandFixture(markerPath, 4000, true); const exactCommand = () => remoteCommand ?? command; let watchReceipt: CopilotDeniedWriteEvidence["mutationObservation"] | undefined; const check = (id: string, passed: boolean, detail: string) => { checks.push({ id, passed, detail }); expect(passed, detail).toBe(true); }; @@ -246,6 +246,15 @@ export async function runCopilotProtectionFlow(input: { check("negative-task-unfinished", issue.status === "in_progress", "The negative test does not claim the task is done"); check("no-extra-native-operation", countCopilotToolOrigins(copilotActionNotices(notices, notices.find(n => n.stage === "tool" && n.toolCallId === request.toolCallId)!, remoteFixture ? { actionFile: remoteFixture.actionFile, events: runEvents } : undefined, runEvents)) === 1, "No alternate native edit, command or delegated operation is permitted"); } else { + let finishOrigin: CopilotToolNotice | undefined; + await wait("native finish while the exact attached command is live", state => { + const origins = state.notices.filter(n => n.stage === "tool" && n.status === "pending" && n.operation === "execute" && n.commandSha256 === exactCommand()!.commandSha256); + if (origins.length !== 1) return false; + finishOrigin = copilotAttachedFinishReady(state.notices, origins[0]!); + return Boolean(finishOrigin); + }); + const release = remote ? await remoteFixture!.releaseAttachedCommand() : command!.releaseAfterFinish(); + await input.evidence("copilot-attached-release.json", { finishOrigin, release, source: "fixture controller releases only its own finite child after the actual native finish receipt; final grading unchanged" }); await wait("attached command and task settlement", s => s.issue.status === "done" && s.runs[0]?.status === "succeeded" && (remote || (s.processes.captured && s.processes.live.length === 0))); // Preserve independent local proof before any command matcher can abort. const { external: localExternal, markerMatches: localMarkerMatches, afterCleanupMarkerMatches: localAfterCleanupMarkerMatches, matched } = diff --git a/tests/runner-e2e/copilot-semantic-evidence.test.ts b/tests/runner-e2e/copilot-semantic-evidence.test.ts index e832c1bcb04..724aa65b860 100644 --- a/tests/runner-e2e/copilot-semantic-evidence.test.ts +++ b/tests/runner-e2e/copilot-semantic-evidence.test.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; import { describe, expect, it } from "vitest"; -import { copilotFinishAttemptBeforeCommandExit, onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js"; +import { copilotAttachedFinishReady, copilotFinishAttemptBeforeCommandExit, onlyCopilotAttachedOperations, readCopilotSemanticCompletion } from "./copilot-semantic-evidence.js"; import { readCopilotToolEvidence } from "./copilot-evidence.js"; import { copilotActionNotices } from "./copilot-protection-evidence.js"; import { validatePrpStructuredRunResult } from "../../packages/paperclip-runner/src/protocol/replay-contract.js"; @@ -70,7 +70,7 @@ describe("Copilot semantic completion public-event oracle", () => { }); it("versions semantic completion and current-mainline denial settlement independently", () => { const suite = runnerSuites.find(s => s.id === "copilot-protection")!; - expect(suite.definitionMetadata).toMatchObject({ version: 20, titleCreation: "explicit-search-composer", taskBinding: "browser-creation-response-id", remoteBootstrapContext: "already-exposed-dedicated-tool-no-http-substitution", semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v4", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v4" }); + expect(suite.definitionMetadata).toMatchObject({ version: 22, titleCreation: "explicit-search-composer", taskBinding: "browser-creation-response-id", remoteBootstrapContext: "already-exposed-dedicated-tool-no-http-substitution", semanticCompletionEvidence: "paperclip.e2e.copilot-semantic-completion.v4", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v4" }); expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 8 } })); }); it("joins exact native lifecycle, authoritative callback, proposed content and accepted control-plane result", () => { @@ -173,6 +173,16 @@ describe("Copilot semantic completion public-event oracle", () => { expect(copilotFinishAttemptBeforeCommandExit(f.proof, at + 1)).toBe(true); for (const exitAt of [at - 1, at, NaN, Infinity]) expect(copilotFinishAttemptBeforeCommandExit(f.proof, exitAt)).toBe(false); }); + it("releases the fixture only after a matching native finish receipt", () => { + const f = attachedFixture(); + expect(copilotAttachedFinishReady(f.notices, f.command)).toMatchObject({ toolCallId: f.proof.nativeToolCallId, status: "pending" }); + for (const changed of [{ turnId: "foreign" }, { sessionId: "foreign" }, { semanticOutcome: "error" }]) { + const partial = f.notices.map(n => n.semanticOperationId === "paperclip_finish" ? { ...n, ...changed } : n); + expect(copilotAttachedFinishReady(partial as any, f.command)).toBeUndefined(); + } + const incomplete = f.notices.filter(n => !(n.toolCallId === f.proof.nativeToolCallId && n.status === "pending")); + expect(() => copilotAttachedFinishReady(incomplete, f.command)).toThrow("matching finish origin"); + }); it("accepts bounded in-progress shell updates without inventing terminal command identity", () => { const f = attachedFixture(), pending = f.notices.find(n => n.toolCallId === "shell-read" && n.status === "pending")!; f.notices.push({ ...pending, status: "in_progress", seq: 53 }); diff --git a/tests/runner-e2e/copilot-semantic-evidence.ts b/tests/runner-e2e/copilot-semantic-evidence.ts index 40fa0af95f6..2dad7976f33 100644 --- a/tests/runner-e2e/copilot-semantic-evidence.ts +++ b/tests/runner-e2e/copilot-semantic-evidence.ts @@ -158,3 +158,15 @@ export function copilotFinishAttemptBeforeCommandExit(proof: CopilotSemanticComp return Number.isFinite(proof.nativePendingObservedAtMs) && proof.nativePendingObservedAtMs >= 0 && Number.isFinite(commandExitAtMs) && commandExitAtMs > proof.nativePendingObservedAtMs; } + +/** Fixture scheduling only; final grading still verifies the complete durable receipt and acceptance. */ +export function copilotAttachedFinishReady(notices: readonly CopilotToolNotice[], command: CopilotToolNotice): CopilotToolNotice | undefined { + const matches = notices.filter(n => n.semanticOperationId === "paperclip_finish" && n.semanticOutcome === "returned" && n.stage === "tool" && n.status === "completed" + && n.runId === command.runId && n.turnId === command.turnId && n.sessionId === command.sessionId && n.seq > command.seq); + if (!matches.length) return undefined; + if (matches.length !== 1) throw new Error("Attached fixture has ambiguous finish receipts"); + const finish = matches[0]!, pending = notices.filter(n => n.toolCallId === finish.toolCallId && n.stage === "tool" && n.status === "pending" + && n.runId === command.runId && n.turnId === command.turnId && n.sessionId === command.sessionId && n.seq > command.seq && n.seq < finish.seq); + if (pending.length !== 1) throw new Error("Attached fixture lacks the matching finish origin"); + return pending[0]; +} diff --git a/tests/runner-e2e/native-active-stop.test.ts b/tests/runner-e2e/native-active-stop.test.ts index e36e0408c5a..cc931d7064b 100644 --- a/tests/runner-e2e/native-active-stop.test.ts +++ b/tests/runner-e2e/native-active-stop.test.ts @@ -375,7 +375,7 @@ describe("explicit active Stop discovery", () => { expect(cells).toHaveLength(2); expect(suite.manualOnly).toBe(true); expect(cells.map(e => `${e.profile.qualificationCandidate}/${e.environment.id}`).sort()).toEqual(["cursor/daytona", "cursor/local"]); expect(cells.every(e => e.task.expectedRunCount === 1 && e.task.flow === "native_active_stop" && e.task.expectedTerminalState?.run === "cancelled")).toBe(true); - expect(suite.definitionMetadata).toMatchObject({ version: 19, evidence: "paperclip.e2e.native-active-stop-settlement.v2", normalCompletionAccepted: false, providerDeath: "not-covered" }); + expect(suite.definitionMetadata).toMatchObject({ version: 20, evidence: "paperclip.e2e.native-active-stop-settlement.v2", normalCompletionAccepted: false, providerDeath: "not-covered" }); expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, version: 3 } })); expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, normalCompletionAccepted: true } })); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(e => e.suite.id === suite.id)).toBe(false); diff --git a/tests/runner-e2e/remote-native-fixtures.test.ts b/tests/runner-e2e/remote-native-fixtures.test.ts index b2543368c63..df0d25c8f81 100644 --- a/tests/runner-e2e/remote-native-fixtures.test.ts +++ b/tests/runner-e2e/remote-native-fixtures.test.ts @@ -460,7 +460,7 @@ describe("actual generated observer state machine", () => { }; const net = { createServer(fn: (socket: any) => void) { handlers.push(fn); return server; } }; const context = { - require(name: string) { if (name === "node:fs") return fs; if (name === "node:net") return net; if (name === "node:child_process") return { spawn: vi.fn(() => { const child = Object.assign(new EventEmitter(), { pid: 88, exitCode: null, signalCode: null, kill: vi.fn() }); children.push(child); return child; }) }; if (name === "node:path") return { join: (...paths: string[]) => paths.join("/"), dirname: (path: string) => path.slice(0, path.lastIndexOf("/")), basename: (path: string) => path.slice(path.lastIndexOf("/") + 1) }; if (name === "node:crypto") return { createHash }; throw new Error("unexpected module"); }, + require(name: string) { if (name === "node:fs") return fs; if (name === "node:net") return net; if (name === "node:child_process") return { spawn: vi.fn((_node: string, argv: string[]) => { const child = Object.assign(new EventEmitter(), { argv, stdin: { end: vi.fn() }, pid: 88, exitCode: null, signalCode: null, kill: vi.fn() }); children.push(child); return child; }) }; if (name === "node:path") return { join: (...paths: string[]) => paths.join("/"), dirname: (path: string) => path.slice(0, path.lastIndexOf("/")), basename: (path: string) => path.slice(path.lastIndexOf("/") + 1) }; if (name === "node:crypto") return { createHash }; throw new Error("unexpected module"); }, process: { argv: ["node", `${config.root}/observer.cjs`, Buffer.from(JSON.stringify(config)).toString("base64")], execPath: "/node", hrtime: { bigint: () => 12345n }, exit: vi.fn(), kill: vi.fn() }, Buffer, __filename: `${config.root}/observer.cjs`, setInterval(fn: () => void) { intervals.push(fn); return 1; }, clearInterval: vi.fn(), @@ -710,6 +710,25 @@ describe("actual generated observer state machine", () => { expect(o.children).toHaveLength(0); expect(socket.destroy).toHaveBeenCalled(); expect(o.request("snapshot")[0].result.attached.failure).toBe("client_rejected"); }); + it("releases a gated remote child once, through the owned controller after publication", async () => { + const o = await observerHarness(); o.request("snapshot"); + const config = { marker: "result.txt", markerText: "settled", delayMs: 100, clientNonce: "test-gated-client", waitForFinishAttempt: true }; + const fixture = o.request("attached", config)[0].result; + expect(o.request("release-attached")[0].ok).toBe(false); + o.request("publish", { path: o.config.actionFile, text: "exact action" }); + expect(o.request("release-attached")[0].ok).toBe(false); + o.proc.set(23, { ppid: 21, group: 21, ticks: "300", argv: ["/node", fixture.clientScript, fixture.clientSocket, config.clientNonce] }); + const replies: any[] = [], socket = Object.assign(new EventEmitter(), { end: (value: string) => replies.push(JSON.parse(value)), destroy: vi.fn() }); + o.handlers[1]!(socket); socket.emit("data", Buffer.from(JSON.stringify({ nonce: config.clientNonce, pid: 23 }) + "\n")); + expect(o.children).toHaveLength(1); expect(o.files.has("/workspace/result.txt")).toBe(false); + expect(() => new Script(o.children[0].argv[1])).not.toThrow(); + expect(o.request("release-attached")[0]).toMatchObject({ ok: true, result: { releasedAtMs: expect.any(Number) } }); + expect(o.children[0].stdin.end).toHaveBeenCalledExactlyOnceWith("release\n"); + expect(o.request("release-attached")[0].ok).toBe(false); + expect(replies).toHaveLength(0); + o.children[0].exitCode = 0; o.children[0].emit("exit", 0, null); + expect(replies).toEqual([{ code: 0 }]); expect(o.files.get("/workspace/result.txt")?.toString()).toBe("settled"); + }); it("scopes actual runtime symlinks/state churn out while retaining sentinel and sibling coverage", async () => { const o = await observerHarness(); o.symbolicLinks.add("/workspace/.paperclip-runtime/paperclip-runner/provider-pack"); diff --git a/tests/runner-e2e/remote-native-fixtures.ts b/tests/runner-e2e/remote-native-fixtures.ts index 7c113fd11a5..037fb05f955 100644 --- a/tests/runner-e2e/remote-native-fixtures.ts +++ b/tests/runner-e2e/remote-native-fixtures.ts @@ -31,7 +31,7 @@ function relative(value: string): string { } // Only closed diagnostic enums cross the remote boundary; SDK errors and output // are never retained. These diagnostics explain failed evidence, not qualification. -const RPC_PHASES = ["runtime-ready", "install", "wait", "close", "arm", "publish", "snapshot", "attached", "read", "inject-loss", "provider-death"] as const; +const RPC_PHASES = ["runtime-ready", "install", "wait", "close", "arm", "publish", "snapshot", "attached", "release-attached", "read", "inject-loss", "provider-death"] as const; const RPC_CODES = ["node_identity", "sentinel_type", "sentinel", "cwd", "runtime_root_identity", "runtime_binary_identity", "proc_bound", "ambiguous_run_root", "runtime_not_ready", "runtime_identity_changed", "invalid_proc_identity", "invalid_proc_fields", "socket_error", "socket_timeout", "output_bound", "rpc_deadline", "remote_unknown", "transport_failure", "invalid_response", "readiness_deadline", "terminal_evidence_incomplete"] as const; type RpcPhase = typeof RPC_PHASES[number]; type RpcCode = typeof RPC_CODES[number]; @@ -172,14 +172,15 @@ const server=net.createServer(socket=>{sockets.add(socket);socket.on('close',()= else if(r.op==='arm'){if(waiters.size)result={armed:true,sealed};else{armWaiters.add(socket);socket.on('close',()=>armWaiters.delete(socket));return}} else if(r.op==='publish'){if(sealed||publishedHash||r.path!==config.actionFile||typeof r.text!=='string'||Buffer.byteLength(r.text)>16384)throw Error('publish_bound');const p=path.join(config.binding.remoteCwd,config.actionFile);if(fs.realpathSync(path.dirname(p))!==path.dirname(p))throw Error('publish_parent');publishedHash=hash(r.text);try{fs.writeFileSync(p,r.text,{flag:'wx',mode:0o600})}catch(e){publishedHash=null;throw e}result={path:r.path,sha256:publishedHash,published:true};} else if(r.op==='read'){const p=r.path==='@cross-root'&&config.crossRoot?path.join(config.root,'cross-root-target'):path.join(config.binding.remoteCwd,r.path);if(r.path!=='@cross-root'&&!config.targets.includes(r.path))throw Error('unregistered_read');const status=file(p);if(status.absent)throw Error('file_missing');result={...status,base64:readSafe(p).toString('base64')};} - else if(r.op==='attached'){if(attached||sealed)throw Error('attached_already_configured');if(!config.targets.includes(r.marker)||!Number.isInteger(r.delayMs)||r.delayMs<100||r.delayMs>8000)throw Error('attached_bounds'); - attached={connections:0,failure:null,commandExit:null,markerWrittenAtMs:null,markerWrittenMonotonicNs:null,clientExitedAtMs:null,clientExitedMonotonicNs:null}; + else if(r.op==='attached'){if(attached||sealed)throw Error('attached_already_configured');if(!config.targets.includes(r.marker)||!Number.isInteger(r.delayMs)||r.delayMs<100||r.delayMs>8000||(r.waitForFinishAttempt!==undefined&&typeof r.waitForFinishAttempt!=='boolean'))throw Error('attached_bounds'); + attached={waitForFinishAttempt:r.waitForFinishAttempt===true,releasedAtMs:null,connections:0,failure:null,commandExit:null,markerWrittenAtMs:null,markerWrittenMonotonicNs:null,clientExitedAtMs:null,clientExitedMonotonicNs:null}; const clientScript=path.join(config.root,'client.cjs'),clientSocket=path.join(config.root,'attached.sock'); - fs.writeFileSync(clientScript,ATTACHED_CLIENT,{flag:'wx',mode:0o400}); + fs.writeFileSync(clientScript,ATTACHED_CLIENT.replace('15000',attached.waitForFinishAttempt?'45000':'15000'),{flag:'wx',mode:0o400}); const srv=net.createServer(s=>{sockets.add(s);s.on('close',()=>sockets.delete(s));s.on('error',()=>{});let b='';s.on('data',d=>{b+=d;if(b.length>1024){s.destroy();attached.failure='client_bound';return}if(!b.includes('\n'))return;s.removeAllListeners('data');try{const q=JSON.parse(b);attached.connections++;if(q.nonce!==r.clientNonce||attached.connections!==1)throw Error('client_identity');client=identity(q.pid);sample();if(journal.get(q.pid)?.startTicks!==client.startTicks)throw Error('client_not_owned_by_run');const argv=fs.readFileSync('/proc/'+q.pid+'/cmdline').toString().split('\0');if(argv[1]!==clientScript||argv[2]!==clientSocket||argv[3]!==r.clientNonce)throw Error('client_argv'); - child=cp.spawn(process.execPath,['-e','setTimeout(()=>process.exit(0),'+r.delayMs+')'],{env:{PATH:'/usr/bin:/bin'},stdio:'ignore'});child.once('error',()=>{attached.failure='child_start';s.destroy()});child.once('exit',(code,signal)=>{attached.commandExit={code:code??-1,observedAtMs:Date.now(),observedMonotonicNs:process.hrtime.bigint().toString()};if(code!==0||signal){attached.failure='child_failed';s.destroy();return}try{fs.writeFileSync(path.join(config.binding.remoteCwd,r.marker),r.markerText,{flag:'wx'});attached.markerWrittenAtMs=Date.now();attached.markerWrittenMonotonicNs=process.hrtime.bigint().toString();s.end(JSON.stringify({code:0})+'\n')}catch{attached.failure='marker_failed';s.destroy()}}); + const childScript=attached.waitForFinishAttempt?"const deadline=setTimeout(()=>process.exit(3),30000);process.stdin.once('data',b=>{if(b.toString()!=='release\\n')process.exit(2);clearTimeout(deadline);setTimeout(()=>process.exit(0),"+r.delayMs+")});":'setTimeout(()=>process.exit(0),'+r.delayMs+')';child=cp.spawn(process.execPath,['-e',childScript],{env:{PATH:'/usr/bin:/bin'},stdio:attached.waitForFinishAttempt?['pipe','ignore','ignore']:'ignore'});child.once('error',()=>{attached.failure='child_start';s.destroy()});child.once('exit',(code,signal)=>{attached.commandExit={code:code??-1,observedAtMs:Date.now(),observedMonotonicNs:process.hrtime.bigint().toString()};if(code!==0||signal){attached.failure='child_failed';s.destroy();return}try{fs.writeFileSync(path.join(config.binding.remoteCwd,r.marker),r.markerText,{flag:'wx'});attached.markerWrittenAtMs=Date.now();attached.markerWrittenMonotonicNs=process.hrtime.bigint().toString();s.end(JSON.stringify({code:0})+'\n')}catch{attached.failure='marker_failed';s.destroy()}}); }catch{attached.failure='client_rejected';s.destroy()}})});srv.listen(clientSocket);attached.server=srv;result={clientScript,clientSocket}; } + else if(r.op==='release-attached'){if(sealed||!publishedHash||!attached?.waitForFinishAttempt||attached.releasedAtMs!==null||!child?.stdin||child.exitCode!==null||child.signalCode!==null)throw Error('attached_release_boundary');attached.releasedAtMs=Date.now();child.stdin.end('release\n');result={releasedAtMs:attached.releasedAtMs};} else if(r.op==='provider-death'){ if(providerDeathDispatched||!publishedHash||sealed)throw Error('death_boundary'); const observed=sample();if(!complete||!observed.captured)throw Error('death_owner'); @@ -235,7 +236,8 @@ export interface RemoteNativeFixture { readFile(path: string): Promise; publishAction(path: string, text: string): Promise; killOwnedCopilot(): Promise>; - setupAttachedCommand(input: { marker: string; markerText: string; delayMs: number }): Promise<{ command: string; commandSha256: string }>; + setupAttachedCommand(input: { marker: string; markerText: string; delayMs: number; waitForFinishAttempt?: boolean }): Promise<{ command: string; commandSha256: string }>; + releaseAttachedCommand(): Promise<{ releasedAtMs: number }>; /** Consumes the host-held terminal receipt; never queries a deleted lease. */ finish(): Promise; /** Only fixture-owned observer cleanup. Never deletes a sandbox or lease. */ @@ -495,6 +497,12 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption const command = `${quote(NODE)} ${quote(result.clientScript as string)} ${quote(result.clientSocket as string)} ${quote(clientNonce)}`; return { command, commandSha256: digest(command) }; }, + async releaseAttachedCommand() { + fail(!closed && published && !finished, "attached_release_boundary"); + const result = record(await rpc({ op: "release-attached" })); + fail(Number.isSafeInteger(result.releasedAtMs) && (result.releasedAtMs as number) >= 0, "attached_release_ack"); + return { releasedAtMs: result.releasedAtMs as number }; + }, async finish() { if (finished) return finished; const receipt = await terminal;