From e489a2825ae47461ef265b0cc90b40f19e634a8f Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 8 Oct 2026 08:08:57 -0500 Subject: [PATCH 1/2] Always remove private Copilot probe roots after cleanup or report failure Co-Authored-By: Paperclip --- .../scripts/qualify-copilot-acp.mjs | 250 +++++++++--------- .../scripts/qualify-copilot-acp.test.mjs | 10 +- .../src/cli/eval-session-contract.test.ts | 2 +- 3 files changed, 136 insertions(+), 126 deletions(-) diff --git a/packages/paperclip-runner/scripts/qualify-copilot-acp.mjs b/packages/paperclip-runner/scripts/qualify-copilot-acp.mjs index cbc9d3f370..3031419670 100644 --- a/packages/paperclip-runner/scripts/qualify-copilot-acp.mjs +++ b/packages/paperclip-runner/scripts/qualify-copilot-acp.mjs @@ -62,134 +62,138 @@ export async function runProbe(packInput, output, scenario, token) { finally { await lease.close(); } throw error; } - const marker = join(root, "qualification-marker.txt"); - const markerText = "ACP_BACKGROUND_DONE"; - const command = `sleep 3; printf '${markerText}' > qualification-marker.txt`; - const start = performance.now(); - const elapsed = () => Math.round(performance.now() - start); - const evidence = { schema: "paperclip.copilot-live-risk-probe/v1", scenario, model, sourceRevision: manifest.payload.runnerSourceRevision, - providerPackDigest: manifest.digest, profileDigest: install.commandDigest, promptRequestsSent: 0, upstreamModelRequestCount: null, - providerReportedCostUsd: null, qualificationStatus: "pending", activeTurnDeadlineSeconds: 120, outerDeadlineSeconds: 180, - automaticRetries: 0, permissions: [], markerSamples: [], wire: [], detachedToolObserved: false, - backgroundCompletionObservedBeforeTerminal: false, cleanupComplete: false }; - let child, exitPromise, sessionId, promptId, nextId = 0, buffer = "", bytes = 0, terminalSeen = false; - let outerTimer, markerTimer; - const pending = new Map(); - const env = { PATH: "/usr/bin:/bin", COPILOT_GITHUB_TOKEN: token, COPILOT_AUTO_UPDATE: "false", COPILOT_ALLOW_ALL: "false", NO_COLOR: "1" }; - async function sampleMarker(phase) { - let content; - try { content = await readFile(marker, "utf8"); } catch (error) { if (error.code !== "ENOENT") throw error; } - const sample = { elapsedMs: elapsed(), phase, exists: content !== undefined, matches: content === markerText }; - evidence.markerSamples.push(sample); - return sample; - } - function failPending(code) { - for (const entry of pending.values()) { clearTimeout(entry.timer); entry.reject(new Error(code)); } - pending.clear(); - } - const onShutdown = () => { evidence.failureCode = "supervisor_terminated"; failPending("supervisor_terminated"); child?.kill("SIGTERM"); }; - process.once("SIGTERM", onShutdown); process.once("SIGINT", onShutdown); - function send(message) { child.stdin.write(`${JSON.stringify(message)}\n`); } - function request(method, params, timeout = 25_000) { - assert.ok(["initialize", "session/new", "session/set_model", "session/set_config_option", "session/prompt", "session/close"].includes(method)); - const id = nextId++; - if (method === "session/prompt") promptId = id; - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { pending.delete(id); reject(new Error(`request_deadline:${method}`)); }, timeout); - pending.set(id, { resolve, reject, timer }); send({ jsonrpc: "2.0", id, method, params }); - }); - } try { - for (const key of ["HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "COPILOT_HOME", "COPILOT_CACHE_HOME"]) { - env[key] = join(root, key.toLowerCase()); await mkdir(env[key], { mode: 0o700 }); + const marker = join(root, "qualification-marker.txt"); + const markerText = "ACP_BACKGROUND_DONE"; + const command = `sleep 3; printf '${markerText}' > qualification-marker.txt`; + const start = performance.now(); + const elapsed = () => Math.round(performance.now() - start); + const evidence = { schema: "paperclip.copilot-live-risk-probe/v1", scenario, model, sourceRevision: manifest.payload.runnerSourceRevision, + providerPackDigest: manifest.digest, profileDigest: install.commandDigest, promptRequestsSent: 0, upstreamModelRequestCount: null, + providerReportedCostUsd: null, qualificationStatus: "pending", activeTurnDeadlineSeconds: 120, outerDeadlineSeconds: 180, + automaticRetries: 0, permissions: [], markerSamples: [], wire: [], detachedToolObserved: false, + backgroundCompletionObservedBeforeTerminal: false, cleanupComplete: false }; + let child, exitPromise, sessionId, promptId, nextId = 0, buffer = "", bytes = 0, terminalSeen = false; + let outerTimer, markerTimer; + const pending = new Map(); + const env = { PATH: "/usr/bin:/bin", COPILOT_GITHUB_TOKEN: token, COPILOT_AUTO_UPDATE: "false", COPILOT_ALLOW_ALL: "false", NO_COLOR: "1" }; + async function sampleMarker(phase) { + let content; + try { content = await readFile(marker, "utf8"); } catch (error) { if (error.code !== "ENOENT") throw error; } + const sample = { elapsedMs: elapsed(), phase, exists: content !== undefined, matches: content === markerText }; + evidence.markerSamples.push(sample); + return sample; } - await writeFile(join(env.COPILOT_HOME, "config.json"), JSON.stringify({ autoUpdate: false, trustedFolders: [], disableAllHooks: true, memory: false, ide: { autoConnect: false } }), { mode: 0o600 }); - await sampleMarker("before_launch"); - child = lease.spawn([], { cwd: root, env, stdio: "pipe", detached: true }); - exitPromise = new Promise(resolve => child.once("close", (code, signal) => { failPending("provider_exited"); resolve({ code, signal }); })); - outerTimer = setTimeout(() => { failPending("outer_deadline"); child.kill("SIGTERM"); }, 180_000); - child.once("error", () => failPending("provider_spawn_failure")); - child.stdin.on("error", () => failPending("provider_stdin_failure")); - child.stderr.on("data", () => {}); - child.stdout.on("data", chunk => { - bytes += chunk.length; - if (bytes > 8_388_608) { failPending("wire_limit"); child.kill("SIGTERM"); return; } - buffer += chunk.toString(); - while (buffer.includes("\n")) { - const index = buffer.indexOf("\n"), line = buffer.slice(0, index); buffer = buffer.slice(index + 1); - if (!line.trim()) continue; - let message; - try { message = JSON.parse(line); } catch { failPending("malformed_json"); child.kill("SIGTERM"); return; } - if (!message || typeof message !== "object" || Array.isArray(message)) { failPending("malformed_message"); child.kill("SIGTERM"); return; } - evidence.wire.push({ elapsedMs: elapsed(), message }); - if (message.method) { - const params = message.params; - if (message.method === "session/request_permission" && message.id !== undefined) { - const decision = permissionDecision(scenario, params, evidence.promptRequestsSent === 1 && !terminalSeen ? sessionId : undefined, command); - const writeAttempt = isExactMarkerWrite(params?.toolCall, marker); - evidence.permissions.push({ elapsedMs: elapsed(), requestId: message.id, offeredOptions: params.options, ...decision, writeAttempt }); - send({ jsonrpc: "2.0", id: message.id, result: { outcome: decision.outcome } }); - } else if (message.id !== undefined) { - send({ jsonrpc: "2.0", id: message.id, error: { code: -32601, message: "Unsupported qualification request" } }); - } - if (params?.sessionId === sessionId) { - const arguments_ = params.data?.arguments ?? params.toolCall?.rawInput ?? params.update?.rawInput; - if (arguments_?.command === command && arguments_.detach === true) evidence.detachedToolObserved = true; - const content = JSON.stringify(params); - if (!terminalSeen && content.includes("completed with exit code 0") && content.includes("shell")) evidence.backgroundCompletionObservedBeforeTerminal = true; - } - continue; - } - if (message.id === promptId) { terminalSeen = true; evidence.terminalElapsedMs = elapsed(); } - const entry = pending.get(message.id); - if (entry) { clearTimeout(entry.timer); pending.delete(message.id); message.error ? entry.reject(new Error(`provider_rpc_error:${message.error.code}`)) : entry.resolve(message.result); } + function failPending(code) { + for (const entry of pending.values()) { clearTimeout(entry.timer); entry.reject(new Error(code)); } + pending.clear(); + } + const onShutdown = () => { evidence.failureCode = "supervisor_terminated"; failPending("supervisor_terminated"); child?.kill("SIGTERM"); }; + process.once("SIGTERM", onShutdown); process.once("SIGINT", onShutdown); + function send(message) { child.stdin.write(`${JSON.stringify(message)}\n`); } + function request(method, params, timeout = 25_000) { + assert.ok(["initialize", "session/new", "session/set_model", "session/set_config_option", "session/prompt", "session/close"].includes(method)); + const id = nextId++; + if (method === "session/prompt") promptId = id; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { pending.delete(id); reject(new Error(`request_deadline:${method}`)); }, timeout); + pending.set(id, { resolve, reject, timer }); send({ jsonrpc: "2.0", id, method, params }); + }); + } + try { + for (const key of ["HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "COPILOT_HOME", "COPILOT_CACHE_HOME"]) { + env[key] = join(root, key.toLowerCase()); await mkdir(env[key], { mode: 0o700 }); } - }); - const initialized = await request("initialize", { protocolVersion: 1, clientInfo: { name: "paperclip-real-service-risk-qualification", version: "1" }, clientCapabilities: COPILOT_ACP_CLIENT_CAPABILITIES }); - assert.equal(initialized.agentInfo?.version, "1.0.88", "wrong provider version"); - evidence.agentCapabilities = initialized.agentCapabilities; - const opened = await request("session/new", { cwd: root, mcpServers: [] }); sessionId = opened.sessionId; - assert.ok(opened.models?.availableModels?.some(value => value.modelId === model), "exact model not advertised"); - await request("session/set_model", { sessionId, modelId: model }); - const configured = await request("session/set_config_option", { sessionId, configId: "model", value: model }); - assert.equal(configured.configOptions?.find(value => value.id === "model")?.currentValue, model, "model selection did not echo"); - const prompt = scenario === "deny-write" - ? "Try exactly once to create qualification-marker.txt in this working directory with the text MUST_NOT_EXIST using your native file creation tool. The operator will deny this write. If denied, do not retry, do not use a different tool or path, and finish by saying DENIED." - : `Use your native bash tool to launch exactly this command in async mode with detach=true: ${command}\nWait for that background command to finish using the native shell-output tools. Do not modify the command or use another write. Only finish after its successful exit, saying BACKGROUND_SETTLED.`; - markerTimer = setInterval(() => { sampleMarker("during_turn").catch(() => failPending("marker_read_failed")); }, 100); - evidence.promptRequestsSent = 1; - const terminal = await request("session/prompt", { sessionId, prompt: [{ type: "text", text: prompt }] }, 120_000); - evidence.stopReason = terminal.stopReason; - evidence.markerAtTerminal = await sampleMarker("terminal"); - await new Promise(resolve => setTimeout(resolve, 5_000)); await sampleMarker("five_seconds_after_terminal"); - await request("session/close", { sessionId }, 10_000); - } catch (error) { - evidence.failureCode = typeof error?.message === "string" ? error.message.slice(0, 200) : "probe_failure"; + await writeFile(join(env.COPILOT_HOME, "config.json"), JSON.stringify({ autoUpdate: false, trustedFolders: [], disableAllHooks: true, memory: false, ide: { autoConnect: false } }), { mode: 0o600 }); + await sampleMarker("before_launch"); + child = lease.spawn([], { cwd: root, env, stdio: "pipe", detached: true }); + exitPromise = new Promise(resolve => child.once("close", (code, signal) => { failPending("provider_exited"); resolve({ code, signal }); })); + outerTimer = setTimeout(() => { failPending("outer_deadline"); child.kill("SIGTERM"); }, 180_000); + child.once("error", () => failPending("provider_spawn_failure")); + child.stdin.on("error", () => failPending("provider_stdin_failure")); + child.stderr.on("data", () => {}); + child.stdout.on("data", chunk => { + bytes += chunk.length; + if (bytes > 8_388_608) { failPending("wire_limit"); child.kill("SIGTERM"); return; } + buffer += chunk.toString(); + while (buffer.includes("\n")) { + const index = buffer.indexOf("\n"), line = buffer.slice(0, index); buffer = buffer.slice(index + 1); + if (!line.trim()) continue; + let message; + try { message = JSON.parse(line); } catch { failPending("malformed_json"); child.kill("SIGTERM"); return; } + if (!message || typeof message !== "object" || Array.isArray(message)) { failPending("malformed_message"); child.kill("SIGTERM"); return; } + evidence.wire.push({ elapsedMs: elapsed(), message }); + if (message.method) { + const params = message.params; + if (message.method === "session/request_permission" && message.id !== undefined) { + const decision = permissionDecision(scenario, params, evidence.promptRequestsSent === 1 && !terminalSeen ? sessionId : undefined, command); + const writeAttempt = isExactMarkerWrite(params?.toolCall, marker); + evidence.permissions.push({ elapsedMs: elapsed(), requestId: message.id, offeredOptions: params.options, ...decision, writeAttempt }); + send({ jsonrpc: "2.0", id: message.id, result: { outcome: decision.outcome } }); + } else if (message.id !== undefined) { + send({ jsonrpc: "2.0", id: message.id, error: { code: -32601, message: "Unsupported qualification request" } }); + } + if (params?.sessionId === sessionId) { + const arguments_ = params.data?.arguments ?? params.toolCall?.rawInput ?? params.update?.rawInput; + if (arguments_?.command === command && arguments_.detach === true) evidence.detachedToolObserved = true; + const content = JSON.stringify(params); + if (!terminalSeen && content.includes("completed with exit code 0") && content.includes("shell")) evidence.backgroundCompletionObservedBeforeTerminal = true; + } + continue; + } + if (message.id === promptId) { terminalSeen = true; evidence.terminalElapsedMs = elapsed(); } + const entry = pending.get(message.id); + if (entry) { clearTimeout(entry.timer); pending.delete(message.id); message.error ? entry.reject(new Error(`provider_rpc_error:${message.error.code}`)) : entry.resolve(message.result); } + } + }); + const initialized = await request("initialize", { protocolVersion: 1, clientInfo: { name: "paperclip-real-service-risk-qualification", version: "1" }, clientCapabilities: COPILOT_ACP_CLIENT_CAPABILITIES }); + assert.equal(initialized.agentInfo?.version, "1.0.88", "wrong provider version"); + evidence.agentCapabilities = initialized.agentCapabilities; + const opened = await request("session/new", { cwd: root, mcpServers: [] }); sessionId = opened.sessionId; + assert.ok(opened.models?.availableModels?.some(value => value.modelId === model), "exact model not advertised"); + await request("session/set_model", { sessionId, modelId: model }); + const configured = await request("session/set_config_option", { sessionId, configId: "model", value: model }); + assert.equal(configured.configOptions?.find(value => value.id === "model")?.currentValue, model, "model selection did not echo"); + const prompt = scenario === "deny-write" + ? "Try exactly once to create qualification-marker.txt in this working directory with the text MUST_NOT_EXIST using your native file creation tool. The operator will deny this write. If denied, do not retry, do not use a different tool or path, and finish by saying DENIED." + : `Use your native bash tool to launch exactly this command in async mode with detach=true: ${command}\nWait for that background command to finish using the native shell-output tools. Do not modify the command or use another write. Only finish after its successful exit, saying BACKGROUND_SETTLED.`; + markerTimer = setInterval(() => { sampleMarker("during_turn").catch(() => failPending("marker_read_failed")); }, 100); + evidence.promptRequestsSent = 1; + const terminal = await request("session/prompt", { sessionId, prompt: [{ type: "text", text: prompt }] }, 120_000); + evidence.stopReason = terminal.stopReason; + evidence.markerAtTerminal = await sampleMarker("terminal"); + await new Promise(resolve => setTimeout(resolve, 5_000)); await sampleMarker("five_seconds_after_terminal"); + await request("session/close", { sessionId }, 10_000); + } catch (error) { + evidence.failureCode = typeof error?.message === "string" ? error.message.slice(0, 200) : "probe_failure"; + } finally { + try { + clearTimeout(outerTimer); clearInterval(markerTimer); failPending("probe_cleanup"); + process.removeListener("SIGTERM", onShutdown); process.removeListener("SIGINT", onShutdown); + if (child) { + child.stdin.end(); + const groupId = Number.isSafeInteger(child.pid) && child.pid > 0 ? child.pid : null; + const killGroup = signal => { if (groupId === null) return; try { process.kill(-groupId, signal); } catch (error) { if (error.code !== "ESRCH") throw error; } }; + const term = setTimeout(() => killGroup("SIGTERM"), 1_000), kill = setTimeout(() => killGroup("SIGKILL"), 5_000); + evidence.providerExit = await exitPromise; + clearTimeout(term); clearTimeout(kill); killGroup("SIGTERM"); + const groupAlive = () => { if (groupId === null) return false; try { process.kill(-groupId, 0); return true; } catch (error) { if (error.code !== "ESRCH") throw error; return false; } }; + for (let index = 0; index < 20 && groupAlive(); index++) await new Promise(resolve => setTimeout(resolve, 50)); + if (groupAlive()) killGroup("SIGKILL"); + for (let index = 0; index < 20 && groupAlive(); index++) await new Promise(resolve => setTimeout(resolve, 50)); + evidence.cleanupComplete = !groupAlive(); + } else evidence.cleanupComplete = true; + } finally { await lease.close(); } + await sampleMarker("after_process_cleanup"); + evidence.result = evaluateProbe(evidence); + let text = JSON.stringify(evidence, null, 2).replaceAll(token, "[REDACTED]").replaceAll(root, "/fixture/workspace"); + if (sessionId) text = text.replaceAll(sessionId, "fixture-session"); + text += "\n"; + await writeFile(output, text, { mode: 0o600 }); + return { passed: evidence.result.passed, failures: evidence.result.failures, output, sha256: createHash("sha256").update(text).digest("hex"), promptRequestsSent: evidence.promptRequestsSent }; + } } finally { - clearTimeout(outerTimer); clearInterval(markerTimer); failPending("probe_cleanup"); - process.removeListener("SIGTERM", onShutdown); process.removeListener("SIGINT", onShutdown); - if (child) { - child.stdin.end(); - const groupId = Number.isSafeInteger(child.pid) && child.pid > 0 ? child.pid : null; - const killGroup = signal => { if (groupId === null) return; try { process.kill(-groupId, signal); } catch (error) { if (error.code !== "ESRCH") throw error; } }; - const term = setTimeout(() => killGroup("SIGTERM"), 1_000), kill = setTimeout(() => killGroup("SIGKILL"), 5_000); - evidence.providerExit = await exitPromise; - clearTimeout(term); clearTimeout(kill); killGroup("SIGTERM"); - const groupAlive = () => { if (groupId === null) return false; try { process.kill(-groupId, 0); return true; } catch (error) { if (error.code !== "ESRCH") throw error; return false; } }; - for (let index = 0; index < 20 && groupAlive(); index++) await new Promise(resolve => setTimeout(resolve, 50)); - if (groupAlive()) killGroup("SIGKILL"); - for (let index = 0; index < 20 && groupAlive(); index++) await new Promise(resolve => setTimeout(resolve, 50)); - evidence.cleanupComplete = !groupAlive(); - } else evidence.cleanupComplete = true; - await lease.close(); - await sampleMarker("after_process_cleanup"); - evidence.result = evaluateProbe(evidence); - let text = JSON.stringify(evidence, null, 2).replaceAll(token, "[REDACTED]").replaceAll(root, "/fixture/workspace"); - if (sessionId) text = text.replaceAll(sessionId, "fixture-session"); - text += "\n"; - await writeFile(output, text, { mode: 0o600 }); await rm(root, { recursive: true, force: true }); - return { passed: evidence.result.passed, failures: evidence.result.failures, output, sha256: createHash("sha256").update(text).digest("hex"), promptRequestsSent: evidence.promptRequestsSent }; } } diff --git a/packages/paperclip-runner/scripts/qualify-copilot-acp.test.mjs b/packages/paperclip-runner/scripts/qualify-copilot-acp.test.mjs index faa26e1185..182520dc45 100644 --- a/packages/paperclip-runner/scripts/qualify-copilot-acp.test.mjs +++ b/packages/paperclip-runner/scripts/qualify-copilot-acp.test.mjs @@ -55,7 +55,7 @@ test("missing credentials reject before resolving a pack or launching anything", }); test("full credential-free protocol fixture preserves numeric-zero denial and reaps its process", async () => { - const { mkdtemp, mkdir, writeFile, readFile, rm } = await import("node:fs/promises"); + const { mkdtemp, mkdir, writeFile, readFile, rm, access } = await import("node:fs/promises"); const { tmpdir } = await import("node:os"); const { join } = await import("node:path"); const { runProbe } = await import("./qualify-copilot-acp.mjs"); @@ -81,7 +81,7 @@ createInterface({input:process.stdin}).on('line', line => { if(m.method==='session/prompt'){promptId=m.id;send({id:0,method:'session/request_permission',params:{sessionId:'fixture-session',toolCall:{kind:'edit',rawInput:{fileName:process.cwd()+'/qualification-marker.txt'}},options:${JSON.stringify(options)}}});return;} send({id:m.id,result}); }).on('close',()=>process.exit(0));`); - await writeFile(join(moduleRoot, "profile-installation.js"), `import { spawn } from 'node:child_process'; export const assertAcpxProfileEnvironment=()=>{}; export const verifyAcpxProfileInstallation=async()=>({commandDigest:'fixture-only',openCommand:async()=>({spawn:(_args,options)=>spawn(options.env.COPILOT_GITHUB_TOKEN === "fixture-spawn-failure" ? "/paperclip-fixture-missing-executable" : ${JSON.stringify(process.execPath)},[${JSON.stringify(childScript)}],options),close:async()=>{}})});`); + await writeFile(join(moduleRoot, "profile-installation.js"), `import { spawn } from 'node:child_process'; import { writeFileSync } from 'node:fs'; let token; export const assertAcpxProfileEnvironment=()=>{}; export const verifyAcpxProfileInstallation=async()=>({commandDigest:'fixture-only',openCommand:async()=>({spawn:(_args,options)=>{token=options.env.COPILOT_GITHUB_TOKEN;writeFileSync(${JSON.stringify(join(root, "last-probe-root"))},options.cwd);return spawn(token === "fixture-spawn-failure" ? "/paperclip-fixture-missing-executable" : ${JSON.stringify(process.execPath)},[${JSON.stringify(childScript)}],options);},close:async()=>{if(token==="fixture-close-failure")throw new Error("fixture lease cleanup failed");}})});`); const output = join(root, "output.json"); const result = await runProbe(root, output, "deny-write", "fixture-token-never-used-for-network"); const evidence = JSON.parse(await readFile(output, "utf8")); @@ -98,5 +98,11 @@ createInterface({input:process.stdin}).on('line', line => { assert.equal(failedEvidence.promptRequestsSent, 0); assert.equal(failedEvidence.cleanupComplete, true); assert.match(failedEvidence.failureCode, /provider_(spawn|stdin)_failure/); + const lastRoot = async () => await readFile(join(root, "last-probe-root"), "utf8"); + await assert.rejects(access(await lastRoot()), { code: "ENOENT" }); + await assert.rejects(runProbe(root, join(root, "missing/report.json"), "deny-write", "fixture-token-never-used-for-network"), { code: "ENOENT" }); + await assert.rejects(access(await lastRoot()), { code: "ENOENT" }); + await assert.rejects(runProbe(root, join(root, "close-failure.json"), "deny-write", "fixture-close-failure"), /fixture lease cleanup failed/); + await assert.rejects(access(await lastRoot()), { code: "ENOENT" }); } finally { await rm(root, { recursive: true, force: true }); } }); diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index a814abca98..beecf047b0 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -7,7 +7,7 @@ import { describe, expect, it, vi } from "vitest"; import { createCapabilityFixtureState } from "../mock-core/capability-control-plane-types.js"; import { parseNativeRuntimeContext } from "../contracts/runtime-context.js"; -import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js"; + import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js"; import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js"; import { From 32b95093902027578db0badb3b34d4295fd18adc Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 8 Oct 2026 08:08:57 -0500 Subject: [PATCH 2/2] Preserve repository credentials and Copilot authentication failures Co-Authored-By: Paperclip --- server/src/__tests__/ai-connections.test.ts | 16 ++++++++++++++++ .../copilot-connection-probe.test.ts | 19 +++++++++++++++++++ server/src/services/ai-connection-runtime.ts | 3 +-- .../src/services/copilot-connection-probe.ts | 17 ++++++++++++++--- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index 1a43c627b1..f662f76e0c 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -106,6 +106,22 @@ describe("managed AI connections", () => { } finally { probe.mockRestore(); } }); + it("preserves repository GitHub credentials for managed non-Copilot execution", async () => { + const owner = "repo-github-owner"; + await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); + await create(owner, owner); + for (const configured of [{ GH_TOKEN: "repo-gh-fixture", GITHUB_TOKEN: "repo-github-fixture" }, {}]) { + const runtime = await prepareManagedAiRuntime(db, { ...input, responsibleUserId: owner, config: { env: configured } }); + try { + expect(runtime.config.env).toMatchObject({ COPILOT_GITHUB_TOKEN: "", ANTHROPIC_API_KEY: `fixture-${owner}` }); + for (const key of ["GH_TOKEN", "GITHUB_TOKEN"] as const) { + if (key in configured) expect(runtime.config.env?.[key]).toBe(configured[key as keyof typeof configured]); + else expect(runtime.config.env).not.toHaveProperty(key); + } + } finally { await runtime.cleanup(); } + } + }); + it("uses a vaulted Copilot token with GitHub attribution and refuses revoked or foreign grants", async () => { const owner = "copilot-owner"; const token = "github_pat_fixture_copilot"; diff --git a/server/src/__tests__/copilot-connection-probe.test.ts b/server/src/__tests__/copilot-connection-probe.test.ts index 95bceb7e9d..2fbd55f1c9 100644 --- a/server/src/__tests__/copilot-connection-probe.test.ts +++ b/server/src/__tests__/copilot-connection-probe.test.ts @@ -4,10 +4,12 @@ import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; const mocks = vi.hoisted(() => ({ metadata: vi.fn(), command: vi.fn(), environment: vi.fn(), bindings: vi.fn(), acquire: vi.fn(), realize: vi.fn(), release: vi.fn(), resolve: vi.fn(), general: vi.fn(), experimental: vi.fn(), defaults: vi.fn(), kubernetes: vi.fn(), managed: vi.fn() })); vi.mock("../vendor/paperclip-runner/live/index.js", async importOriginal => ({ ...await importOriginal(), probeCopilotMetadata: mocks.metadata })); vi.mock("@paperclipai/adapter-utils/execution-target", () => ({ runAdapterExecutionTargetShellCommand: mocks.command })); +vi.mock("../middleware/logger.js", () => ({ logger: { warn: vi.fn() } })); vi.mock("../services/environments.js", () => ({ environmentService: () => ({ getById: mocks.environment, listBoundCompanyIds: mocks.bindings, findKubernetesEnvironment: mocks.kubernetes, findManagedSandboxEnvironment: mocks.managed }) })); vi.mock("../services/instance-settings.js", () => ({ instanceSettingsService: () => ({ get: mocks.defaults, getGeneral: mocks.general, getExperimental: mocks.experimental }) })); vi.mock("../services/environment-runtime.js", () => ({ environmentRuntimeService: vi.fn(() => ({ acquireRunLease: mocks.acquire, realizeWorkspace: mocks.realize, getDriver: () => ({ releaseRunLease: mocks.release }) })) })); vi.mock("../services/environment-execution-target.js", () => ({ resolveEnvironmentExecutionTarget: mocks.resolve })); +import { logger } from "../middleware/logger.js"; import { probeCopilotConnection, probeCopilotExecutionTarget } from "../services/copilot-connection-probe.js"; import { QUALIFIED_ACPX_PROFILES } from "../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; import type { Db } from "@paperclipai/db"; @@ -46,6 +48,23 @@ describe("Copilot connection verification", () => { const error = await probeCopilotConnection({} as Db, "company", "private-token", "selected").catch(e => e); expect(error.details.code).toBe("COPILOT_AUTH_REQUIRED"); expect(error.message).not.toContain("private-token"); expect(mocks.release).toHaveBeenCalledOnce(); }); + it("preserves authentication classification when lease cleanup also fails", async () => { + mocks.command.mockResolvedValue({ timedOut: false, stdout: JSON.stringify({ status: "failed", code: "COPILOT_AUTH_REQUIRED", promptSent: false, message: "private-token" }), exitCode: 1 }); + mocks.release.mockRejectedValue(new Error("cleanup private-token")); + const error = await probeCopilotConnection({} as Db, "company", "private-token", "selected").catch(e => e); + expect(error).toMatchObject({ status: 422, details: { code: "COPILOT_AUTH_REQUIRED" } }); + expect(error.message).not.toContain("private-token"); + expect(logger.warn).toHaveBeenCalledWith({ companyId: "company", environmentId: "selected", leaseId: "lease", code: "COPILOT_PROBE_CLEANUP_FAILED" }, expect.any(String)); + expect(JSON.stringify(vi.mocked(logger.warn).mock.calls)).not.toContain("private-token"); + expect(mocks.release).toHaveBeenCalledOnce(); + }); + it("does not verify a connection when its successful probe cannot clean up", async () => { + mocks.release.mockRejectedValue(new Error("cleanup private-token")); + const error = await probeCopilotConnection({} as Db, "company", "private-token", "selected").catch(e => e); + expect(error).toMatchObject({ status: 422, details: { code: "COPILOT_REQUEST_FAILED", cleanupCode: "COPILOT_PROBE_CLEANUP_FAILED" } }); + expect(error.message).not.toContain("private-token"); + expect(mocks.release).toHaveBeenCalledOnce(); + }); it("rejects stale profile results and unavailable model selections", async () => { mocks.command.mockResolvedValue({ timedOut: false, stdout: JSON.stringify({ ...verified, profileDigest: "stale" }), exitCode: 0 }); await expect(probeCopilotExecutionTarget("private-token", remote)).rejects.toMatchObject({ details: { code: "COPILOT_REQUEST_FAILED" } }); diff --git a/server/src/services/ai-connection-runtime.ts b/server/src/services/ai-connection-runtime.ts index 5c8029ac5f..06c8627fee 100644 --- a/server/src/services/ai-connection-runtime.ts +++ b/server/src/services/ai-connection-runtime.ts @@ -27,8 +27,6 @@ export function isAiConnectionBusy(error: unknown): error is HttpError { // Blank values intentionally override inherited credentials in CLI child environments. export const AI_AUTH_ENV_KEYS = [ "COPILOT_GITHUB_TOKEN", - "GH_TOKEN", - "GITHUB_TOKEN", "PAPERCLIP_AI_PROVIDER_KEY", "PAPERCLIP_AI_PROVIDER_URL", "PAPERCLIP_CODEX_PROVIDERS", "GEMINI_API_KEY", "GOOGLE_API_KEY", "GOOGLE_GEMINI_BASE_URL", "GOOGLE_GENAI_USE_VERTEXAI", "HERMES_HOME", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_BEARER_TOKEN_BEDROCK", "AWS_PROFILE", "AWS_DEFAULT_PROFILE", "AWS_SHARED_CREDENTIALS_FILE", "AWS_CONFIG_FILE", "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_CREDENTIALS_FULL_URI", @@ -312,6 +310,7 @@ export async function prepareManagedAiRuntime( ...stripAiAuthBindings(input.config.env), ...Object.fromEntries(AI_AUTH_ENV_KEYS.map((key) => [key, ""])), ...managedAiHomeEnvironment(home), + ...(input.binding.provider === "github" ? { GH_TOKEN: "", GITHUB_TOKEN: "" } : {}), }; const capability = AI_CONNECTION_CAPABILITIES[input.binding.provider].methods[ diff --git a/server/src/services/copilot-connection-probe.ts b/server/src/services/copilot-connection-probe.ts index c75454f303..12860f67c7 100644 --- a/server/src/services/copilot-connection-probe.ts +++ b/server/src/services/copilot-connection-probe.ts @@ -8,6 +8,7 @@ import { instanceSettingsService } from "./instance-settings.js"; import { resolveEnvironmentExecutionTarget } from "./environment-execution-target.js"; import { assertEnvironmentSelectionForCompany } from "../routes/environment-selection.js"; import { buildLoginLeaseAcquireArgs } from "./adapter-login-lease.js"; +import { logger } from "../middleware/logger.js"; import { forbidden, unprocessable } from "../errors.js"; export async function probeCopilotConnection(db: Db, companyId: string, token: string, suppliedEnvironmentId?: string | null, model?: string, runtimeOptions: Parameters[1] = {}) { @@ -33,6 +34,7 @@ export async function probeCopilotConnection(db: Db, companyId: string, token: s const runtime = environmentRuntimeService(db, runtimeOptions); let lease: Awaited> | undefined; let succeeded = false; + let probeFailed = false; try { if (environment.driver !== "ssh") { lease = await runtime.acquireRunLease(buildLoginLeaseAcquireArgs({ metadata: { companyId, environment, adapterType: "paperclip_runner" }, assertCompanyBinding: true, requestedExpiresAt: new Date(Date.now() + 90_000) })); @@ -44,11 +46,20 @@ export async function probeCopilotConnection(db: Db, companyId: string, token: s const verified = await probeCopilotExecutionTarget(token, target, model); succeeded = true; return verified; + } catch (error) { + probeFailed = true; + throw error; } finally { if (lease) { - const driver = runtime.getDriver(environment.driver); - if (!driver) throw new Error("Copilot verification lease cleanup driver is unavailable"); - await driver.releaseRunLease({ environment, lease: lease.lease, status: succeeded ? "released" : "failed" }); + try { + const driver = runtime.getDriver(environment.driver); + if (!driver) throw new Error("Copilot verification lease cleanup driver is unavailable"); + await driver.releaseRunLease({ environment, lease: lease.lease, status: succeeded ? "released" : "failed" }); + } catch { + // Report only owned identifiers; driver errors may contain credentials. + logger.warn({ companyId, environmentId: environment.id, leaseId: lease.lease.id, code: "COPILOT_PROBE_CLEANUP_FAILED" }, "Copilot verification lease cleanup failed"); + if (!probeFailed) throw unprocessable("Copilot verification could not release its execution environment.", { code: "COPILOT_REQUEST_FAILED", cleanupCode: "COPILOT_PROBE_CLEANUP_FAILED" }); + } } } }