diff --git a/doc/execution-github-identity.md b/doc/execution-github-identity.md index 8fcf22ccb00..867e873b2b2 100644 --- a/doc/execution-github-identity.md +++ b/doc/execution-github-identity.md @@ -56,7 +56,7 @@ including the native input required by project creation authorization. Native runners retain a session-owned broker and launcher path across warm turns. The provider keeps an opaque transport token, not a GitHub credential or the previous run's signed capability. After acquiring exclusive session ownership, the controller binds the broker to the current company, agent, task, and run. Requests cannot choose another run or responsible person. The broker rejects requests while idle and discards credential responses if their run binding changed during acquisition. Each operation still rechecks the live run, accepted identity, grants, and trust policy through the shared credential resolver. Changing run IDs alone no longer replaces the provider process; changes to authentication mode, provider credentials, permissions, or other session configuration retain their existing retirement rules. -The session broker listens only on controller loopback and accepts only its authenticated GitHub credential operation. Remote executions reach it through the existing authenticated callback bridge. Its transport and launchers are retired with the provider session. If remote bridge startup fails, anonymous launchers keep ordinary work available; the next run retries setup with a fresh session. Controller restart/cold recovery still uses the existing checkpoint and process-recovery rules; the broker's in-memory authority is not persisted for adoption. +The session broker listens only on controller loopback and accepts only its authenticated GitHub credential operation. Remote executions reach it through the existing authenticated callback bridge. Its transport and launchers are retired with the provider session. If remote bridge startup fails, anonymous launchers keep ordinary work available; the next run retries setup with a fresh session. Controller restart/cold recovery still uses the existing checkpoint and process-recovery rules; the broker's in-memory authority is not persisted for adoption. A remote launcher reads its broker file again while it retries an unreachable broker, for up to 60 seconds. A command that runs during a controller restart thus uses the replacement broker when the restored controller publishes it. A removed broker file ends the wait at once. The native GitHub callback bridge uses HTTP/2 only when the server-selected `enableSandboxDuplexBridge` flag and the sandbox's verified duplex capability diff --git a/packages/adapter-utils/src/github-launcher.test.ts b/packages/adapter-utils/src/github-launcher.test.ts index 3bc2f4dbfd7..de8bc7ebc22 100644 --- a/packages/adapter-utils/src/github-launcher.test.ts +++ b/packages/adapter-utils/src/github-launcher.test.ts @@ -440,3 +440,89 @@ it("captures the current task broker file once per invocation and never falls ba expect((await run()).stdout).toBe("anonymous"); expect(captured).toEqual(["Bearer first-run", "Bearer next-run"]); }); + +// A control-plane restart stops the task's broker while its provider keeps +// running. The replacement broker is published to broker.json a few seconds later. +// An HTTP/2 gateway can outlive its controller and answer 503 instead of refusing. +// The replacement file can also be read while it is only partly written. +it.each(["closed port", "bridge unavailable", "partly written broker file"])( + "waits for a replacement task broker instead of running without credentials (%s)", async (outage) => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-task-github-outage-")); + cleanups.push(() => rm(root, { recursive: true, force: true })); + const bin = path.join(root, "managed"), real = path.join(root, "real"); + await mkdir(bin); await mkdir(real); + await writeFile(path.join(bin, "gh"), githubLauncherSource({ brokerFile: true }), { mode: 0o700 }); + await writeFile(path.join(real, "gh"), '#!/usr/bin/env node\nprocess.stdout.write(process.env.GH_TOKEN || "anonymous");\n', { mode: 0o700 }); + const stopped = createServer((_req, res) => { + res.writeHead(503, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "bridge_unavailable" })); + }); + await new Promise(resolve => stopped.listen(0, "127.0.0.1", resolve)); + const stoppedUrl = `http://127.0.0.1:${(stopped.address() as { port: number }).port}`; + if (outage === "bridge unavailable") cleanups.push(() => new Promise(resolve => stopped.close(() => resolve()))); + else await new Promise(resolve => stopped.close(() => resolve())); + const replacement = createServer((req, res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ status: "available", env: { GH_TOKEN: req.headers.authorization!.slice(7) + "-credential" } })); + }); + await new Promise(resolve => replacement.listen(0, "127.0.0.1", resolve)); + cleanups.push(() => new Promise(resolve => replacement.close(() => resolve()))); + const replacementUrl = `http://127.0.0.1:${(replacement.address() as { port: number }).port}`; + await writeFile(path.join(bin, "broker.json"), JSON.stringify({ url: stoppedUrl, token: "stopped-run" })); + const operation = exec(path.join(bin, "gh"), [], { env: { ...process.env, PATH: `${bin}:${real}:${process.env.PATH}` } }); + if (outage === "partly written broker file") { + await new Promise(resolve => setTimeout(resolve, 1000)); + await writeFile(path.join(bin, "broker.json"), '{"url":'); + } + await new Promise(resolve => setTimeout(resolve, outage === "partly written broker file" ? 2000 : 3000)); + await writeFile(path.join(bin, "broker.json"), JSON.stringify({ url: replacementUrl, token: "reattached-run" })); + const result = await operation; + expect(result.stdout).toBe("reattached-run-credential"); + expect(result.stderr).not.toContain("continuing without managed credentials"); + expect(result.stderr).not.toMatch(/stopped-run|reattached-run/); +}, 20_000); + +// A failed replacement bridge publishes empty broker values instead of removing the file. +it.each(["removed", "emptied"])("stops waiting for an unreachable task broker once its broker file is %s", async (ending) => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-task-github-ended-")); + cleanups.push(() => rm(root, { recursive: true, force: true })); + const bin = path.join(root, "managed"), real = path.join(root, "real"); + await mkdir(bin); await mkdir(real); + await writeFile(path.join(bin, "gh"), githubLauncherSource({ brokerFile: true }), { mode: 0o700 }); + await writeFile(path.join(real, "gh"), '#!/usr/bin/env node\nprocess.stdout.write(process.env.GH_TOKEN || "anonymous");\n', { mode: 0o700 }); + const stopped = createServer(); + await new Promise(resolve => stopped.listen(0, "127.0.0.1", resolve)); + const stoppedUrl = `http://127.0.0.1:${(stopped.address() as { port: number }).port}`; + await new Promise(resolve => stopped.close(() => resolve())); + await writeFile(path.join(bin, "broker.json"), JSON.stringify({ url: stoppedUrl, token: "ended-run" })); + const started = Date.now(); + const operation = exec(path.join(bin, "gh"), [], { env: { ...process.env, PATH: `${bin}:${real}:${process.env.PATH}` } }); + await new Promise(resolve => setTimeout(resolve, 2500)); + if (ending === "removed") await rm(path.join(bin, "broker.json")); + else await writeFile(path.join(bin, "broker.json"), JSON.stringify({ url: "", token: "" })); + const result = await operation; + expect(result.stdout).toBe("anonymous"); + expect(result.stderr).toContain("broker_transport_unavailable"); + expect(Date.now() - started).toBeLessThan(10_000); +}, 20_000); + +it.each([401, 403])("does not wait when the task broker denies the capability (%s)", async (status) => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-task-github-denied-")); + cleanups.push(() => rm(root, { recursive: true, force: true })); + const bin = path.join(root, "managed"), real = path.join(root, "real"); + await mkdir(bin); await mkdir(real); + await writeFile(path.join(bin, "gh"), githubLauncherSource({ brokerFile: true }), { mode: 0o700 }); + await writeFile(path.join(real, "gh"), '#!/usr/bin/env node\nprocess.stdout.write(process.env.GH_TOKEN || "anonymous");\n', { mode: 0o700 }); + let requests = 0; + const server = createServer((_req, res) => { requests++; res.writeHead(status); res.end(); }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + cleanups.push(() => new Promise(resolve => server.close(() => resolve()))); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + await writeFile(path.join(bin, "broker.json"), JSON.stringify({ url, token: "denied-run" })); + const started = Date.now(); + const result = await exec(path.join(bin, "gh"), [], { env: { ...process.env, PATH: `${bin}:${real}:${process.env.PATH}` } }); + expect(result.stdout).toBe("anonymous"); + expect(result.stderr).toContain("capability_rejected"); + expect(requests).toBe(1); + expect(Date.now() - started).toBeLessThan(3000); +}); diff --git a/packages/adapter-utils/src/github-launcher.ts b/packages/adapter-utils/src/github-launcher.ts index c0da0d578ca..88583516326 100644 --- a/packages/adapter-utils/src/github-launcher.ts +++ b/packages/adapter-utils/src/github-launcher.ts @@ -6,6 +6,10 @@ export function githubLauncherSource(options: { brokerFile?: boolean; interprete || !isAbsolute(options.interpreterPath) || /[\s\0]/u.test(options.interpreterPath))) { throw new Error("GitHub launcher interpreter must be an absolute path without whitespace"); } + // A controller restart stops a task's broker while its provider keeps running, + // then publishes a replacement to broker.json. Only that file can change, so + // only broker-file launchers wait for it; an inherited URL never recovers. + const brokerRecoveryMs = options.brokerFile ? 60_000 : 0; return String.raw`#!${options.interpreterPath ?? "/usr/bin/env node"} const fs = require('node:fs'); const path = require('node:path'); @@ -58,14 +62,22 @@ ${options.brokerFile ? String.raw` delete env.PAPERCLIP_GITHUB_BRIDGE_TOKEN; env.PAPERCLIP_GITHUB_BROKER_URL = ''; env.PAPERCLIP_GITHUB_BROKER_TOKEN = ''; - try { - const broker = JSON.parse(fs.readFileSync(path.join(directory, 'broker.json'), 'utf8')); - if (typeof broker.url === 'string' && typeof broker.token === 'string') { - env.PAPERCLIP_GITHUB_BROKER_URL = broker.url; - env.PAPERCLIP_GITHUB_BROKER_TOKEN = broker.token; - } - } catch {} -` : ''} + // An unreadable file (for example during a rewrite) keeps the last location. + // Only a missing file, or one without a usable broker, means the task's access ended. + const readBroker = () => { + let broker; + try { broker = JSON.parse(fs.readFileSync(path.join(directory, 'broker.json'), 'utf8')); } + catch (error) { return error && error.code === 'ENOENT' ? 'absent' : 'unreadable'; } + // A failed replacement bridge publishes empty values, which also means no broker. + if (!broker || typeof broker.url !== 'string' || typeof broker.token !== 'string' || !broker.url || !broker.token) return 'absent'; + env.PAPERCLIP_GITHUB_BROKER_URL = broker.url; + env.PAPERCLIP_GITHUB_BROKER_TOKEN = broker.token; + return 'present'; + }; + readBroker(); +` : String.raw` + const readBroker = () => 'present'; +`} const diagnostic = (code) => process.stderr.write('Paperclip: GitHub ' + code + '; continuing without managed credentials.\n'); let connected = false; const configRoot = env.GH_CONFIG_DIR || os.tmpdir(); @@ -102,20 +114,30 @@ ${options.brokerFile ? String.raw` try { let response; if (askBroker && base && env.PAPERCLIP_GITHUB_BROKER_TOKEN) { - const url = base.replace(/\/+$/, '').replace(/\/api$/, '') + '/runtime-tools/github/credentials'; // A slow or restarting control plane must not cost the operation its // managed identity, so a failed request is retried before giving up. // Busy (409) responses and transport failures keep separate budgets, and // the body is read inside the retry so a failed read is retried too. + // Transport failures and 503 responses retry until the recovery deadline, + // reading the broker location again so a replacement is used once published. + // Authentication denials (401/403) still end the request at once. + const recoveryDeadline = Date.now() + ${brokerRecoveryMs}; let transportFailures = 0, conflicts = 0, result; for (;;) { try { + const url = (env.PAPERCLIP_GITHUB_BROKER_URL || env.PAPERCLIP_API_URL).replace(/\/+$/, '').replace(/\/api$/, '') + '/runtime-tools/github/credentials'; response = await fetch(url, { method: 'POST', redirect: 'error', signal: AbortSignal.timeout(10000), headers: { authorization: 'Bearer ' + (env.PAPERCLIP_GITHUB_BRIDGE_TOKEN || env.PAPERCLIP_API_KEY || env.PAPERCLIP_GITHUB_BROKER_TOKEN), 'x-paperclip-github-capability': env.PAPERCLIP_GITHUB_BROKER_TOKEN, 'content-type': 'application/json' }, body: '{}', }); + // A sandbox gateway that outlived its controller answers 503 until + // the replacement broker is published, so it waits like a refusal. + if (response.status === 503 && Date.now() < recoveryDeadline) { + await response.arrayBuffer().catch(() => {}); + throw new Error('broker unavailable'); + } if (response.status === 409 && conflicts < 29) { conflicts += 1; await response.arrayBuffer(); @@ -126,8 +148,14 @@ ${options.brokerFile ? String.raw` break; } catch (error) { transportFailures += 1; - if (transportFailures >= 3) throw error; - await new Promise(resolve => setTimeout(resolve, 500 * transportFailures)); + if (transportFailures >= 3 && Date.now() >= recoveryDeadline) throw error; + if (transportFailures === 3) { + process.stderr.write('Paperclip: GitHub credential service is not reachable; waiting up to ' + + Math.ceil((recoveryDeadline - Date.now()) / 1000) + ' seconds for it to return.\n'); + } + await new Promise(resolve => setTimeout(resolve, Math.min(500 * transportFailures, 2000))); + // A removed broker file means the task's access ended; nothing will return. + if (readBroker() === 'absent') throw error; } } if (!response.ok) {