From 2a19f3721de94111d63e0bfaab00589953c0c083 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 12 Oct 2026 16:14:34 -0500 Subject: [PATCH] fix(runner): close review gaps in setup and platform qualification Read inert exact-source target data before selecting hosted native jobs. Bound registration teardown and retain unconfirmed recovery state. Record company-scoped sandbox platform facts only for immutable OCI images after confirmed cleanup; preserve existing connection and runner behavior. Co-Authored-By: Paperclip --- .github/workflows/release-verify.yml | 35 +++++- doc/plans/2026-10-06-default-agent-runner.md | 59 ++++++++++ packages/paperclip-runner/README.md | 10 ++ .../release-runner-targets.json | 4 + .../scripts/stage-release-runner-binaries.mjs | 14 ++- .../scripts/stage-runner-binary.test.mjs | 14 ++- .../src/live/runnerd-codex-transport.test.ts | 87 +++++++++++++++ .../src/live/runnerd-codex-transport.ts | 45 ++++++-- .../release-verify-workflow.test.mjs | 95 ++++++++++++---- .../environment-instance-routes.test.ts | 10 ++ .../src/__tests__/environment-probe.test.ts | 93 +++++++++++++++- server/src/routes/environments.ts | 16 +++ server/src/services/environment-config.ts | 4 + .../environment-custom-image-runtime.ts | 7 +- server/src/services/environment-probe.ts | 102 ++++++++++++++++-- server/src/services/environments.ts | 17 +++ 16 files changed, 568 insertions(+), 44 deletions(-) create mode 100644 packages/paperclip-runner/release-runner-targets.json diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 96402b6abe..b1c3d61694 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -544,14 +544,41 @@ jobs: [[ "$(git rev-parse HEAD)" == "$SOURCE_SHA" ]] || exit 1 node --input-type=module <<'NODE' import assert from 'node:assert/strict'; + import { execFileSync } from 'node:child_process'; import { appendFileSync } from 'node:fs'; - import { RELEASE_RUNNER_TARGETS } from './packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs'; + const source = process.env.SOURCE_SHA; + const readBlob = file => { + const entries = execFileSync('git', ['ls-tree', '-z', source, '--', file], { + encoding: 'utf8', maxBuffer: 4096, + }).split('\0').filter(Boolean); + if (!entries.length) return undefined; + assert.equal(entries.length, 1, 'Ambiguous exact-source target contract'); + assert.ok(entries[0].startsWith('100644 blob ') && entries[0].endsWith('\t' + file), 'Target contract must be a regular source file'); + return execFileSync('git', ['show', source + ':' + file], { encoding: 'utf8', maxBuffer: 64 * 1024 }); + }; + const data = readBlob('packages/paperclip-runner/release-runner-targets.json'); + let declared; + if (data !== undefined) { + const contract = JSON.parse(data); + assert.ok(contract && contract.schema === 'paperclip.runner.release-targets.v1' + && Object.keys(contract).sort().join() === 'schema,targets', 'Unknown exact-source daemon platform contract'); + declared = contract.targets; + } else { + // Frozen pre-contract sources used a literal export. Read that one + // declaration as JSON data; never import or evaluate preview code. + const legacy = readBlob('packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs'); + const declarations = [...(legacy ?? '').matchAll(/^[ \t]*export const RELEASE_RUNNER_TARGETS = (?:Object\.freeze\()?(\[[^\r\n]*\])\)?;[ \t]*$/gm)]; + assert.equal(declarations.length, 1, 'Unknown exact-source daemon platform contract'); + declared = JSON.parse(declarations[0][1]); + } const historical = ['darwin-arm64', 'darwin-x64', 'linux-x64']; const current = [...historical, 'linux-arm64']; - const targets = [...RELEASE_RUNNER_TARGETS].sort(); - assert.ok(JSON.stringify(targets) === JSON.stringify(historical.sort()) || JSON.stringify(targets) === JSON.stringify(current.sort()), 'Unknown exact-source daemon platform contract'); + assert.ok(Array.isArray(declared), 'Unknown exact-source daemon platform contract'); + const matches = expected => JSON.stringify([...declared].sort()) === JSON.stringify([...expected].sort()); + assert.ok(matches(historical) || matches(current), 'Unknown exact-source daemon platform contract'); + const targets = matches(current) ? current : historical; const hosts = { 'linux-x64': targets.length === current.length ? 'ubuntu-24.04' : 'ubuntu-latest', 'linux-arm64': 'ubuntu-24.04-arm', 'darwin-arm64': 'macos-15', 'darwin-x64': 'macos-15-intel' }; - const include = RELEASE_RUNNER_TARGETS.map(target => ({ target, host: hosts[target] })); + const include = targets.map(target => ({ target, host: hosts[target] })); appendFileSync(process.env.GITHUB_OUTPUT, `matrix=${JSON.stringify({ include })}\nnative_consumers=${JSON.stringify({ include: include.filter(row => row.target !== 'linux-x64') })}\n`); NODE diff --git a/doc/plans/2026-10-06-default-agent-runner.md b/doc/plans/2026-10-06-default-agent-runner.md index 0e00c147a0..99feb3ddb9 100644 --- a/doc/plans/2026-10-06-default-agent-runner.md +++ b/doc/plans/2026-10-06-default-agent-runner.md @@ -2,6 +2,65 @@ Updated 2026-10-12. +## Final review checkpoint: shared fixes and sandbox qualification + +The user authorizes normal sequential merges after the applicable gates pass. +Runtime prerequisite [#16028](https://github.com/paperclipai/paperclip/pull/16028) +and Codex defaults [#15683](https://github.com/paperclipai/paperclip/pull/15683) +remain draft. Their pre-fix sources are `00ac509b0a` and `8de43be4cd`. +The shared SSH argv and verified cold-resume corrections are already composed. +Six argv controls, fourteen recovery controls, and actual SSH subprocess checks +pass. The separate Claude owner reports real ARM and Linux x64 task/follow-up +pairs across controller restarts on its literal installed-package overlays; +those results do not qualify an unmodified final Codex package. + +The Mac test-drive serves `8de43be4cd` at port 3125. A read-only restart comparison +preserves four full legacy/native execution configurations and two historical +native runs with the deprecated native flag false. Zero provider turns and +zero local native/Docker builds were used. The canonical server no-emit passes +on the composed root and actual prerequisite worktree. + +Both shared fixes are implemented and frozen. The actual workflow selector +rejects candidate execution/output injection in real Git fixtures. Its full +existing release-workflow suite passes 70 tests; existing target assembly passes +7 tests. Registration passes 15 focused controls and canonical runner no-emit. +An independent scoped review confirms bounded teardown and retained private +state on unconfirmed release. The first broader local workflow attempt failed +only because mock loopback listeners were sandbox-blocked; its loopback-enabled +rerun passes. No daemon/provider execution was used by these controls. + +The generic sandbox qualification support passes 30 existing probe and route +controls plus the canonical server no-emit command. It accepts only OCI digest +images, checks the returned cleanup receipt, and reports conditional-update races +without claiming a record. Mutable or name-only configurations retain existing +connection behavior and receive no native compatibility credit. The default +consumer and explicit-permission/import controls are being composed next. + +Fresh review found four source defects: candidate-code execution during CI runner +selection, unbounded registration teardown, a lost explicit Codex permission +setting on automatic Legacy selection, and missing sandbox platform/libc facts. +Existing subagents own narrow fixes in parallel. Use inert target data and +workflow-owned public labels; preserve primary failures and private state on +unconfirmed cleanup; preserve explicit permissions; and use server-measured +sandbox qualification bound to its effective image/config and company scope. +The existing explicit environment-probe/setup flow owns qualification. Saving +and discovery must not provision resources or accept client-authored evidence. +Unknown targets retain automatic Legacy with an actionable explicit-native error. +No new product API or database migration is planned. + +Preview runs [38303434211](https://github.com/paperclipai/paperclip/actions/runs/38303434211), +[38304889308](https://github.com/paperclipai/paperclip/actions/runs/38304889308), and +[38306885592](https://github.com/paperclipai/paperclip/actions/runs/38306885592) +were cancelled as shared source defects changed their candidates. They are +excluded from final qualification. Preserve historical evidence and all cost +and cleanup holds. The next action is to finish the four fixes, run affected +controls, obtain fresh exact-source checks/review, then build the final immutable +preview and matching DB/shared pair remotely for the pinned QA tenant. +Managed Codex retains its $2.50 setup/task envelope and $2 cleanup within the +original $250 ceiling. Claude retains its separate $50 ceiling. No merge or +production deployment has occurred. + + ## Latest checkpoint: runtime prerequisite and authorized merges The user authorizes preparing and merging ready PRs in dependency order. diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index 09fce4eb82..675d199435 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -203,6 +203,16 @@ Apple Silicon and Intel hosts. Actual task and follow-up acceptance is performed on Apple Silicon; Intel qualification covers build, installation, and protocol startup. +For sandbox environments, the existing explicit environment probe can measure +OS, architecture and libc from its allocated lease. It records those facts only +for an image pinned with `@sha256`, after confirmed lease cleanup. The record is +server-owned and bound to the company, effective provider/image configuration +and active custom-image identity. Creation and discovery read this record without +allocating a sandbox. Configuration or image changes invalidate it. Mutable tags, +provider defaults and name-only snapshots do not establish compatibility; they +keep automatic Legacy selection until there is verifiable immutable image evidence. +A connection-only success does not establish native readiness. + On Linux, Codex's sandboxed filesystem modes also require the host to permit its unprivileged namespaces. A no-privilege Docker container can prohibit nested namespace creation even when installation and binary compatibility pass. diff --git a/packages/paperclip-runner/release-runner-targets.json b/packages/paperclip-runner/release-runner-targets.json new file mode 100644 index 0000000000..8ffbc7ffa6 --- /dev/null +++ b/packages/paperclip-runner/release-runner-targets.json @@ -0,0 +1,4 @@ +{ + "schema": "paperclip.runner.release-targets.v1", + "targets": ["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"] +} diff --git a/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs b/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs index c36e79e2aa..61bdd2c2ac 100644 --- a/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs +++ b/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs @@ -9,7 +9,19 @@ import { fileURLToPath } from "node:url"; import { runnerBinaryTarget } from "../src/live/runner-binary.ts"; import { verifyReleaseProviderPack } from "./release-provider-pack.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -export const RELEASE_RUNNER_TARGETS = Object.freeze(["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"]); +export function releaseRunnerTargets(contract) { + const supported = ["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"]; + if (!contract || contract.schema !== "paperclip.runner.release-targets.v1" + || Object.keys(contract).sort().join() !== "schema,targets" + || !Array.isArray(contract.targets) + || JSON.stringify([...contract.targets].sort()) !== JSON.stringify([...supported].sort())) { + throw Error("Unknown release daemon target contract"); + } + return Object.freeze([...contract.targets]); +} +export const RELEASE_RUNNER_TARGETS = releaseRunnerTargets(JSON.parse( + readFileSync(new URL("../release-runner-targets.json", import.meta.url), "utf8"), +)); const required = RELEASE_RUNNER_TARGETS; const digest = bytes => "sha256:" + createHash("sha256").update(bytes).digest("hex"); function regularFile(path, maximumSize) { diff --git a/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs b/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs index 3efc5bcf7e..9c0c565637 100644 --- a/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs +++ b/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs @@ -7,7 +7,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; import { stageBuiltRunnerBinary, stageRunnerBinary } from "./stage-runner-binary.mjs"; -import { RELEASE_RUNNER_TARGETS, releaseDaemonManifest, verifyReleaseRunnerMetadata } from "./stage-release-runner-binaries.mjs"; +import { RELEASE_RUNNER_TARGETS, releaseRunnerTargets, releaseDaemonManifest, verifyReleaseRunnerMetadata } from "./stage-release-runner-binaries.mjs"; import { paperclipRunnerBinaryNeedsBuild, resolveNativeRunnerDevBinary } from "../../../scripts/dev-runner-native-binary.mjs"; import { resolvePackagedRunnerBinary } from "../src/live/runner-binary.ts"; @@ -16,6 +16,18 @@ const contract = { package: { name: "@paperclipai/paperclip-runner", version: "0 const metadata = { binaryContractVersion: 2, nativeExecutionVersion: 1, harnessDriverVersion: 1, packageName: "@paperclipai/paperclip-runner", packageVersion: "0.0.0", prp: { name: "paperclip.runner", minimumVersion: 1, maximumVersion: 2 } }; +test("release assembly uses the inert four-target contract and rejects unexpected selection data", async () => { + const contract = JSON.parse(await readFile(new URL("../release-runner-targets.json", import.meta.url), "utf8")); + assert.deepEqual(RELEASE_RUNNER_TARGETS, ["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"]); + assert.deepEqual(releaseRunnerTargets(contract), RELEASE_RUNNER_TARGETS); + assert.ok(Object.isFrozen(RELEASE_RUNNER_TARGETS)); + for (const invalid of [null, {}, { ...contract, schema: "other" }, { ...contract, host: "trusted-fleet" }, + { ...contract, targets: contract.targets.slice(0, 3) }, { ...contract, targets: [...contract.targets, "linux-arm64"] }, + { ...contract, targets: [...contract.targets.slice(0, 3), "win32-x64"] }, { ...contract, targets: "linux-arm64" }]) { + assert.throws(() => releaseRunnerTargets(invalid), /Unknown release daemon target contract/); + } +}); + test("release assembly binds all four actual targets, source, hashes and compatible metadata", async () => { const root = await realpath(await mkdtemp(path.join(tmpdir(), "runner-release-inputs-"))), source = "a".repeat(40); const verify = value => verifyReleaseRunnerMetadata(value, contract); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index d33ff858a0..5ff210ad59 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -2716,6 +2716,93 @@ const fakeCodex = resolve( "../../runner/target/debug/fake-codex-app-server", ); +it.each(["stuck", "late-release", "late-release-error", "stuck-release", "cooperative-release", "launch-failure"] as const)( + "bounds setup registration teardown and retains unconfirmed state (%s)", + async mode => { + const root = await mkdtemp(join(tmpdir(), "runnerd-setup-registration-")); + const runtimeDirectory = join(root, "native-state"); + await mkdir(runtimeDirectory, { mode: 0o700 }); + let registrationStarted!: () => void; + const started = new Promise(resolve => { registrationStarted = resolve; }); + let returnRegistration!: (registration: { connection: { mode: "listen"; listenAddress: string; listenPort: number; listenPath: string }; release: () => Promise }) => void; + const registration = new Promise[0]>(resolve => { returnRegistration = resolve; }); + let finishRelease!: () => void; + const releasing = new Promise(resolve => { finishRelease = resolve; }); + const releaseError = new Error("setup registration release failed"); + const release = vi.fn(async () => { + if (mode === "stuck-release") await releasing; + if (mode === "late-release-error") throw releaseError; + }); + const registered = { + connection: { mode: "listen" as const, listenAddress: "127.0.0.1", listenPort: 43127, listenPath: "/setup-probe" }, + release, + }; + let registrationSignal: AbortSignal | undefined; + const cleanup = vi.fn(async () => { await rm(runtimeDirectory, { recursive: true, force: true }); }); + const launch = vi.fn(() => { throw new Error("owned launcher failed before execution"); }); + let settled = false; + vi.useFakeTimers(); + const result = probeNativeRunnerEnvironment({ + runtimeDirectory, provider: "codex", model: "gpt-test", timeoutMs: 1_000, + environment: { PATH: "", OPENAI_API_KEY: "inert-test-key" }, + transportOptions: { + // Registration and launch are external owners; hash real inert bytes + // without executing a daemon, Codex, or an ambient developer binary. + runnerBinary: resolve(import.meta.dirname, "../../package.json"), + codexCommand: resolve(import.meta.dirname, "../../package.json"), + sourceCodexHome: "", runnerProcessLauncher: launch, + controlPlaneRegistration: async (_authority, _identity, signal) => { + registrationSignal = signal; + if (mode === "cooperative-release") signal?.addEventListener("abort", () => returnRegistration(registered), { once: true }); + registrationStarted(); + return registration; + }, + }, + onCleanupConfirmed: cleanup, + }).then(value => { settled = true; return value; }, (error: unknown) => { settled = true; return error; }); + try { + await started; + if (mode === "launch-failure") returnRegistration(registered); + await vi.advanceTimersByTimeAsync(1_000); + if (mode === "late-release" || mode === "late-release-error" || mode === "stuck-release") returnRegistration(registered); + await vi.advanceTimersByTimeAsync(10_000); + if (mode === "late-release" || mode === "cooperative-release" || mode === "launch-failure") await result; + expect(settled).toBe(true); + expect(registrationSignal?.aborted).toBe(true); + const failure = await result; + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain(mode === "launch-failure" ? "owned launcher failed before execution" : "Native provider hello probe timed out."); + if (mode === "stuck" || mode === "stuck-release" || mode === "late-release-error") { + expect(failure).toBeInstanceOf(AggregateError); + expect((failure as AggregateError).cause).toMatchObject({ message: "Native provider hello probe timed out." }); + expect(cleanup).not.toHaveBeenCalled(); + expect((await stat(runtimeDirectory)).isDirectory()).toBe(true); + if (mode === "late-release-error") expect((failure as AggregateError).errors).toContain(releaseError); + // A registration that finishes after the bounded return still releases + // its authority, without retroactively certifying or deleting state. + returnRegistration(registered); + finishRelease(); + await vi.advanceTimersByTimeAsync(0); + expect(release).toHaveBeenCalledOnce(); + expect(cleanup).not.toHaveBeenCalled(); + } else { + expect(release).toHaveBeenCalledOnce(); + expect(cleanup).toHaveBeenCalledOnce(); + await expect(stat(runtimeDirectory)).rejects.toThrow(); + } + expect(launch).toHaveBeenCalledTimes(mode === "launch-failure" ? 1 : 0); + expect(vi.getTimerCount()).toBe(0); + } finally { + returnRegistration(registered); + finishRelease(); + await vi.advanceTimersByTimeAsync(0); + await result; + vi.useRealTimers(); + await rm(root, { recursive: true, force: true }); + } + }, +); + function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { return [ "--state-file", diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index ff7dc3e8d5..20c94e11cc 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1335,6 +1335,8 @@ export interface CapabilityRunnerdCodexTransportOptions { controlPlaneRegistration?: ( authority: DurablePrpControlPlane, identity?: DurableRecoveryIdentity, + /** Setup owners abort preparation before draining or releasing its result. */ + signal?: AbortSignal, ) => Promise<{ connectUrl?: string; connection?: RunnerProcessConnection; @@ -7094,6 +7096,7 @@ export async function probeNativeRunnerEnvironment(options: { const controller = new AbortController(); let registrationTeardownError: unknown; let registrationOperation: Promise | undefined; + let registrationCleanupConfirmed = true; const sourceCodexHome = options.transportOptions?.sourceCodexHome ?? resolveSourceCodexHome(environment); // Never let an unbound home or an external-sandbox flag admit host state or // weaken this read-only probe. The normal materializer copies bound auth and @@ -7107,13 +7110,20 @@ export async function probeNativeRunnerEnvironment(options: { ...(options.transportOptions?.controlPlaneRegistration ? { controlPlaneRegistration: (authority, identity) => { const operation = (async () => { controller.signal.throwIfAborted(); - const registration = await options.transportOptions!.controlPlaneRegistration!(authority, identity); + registrationCleanupConfirmed = false; + const registration = await options.transportOptions!.controlPlaneRegistration!(authority, identity, controller.signal); + let releaseOperation: Promise | undefined; + const release = () => releaseOperation ??= Promise.resolve() + .then(() => registration.release()) + .then(() => { registrationCleanupConfirmed = true; }, error => { + registrationTeardownError = error; + throw error; + }); if (controller.signal.aborted) { - try { await registration.release(); } - catch (error) { registrationTeardownError = error; throw error; } + await release(); controller.signal.throwIfAborted(); } - return registration; + return { ...registration, release }; })(); registrationOperation = operation; return operation; @@ -7208,17 +7218,34 @@ export async function probeNativeRunnerEnvironment(options: { failure = error; } finally { clearTimeout(timer); + controller.abort(failure ?? new Error("Native provider hello probe complete.")); // Preserve private recovery state if the production close cannot confirm // teardown; callers remove it only after an authoritative receipt. const teardownErrors: unknown[] = []; + const teardownDeadline = Date.now() + 10_000; + const awaitTeardown = async (pending: Promise, detail: string): Promise => { + let timeout: ReturnType | undefined; + try { + await Promise.race([ + pending, + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error(`Native setup probe cleanup timed out while ${detail}; recovery state retained.`)), Math.max(0, teardownDeadline - Date.now())); + }), + ]); + } finally { clearTimeout(timeout); } + }; try { - if (session) await session.close({ reason: "environment probe complete" }); - else await bundle.transport.close("environment probe complete"); + await awaitTeardown(session ? session.close({ reason: "environment probe complete" }) : bundle.transport.close("environment probe complete"), "closing the selected runtime"); if (bundle.evidence().runnerPid !== null && !bundle.evidence().runnerExited) throw new Error("Native setup probe cleanup is incomplete."); } catch (error) { teardownErrors.push(error); } - await operation.catch(() => undefined); - await registrationOperation?.catch(() => undefined); - if (registrationTeardownError !== undefined) teardownErrors.push(registrationTeardownError); + try { await awaitTeardown(operation.catch(() => undefined), "draining native setup"); } + catch (error) { teardownErrors.push(error); } + if (registrationOperation) { + try { await awaitTeardown(registrationOperation.catch(() => undefined), "draining authority registration"); } + catch (error) { teardownErrors.push(error); } + } + if (registrationTeardownError !== undefined && !teardownErrors.includes(registrationTeardownError)) teardownErrors.push(registrationTeardownError); + if (!registrationCleanupConfirmed) teardownErrors.push(new Error("Native setup probe authority release is unconfirmed; recovery state retained.")); if (bundle.evidence().runnerPid !== null && bundle.evidence().runnerExited && options.provider === "codex" && options.onCodexCredentialRefresh) { try { await options.onCodexCredentialRefresh(resolve(remoteFilesystemRoot ?? options.runtimeDirectory, "codex-home", "auth.json")); } catch (error) { teardownErrors.push(error); } diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 2aa415cd20..152cf00edc 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { tmpdir } from "node:os"; @@ -753,7 +753,9 @@ test("release daemon production uses exact-source native hosts and no publisher } assert.match(producer, /matrix: \$\{\{ fromJSON\(needs.runner_binary_targets.outputs.matrix\) \}\}/); assert.match(targets, /Unknown exact-source daemon platform contract/); - assert.match(targets, /RELEASE_RUNNER_TARGETS/); + assert.match(targets, /release-runner-targets\.json/); + assert.match(targets, /execFileSync\('git', \['show', source \+ ':' \+ file\]/); + assert.doesNotMatch(targets, /import .*from '\.\/|await import\(|\beval\(|new Function/); assert.doesNotMatch(targets, /secrets\.|id-token: write|contents: write|npm publish|environment:/); assert.match(producer, /persist-credentials: false/); assert.match(producer, /contents: read/); @@ -1064,39 +1066,96 @@ test("Linux ARM prerequisite setup rejects other hosts before invoking tools and } finally { rmSync(root, { recursive: true, force: true }); } }); -test("exact-source daemon matrices preserve historical three-target sources and add native ARM only for new sources", () => { +test("exact-source daemon matrices read inert data and never execute preview runner selection", () => { const workflow = readWorkflow("release-verify.yml"); const step = workflow.split(' - name: Preserve the immutable source\'s platform contract')[1].split('\n runner_binaries:')[0]; const script = step.split(' run: |\n')[1].split('\n').map(line => line.startsWith(' ') ? line.slice(10) : line).join('\n'); const root = mkdtempSync(path.join(tmpdir(), 'release-target-contract-')); try { - const tools = path.join(root, 'tools'); mkdirSync(tools); - writeFileSync(path.join(tools, 'git'), '#!/bin/sh\nprintf "%s\\n" "$FIXTURE_HEAD"\n', { mode: 0o755 }); + const environment = { + PATH: `${path.dirname(process.execPath)}:/usr/bin:/bin`, HOME: root, + GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', + }; + const git = args => { + const result = spawnSync('git', args, { cwd: root, env: environment, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return result.stdout.trim(); + }; + const template = path.join(root, 'empty-template'); mkdirSync(template); + git(['init', '-q', '--template=' + template]); const source = path.join(root, 'packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs'); + const dataFile = path.join(root, 'packages/paperclip-runner/release-runner-targets.json'); mkdirSync(path.dirname(source), { recursive: true }); - const output = path.join(root, 'outputs'); + const output = path.join(root, 'outputs'), marker = path.join(root, 'candidate-executed'); const historical = ['darwin-arm64', 'darwin-x64', 'linux-x64']; - for (const targets of [historical, [...historical, 'linux-arm64']]) { - writeFileSync(source, `export const RELEASE_RUNNER_TARGETS = ${JSON.stringify(targets)};`); + const current = [...historical, 'linux-arm64']; + const contract = targets => JSON.stringify({ schema: 'paperclip.runner.release-targets.v1', targets }); + const malicious = `import { writeFileSync } from 'node:fs';\nwriteFileSync(${JSON.stringify(marker)}, 'executed');\nwriteFileSync(process.env.GITHUB_OUTPUT, 'matrix={"include":[{"target":"linux-x64","host":"runs-on/fleet=paperclip-post-merge-x64/env=public-ci"}]}\\n');\nprocess.exit(0);\n`; + const pin = (module, data) => { + rmSync(dataFile, { force: true }); + writeFileSync(source, module); + if (data !== undefined) writeFileSync(dataFile, data); + git(['add', 'packages']); + git(['-c', 'user.name=Release fixture', '-c', 'user.email=fixture@example.invalid', '-c', 'commit.gpgsign=false', + 'commit', '-qm', 'Fixture', '--allow-empty']); + return git(['rev-parse', 'HEAD']); + }; + const run = sha => { writeFileSync(output, ''); const result = spawnSync('bash', ['-euo', 'pipefail', '-c', script], { cwd: root, encoding: 'utf8', env: { - PATH: `${tools}:${path.dirname(process.execPath)}:/usr/bin:/bin`, SOURCE_SHA: 'a'.repeat(40), FIXTURE_HEAD: 'a'.repeat(40), GITHUB_OUTPUT: output, + ...environment, SOURCE_SHA: sha, GITHUB_OUTPUT: output, } }); - assert.equal(result.status, 0, result.stderr); + assert.equal(existsSync(marker), false, 'Candidate code must never execute while selecting runners'); + return result; + }; + const assertMatrix = targets => { const data = Object.fromEntries(readFileSync(output, 'utf8').trim().split('\n').map(line => { const index = line.indexOf('='); return [line.slice(0, index), JSON.parse(line.slice(index + 1))]; })); assert.deepEqual(data.matrix.include.map(row => row.target), targets); assert.deepEqual(data.native_consumers.include.map(row => row.target), targets.filter(target => target !== 'linux-x64')); - assert.ok(data.matrix.include.every(row => row.target !== 'linux-arm64' || row.host === 'ubuntu-24.04-arm')); - assert.equal(data.matrix.include.find(row => row.target === 'linux-x64').host, targets.length === 4 ? 'ubuntu-24.04' : 'ubuntu-latest', - 'new GNU libc baseline stays pinned while historical qualification keeps its producer mode'); + const hosts = { 'linux-x64': targets.length === 4 ? 'ubuntu-24.04' : 'ubuntu-latest', + 'linux-arm64': 'ubuntu-24.04-arm', 'darwin-arm64': 'macos-15', 'darwin-x64': 'macos-15-intel' }; + for (const row of data.matrix.include) assert.equal(row.host, hosts[row.target], 'Only workflow-owned public hosts may be selected'); + }; + for (const [targets, module, data] of [ + [historical, `export const RELEASE_RUNNER_TARGETS = ${JSON.stringify(historical)};`, undefined], + [current, `${malicious}export const RELEASE_RUNNER_TARGETS = Object.freeze(${JSON.stringify(current)});`, undefined], + [historical, malicious, contract(historical)], + [current, malicious, contract(current)], + ]) { + const result = run(pin(module, data)); + assert.equal(result.status, 0, result.stderr); + assertMatrix(targets); } - writeFileSync(source, 'export const RELEASE_RUNNER_TARGETS = ["linux-arm64"];'); writeFileSync(output, ''); - const result = spawnSync('bash', ['-euo', 'pipefail', '-c', script], { cwd: root, encoding: 'utf8', env: { - PATH: `${tools}:${path.dirname(process.execPath)}:/usr/bin:/bin`, SOURCE_SHA: 'a'.repeat(40), FIXTURE_HEAD: 'a'.repeat(40), GITHUB_OUTPUT: output, - } }); - assert.equal(result.status, 1); assert.match(result.stderr, /Unknown exact-source daemon platform contract/); + const sha = pin(malicious, contract(current)); + writeFileSync(dataFile, contract(['runs-on/fleet=paperclip-post-merge-x64/env=public-ci'])); + assert.equal(run(sha).status, 0, 'Read the immutable git blob, not a changed checkout file'); + assertMatrix(current); + assert.equal(run('a'.repeat(40)).status, 1, 'Wrong checked-out source must fail before output'); + assert.equal(readFileSync(output, 'utf8'), ''); + for (const data of [ + contract(['linux-arm64']), contract([...current, 'linux-arm64']), contract([...current.slice(0, 3), 'win32-x64']), + JSON.stringify({ schema: 'paperclip.runner.release-targets.v1', targets: current, host: 'trusted-fleet' }), + JSON.stringify({ schema: 'other', targets: current }), contract('linux-arm64'), 'null', '{invalid', ' '.repeat(65 * 1024), + ]) { + const result = run(pin(`export const RELEASE_RUNNER_TARGETS = ${JSON.stringify(historical)};`, data)); + assert.notEqual(result.status, 0, 'Malformed present data must never fall back to a legacy module'); + assert.equal(readFileSync(output, 'utf8'), ''); + } + for (const module of [malicious, + 'export const RELEASE_RUNNER_TARGETS = getTargets();', + `export const RELEASE_RUNNER_TARGETS = ${JSON.stringify(current)};\nexport const RELEASE_RUNNER_TARGETS = ${JSON.stringify(historical)};`, + 'export const RELEASE_RUNNER_TARGETS = ["linux-arm64"];']) { + const result = run(pin(module, undefined)); + assert.notEqual(result.status, 0, 'Unknown legacy declarations must fail closed'); + assert.equal(readFileSync(output, 'utf8'), ''); + } + pin(malicious, undefined); + symlinkSync('scripts/stage-release-runner-binaries.mjs', dataFile); + git(['add', 'packages']); + git(['-c', 'user.name=Release fixture', '-c', 'user.email=fixture@example.invalid', '-c', 'commit.gpgsign=false', 'commit', '-qm', 'Symlink fixture']); + assert.notEqual(run(git(['rev-parse', 'HEAD'])).status, 0, 'Source contract symlinks must not be followed'); assert.equal(readFileSync(output, 'utf8'), ''); } finally { rmSync(root, { recursive: true, force: true }); } }); diff --git a/server/src/__tests__/environment-instance-routes.test.ts b/server/src/__tests__/environment-instance-routes.test.ts index 28f24be2fb..f2b26355ed 100644 --- a/server/src/__tests__/environment-instance-routes.test.ts +++ b/server/src/__tests__/environment-instance-routes.test.ts @@ -129,6 +129,16 @@ function createApp(actor: Record) { } describe("environment instance routes", () => { + it.each(["post", "patch"])("rejects client-written sandbox platform evidence on %s", async method => { + mockEnvironmentService.getById.mockResolvedValue(createEnvironment()); + const app = createApp({ type: "board", source: "local_implicit", isInstanceAdmin: true }); + const payload = { ...(method === "post" ? { name: "Fixture", driver: "local", config: {} } : {}), metadata: { paperclipSandboxPlatformProbes: { "company-1": { v: 1 } } } }; + const res = method === "post" ? await request(app).post("/api/companies/company-1/environments").send(payload) + : await request(app).patch("/api/environments/env-1").send(payload); + expect(res.status).toBe(422); + expect(res.body.details?.code ?? res.body.code).toBe("environment_platform_probe_reserved"); + expect(mockEnvironmentService.create).not.toHaveBeenCalled(); + }); beforeEach(() => { mockIssueService.clearExecutionWorkspaceEnvironmentSelection.mockReset(); mockProjectService.clearExecutionWorkspaceEnvironmentSelection.mockReset(); diff --git a/server/src/__tests__/environment-probe.test.ts b/server/src/__tests__/environment-probe.test.ts index 9a29883219..125e8838fd 100644 --- a/server/src/__tests__/environment-probe.test.ts +++ b/server/src/__tests__/environment-probe.test.ts @@ -6,12 +6,16 @@ const mockProbePluginSandboxProviderDriver = vi.hoisted(() => vi.fn()); const mockResolvePluginSandboxProviderDriverByKey = vi.hoisted(() => vi.fn()); const mockRuntimeAcquireRunLease = vi.hoisted(() => vi.fn()); const mockRuntimeReleaseRunLease = vi.hoisted(() => vi.fn()); +const mockRuntimeExecute = vi.hoisted(() => vi.fn()); +const mockRecordPlatformProbe = vi.hoisted(() => vi.fn()); const mockEnvironmentRuntimeService = vi.hoisted(() => vi.fn(() => ({ acquireRunLease: mockRuntimeAcquireRunLease, + execute: mockRuntimeExecute, getDriver: vi.fn(() => ({ releaseRunLease: mockRuntimeReleaseRunLease, })), }))); +vi.mock("../services/environments.js", () => ({ environmentService: () => ({ recordSandboxPlatformProbe: mockRecordPlatformProbe }) })); vi.mock("@paperclipai/adapter-utils/ssh", () => ({ ensureSshWorkspaceReady: mockEnsureSshWorkspaceReady, @@ -27,20 +31,105 @@ vi.mock("../services/environment-runtime.js", () => ({ environmentRuntimeService: mockEnvironmentRuntimeService, })); -import { probeEnvironment } from "../services/environment-probe.ts"; +import { probeEnvironment, readSandboxPlatformQualification, SANDBOX_PLATFORM_PROBE_COMMAND } from "../services/environment-probe.ts"; +import * as configRuntime from "../services/environment-config.js"; +import { SANDBOX_PLATFORM_PROBE_METADATA_KEY, resolveActiveEnvironmentCustomImageTemplateForRuntime } from "../services/environment-custom-image-runtime.js"; describe("probeEnvironment", () => { beforeEach(() => { + vi.restoreAllMocks(); + mockRecordPlatformProbe.mockReset().mockResolvedValue(true); + mockRuntimeExecute.mockReset(); mockEnsureSshWorkspaceReady.mockReset(); mockProbePluginEnvironmentDriver.mockReset(); mockProbePluginSandboxProviderDriver.mockReset(); mockResolvePluginSandboxProviderDriverByKey.mockReset(); mockResolvePluginSandboxProviderDriverByKey.mockResolvedValue(null); mockRuntimeAcquireRunLease.mockReset(); - mockRuntimeReleaseRunLease.mockReset(); + mockRuntimeReleaseRunLease.mockReset().mockResolvedValue({ id: "lease", companyId: "company", environmentId: "env", status: "released", cleanupStatus: "success" }); mockEnvironmentRuntimeService.mockClear(); }); + it.each(["2.38", "2.39", "musl"])("records actual immutable-image facts for %s and reads them without another lease", async libc => { + const image = `registry.example.test/runtime@sha256:${"a".repeat(64)}`; + const environment = { id: "env", driver: "sandbox", status: "active", config: { provider: "daytona", image, reuseLease: false }, metadata: null } as any; + vi.spyOn(configRuntime, "resolveEnvironmentDriverConfigForRuntime").mockResolvedValue({ driver: "sandbox", config: environment.config }); + mockRuntimeAcquireRunLease.mockResolvedValue({ lease: { id: "lease", provider: "daytona", metadata: { image, remoteCwd: "/workspace" } } }); + mockRuntimeExecute.mockResolvedValue({ exitCode: 0, stdout: `Linux\naarch64\n${libc === "musl" ? "musl" : `glibc\nglibc ${libc}`}\n` }); + const result = await probeEnvironment({} as any, environment, { companyId: "company", acquireSandboxRuntimeLease: true }); + expect(result.ok).toBe(true); + expect(result.details?.platformQualificationRecorded).toBe(true); + expect(mockRuntimeExecute).toHaveBeenCalledWith(expect.objectContaining({ command: "sh", args: ["-c", SANDBOX_PLATFORM_PROBE_COMMAND], timeoutMs: 10_000, bypassSession: true })); + const proof = mockRecordPlatformProbe.mock.calls[1]![0].proof; + const measured = { platform: "linux", architecture: "arm64", libc: libc === "musl" ? "musl" : "glibc", ...(libc === "musl" ? {} : { glibcVersion: `glibc ${libc}` }) }; + expect(proof.facts).toEqual(measured); + environment.metadata = { [SANDBOX_PLATFORM_PROBE_METADATA_KEY]: { company: proof } }; + expect(await readSandboxPlatformQualification({} as any, environment, "company")).toEqual(measured); + expect(mockRuntimeAcquireRunLease).toHaveBeenCalledTimes(1); + expect(await readSandboxPlatformQualification({} as any, environment, "foreign-company")).toBeNull(); + expect(await readSandboxPlatformQualification({} as any, { ...environment, config: { ...environment.config, image: `registry.example.test/runtime@sha256:${"b".repeat(64)}` } }, "company")).toBeNull(); + vi.mocked(configRuntime.resolveEnvironmentDriverConfigForRuntime).mockImplementation(async (_db, _company, _env, options) => { + options?.onResolvedCustomImageTemplate?.({ id: "changed-template", capturedAt: null }); + return { driver: "sandbox", config: environment.config }; + }); + expect(await readSandboxPlatformQualification({} as any, environment, "company")).toBeNull(); + expect(mockRuntimeAcquireRunLease).toHaveBeenCalledTimes(1); + }); + it.each(["measurement", "cleanup", "config-race"])("does not claim qualification after %s failure while retaining connection readiness", async failure => { + const image = `registry.example.test/runtime@sha256:${"a".repeat(64)}`; + const environment = { id: "env", driver: "sandbox", status: "active", config: { provider: "daytona", image }, metadata: null } as any; + vi.spyOn(configRuntime, "resolveEnvironmentDriverConfigForRuntime").mockResolvedValue({ driver: "sandbox", config: environment.config }); + mockRuntimeAcquireRunLease.mockResolvedValue({ lease: { id: "lease", provider: "daytona", metadata: { image } } }); + mockRuntimeExecute.mockResolvedValue({ exitCode: failure === "measurement" ? 1 : 0, stdout: "Linux\nx86_64\nglibc\nglibc 2.39\n" }); + if (failure === "cleanup") mockRuntimeReleaseRunLease.mockRejectedValueOnce(new Error("owned cleanup failed")); + if (failure === "config-race") mockRecordPlatformProbe.mockResolvedValue(false); + const result = await probeEnvironment({} as any, environment, { companyId: "company", acquireSandboxRuntimeLease: true }); + expect(result.ok).toBe(true); + expect(result.details?.platformQualificationRecorded).toBe(false); + expect(mockRecordPlatformProbe.mock.calls[0]![0].proof).toBeNull(); + if (failure !== "config-race") expect(mockRecordPlatformProbe).toHaveBeenCalledTimes(1); + }); + it.each([ + null, + { id: "lease", companyId: "company", environmentId: "env", status: "released", cleanupStatus: "failed" }, + { id: "lease", companyId: "company", environmentId: "env", status: "pending_cleanup", cleanupStatus: "failed" }, + { id: "other-lease", companyId: "company", environmentId: "env", status: "released", cleanupStatus: "success" }, + { id: "lease", companyId: "foreign-company", environmentId: "env", status: "released", cleanupStatus: "success" }, + { id: "lease", companyId: "company", environmentId: "other-environment", status: "released", cleanupStatus: "success" }, + ])("does not record platform evidence without an owned successful cleanup receipt (%j)", async receipt => { + const image = `registry.example.test/runtime@sha256:${"a".repeat(64)}`; + const environment = { id: "env", driver: "sandbox", status: "active", config: { provider: "daytona", image }, metadata: null } as any; + vi.spyOn(configRuntime, "resolveEnvironmentDriverConfigForRuntime").mockResolvedValue({ driver: "sandbox", config: environment.config }); + mockRuntimeAcquireRunLease.mockResolvedValue({ lease: { id: "lease", provider: "daytona", metadata: { image } } }); + mockRuntimeExecute.mockResolvedValue({ exitCode: 0, stdout: "Linux\nx86_64\nglibc\nglibc 2.39\n" }); + mockRuntimeReleaseRunLease.mockResolvedValue(receipt); + const result = await probeEnvironment({} as any, environment, { companyId: "company", acquireSandboxRuntimeLease: true }); + expect(result.ok).toBe(true); + expect(result.details?.platformQualificationRecorded).toBe(false); + expect(mockRecordPlatformProbe).toHaveBeenCalledTimes(1); + expect(mockRecordPlatformProbe.mock.calls[0]![0].proof).toBeNull(); + }); + it("reads an effective custom image without touching its usage timestamp", async () => { + const update = vi.fn(); + const capturedAt = new Date("2026-10-12T00:00:00Z"); + const row = { id: "template", provider: "daytona", templateKind: "image", templateRef: `registry.example.test/runtime@sha256:${"a".repeat(64)}`, capturedAt, metadata: null }; + const db = { select: () => ({ from: () => ({ where: () => ({ orderBy: () => Promise.resolve([row]) }) }) }), update }; + const identity = vi.fn(); + const result = await resolveActiveEnvironmentCustomImageTemplateForRuntime(db as never, { environmentId: "env", baseConfig: { provider: "daytona", reuseLease: false }, runtimeConfig: { provider: "daytona", reuseLease: false }, readOnly: true, onResolvedTemplate: identity }); + expect(result).toMatchObject({ image: row.templateRef }); + expect(identity).toHaveBeenCalledWith({ id: "template", capturedAt: capturedAt.toISOString() }); + expect(update).not.toHaveBeenCalled(); + }); + it("returns false when an atomic probe record loses its config comparison", async () => { + const { environmentService } = await vi.importActual("../services/environments.js"); + const returning = vi.fn().mockResolvedValue([]); + const where = vi.fn(() => ({ returning })); + const db = { update: vi.fn(() => ({ set: vi.fn(() => ({ where })) })) }; + expect(await environmentService(db as never).recordSandboxPlatformProbe({ environmentId: "env", companyId: "company", config: { provider: "daytona" }, proof: { v: 1 } })).toBe(false); + expect(where).toHaveBeenCalledOnce(); + expect(returning).toHaveBeenCalledOnce(); + }); + it("reports local environments as immediately available", async () => { const result = await probeEnvironment({} as any, { id: "env-1", diff --git a/server/src/routes/environments.ts b/server/src/routes/environments.ts index 68d51234cb..1c253b4514 100644 --- a/server/src/routes/environments.ts +++ b/server/src/routes/environments.ts @@ -57,6 +57,7 @@ import { getConfiguredSecretProvider } from "../secrets/configured-provider.js"; import { assertBoardOrgAccess, getActorInfo } from "./authz.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { environmentService } from "../services/environments.js"; +import { SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "../services/environment-custom-image-runtime.js"; import { environmentRuntimeService } from "../services/environment-runtime.js"; import { executionWorkspaceService } from "../services/execution-workspaces.js"; import { closeWarmNativeSessionsForEnvironment } from "../services/native-runtime/native-session-executor.js"; @@ -311,6 +312,9 @@ async function assertPlatformProvisionedEnvironmentWritable( * Clearing (null/false) is allowed so stale markers can be removed. */ function assertNoClientPlatformProvisionedMarkers(metadata: unknown): void { + if (isPlainRecord(metadata) && Object.prototype.hasOwnProperty.call(metadata, SANDBOX_PLATFORM_PROBE_METADATA_KEY)) { + throw unprocessable("Sandbox platform qualification is recorded only by an explicit environment probe.", { code: "environment_platform_probe_reserved" }); + } if (!isCloudManagedInstance() || !isPlainRecord(metadata)) return; for (const key of PLATFORM_PROVISIONED_MARKER_KEYS) { if (metadata[key] !== undefined && metadata[key] !== null && metadata[key] !== false) { @@ -396,6 +400,10 @@ export function environmentRoutes( envVars?: Record | null; metadata: Record | null; }>(req: Request, environment: T): T { + if (environment.metadata) { + const { [SANDBOX_PLATFORM_PROBE_METADATA_KEY]: _probe, ...metadata } = environment.metadata; + environment = { ...environment, metadata }; + } // Floor: on cloud-managed instances, platform-provisioned rows use the // floored view — credential-shaped config keys are never echoed to any // reader, structural config stays visible, and tenant env vars on the @@ -1208,6 +1216,14 @@ export function environmentRoutes( // Persist the config change and its binding rows atomically: a binding // ref that fails validation (e.g. a deleted secret) must roll the whole // save back instead of leaving the config re-pointed with stale bindings. + if (changesProviderTarget || patch.metadata !== undefined) { + const metadata = { ...(patch.metadata ?? existing.metadata ?? {}) }; + if (changesProviderTarget) delete metadata[SANDBOX_PLATFORM_PROBE_METADATA_KEY]; + else if (existing.metadata?.[SANDBOX_PLATFORM_PROBE_METADATA_KEY] !== undefined) { + metadata[SANDBOX_PLATFORM_PROBE_METADATA_KEY] = existing.metadata[SANDBOX_PLATFORM_PROBE_METADATA_KEY]; + } + patch.metadata = metadata; + } const environment = await db.transaction(async (tx) => { const updated = await svc.update(existing.id, patch, { db: tx }); if (!updated) return null; diff --git a/server/src/services/environment-config.ts b/server/src/services/environment-config.ts index d1d1dc184c..27ae4e876f 100644 --- a/server/src/services/environment-config.ts +++ b/server/src/services/environment-config.ts @@ -660,6 +660,8 @@ export async function resolveEnvironmentDriverConfigForRuntime( // but must still resolve the active custom image as prepared runtime // configuration and tooling so the test reflects what real agent runs use. applyCustomImageTemplate?: boolean; + readOnlyCustomImageTemplate?: boolean; + onResolvedCustomImageTemplate?: (identity: { id: string; capturedAt: string | null }) => void; }, ): Promise { const parsed = parseEnvironmentDriverConfig(environment); @@ -729,6 +731,8 @@ export async function resolveEnvironmentDriverConfigForRuntime( // are excluded when the template's source fingerprint is computed, // so they must be excluded when re-checking it here. secretRefExcludePaths: collectSecretRefPaths(schema), + readOnly: context?.readOnlyCustomImageTemplate, + onResolvedTemplate: context?.onResolvedCustomImageTemplate, }) : runtimeConfig, }; diff --git a/server/src/services/environment-custom-image-runtime.ts b/server/src/services/environment-custom-image-runtime.ts index 996770ef3f..1ed7dc1a18 100644 --- a/server/src/services/environment-custom-image-runtime.ts +++ b/server/src/services/environment-custom-image-runtime.ts @@ -13,6 +13,8 @@ import { readConfigValueAtPath, writeConfigValueAtPath } from "./json-schema-sec type TemplateRow = typeof environmentCustomImageTemplates.$inferSelect; export const ENVIRONMENT_CUSTOM_IMAGE_RUNTIME_CONFIG_BINDING_METADATA_KEY = "runtimeConfigBinding"; +/** Measured platform evidence is reserved to explicit server-side environment probes. */ +export const SANDBOX_PLATFORM_PROBE_METADATA_KEY = "paperclipSandboxPlatformProbes"; export const ENVIRONMENT_CUSTOM_IMAGE_CONFIG_FINGERPRINT_EXCLUDED_PATHS = [ "timeoutMs", "reuseLease", @@ -557,6 +559,8 @@ export async function resolveActiveEnvironmentCustomImageTemplateForRuntime( runtimeConfig: SandboxEnvironmentConfig; secretRefExcludePaths?: Iterable; now?: Date; + readOnly?: boolean; + onResolvedTemplate?: (identity: { id: string; capturedAt: string | null }) => void; }, ): Promise { const row = await db @@ -587,7 +591,8 @@ export async function resolveActiveEnvironmentCustomImageTemplateForRuntime( // match. Runtime-only knobs such as lease reuse, timeouts, and resource hints // are excluded from the fingerprint so those edits do not discard the capture. const now = input.now ?? new Date(); - await db + input.onResolvedTemplate?.({ id: row.id, capturedAt: row.capturedAt?.toISOString() ?? null }); + if (!input.readOnly) await db .update(environmentCustomImageTemplates) .set({ lastUsedAt: now, updatedAt: now }) .where(eq(environmentCustomImageTemplates.id, active.id)); diff --git a/server/src/services/environment-probe.ts b/server/src/services/environment-probe.ts index ee1bce5d4c..bd1b2dafbc 100644 --- a/server/src/services/environment-probe.ts +++ b/server/src/services/environment-probe.ts @@ -11,6 +11,54 @@ import { isBuiltinSandboxProvider, probeSandboxProvider } from "./sandbox-provid import { probePluginEnvironmentDriver, probePluginSandboxProviderDriver } from "./plugin-environment-driver.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; import { environmentRuntimeService } from "./environment-runtime.js"; +import { environmentService } from "./environments.js"; +import { fingerprintEnvironmentSandboxProviderConfig, SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "./environment-custom-image-runtime.js"; +import type { SandboxEnvironmentConfig } from "@paperclipai/shared"; + +export const SANDBOX_PLATFORM_PROBE_COMMAND = "uname -s; uname -m; if version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$version\" ]; then printf 'glibc\\n%s\\n' \"$version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi"; +type PlatformFacts = { platform: string; architecture: string; libc: "glibc" | "musl" | "unknown"; glibcVersion?: string }; +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function fingerprint(config: Record): string { + return fingerprintEnvironmentSandboxProviderConfig(config as SandboxEnvironmentConfig); +} +function isImmutableImage(value: unknown): value is string { + return typeof value === "string" && /^[^\s@]+@sha256:[a-f0-9]{64}$/.test(value); +} +export function parseSandboxPlatformFacts(output: string): PlatformFacts { + const [os, arch, libc, version] = output.trim().split(/\r?\n/); + return { platform: os === "Linux" ? "linux" : os === "Darwin" ? "darwin" : "unknown", + architecture: arch === "x86_64" ? "x64" : arch === "aarch64" || arch === "arm64" ? "arm64" : "unknown", + libc: libc === "glibc" || libc === "musl" ? libc : "unknown", + ...(libc === "glibc" ? { glibcVersion: version?.trim() ?? "unknown" } : {}) }; +} +async function sandboxBinding(db: Db, environment: Environment, companyId: string) { + let template: { id: string; capturedAt: string | null } | null = null; + const parsed = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment, { + applyCustomImageTemplate: true, readOnlyCustomImageTemplate: true, + onResolvedCustomImageTemplate: identity => { template = identity; }, + }); + if (parsed.driver !== "sandbox") return null; + const image = parsed.config.image; + // Names, mutable tags, provider defaults and snapshot names are not immutable identities. + if (!isImmutableImage(image) || record(parsed.config).snapshot) return null; + return { baseConfigFingerprint: fingerprint(environment.config), runtimeConfigFingerprint: fingerprint(parsed.config as unknown as Record), image, template }; +} + +/** Read measured, company-scoped image evidence without leasing or contacting a provider. */ +export async function readSandboxPlatformQualification(db: Db, environment: Environment, companyId: string): Promise { + const proof = record(record(environment.metadata?.[SANDBOX_PLATFORM_PROBE_METADATA_KEY])[companyId]); + if (proof.v !== 1 || proof.companyId !== companyId || proof.baseConfigFingerprint !== fingerprint(environment.config)) return null; + const binding = await sandboxBinding(db, environment, companyId); + if (!binding || proof.runtimeConfigFingerprint !== binding.runtimeConfigFingerprint || proof.image !== binding.image + || JSON.stringify(proof.template) !== JSON.stringify(binding.template)) return null; + const facts = record(proof.facts); + if (typeof facts.platform !== "string" || typeof facts.architecture !== "string" + || !["glibc", "musl", "unknown"].includes(String(facts.libc))) return null; + return { platform: facts.platform, architecture: facts.architecture, libc: facts.libc as PlatformFacts["libc"], + ...(typeof facts.glibcVersion === "string" ? { glibcVersion: facts.glibcVersion } : {}) }; +} export async function probeEnvironment( db: Db, @@ -24,6 +72,10 @@ export async function probeEnvironment( } = {}, ): Promise { const resolvedCompanyId = options.companyId ?? null; + const savedSandboxProbe = environment.driver === "sandbox" && options.acquireSandboxRuntimeLease === true && resolvedCompanyId; + const environments = environmentService(db); + if (savedSandboxProbe) await environments.recordSandboxPlatformProbe({ environmentId: environment.id, + companyId: savedSandboxProbe, config: environment.config, proof: null }); const parsed = options.resolvedConfig ?? ( options.acquireSandboxRuntimeLease === true ? parseEnvironmentDriverConfig(environment) @@ -77,6 +129,8 @@ export async function probeEnvironment( }; let leaseRecord: Awaited> | null = null; let releaseStatus: "released" | "failed" = "released"; + let qualification: Record | null = null; + let details: Record | null = null; try { leaseRecord = await runtime.acquireRunLease({ companyId: resolvedCompanyId, @@ -93,19 +147,37 @@ export async function probeEnvironment( const sandboxName = typeof metadata.sandboxName === "string" && metadata.sandboxName.trim().length > 0 ? metadata.sandboxName.trim() : null; + details = { provider, providerLeaseId: leaseRecord.lease.providerLeaseId, leaseId: leaseRecord.lease.id, + leasePolicy: leaseRecord.lease.leasePolicy, metadata, platformQualificationRecorded: false, + platformQualificationDiagnostic: "An immutable OCI image digest is required; this connection probe does not qualify a mutable image or snapshot name." }; + if (isImmutableImage(metadata.image) && !metadata.snapshot) { + try { + const binding = await sandboxBinding(db, environment, resolvedCompanyId); + if (binding && metadata.image === binding.image) { + const platform = await runtime.execute({ environment: probeEnvironmentConfig, lease: leaseRecord.lease, + command: "sh", args: ["-c", SANDBOX_PLATFORM_PROBE_COMMAND], + bypassSession: true, timeoutMs: 10_000, commandTimeoutMs: 10_000 }); + if (platform.exitCode !== 0) throw new Error("platform_probe_failed"); + const facts = parseSandboxPlatformFacts(platform.stdout); + details.platform = facts; + if (facts.platform !== "unknown" && facts.architecture !== "unknown" + && (facts.platform !== "linux" || facts.libc !== "unknown") + && (facts.libc !== "glibc" || /^(?:glibc\s+)?\d+\.\d+(?:\.\d+)?$/.test(facts.glibcVersion ?? ""))) { + qualification = { v: 1, companyId: resolvedCompanyId, ...binding, facts, probedAt: new Date().toISOString() }; + delete details.platformQualificationDiagnostic; + } else details.platformQualificationDiagnostic = "The image's OS, architecture or libc could not be identified. Legacy remains the creation default."; + } else details.platformQualificationDiagnostic = "The live lease does not match the selected immutable image configuration. Probe again."; + } catch { + details.platformQualificationDiagnostic = "Platform qualification could not complete; connection readiness does not establish native compatibility. Check the image and probe again."; + } + } return { ok: true, driver: "sandbox", summary: sandboxName ? `Connected to ${provider} sandbox ${sandboxName}.` : `Connected to ${provider} sandbox environment.`, - details: { - provider, - providerLeaseId: leaseRecord.lease.providerLeaseId, - leaseId: leaseRecord.lease.id, - leasePolicy: leaseRecord.lease.leasePolicy, - metadata, - }, + details, }; } catch (error) { releaseStatus = "failed"; @@ -122,11 +194,25 @@ export async function probeEnvironment( if (leaseRecord) { const driver = runtime.getDriver(environment.driver); try { - await driver?.releaseRunLease({ + const releasedLease = await driver?.releaseRunLease({ environment: probeEnvironmentConfig, lease: leaseRecord.lease, status: releaseStatus, }); + if (qualification && driver && savedSandboxProbe + && releasedLease?.id === leaseRecord.lease.id + && releasedLease.companyId === resolvedCompanyId + && releasedLease.environmentId === environment.id + && releasedLease.status === "released" && releasedLease.cleanupStatus === "success") { + const recorded = await environments.recordSandboxPlatformProbe({ environmentId: environment.id, companyId: savedSandboxProbe, + config: environment.config, proof: qualification }); + if (details) { + details.platformQualificationRecorded = recorded; + if (!recorded) details.platformQualificationDiagnostic = "The environment changed during the probe. Probe its current configuration again."; + } + } else if (qualification && details) { + details.platformQualificationDiagnostic = "The probe lease cleanup could not be verified. Resolve pending cleanup and probe again."; + } } catch (releaseError) { // Cleanup failures must not mask the connection result shown to // the operator, but a leaked sandbox should still be traceable. diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index 4170332cc3..74e96baaf8 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1,4 +1,6 @@ import { randomUUID } from "node:crypto"; + +import { SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "./environment-custom-image-runtime.js"; import { and, desc, eq, inArray, isNull, ne, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { @@ -859,6 +861,21 @@ export function environmentService(db: Db) { }; return { + recordSandboxPlatformProbe: async (input: { + environmentId: string; + companyId: string; + config: Record; + proof: Record | null; + }): Promise => { + const existing = sql`coalesce(${environments.metadata}->${SANDBOX_PLATFORM_PROBE_METADATA_KEY}, '{}'::jsonb)`; + const probes = input.proof === null + ? sql`${existing} - ${input.companyId}` + : sql`${existing} || ${JSON.stringify({ [input.companyId]: input.proof })}::jsonb`; + const rows = await db.update(environments).set({ + metadata: sql`jsonb_set(coalesce(${environments.metadata}, '{}'::jsonb), ARRAY[${SANDBOX_PLATFORM_PROBE_METADATA_KEY}]::text[], ${probes}, true)`, + }).where(and(eq(environments.id, input.environmentId), eq(environments.driver, "sandbox"), eq(environments.config, input.config))).returning({ id: environments.id }); + return rows.length > 0; + }, list: async ( companyIdOrFilters?: string | EnvironmentListFilters, maybeFilters?: EnvironmentListFilters,