diff --git a/.github/workflows/docker-merge-group.yml b/.github/workflows/docker-merge-group.yml index 24c83b01808..d04db07ceb9 100644 --- a/.github/workflows/docker-merge-group.yml +++ b/.github/workflows/docker-merge-group.yml @@ -22,7 +22,14 @@ concurrency: cancel-in-progress: true jobs: + macos_runner_runtime: + uses: ./.github/workflows/macos-runner-runtime.yml + permissions: + contents: read + actions: read + build: + needs: macos_runner_runtime if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event.merge_group.base_ref == 'refs/heads/master' strategy: fail-fast: false @@ -49,6 +56,12 @@ jobs: fetch-depth: 0 filter: blob:none + - name: Download the native macOS SSH runtime + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ${{ needs.macos_runner_runtime.outputs.artifact_name }} + path: docker/remote-runtimes/ + # The merge queue fast-forwards master to this exact commit, so these # stamps match what a master build of the same commit would carry. - name: Compute build version @@ -133,6 +146,7 @@ jobs: build-args: | PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }} PAPERCLIP_BUILD_COMMIT=${{ github.sha }} + PAPERCLIP_MACOS_RUNTIME_INPUTS_DIGEST=${{ needs.macos_runner_runtime.outputs.inputs_digest }} CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }} platforms: ${{ matrix.platform }} push: false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1f9443c596b..2fa4d84e041 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,12 @@ concurrency: cancel-in-progress: false jobs: + macos_runner_runtime: + uses: ./.github/workflows/macos-runner-runtime.yml + permissions: + contents: read + actions: read + runner_image: name: Resolve the Runner image of this commit if: vars.RUNNER_IMAGE_CONSUME_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' @@ -196,8 +202,8 @@ jobs: # digest; `merge` assembles the manifest list. Both architectures remain # available to self-hosted installations and downstream image composers. build-and-push: - needs: [reuse_canary_image, runner_image] - if: ${{ !cancelled() && (needs.reuse_canary_image.result == 'skipped' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'false')) }} + needs: [reuse_canary_image, runner_image, macos_runner_runtime] + if: ${{ !cancelled() && needs.macos_runner_runtime.result == 'success' && (needs.reuse_canary_image.result == 'skipped' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'false')) }} strategy: # Independent legs: one architecture failing should still publish # nothing, but it must not also hide the other's logs behind a @@ -230,6 +236,12 @@ jobs: # from every historical revision of this large repository. filter: blob:none + - name: Download the native macOS SSH runtime + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ${{ needs.macos_runner_runtime.outputs.artifact_name }} + path: docker/remote-runtimes/ + # `.git` is dockerignored, so a running image cannot derive its own # version and otherwise reports the source package.json placeholder in # analytics and the debug panel. Compute it here from the pristine @@ -456,6 +468,7 @@ jobs: build-args: | PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }} PAPERCLIP_BUILD_COMMIT=${{ github.sha }} + PAPERCLIP_MACOS_RUNTIME_INPUTS_DIGEST=${{ needs.macos_runner_runtime.outputs.inputs_digest }} CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }} ${{ needs.runner_image.outputs.build_args }} build-contexts: ${{ needs.runner_image.outputs.build_contexts }} diff --git a/.github/workflows/macos-runner-runtime.yml b/.github/workflows/macos-runner-runtime.yml new file mode 100644 index 00000000000..571d3fc1260 --- /dev/null +++ b/.github/workflows/macos-runner-runtime.yml @@ -0,0 +1,139 @@ +name: macOS OpenCode SSH runtime + +on: + workflow_call: + outputs: + artifact_name: + value: ${{ jobs.build.outputs.artifact_name }} + inputs_digest: + value: ${{ jobs.build.outputs.inputs_digest }} + pull_request: + paths: + - .github/workflows/macos-runner-runtime.yml + - .github/workflows/docker.yml + - .github/workflows/docker-merge-group.yml + - Dockerfile + - packages/paperclip-runner/** + - scripts/macos-runner-runtime.mjs + - scripts/macos-runner-runtime-cache.mjs + - server/src/services/native-runtime/remote-opencode-artifacts* + - server/src/services/native-runtime/remote-opencode-runtime* + +permissions: + contents: read + actions: read + +jobs: + build: + runs-on: macos-15 + timeout-minutes: 45 + outputs: + artifact_name: ${{ steps.cached.outputs.artifact_name }} + inputs_digest: ${{ steps.cached.outputs.digest }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 + with: + version: 9.15.4 + run_install: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24.21.0 + package-manager-cache: false + - name: Find an identical runtime from a successful master image build + id: cached + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/macos-runner-runtime-cache.mjs + - name: Reuse the verified runtime artifact + if: steps.cached.outputs.run_id != '' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ${{ steps.cached.outputs.artifact_name }} + run-id: ${{ steps.cached.outputs.run_id }} + github-token: ${{ github.token }} + path: ${{ runner.temp }}/macos-runner-runtime + - run: pnpm install --frozen-lockfile + if: steps.cached.outputs.run_id == '' + - run: pnpm --filter @paperclipai/paperclip-runner build:typescript + if: steps.cached.outputs.run_id == '' + - run: pnpm --filter @paperclipai/paperclip-runner build:binary + if: steps.cached.outputs.run_id == '' + - name: Build the pinned native provider pack + if: steps.cached.outputs.run_id == '' + env: + PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ github.sha }} + run: node packages/paperclip-runner/scripts/build-provider-pack.mjs "$RUNNER_TEMP/macos-provider-pack" + - name: Qualify the actual OpenCode server + if: steps.cached.outputs.run_id == '' + env: + PAPERCLIP_OPENCODE_QUALIFY: "1" + run: | + export PAPERCLIP_TEST_OPENCODE_BINARY="$RUNNER_TEMP/macos-provider-pack/node_modules/opencode-ai/bin/opencode.exe" + node --test packages/paperclip-runner/scripts/qualify-opencode-runtime.test.mjs + - name: Package the macOS SSH runtime + if: steps.cached.outputs.run_id == '' + env: + RUNTIME_INPUTS_DIGEST: ${{ steps.cached.outputs.digest }} + run: | + node scripts/macos-runner-runtime.mjs package \ + "$RUNNER_TEMP/macos-provider-pack" \ + packages/paperclip-runner/dist/bin/paperclip-runnerd \ + "$GITHUB_SHA" "$RUNNER_TEMP/macos-runner-runtime" "$RUNTIME_INPUTS_DIGEST" + - name: Verify the unpacked pack with the remote startup verifier + if: steps.cached.outputs.run_id == '' + env: + RUNTIME_INPUTS_DIGEST: ${{ steps.cached.outputs.digest }} + run: | + node scripts/macos-runner-runtime.mjs install \ + "$RUNNER_TEMP/macos-runner-runtime" "$RUNNER_TEMP/qualified-runtimes" "$GITHUB_SHA" "$RUNTIME_INPUTS_DIGEST" + node --import ./server/node_modules/tsx/dist/loader.mjs --input-type=module <<'JS' + import { execFile } from 'node:child_process'; + import { readFileSync } from 'node:fs'; + import { join } from 'node:path'; + import { promisify } from 'node:util'; + import { verifyRemoteProviderPack } from './server/src/services/native-runtime/remote-provider-pack-verification.ts'; + const packRoot = join(process.env.RUNNER_TEMP, 'qualified-runtimes/darwin-arm64/provider-pack'); + const manifest = JSON.parse(readFileSync(join(packRoot, 'provider-pack.json'), 'utf8')); + const execute = async request => { + try { + const result = await promisify(execFile)(request.command, request.args, { cwd: request.cwd, timeout: request.timeoutMs }); + return { ...result, exitCode: 0, signal: null, timedOut: false, pid: null, startedAt: null }; + } catch (error) { + return { stdout: error.stdout ?? '', stderr: error.stderr ?? String(error), exitCode: error.code, + signal: error.signal ?? null, timedOut: Boolean(error.killed), pid: null, startedAt: null }; + } + }; + await verifyRemoteProviderPack({ runner: { execute }, cwd: packRoot, packRoot, manifest }); + JS + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ${{ steps.cached.outputs.artifact_name }} + path: ${{ runner.temp }}/macos-runner-runtime/ + if-no-files-found: error + retention-days: 7 + + verify_linux_controller: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ${{ needs.build.outputs.artifact_name }} + path: ${{ runner.temp }}/macos-runner-runtime + - name: Verify that Linux can install the Mac runtime without executing it + env: + RUNTIME_INPUTS_DIGEST: ${{ needs.build.outputs.inputs_digest }} + run: | + node scripts/macos-runner-runtime.mjs install \ + "$RUNNER_TEMP/macos-runner-runtime" "$RUNNER_TEMP/installed-runtimes" "$GITHUB_SHA" "$RUNTIME_INPUTS_DIGEST" diff --git a/.gitignore b/.gitignore index e985a535a12..e914f0c43e8 100644 --- a/.gitignore +++ b/.gitignore @@ -94,6 +94,8 @@ test-results/ # Copilot assets are generated from pinned platform packages during installation. /packages/paperclip-runner/provider-assets/ /server/provider-assets/ +/docker/remote-runtimes/*.json +/docker/remote-runtimes/*.tar.gz # Local native Runner test/session state .paperclip-runner-prp/ diff --git a/Dockerfile b/Dockerfile index e38ac7bed68..88aa92814d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -314,7 +314,7 @@ COPY --link --from=runner-artifacts-build /out-transport/ / FROM base AS build WORKDIR /app COPY --from=deps /app /app -COPY . . +COPY --exclude=docker/remote-runtimes . . COPY --from=runner-artifacts /runner-package/dist /app/packages/paperclip-runner/dist COPY --from=runner-artifacts /runner-package/node_modules/.cache/paperclip-runner-build-inputs.json /app/packages/paperclip-runner/node_modules/.cache/paperclip-runner-build-inputs.json ARG PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST="" @@ -406,6 +406,13 @@ COPY --chown=node:node --exclude=node_modules --from=build /app /app COPY --link --chown=0:0 --from=runner-provider-pack /provider-pack-large/ /opt/paperclip-runner/provider-pack/ COPY --chown=0:0 --from=runner-provider-pack /provider-pack /opt/paperclip-runner/provider-pack COPY --from=runner-provider-pack /provider-pack-transport/ /opt/paperclip-runner/ +ARG PAPERCLIP_BUILD_COMMIT="" +ARG PAPERCLIP_MACOS_RUNTIME_INPUTS_DIGEST="" +RUN --mount=type=bind,source=docker/remote-runtimes,target=/tmp/paperclip-remote-runtimes \ + mkdir -p /opt/paperclip-runner/remote-runtimes \ + && node scripts/macos-runner-runtime.mjs install /tmp/paperclip-remote-runtimes /opt/paperclip-runner/remote-runtimes "${PAPERCLIP_BUILD_COMMIT}" "${PAPERCLIP_MACOS_RUNTIME_INPUTS_DIGEST}" \ + && chmod -R a+rX /opt/paperclip-runner/remote-runtimes \ + && find /opt/paperclip-runner/remote-runtimes -exec touch -h -d @0 {} + # Managed deployments can remap node's UID at startup. This immutable pack # contains public code and integrity metadata, never credentials; it must remain # readable afterward. The Runner build grants a+rX, and every copy up to this @@ -422,7 +429,6 @@ ENV PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH=/opt/paperclip-runner/provider-pa # The build stage still receives the commit before writing dist/build-info.json. # Empty for local builds, preserving the server's normal version fallbacks. ARG PAPERCLIP_BUILD_VERSION="" -ARG PAPERCLIP_BUILD_COMMIT="" ENV NODE_ENV=production \ HOME=/paperclip \ HOST=0.0.0.0 \ diff --git a/doc/architecture/paperclip-runner.md b/doc/architecture/paperclip-runner.md index 49bf3ceac31..76fbe81ffeb 100644 --- a/doc/architecture/paperclip-runner.md +++ b/doc/architecture/paperclip-runner.md @@ -228,6 +228,41 @@ Disabling the rollout flag blocks fresh runner starts. It does not make an already persisted native run unreadable or prevent bounded recovery and finalization of that run. +## OpenCode on macOS SSH hosts + +A Linux controller can launch native OpenCode on an Apple Silicon Mac over +SSH. This uses the existing SSH environment and its remote account. It does +not require a sandbox image or change an existing legacy agent. + +The standard image workflow builds the macOS daemon and provider pack on a +native Apple Silicon host. It tests the pinned OpenCode 1.18.34 server, then +ships the daemon, Node, proxy and OpenCode executable as a separate runtime. +The runtime omits other harness distributions and their optional inventories. +The image verifies the archive digest and source revision before installation. +Merge-queue images use the same producer, so promotion retains these artifacts. +Unchanged runtime inputs can reuse an artifact from a successful protected +master image build. The input digest covers the Runner build inputs and the +macOS build recipe. Pull request artifacts never supply this reuse authority. + +For a macOS SSH target, Core selects the `darwin-arm64` runtime under +`/opt/paperclip-runner/remote-runtimes`. Operators can set +`PAPERCLIP_RUNNER_REMOTE_RUNTIMES_PATH` to another absolute runtime parent. +The existing default Linux provider pack remains in place for Linux targets. +Explicit custom provider-pack or daemon overrides keep their existing meaning. + +Runner verifies the selected daemon and pack, then stages private launch files +under the SSH account's home. Node and OpenCode must have Mach-O headers for +the admitted architecture. Saved artifact and account bindings still apply to +continuation and recovery. The agent uses the model endpoint from its own +execution environment, such as a loopback SSH tunnel. + +A manual Docker build can omit the optional runtime inputs. Such an image +continues to support its existing targets, but a macOS OpenCode SSH run fails +with a missing-runtime diagnostic. To include the runtime, place the +same-source producer's `darwin-arm64.json` and `darwin-arm64.tar.gz` in +`docker/remote-runtimes/` before the image build. Keep these generated files +out of Git. Intel Mac SSH runtime distribution is outside this release. + ## Rollout The rollout has three gates: diff --git a/docker/remote-runtimes/.gitkeep b/docker/remote-runtimes/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/package.json b/package.json index 0997306dd5a..416070f08a9 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:notion-generic-live": "node scripts/smoke/notion-generic-live.mjs", "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/ci/stalled-file-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs scripts/__tests__/ci-typecheck-lane.test.mjs scripts/scale-ramp/scale-ramp.test.ts scripts/scale-ramp/scale-ramp-spend.test.ts", + "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/macos-runner-runtime.test.mjs scripts/macos-runner-runtime-cache.test.mjs scripts/macos-runner-runtime-workflow.test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/ci/stalled-file-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs scripts/__tests__/ci-typecheck-lane.test.mjs scripts/scale-ramp/scale-ramp.test.ts scripts/scale-ramp/scale-ramp-spend.test.ts", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/check-node-version-policy.mjs b/scripts/check-node-version-policy.mjs index 06c513b63b7..dff2d00d8dc 100644 --- a/scripts/check-node-version-policy.mjs +++ b/scripts/check-node-version-policy.mjs @@ -40,8 +40,9 @@ const workflowRoot = path.join(repoRoot, ".github", "workflows"); walk(workflowRoot, (filePath) => { if (!/\.ya?ml$/.test(filePath)) return; const source = fs.readFileSync(filePath, "utf8"); - const packProducer = relative(filePath) === ".github/workflows/release-verify.yml" - ? /(?:^|\n) runner_binaries:\n[\s\S]*?(?=\n [a-z_]+:\n|$)/.exec(source) + const packJob = { ".github/workflows/release-verify.yml": "runner_binaries", ".github/workflows/macos-runner-runtime.yml": "build" }[relative(filePath)]; + const packProducer = packJob + ? new RegExp(`(?:^|\\n) ${packJob}:\\n[\\s\\S]*?(?=\\n [a-z_]+:\\n|$)`).exec(source) : null; for (const match of source.matchAll(/node-version:\s*["']?([^\s"'#]+)/g)) { if (match[1] === "24.21.0" && packProducer diff --git a/scripts/macos-runner-runtime-cache.mjs b/scripts/macos-runner-runtime-cache.mjs new file mode 100644 index 00000000000..1c47342f18a --- /dev/null +++ b/scripts/macos-runner-runtime-cache.mjs @@ -0,0 +1,50 @@ +import { createHash } from "node:crypto"; +import { appendFileSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { computeBuildInputsDigest } from "../packages/paperclip-runner/scripts/build-inputs.mjs"; + +export function macRuntimeInputsDigest(root) { + const hash = createHash("sha256").update(computeBuildInputsDigest(root).digest); + for (const file of [".github/workflows/macos-runner-runtime.yml", "scripts/macos-runner-runtime.mjs", "scripts/macos-runner-runtime-cache.mjs"]) { + hash.update(`\0${file}\0`).update(readFileSync(resolve(root, file))); + } + return `sha256:${hash.digest("hex")}`; +} + +export function isTrustedMacRuntimeProducer(run, repository) { + return run?.repository?.full_name === repository && run.path === ".github/workflows/docker.yml" + && run.event === "push" && run.head_branch === "master" && run.conclusion === "success" + && /^[a-f0-9]{40}$/.test(run.head_sha ?? ""); +} + +export async function findMacRuntimeArtifact({ digest, repository, token, fetchImpl = fetch }) { + if (!token || repository !== "paperclipai/paperclip") return null; + const name = `macos-runner-runtime-${digest.slice(7)}`; + const get = async path => { + const response = await fetchImpl(`https://api.github.com/repos/${repository}/${path}`, { + headers: { authorization: `Bearer ${token}`, accept: "application/vnd.github+json" }, + redirect: "error", signal: AbortSignal.timeout(10_000), + }); + if (!response.ok) throw Error(`Runtime artifact lookup failed: ${response.status}`); + return response.json(); + }; + const result = await get(`actions/artifacts?name=${name}&per_page=20`); + for (const artifact of (result.artifacts ?? []).filter(item => item.name === name && !item.expired).slice(0, 4)) { + if (!Number.isSafeInteger(artifact.workflow_run?.id) || !/^sha256:[a-f0-9]{64}$/.test(artifact.digest ?? "")) continue; + const run = await get(`actions/runs/${artifact.workflow_run.id}`); + if (isTrustedMacRuntimeProducer(run, repository)) return { runId: run.id, name }; + } + return null; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const root = fileURLToPath(new URL("..", import.meta.url)); + const digest = macRuntimeInputsDigest(root); + const name = `macos-runner-runtime-${digest.slice(7)}`; + let cached; + try { cached = await findMacRuntimeArtifact({ digest, repository: process.env.GITHUB_REPOSITORY, token: process.env.GITHUB_TOKEN }); } + catch { process.stderr.write("No verified runtime artifact could be reused; building it from source.\n"); } + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `digest=${digest}\nartifact_name=${name}\nrun_id=${cached?.runId ?? ""}\n`); + console.log(JSON.stringify({ digest, artifactName: name, reusedFromRun: cached?.runId ?? null })); +} diff --git a/scripts/macos-runner-runtime-cache.test.mjs b/scripts/macos-runner-runtime-cache.test.mjs new file mode 100644 index 00000000000..270a7cf1bbc --- /dev/null +++ b/scripts/macos-runner-runtime-cache.test.mjs @@ -0,0 +1,25 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { findMacRuntimeArtifact, isTrustedMacRuntimeProducer } from "./macos-runner-runtime-cache.mjs"; + +const repository = "paperclipai/paperclip", digest = `sha256:${"a".repeat(64)}`; +const good = { id: 123, repository: { full_name: repository }, path: ".github/workflows/docker.yml", event: "push", head_branch: "master", conclusion: "success", head_sha: "b".repeat(40) }; + +test("reuses only successful master image builds from the protected repository", () => { + assert.equal(isTrustedMacRuntimeProducer(good, repository), true); + for (const change of [{ repository: { full_name: "fork/project" } }, { event: "pull_request" }, { head_branch: "feature" }, { conclusion: "failure" }, { path: ".github/workflows/macos-runner-runtime.yml" }]) { + assert.equal(isTrustedMacRuntimeProducer({ ...good, ...change }, repository), false); + } +}); + +test("a matching artifact from a pull request cannot replace the master runtime", async () => { + const name = `macos-runner-runtime-${digest.slice(7)}`; + const records = [12, 123].map(id => ({ name, workflow_run: { id }, digest: `sha256:${"c".repeat(64)}` })); + const fetchImpl = async url => ({ ok: true, json: async () => url.includes("actions/artifacts") + ? { artifacts: records } : url.endsWith("/12") ? { ...good, id: 12, event: "pull_request" } : good }); + assert.deepEqual(await findMacRuntimeArtifact({ digest, repository, token: "fixture", fetchImpl }), { runId: 123, name }); +}); + +test("fork builds do not look up or trust another repository's artifacts", async () => { + assert.equal(await findMacRuntimeArtifact({ digest, repository: "fork/project", token: "fixture", fetchImpl: () => { throw Error("network forbidden"); } }), null); +}); diff --git a/scripts/macos-runner-runtime-workflow.test.mjs b/scripts/macos-runner-runtime-workflow.test.mjs new file mode 100644 index 00000000000..779ae54de11 --- /dev/null +++ b/scripts/macos-runner-runtime-workflow.test.mjs @@ -0,0 +1,46 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import test from "node:test"; +import YAML from "yaml"; + +const read = path => readFileSync(new URL(`../${path}`, import.meta.url), "utf8"); + +test("the macOS producer builds the pinned executable on its native target and tests the real server", () => { + const workflow = YAML.parse(read(".github/workflows/macos-runner-runtime.yml")); + assert.equal(workflow.jobs.build["runs-on"], "macos-15"); + assert.deepEqual(workflow.permissions, { contents: "read", actions: "read" }); + const steps = workflow.jobs.build.steps; + assert.equal(steps.find(step => step.uses?.startsWith("actions/setup-node@")).with["node-version"], "24.21.0"); + assert.equal(workflow.jobs.verify_linux_controller.steps.find(step => step.uses?.startsWith("actions/setup-node@")).with["node-version"], 24); + assert.ok(steps.some(step => step.run?.includes("build:binary"))); + assert.ok(steps.some(step => step.run?.includes("qualify-opencode-runtime.test.mjs") && step.env?.PAPERCLIP_OPENCODE_QUALIFY === "1")); + assert.ok(steps.some(step => step.run?.includes("macos-runner-runtime.mjs package") && step.run.includes('"$GITHUB_SHA"'))); + assert.ok(steps.some(step => step.run?.includes("macos-runner-runtime.mjs install") && step.run.includes("await verifyRemoteProviderPack"))); + assert.equal(JSON.parse(read("packages/paperclip-runner/package.json")).dependencies["opencode-ai"], "1.18.34"); +}); + +test("downloaded runtime inputs keep the image checkout clean", () => { + const paths = ["docker/remote-runtimes/darwin-arm64.json", "docker/remote-runtimes/darwin-arm64.tar.gz"]; + const ignored = execFileSync("git", ["check-ignore", "--no-index", ...paths], { cwd: new URL("..", import.meta.url), encoding: "utf8" }); + assert.deepEqual(ignored.trim().split("\n"), paths); +}); + +for (const [path, job] of [[".github/workflows/docker.yml", "build-and-push"], [".github/workflows/docker-merge-group.yml", "build"]]) { + test(`${path} includes the native runtime in the source image and merge-queue image`, () => { + const workflow = YAML.parse(read(path)); + assert.equal(workflow.jobs.macos_runner_runtime.uses, "./.github/workflows/macos-runner-runtime.yml"); + const build = workflow.jobs[job]; + assert.ok([build.needs].flat().includes("macos_runner_runtime")); + const download = build.steps.find(step => step.name === "Download the native macOS SSH runtime"); + assert.equal(download.with.name, "${{ needs.macos_runner_runtime.outputs.artifact_name }}"); + assert.equal(download.with.path, "docker/remote-runtimes/"); + }); +} + +test("Core installs the additional Mac runtime without replacing the Linux pack contract", () => { + const docker = read("Dockerfile"); + assert.match(docker, /COPY --exclude=docker\/remote-runtimes \. \./); + assert.match(docker, /macos-runner-runtime\.mjs install .* "\$\{PAPERCLIP_BUILD_COMMIT\}"/); + assert.match(docker, /ENV PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH=\/opt\/paperclip-runner\/provider-pack/); +}); diff --git a/scripts/macos-runner-runtime.mjs b/scripts/macos-runner-runtime.mjs new file mode 100644 index 00000000000..e1df4fcdc8d --- /dev/null +++ b/scripts/macos-runner-runtime.mjs @@ -0,0 +1,149 @@ +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { chmodSync, closeSync, copyFileSync, cpSync, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, readSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runnerBinaryFileTarget } from "../packages/paperclip-runner/src/live/runner-binary.ts"; + +const schema = "paperclip.runner.remote-runtime.v1"; +const target = "darwin-arm64"; +const sha = bytes => `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +const canonical = value => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(",")}}` : JSON.stringify(value); + +function shaFile(path, limit = 2 * 1024 ** 3) { + const metadata = lstatSync(path); + if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > limit) throw Error("Invalid remote runtime file"); + const fd = openSync(path, "r"), hash = createHash("sha256"), buffer = Buffer.alloc(1024 * 1024); + let total = 0; + try { + let bytes; + while ((bytes = readSync(fd, buffer, 0, buffer.length, null))) { + total += bytes; + if (total > metadata.size) throw Error("Remote runtime file changed"); + hash.update(buffer.subarray(0, bytes)); + } + const after = fstatSync(fd); + if (total !== metadata.size || after.size !== metadata.size || after.mtimeMs !== metadata.mtimeMs) throw Error("Remote runtime file changed"); + return `sha256:${hash.digest("hex")}`; + } finally { closeSync(fd); } +} + +export function createOpenCodeRuntimePack(source, destination) { + const sourceRoot = realpathSync(source); + const manifest = JSON.parse(readFileSync(join(sourceRoot, "provider-pack.json"), "utf8")); + if (manifest.digest !== sha(canonical(manifest.payload))) throw Error("Invalid source provider pack digest"); + mkdirSync(destination, { recursive: true }); + cpSync(join(sourceRoot, "dist"), join(destination, "dist"), { recursive: true, verbatimSymlinks: true }); + for (const artifact of Object.values(manifest.payload.artifacts)) { + const path = artifact.path; + if (typeof path !== "string" || isAbsolute(path) || path.includes("\\") || path.split("/").some(part => !part || part === "." || part === "..")) throw Error("Invalid runtime artifact path"); + const file = realpathSync(join(sourceRoot, path)); + if (relative(sourceRoot, file).startsWith("..") || shaFile(file) !== artifact.sha256) throw Error("Invalid runtime artifact identity"); + const target = join(destination, path); + mkdirSync(dirname(target), { recursive: true }); + copyFileSync(file, target); + chmodSync(target, lstatSync(file).mode & 0o777); + } + for (const [name, pin] of Object.entries({ acpx: "acpx", "@agentclientprotocol/claude-agent-acp": "claudeAcp", + "@agentclientprotocol/codex-acp": "codexAcp", "opencode-ai": "opencode" })) { + const path = join("node_modules", name, "package.json"), file = realpathSync(join(sourceRoot, path)); + if (relative(sourceRoot, file).startsWith("..") || JSON.parse(readFileSync(file, "utf8")).version !== manifest.payload.pins[pin]) { + throw Error(`Invalid runtime package metadata: ${name}`); + } + mkdirSync(dirname(join(destination, path)), { recursive: true }); + copyFileSync(file, join(destination, path)); + } + delete manifest.payload.providers; + delete manifest.payload.candidateProviders; + manifest.digest = sha(canonical(manifest.payload)); + writeFileSync(join(destination, "provider-pack.json"), JSON.stringify(manifest, null, 2) + "\n"); + return manifest; +} + +export function verifyMacRuntime(directory, sourceRevision, inputsDigest) { + const root = resolve(directory); + if (realpathSync(root) !== root) throw Error("Remote runtime must be a canonical directory"); + const metadata = JSON.parse(readFileSync(join(root, "runtime.json"), "utf8")); + const pack = JSON.parse(readFileSync(join(root, "provider-pack/provider-pack.json"), "utf8")); + const binary = join(root, "bin/paperclip-runnerd"), runner = lstatSync(binary); + if (metadata.schema !== schema || metadata.target !== target || !/^[a-f0-9]{40}$/.test(sourceRevision ?? "") + || metadata.sourceRevision !== sourceRevision || pack.payload?.runnerSourceRevision !== sourceRevision + || (inputsDigest !== undefined && metadata.runtimeInputsDigest !== inputsDigest) + || pack.schema !== "paperclip-runner/remote-provider-pack/v1" + || pack.payload?.target?.platform !== "darwin" || pack.payload?.target?.architecture !== "arm64" + || pack.payload?.pins?.opencode !== "1.18.34" + || metadata.providerPackDigest !== pack.digest || pack.digest !== sha(canonical(pack.payload)) + || !runner.isFile() || runner.isSymbolicLink() || (runner.mode & 0o111) === 0 + || runnerBinaryFileTarget(binary) !== target || metadata.runnerSha256 !== shaFile(binary)) { + throw Error("Remote macOS runtime source, target, pin or digest mismatch"); + } + return metadata; +} + +export function packageMacRuntime({ providerPack, runnerBinary, sourceRevision, output, inputsDigest }) { + if (inputsDigest !== undefined && !/^sha256:[a-f0-9]{64}$/.test(inputsDigest)) throw Error("Invalid runtime inputs digest"); + if (`${process.platform}-${process.arch}` !== target) throw Error("Build the macOS runtime on a native Apple Silicon host"); + if (existsSync(output)) throw Error("Remote runtime output already exists"); + const root = join(resolve(output), target); + mkdirSync(join(root, "bin"), { recursive: true }); + try { + createOpenCodeRuntimePack(providerPack, join(root, "provider-pack")); + copyFileSync(runnerBinary, join(root, "bin/paperclip-runnerd")); + chmodSync(join(root, "bin/paperclip-runnerd"), 0o755); + const pack = JSON.parse(readFileSync(join(root, "provider-pack/provider-pack.json"), "utf8")); + writeFileSync(join(root, "runtime.json"), JSON.stringify({ schema, sourceRevision, target, ...(inputsDigest ? { runtimeInputsDigest: inputsDigest } : {}), + providerPackDigest: pack.digest, runnerSha256: shaFile(join(root, "bin/paperclip-runnerd")) }, null, 2) + "\n"); + const metadata = verifyMacRuntime(root, sourceRevision); + const archive = join(resolve(output), `${target}.tar.gz`); + execFileSync("tar", ["-czf", archive, "-C", resolve(output), target], { env: { ...process.env, COPYFILE_DISABLE: "1" } }); + writeFileSync(join(resolve(output), `${target}.json`), JSON.stringify({ ...metadata, archiveSha256: shaFile(archive) }, null, 2) + "\n"); + rmSync(root, { recursive: true }); + return metadata; + } catch (error) { + rmSync(output, { recursive: true, force: true }); + throw error; + } +} + +export function installMacRuntime({ input, output, sourceRevision, inputsDigest }) { + if (inputsDigest !== undefined && !/^sha256:[a-f0-9]{64}$/.test(inputsDigest)) throw Error("Invalid runtime inputs digest"); + const names = readdirSync(input).filter(name => name !== ".gitkeep").sort(); + if (names.length === 0) return false; + if (names.join() !== `${target}.json,${target}.tar.gz`) throw Error("Unexpected remote runtime input files"); + const metadata = JSON.parse(readFileSync(join(input, `${target}.json`), "utf8")); + const archive = join(input, `${target}.tar.gz`); + if ((metadata.sourceRevision !== sourceRevision && (!inputsDigest || metadata.runtimeInputsDigest !== inputsDigest)) + || (inputsDigest && metadata.runtimeInputsDigest !== inputsDigest) + || metadata.archiveSha256 !== shaFile(archive)) throw Error("Remote runtime archive source or digest mismatch"); + if (existsSync(join(output, target))) throw Error("Remote runtime destination already exists"); + mkdirSync(output, { recursive: true }); + if (realpathSync(output) !== resolve(output)) throw Error("Remote runtime output must be canonical"); + execFileSync("python3", ["-c", `import pathlib,sys,tarfile +archive,destination,target=sys.argv[1:] +with tarfile.open(archive,'r:gz') as bundle: + entries=bundle.getmembers() + if len(entries)>100000 or sum(e.size for e in entries)>8*1024**3: raise RuntimeError('Runtime archive exceeds bounds') + names=set() + for entry in entries: + path=pathlib.PurePosixPath(entry.name) + if path.is_absolute() or '..' in path.parts or not path.parts or path.parts[0]!=target or entry.name in names: raise RuntimeError('Unsafe runtime archive path') + if not(entry.isfile() or entry.isdir() or entry.issym() or entry.islnk()): raise RuntimeError('Unsafe runtime archive entry') + names.add(entry.name) + bundle.extractall(destination,filter='data') +`, archive, output, target], { stdio: "pipe", timeout: 300_000 }); + const { archiveSha256, ...expected } = metadata; + if (canonical(verifyMacRuntime(join(output, target), metadata.sourceRevision, inputsDigest)) !== canonical(expected)) throw Error("Remote runtime archive metadata mismatch"); + return true; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const [command, ...args] = process.argv.slice(2); + if (command === "package" && [4, 5].includes(args.length)) { + const [providerPack, runnerBinary, sourceRevision, output, inputsDigest] = args; + console.log(JSON.stringify(packageMacRuntime({ providerPack, runnerBinary, sourceRevision, output, inputsDigest }))); + } else if (command === "install" && [3, 4].includes(args.length)) { + const [input, output, sourceRevision, inputsDigest] = args; + console.log(JSON.stringify({ installed: installMacRuntime({ input, output, sourceRevision, inputsDigest: inputsDigest || undefined }) })); + } else throw Error(`Usage: ${basename(process.argv[1])} package PACK RUNNER SOURCE OUTPUT | install INPUT OUTPUT SOURCE`); +} diff --git a/scripts/macos-runner-runtime.test.mjs b/scripts/macos-runner-runtime.test.mjs new file mode 100644 index 00000000000..1857e231906 --- /dev/null +++ b/scripts/macos-runner-runtime.test.mjs @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { createOpenCodeRuntimePack, installMacRuntime, verifyMacRuntime } from "./macos-runner-runtime.mjs"; + +const sourceRevision = "a".repeat(40), target = "darwin-arm64"; +const sha = value => `sha256:${createHash("sha256").update(value).digest("hex")}`; +const canonical = value => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(",")}}` : JSON.stringify(value); + +function fixture(t) { + const root = realpathSync(mkdtempSync(join(tmpdir(), "mac-runtime-build-"))); t.after(() => rmSync(root, { recursive: true, force: true })); + const runtime = join(root, target), input = join(root, "input"), output = join(root, "output"); + mkdirSync(join(runtime, "bin"), { recursive: true }); mkdirSync(join(runtime, "provider-pack")); mkdirSync(input); + const binary = Buffer.alloc(64); binary.writeUInt32LE(0xfeedfacf, 0); binary.writeUInt32LE(0x100000c, 4); + writeFileSync(join(runtime, "bin/paperclip-runnerd"), binary, { mode: 0o755 }); + const payload = { runnerSourceRevision: sourceRevision, target: { platform: "darwin", architecture: "arm64" }, pins: { opencode: "1.18.34" } }; + const pack = { schema: "paperclip-runner/remote-provider-pack/v1", payload, digest: sha(canonical(payload)) }; + writeFileSync(join(runtime, "provider-pack/provider-pack.json"), JSON.stringify(pack)); + const metadata = { schema: "paperclip.runner.remote-runtime.v1", sourceRevision, target, runtimeInputsDigest: `sha256:${"b".repeat(64)}`, providerPackDigest: pack.digest, runnerSha256: sha(binary) }; + writeFileSync(join(runtime, "runtime.json"), JSON.stringify(metadata)); + execFileSync("tar", ["-czf", join(input, `${target}.tar.gz`), "-C", root, target], { env: { ...process.env, COPYFILE_DISABLE: "1" } }); + writeFileSync(join(input, `${target}.json`), JSON.stringify({ ...metadata, archiveSha256: sha(readFileSync(join(input, `${target}.tar.gz`))) })); + return { runtime, input, output, metadata }; +} + +test("installs the native runtime into a Linux controller layout without executing a Mac binary", t => { + const f = fixture(t); + assert.equal(installMacRuntime({ ...f, sourceRevision }), true); + assert.deepEqual(verifyMacRuntime(join(f.output, target), sourceRevision), f.metadata); +}); + +test("rejects a runtime archive from another Core source", t => { + const f = fixture(t); + assert.throws(() => installMacRuntime({ ...f, sourceRevision: "b".repeat(40) }), /source or digest/); +}); + +test("rejects corrupted archive bytes before extraction", t => { + const f = fixture(t); writeFileSync(join(f.input, `${target}.tar.gz`), "corrupt"); + assert.throws(() => installMacRuntime({ ...f, sourceRevision }), /source or digest/); +}); + +test("keeps an ordinary local Docker build usable without optional runtime inputs", t => { + const f = fixture(t); rmSync(join(f.input, `${target}.json`)); rmSync(join(f.input, `${target}.tar.gz`)); + assert.equal(installMacRuntime({ ...f, sourceRevision }), false); +}); + +test("does not silently drop an incomplete runtime input", t => { + const f = fixture(t); rmSync(join(f.input, `${target}.json`)); + assert.throws(() => installMacRuntime({ ...f, sourceRevision }), /Unexpected remote runtime input/); +}); + +test("reuses a previous source only when its immutable runtime inputs match", t => { + const f = fixture(t); + assert.equal(installMacRuntime({ ...f, sourceRevision: "c".repeat(40), inputsDigest: f.metadata.runtimeInputsDigest }), true); +}); + +test("rejects a cached runtime built from other inputs", t => { + const f = fixture(t); + assert.throws(() => installMacRuntime({ ...f, sourceRevision, inputsDigest: `sha256:${"d".repeat(64)}` }), /source or digest/); +}); + +test("the OpenCode runtime keeps its sealed artifacts and excludes unrelated harness distributions", t => { + const f = fixture(t), source = join(f.runtime, "provider-pack"), destination = join(f.runtime, "minimal-pack"); + mkdirSync(join(source, "dist")); mkdirSync(join(source, "node_modules/unrelated"), { recursive: true }); + writeFileSync(join(source, "dist/proxy.cjs"), "qualified proxy"); + writeFileSync(join(source, "node_modules/unrelated/runtime"), "unrelated harness"); + const file = join(source, "provider-pack.json"), pack = JSON.parse(readFileSync(file, "utf8")); + for (const [name, pin] of Object.entries({ acpx: "acpx", "@agentclientprotocol/claude-agent-acp": "claudeAcp", + "@agentclientprotocol/codex-acp": "codexAcp", "opencode-ai": "opencode" })) { + const directory = join(source, "node_modules", name); + mkdirSync(directory, { recursive: true }); + pack.payload.pins[pin] ??= "1.0.0"; + writeFileSync(join(directory, "package.json"), JSON.stringify({ name, version: pack.payload.pins[pin] })); + } + pack.payload.artifacts = { opencodeProxy: { path: "dist/proxy.cjs", sha256: sha("qualified proxy") } }; + pack.payload.providers = { kimi: { path: "provider-assets/kimi/darwin-arm64" } }; + pack.digest = sha(canonical(pack.payload)); writeFileSync(file, JSON.stringify(pack)); + const result = createOpenCodeRuntimePack(source, destination); + assert.equal(readFileSync(join(destination, "dist/proxy.cjs"), "utf8"), "qualified proxy"); + assert.equal(existsSync(join(destination, "node_modules/unrelated")), false); + assert.equal(result.payload.providers, undefined); + assert.equal(result.digest, sha(canonical(result.payload))); + for (const name of ["acpx", "@agentclientprotocol/claude-agent-acp", "@agentclientprotocol/codex-acp", "opencode-ai"]) { + assert.deepEqual(JSON.parse(readFileSync(join(destination, "node_modules", name, "package.json"), "utf8")), + JSON.parse(readFileSync(join(source, "node_modules", name, "package.json"), "utf8"))); + } +}); diff --git a/scripts/merge-group-image.mjs b/scripts/merge-group-image.mjs index 19f8be48249..93ebf0bc022 100644 --- a/scripts/merge-group-image.mjs +++ b/scripts/merge-group-image.mjs @@ -124,7 +124,7 @@ export function verifyRecord({ record, sha, tree, arch, archiveDigest }) { * check from the copy of this script in the push's `before` commit, so a push * cannot remove the check it is subject to. */ -export const pipelineFiles = [workflowPath, ".github/workflows/docker.yml", "scripts/merge-group-image.mjs", "scripts/assert-orphan-reaping.sh"]; +export const pipelineFiles = [workflowPath, ".github/workflows/docker.yml", ".github/workflows/macos-runner-runtime.yml", "scripts/macos-runner-runtime.mjs", "scripts/macos-runner-runtime-cache.mjs", "scripts/merge-group-image.mjs", "scripts/assert-orphan-reaping.sh"]; export function assertPipelineUnchanged(changedFiles) { const changed = changedFiles.filter(file => pipelineFiles.includes(file)); diff --git a/scripts/merge-group-image.test.mjs b/scripts/merge-group-image.test.mjs index c582abefae6..1f9cf3930a7 100644 --- a/scripts/merge-group-image.test.mjs +++ b/scripts/merge-group-image.test.mjs @@ -169,7 +169,9 @@ test("the merge-queue build has no publishing credentials or paid infrastructure assert.match(workflow, /^permissions: \{\}$/m); const permissions = [...workflow.matchAll(/^ permissions:\n((?: {6}.+\n)+)/gm)].map(match => match[1].trim()); assert.ok(permissions.length > 0); - for (const block of permissions) assert.equal(block, "contents: read"); + for (const block of permissions) { + assert.ok(block.split("\n").map(line => line.trim()).every(line => /^(contents|actions): read$/.test(line))); + } assert.doesNotMatch(workflow, /secrets\.|packages: write|id-token|attestations|docker\/login-action|cache-to:|push: true|environment:|runs-on\//); assert.doesNotMatch(workflow, /pull_request_target|workflow_run|workflow_dispatch/); assert.match(workflow, /persist-credentials: false/); diff --git a/scripts/reuse-canary-image.test.mjs b/scripts/reuse-canary-image.test.mjs index 226589fa24f..f3b85f6beaf 100644 --- a/scripts/reuse-canary-image.test.mjs +++ b/scripts/reuse-canary-image.test.mjs @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import test from "node:test"; +import { runInNewContext } from "node:vm"; import { reuseCanaryImage } from "./reuse-canary-image.mjs"; const sha = "a".repeat(40); @@ -66,8 +67,20 @@ test("canary reuse gates both native builds and still runs convergent promotion" assert.match(reuse, /github.repository_id == '1170821064'.*startsWith\(github.ref, 'refs\/tags\/canary\/v'\)/); assert.match(reuse, /node scripts\/reuse-canary-image.mjs/); const build = workflow.split(" build-and-push:\n")[1].split(" merge-and-push:\n")[0]; - assert.match(build, /needs: \[reuse_canary_image, runner_image\]/); + const dependencies = build.match(/^ needs: \[([^\]]+)\]/m)[1].split(", ").sort(); + for (const dependency of ["reuse_canary_image", "runner_image", "macos_runner_runtime"]) assert.ok(dependencies.includes(dependency)); assert.match(build, /!cancelled\(\).*result == 'skipped'.*result == 'success'.*outputs.reused == 'false'/); + const expression = build.match(/^ if: \$\{\{ (.+) \}\}$/m)[1]; + for (const [mac, reuse, reused, expected] of [ + ["success", "skipped", undefined, true], ["success", "success", "false", true], + ["success", "success", "true", false], ["success", "failure", undefined, false], + ["failure", "skipped", undefined, false], ["cancelled", "success", "false", false], + ["skipped", "skipped", undefined, false], + ]) { + assert.equal(runInNewContext(expression, { cancelled: () => false, needs: { + macos_runner_runtime: { result: mac }, reuse_canary_image: { result: reuse, outputs: { reused } }, + } }), expected); + } assert.match(build, /platform: linux\/amd64/); assert.match(build, /platform: linux\/arm64/); const promotion = workflow.split(" promote_canary_channel:\n")[1]; diff --git a/server/src/__tests__/docker-build-stamp.test.ts b/server/src/__tests__/docker-build-stamp.test.ts index 2f624dbd28b..e3cc4544a04 100644 --- a/server/src/__tests__/docker-build-stamp.test.ts +++ b/server/src/__tests__/docker-build-stamp.test.ts @@ -86,7 +86,7 @@ describe("Application image Runner", () => { it("takes the Runner build and its stamp from runner-artifacts, and checks them before any application build", () => { const build = stageBody(dockerfile, "build"); expect(build).toContain("FROM base AS build"); - const source = build.search(/^COPY \. \.$/m); + const source = build.search(/^COPY --exclude=docker\/remote-runtimes \. \.$/m); const output = build.search(/^COPY --from=runner-artifacts \/runner-package\/dist \/app\/packages\/paperclip-runner\/dist$/m); const stamp = build.search( /^COPY --from=runner-artifacts \/runner-package\/node_modules\/\.cache\/paperclip-runner-build-inputs\.json \/app\/packages\/paperclip-runner\/node_modules\/\.cache\/paperclip-runner-build-inputs\.json$/m, diff --git a/server/src/__tests__/docker-runner-workflows.test.ts b/server/src/__tests__/docker-runner-workflows.test.ts index 33183e2abcd..eae1ec96ae3 100644 --- a/server/src/__tests__/docker-runner-workflows.test.ts +++ b/server/src/__tests__/docker-runner-workflows.test.ts @@ -96,6 +96,7 @@ describe("the Runner image an application image build copies in docker.yml", () buildArgs: [ "PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}", "PAPERCLIP_BUILD_COMMIT=${{ github.sha }}", + "PAPERCLIP_MACOS_RUNTIME_INPUTS_DIGEST=${{ needs.macos_runner_runtime.outputs.inputs_digest }}", "CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}", ], buildContexts: [], @@ -111,7 +112,7 @@ describe("the Runner image an application image build copies in docker.yml", () } }); - it("gives a build with no answer exactly the build arguments, contexts and labels it had before", () => { + it("keeps inline Runner arguments and includes the macOS runtime identity without a published Runner", () => { expect(buildInputs({})).toEqual(inlineBuild); expect(buildInputs({ mode: "inline" })).toEqual(inlineBuild); }); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index ba38b91623b..cc18a0d071c 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -7,6 +7,7 @@ import * as runnerCheckpoint from "./native-runner-checkpoint.js"; import * as harnessStorage from "./native-harness-backup-storage.js"; import * as stagedAttachmentRecovery from "./native-staged-attachment-recovery.js"; import * as controllerRecovery from "./native-controller-recovery.js"; +import * as nativeSshCommandRunner from "./native-ssh-command-runner.js"; import * as ownerProof from "../owner-proof.js"; import { isSettledRemoteWarmControlState } from "./remote-warm-session-recovery.js"; import * as remoteWarmRecoveryRuntime from "./remote-warm-session-recovery.js"; @@ -23584,6 +23585,9 @@ describe("runnerd provider runtime wiring", () => { ])( "requires a build-owned provider pack identity before launching remote %s", async (providerKind, provider, driverKind) => { + vi.spyOn(nativeSshCommandRunner, "createNativeSshCommandRunner").mockReturnValue({ + execute: vi.fn(async () => ({ stdout: "Linux\nx86_64\n", stderr: "", exitCode: 0, timedOut: false })), + } as unknown as ReturnType); state.bundledPackManifestPath = join(tmpdir(), randomUUID(), "provider-pack.json"); const remoteCwd = "/home/daytona/paperclip-workspace"; const remoteProviderExecution = { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 1f4c0fb60cf..6189cee896a 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -333,6 +333,7 @@ import { conflict, HttpError } from "../../errors.js"; import { redactSensitiveText } from "../../redaction.js"; import { classifyRemoteConnectionError } from "../remote-connection-failure.js"; import { resolvePaperclipRunnerBinary, resolvePaperclipRunnerBinaryAsync, resolveRemotePaperclipRunnerBinary, verifiedBundledRemoteRunnerBinary } from "./native-codex-runner.js"; +import { resolveRemoteOpenCodeArtifacts } from "./remote-opencode-runtime.js"; import { createNativeRunTrace, isNativeRunRootHistoricalSpan, @@ -17214,6 +17215,14 @@ async function createRunnerdBackendWithinSessionClaim( const adoptingPersistedRemoteRunner = remoteTarget !== null && input.restartRecovery?.kind === "reattach_remote_runner" && input.controllerPersistence?.runnerArtifact !== undefined; + const remoteOpenCodeArtifacts = remoteTarget?.transport === "ssh" && input.execution.provider.kind === "opencode" && !adoptingPersistedRemoteRunner + ? await timeController("companion_selection", () => resolveRemoteOpenCodeArtifacts({ + runner: remoteCommandRunner!, remoteCwd: remoteTarget.remoteCwd, + configuredProviderPack: input.runnerRemoteProviderPackPath, configuredRunner: input.runnerRemoteBinaryPath, + runtimeRoot: process.env.PAPERCLIP_RUNNER_REMOTE_RUNTIMES_PATH, + })) : null; + const remoteOpenCodeManifest = remoteOpenCodeArtifacts + ? await verifyRemoteProviderPackManifest(remoteOpenCodeArtifacts.providerPack) : null; const remoteClaude = remoteTarget !== null && input.execution.provider.kind === "acpx" && input.execution.provider.agent === "claude"; const remoteClaudeRunnerBinary = remoteClaude && !adoptingPersistedRemoteRunner ? await resolveRemotePaperclipRunnerBinary({ runner: remoteCommandRunner!, remoteCwd: remoteTarget!.remoteCwd, configuredPath: input.runnerRemoteBinaryPath }) @@ -17236,7 +17245,7 @@ async function createRunnerdBackendWithinSessionClaim( workspaceRoot: input.execution.workspace.cwd!, })); const configuredProviderPackRoot = - input.runnerRemoteProviderPackPath?.trim() || remotePiCompanion?.providerPack || bundledClaudeRoot || null; + remoteOpenCodeArtifacts?.providerPack || input.runnerRemoteProviderPackPath?.trim() || remotePiCompanion?.providerPack || bundledClaudeRoot || null; const useBundledRemoteImageAssets = (requiresRemoteProviderPack && !configuredProviderPackRoot && (input.execution.provider.kind === "opencode" || remoteClaude)) || @@ -17260,7 +17269,7 @@ async function createRunnerdBackendWithinSessionClaim( ? readBundledRemoteProviderPackManifest(bundledClaudeManifest, remoteClaudeTarget) : readBundledRemoteProviderPackManifest(); } - return resolveRemoteProviderPackSource(configuredProviderPackRoot).manifest; + return remoteOpenCodeManifest ?? resolveRemoteProviderPackSource(configuredProviderPackRoot).manifest; })(); return currentProviderPackManifest; }; @@ -17283,7 +17292,7 @@ async function createRunnerdBackendWithinSessionClaim( : targetMatchedCodexArtifact ? await timeController("runner_binary", () => resolveRemotePaperclipRunnerBinary({ runner: remoteCommandRunner!, remoteCwd: remoteTarget.remoteCwd, configuredPath: input.runnerRemoteBinaryPath })) : remoteTarget - ? remoteClaudeRunnerBinary || input.runnerRemoteBinaryPath?.trim() || remotePiCompanion?.runnerBinary || (useBundledRemoteImageAssets ? await verifiedBundledRemoteRunnerBinary() : await resolvePaperclipRunnerBinaryAsync()) + ? remoteOpenCodeArtifacts?.runnerBinary || remoteClaudeRunnerBinary || input.runnerRemoteBinaryPath?.trim() || remotePiCompanion?.runnerBinary || (useBundledRemoteImageAssets ? await verifiedBundledRemoteRunnerBinary() : await resolvePaperclipRunnerBinaryAsync()) : await resolvePaperclipRunnerBinaryAsync(); const selectProviderPackAuthority = (reattach: boolean) => requiresRemoteProviderPack ? resolveRemoteProviderPackAuthority({ stateDirectory: root, @@ -17417,7 +17426,7 @@ async function createRunnerdBackendWithinSessionClaim( remoteRuntimeRoot, remoteCodexBinary, runnerRemoteBinaryPath: input.runnerRemoteBinaryPath, runnerRemoteCodexPath: input.runnerRemoteCodexPath, runnerRemoteCodexNpmSpec: input.runnerRemoteCodexNpmSpec, model: input.execution.provider.kind === "codex" ? input.execution.provider.model : null, - reuseRetainedRunner: sandboxLeaseAcquisition?.outcome === "resumed", targetPlatformArtifactVerified: remotePiCompanion !== null || useBundledRemoteImageAssets || targetMatchedCodexArtifact || remoteClaude, + reuseRetainedRunner: sandboxLeaseAcquisition?.outcome === "resumed", targetPlatformArtifactVerified: remoteOpenCodeArtifacts !== null || remotePiCompanion !== null || useBundledRemoteImageAssets || targetMatchedCodexArtifact || remoteClaude, trace: input.trace, onLog: input.onLog, providerPackRoots: requiresRemoteProviderPack && stagedRemoteProviderPackRoot ? [stagedRemoteProviderPackRoot, "/opt/paperclip-runner/provider-pack"] : undefined, diff --git a/server/src/services/native-runtime/provider-pack-selection-harness.test.ts b/server/src/services/native-runtime/provider-pack-selection-harness.test.ts index 8ff7be13870..9432b358380 100644 --- a/server/src/services/native-runtime/provider-pack-selection-harness.test.ts +++ b/server/src/services/native-runtime/provider-pack-selection-harness.test.ts @@ -10,6 +10,9 @@ import { QUALIFIED_ACPX_PROFILES, QUALIFIED_ACPX_VERSION } from "../../vendor/pa import { prepareRemoteProviderPackOnTarget, resolveRemoteProviderPackSource, sha256DirectoryTree } from "./native-session-executor.js"; import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; +const runtimePackScript = new URL("../../../../scripts/macos-runner-runtime.mjs", import.meta.url).href; +const { createOpenCodeRuntimePack } = await import(runtimePackScript); + const exec = promisify(execFile); const EXECUTE_LATENCY_MS = 150; const TRANSFER_LATENCY_MS = 300; @@ -163,6 +166,22 @@ describe("provider pack selection harness", () => { targetPath: join(root, "workspace", "..", "outside"), mode: 0o700 }] }])).rejects.toThrow("escapes the workspace"); }); + it("uploads and verifies an OpenCode-only runtime on a fresh host", async () => { + const sandbox = await mkdtemp(join(root, "fresh-opencode-")); + const packRoot = join(sandbox, "source"), home = join(sandbox, "home"), workspace = join(sandbox, "workspace"); + await mkdir(workspace, { recursive: true }); + createOpenCodeRuntimePack(server.configuredRoot, packRoot); + const selected = resolveRemoteProviderPackSource(packRoot); + const { runner, usage } = simulatedSandbox(home, workspace); + const prepared = await prepareRemoteProviderPackOnTarget({ + target: { kind: "remote", transport: "sandbox", remoteCwd: workspace, runner } as never, + runner, configuredProviderPackRoot: selected.configuredRoot, manifest: selected.manifest, + }); + expect(prepared.source).toBe("uploaded"); + expect(prepared.manifest).toEqual(selected.manifest); + expect(usage.bytesUploaded).toBeGreaterThan(LARGE_ARTIFACT_BYTES); + }, 30_000); + it.each<[Scenario, "preinstalled" | "uploaded", "preinstalled" | "uploaded"]>([ ["exact", "preinstalled", "preinstalled"], ["same_content_other_build", "uploaded", "preinstalled"], diff --git a/server/src/services/native-runtime/remote-opencode-artifacts.test.ts b/server/src/services/native-runtime/remote-opencode-artifacts.test.ts index c1ff0bf8756..1060ed82f10 100644 --- a/server/src/services/native-runtime/remote-opencode-artifacts.test.ts +++ b/server/src/services/native-runtime/remote-opencode-artifacts.test.ts @@ -110,6 +110,32 @@ async function fixture() { }, }; } +it("stages verified Mach-O launch files for an Apple Silicon SSH account", async () => { + const f = await fixture(); + const binary = Buffer.alloc(64); binary.writeUInt32LE(0xfeedfacf, 0); binary.writeUInt32LE(0x0100000c, 4); + f.manifest.payload.target = { platform: "darwin", architecture: "arm64" }; + for (const name of ["nodeCommand", "opencodeExecutable"] as const) { + const entry = f.manifest.payload.artifacts[name]; + await chmod(join(f.sourceRoot, entry.path), 0o700); + await writeFile(join(f.sourceRoot, entry.path), binary); + await chmod(join(f.sourceRoot, entry.path), 0o500); + entry.sha256 = `sha256:${createHash("sha256").update(binary).digest("hex")}`; + } + f.manifest.digest = `sha256:${createHash("sha256").update(canonicalProviderPackJson(f.manifest.payload)).digest("hex")}`; + const first = await prepareRemoteOpenCodeArtifacts({ ...f, bootstrapNode: process.execPath, nonce: randomUUID() }); + expect(await readFile(first.paths.nodeCommand)).toEqual(binary); + expect(await readFile(first.paths.opencodeExecutable)).toEqual(binary); + const cached = await prepareRemoteOpenCodeArtifacts({ ...f, bootstrapNode: process.execPath, nonce: randomUUID() }); + expect(cached).toEqual(first); +}); + +it("rejects Linux launch files when the admitted pack targets macOS", async () => { + const f = await fixture(); + f.manifest.payload.target = { platform: "darwin", architecture: "arm64" }; + f.manifest.digest = `sha256:${createHash("sha256").update(canonicalProviderPackJson(f.manifest.payload)).digest("hex")}`; + await expect(prepareRemoteOpenCodeArtifacts({ ...f, bootstrapNode: process.execPath, nonce: randomUUID() })).rejects.toThrow("remote_unavailable"); +}); + it("restores exact launch paths independently of a replacement lease cwd and upgraded image", async () => { const f = await fixture(); const first = await prepareRemoteOpenCodeArtifacts({ diff --git a/server/src/services/native-runtime/remote-opencode-artifacts.ts b/server/src/services/native-runtime/remote-opencode-artifacts.ts index 8c4937f0b82..aae45876154 100644 --- a/server/src/services/native-runtime/remote-opencode-artifacts.ts +++ b/server/src/services/native-runtime/remote-opencode-artifacts.ts @@ -43,7 +43,7 @@ const copyQualified=(source,target,expected,limit)=>{ return total; }finally{fs.closeSync(input);if(output!==undefined)fs.closeSync(output)} }; -const verify=(root)=>{checkDir(root);let total=0;for(const e of x.entries){let d=root;const parts=e.path.split('/');for(const part of parts.slice(0,-1)){d=p.join(d,part);checkDir(d)}const f=p.join(root,e.path),s=fs.lstatSync(f);if(s.isSymbolicLink()||!s.isFile()||s.uid!==x.uid||(s.mode&0o022)||(total+=s.size)>512*1024*1024)bad();hash(f,e.sha256);if(e.name!=='opencodeProxy'){const fd=fs.openSync(f,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW),b=Buffer.alloc(4);try{if(fs.readSync(fd,b,0,4,0)!==4||b.toString('hex')!=='7f454c46')bad()}finally{fs.closeSync(fd)}}}}; +const verify=(root)=>{checkDir(root);let total=0;for(const e of x.entries){let d=root;const parts=e.path.split('/');for(const part of parts.slice(0,-1)){d=p.join(d,part);checkDir(d)}const f=p.join(root,e.path),s=fs.lstatSync(f);if(s.isSymbolicLink()||!s.isFile()||s.uid!==x.uid||(s.mode&0o022)||(total+=s.size)>512*1024*1024)bad();hash(f,e.sha256);if(e.name!=='opencodeProxy'){const fd=fs.openSync(f,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW),b=Buffer.alloc(8);try{if(fs.readSync(fd,b,0,8,0)!==8)bad();if(x.target.platform==='darwin'){const cpu=x.target.architecture==='arm64'?0x0100000c:x.target.architecture==='x64'?0x01000007:0;if(b.readUInt32LE(0)!==0xfeedfacf||!cpu||b.readUInt32LE(4)!==cpu)bad()}else if(x.target.platform!=='linux'||b.subarray(0,4).toString('hex')!=='7f454c46')bad()}finally{fs.closeSync(fd)}}}}; if(x.mode==='inspect'){try{verify(x.root);process.stdout.write('ready')}catch(e){if(e.code!=='ENOENT')throw e}process.exit(0)} if(x.mode==='publish'){verify(x.pending);try{fs.renameSync(x.pending,x.root)}catch(e){if(e.code!=='EEXIST'&&e.code!=='ENOTEMPTY')throw e;verify(x.root);fs.rmSync(x.pending,{recursive:true})}verify(x.root);process.exit(0)} try{fs.mkdirSync(x.pending,{mode:0o700})}catch(e){if(e.code!=='EEXIST')throw e}checkDir(x.pending); @@ -127,6 +127,7 @@ export async function prepareRemoteOpenCodeArtifacts(input: { pending, digest: input.manifest.digest.slice(7), entries, + target: input.manifest.payload.target, source: input.restored ? null : input.sourceRoot, mode, }), diff --git a/server/src/services/native-runtime/remote-opencode-runtime.test.ts b/server/src/services/native-runtime/remote-opencode-runtime.test.ts new file mode 100644 index 00000000000..56c32d3b2a9 --- /dev/null +++ b/server/src/services/native-runtime/remote-opencode-runtime.test.ts @@ -0,0 +1,70 @@ +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { createHash } from "node:crypto"; +import { chmod, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { DEFAULT_REMOTE_PROVIDER_PACK, resolveRemoteOpenCodeArtifacts } from "./remote-opencode-runtime.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const hash = (bytes: string | Buffer) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +const canonical = (value: unknown): string => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical((value as Record)[key])}`).join(",")}}` : JSON.stringify(value); + +async function fixture(stdout = "Darwin\narm64\n") { + const root = await realpath(await mkdtemp(join(tmpdir(), "opencode-ssh-artifacts-"))); roots.push(root); + const runtime = join(root, "darwin-arm64"), providerPack = join(runtime, "provider-pack"), runnerBinary = join(runtime, "bin/paperclip-runnerd"); + await mkdir(providerPack, { recursive: true }); await mkdir(join(runtime, "bin")); + const binary = Buffer.alloc(64); binary.writeUInt32LE(0xfeedfacf, 0); binary.writeUInt32LE(0x100000c, 4); + await writeFile(runnerBinary, binary, { mode: 0o755 }); + const payload = { runnerSourceRevision: "a".repeat(40), target: { platform: "darwin", architecture: "arm64" } }; + const digest = hash(canonical(payload)); + await writeFile(join(providerPack, "provider-pack.json"), JSON.stringify({ schema: "paperclip-runner/remote-provider-pack/v1", digest, payload })); + await writeFile(join(runtime, "runtime.json"), JSON.stringify({ schema: "paperclip.runner.remote-runtime.v1", target: "darwin-arm64", sourceRevision: "a".repeat(40), providerPackDigest: digest, runnerSha256: hash(binary) })); + const execute = vi.fn().mockResolvedValue({ stdout, stderr: "", exitCode: 0, timedOut: false }); + return { runtime, runnerBinary, providerPack, execute, input: { runner: { execute } as unknown as CommandManagedRuntimeRunner, + remoteCwd: "/workspace", runtimeRoot: root } }; +} + +it.each([undefined, DEFAULT_REMOTE_PROVIDER_PACK])("selects the Mac daemon and pack instead of the inherited Linux default %s", async configuredProviderPack => { + const f = await fixture(); + await expect(resolveRemoteOpenCodeArtifacts({ ...f.input, configuredProviderPack })).resolves.toEqual({ runnerBinary: f.runnerBinary, providerPack: f.providerPack }); + expect(f.execute).toHaveBeenCalledWith(expect.objectContaining({ command: "sh", bypassSession: true, timeoutMs: 10_000 })); +}); + +it("leaves Linux SSH runtime selection unchanged", async () => { + const f = await fixture("Linux\nx86_64\n"); + await expect(resolveRemoteOpenCodeArtifacts(f.input)).resolves.toBeNull(); +}); + +it.each([{ configuredProviderPack: "/operator/pack" }, { configuredRunner: "/operator/runnerd" }])("preserves explicit operator runtime selection %j", async override => { + const f = await fixture(); + await expect(resolveRemoteOpenCodeArtifacts({ ...f.input, ...override })).resolves.toBeNull(); + expect(f.execute).not.toHaveBeenCalled(); +}); + +it.each(["missing", "daemon", "source", "pack", "target", "permissions"])("rejects an unavailable or changed runtime: %s", async failure => { + const f = await fixture(); + if (failure === "missing") await rm(join(f.runtime, "runtime.json")); + if (failure === "daemon") await writeFile(f.runnerBinary, "different bytes"); + if (failure === "permissions") await chmod(f.runnerBinary, 0o644); + if (["source", "pack", "target"].includes(failure)) { + const file = join(f.runtime, "runtime.json"), metadata = JSON.parse(await readFile(file, "utf8")); + if (failure === "source") metadata.sourceRevision = "b".repeat(40); + if (failure === "pack") metadata.providerPackDigest = hash("different pack"); + if (failure === "target") metadata.target = "linux-x64"; + await writeFile(file, JSON.stringify(metadata)); + } + await expect(resolveRemoteOpenCodeArtifacts(f.input)).rejects.toThrow("verified darwin-arm64"); +}); + +it("does not launch with a controller binary when the SSH platform probe times out", async () => { + const f = await fixture(); f.execute.mockResolvedValue({ stdout: "", stderr: "", exitCode: null, timedOut: true }); + await expect(resolveRemoteOpenCodeArtifacts(f.input)).rejects.toThrow("could not identify"); +}); + +it("does not advertise Intel Mac qualification", async () => { + const f = await fixture("Darwin\nx86_64\n"); + await expect(resolveRemoteOpenCodeArtifacts(f.input)).rejects.toThrow("requires Apple Silicon"); +}); diff --git a/server/src/services/native-runtime/remote-opencode-runtime.ts b/server/src/services/native-runtime/remote-opencode-runtime.ts new file mode 100644 index 00000000000..164cea39307 --- /dev/null +++ b/server/src/services/native-runtime/remote-opencode-runtime.ts @@ -0,0 +1,71 @@ +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { createHash } from "node:crypto"; +import { lstat, readFile, realpath } from "node:fs/promises"; +import { isAbsolute, join, resolve } from "node:path"; +import { runnerBinaryTarget } from "../../vendor/paperclip-runner/index.js"; +import { binaryTargetFromHeader, cachedFileSha256 } from "./off-loop-file-work.js"; + +export const DEFAULT_REMOTE_PROVIDER_PACK = "/opt/paperclip-runner/provider-pack"; +export const DEFAULT_REMOTE_RUNTIMES = "/opt/paperclip-runner/remote-runtimes"; + +const canonical = (value: unknown): string => Array.isArray(value) + ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" + ? `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical((value as Record)[key])}`).join(",")}}` + : JSON.stringify(value); + +async function boundedJson(path: string) { + const metadata = await lstat(path); + if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > 1024 * 1024) { + throw new Error("runner_remote_artifact_incompatible: invalid macOS runtime metadata"); + } + return JSON.parse(await readFile(path, "utf8")); +} + +export async function resolveRemoteOpenCodeArtifacts(input: { + runner: CommandManagedRuntimeRunner; + remoteCwd: string; + configuredProviderPack?: string | null; + configuredRunner?: string | null; + runtimeRoot?: string; +}): Promise<{ runnerBinary: string; providerPack: string } | null> { + const configured = input.configuredProviderPack?.trim(); + if ((configured && configured !== DEFAULT_REMOTE_PROVIDER_PACK) || input.configuredRunner?.trim()) return null; + const probe = await input.runner.execute({ command: "sh", args: ["-c", "uname -s; uname -m"], + cwd: input.remoteCwd, bypassSession: true, timeoutMs: 10_000 }); + if (probe.exitCode !== 0 || probe.timedOut) { + throw new Error("runner_remote_artifact_platform_mismatch: could not identify the SSH host platform"); + } + const [platform, architecture] = probe.stdout.trim().split(/\r?\n/); + if (platform !== "Darwin") return null; + if (architecture !== "arm64") { + throw new Error("runner_remote_artifact_unavailable: automatic macOS OpenCode SSH runtime requires Apple Silicon"); + } + const runtimeRoot = input.runtimeRoot?.trim() || DEFAULT_REMOTE_RUNTIMES; + if (!isAbsolute(runtimeRoot)) throw new Error("PAPERCLIP_RUNNER_REMOTE_RUNTIMES_PATH must be an absolute path"); + const root = resolve(runtimeRoot, "darwin-arm64"); + try { + if (await realpath(root) !== root) throw new Error("linked runtime root"); + const metadata = await boundedJson(join(root, "runtime.json")); + const providerPack = join(root, "provider-pack"); + const pack = await boundedJson(join(providerPack, "provider-pack.json")); + const runnerBinary = join(root, "bin", "paperclip-runnerd"); + const runner = await lstat(runnerBinary); + const packDigest = `sha256:${createHash("sha256").update(canonical(pack.payload)).digest("hex")}`; + if (metadata.schema !== "paperclip.runner.remote-runtime.v1" || metadata.target !== "darwin-arm64" + || !/^[a-f0-9]{40}$/.test(metadata.sourceRevision ?? "") + || pack.schema !== "paperclip-runner/remote-provider-pack/v1" + || pack.payload?.runnerSourceRevision !== metadata.sourceRevision + || pack.payload?.target?.platform !== "darwin" || pack.payload?.target?.architecture !== "arm64" + || metadata.providerPackDigest !== pack.digest || pack.digest !== packDigest + || !/^sha256:[a-f0-9]{64}$/.test(metadata.runnerSha256 ?? "") + || !runner.isFile() || runner.isSymbolicLink() || (runner.mode & 0o111) === 0 + || await binaryTargetFromHeader(runnerBinary, runnerBinaryTarget) !== "darwin-arm64" + || await cachedFileSha256(runnerBinary) !== metadata.runnerSha256) { + throw new Error("runtime source, target or digest mismatch"); + } + return { runnerBinary, providerPack }; + } catch (cause) { + throw new Error("runner_remote_artifact_unavailable: install the Core image with its verified darwin-arm64 OpenCode runtime", { cause }); + } +} diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 320c3acddef..aee268d9087 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -1,6 +1,6 @@ { "inputs": { - "package.json": "0997306dd5a8166f862e0eeb9515b936b03f8460", + "package.json": "416070f08a97a51ad73bf8839ec52336ecb7ad4e", "packages/plugins/sdk/package.json": "b7a4936e2803fd0d3b32dd7f17fc1ea86609ead0", "packages/plugins/sdk/src/bundlers.ts": "883bf3d9f2bd42877cfc0c3707cd882a6f6248d0", "packages/plugins/sdk/src/define-plugin.ts": "a99e07ce77ef72fcf1df1347346aacb8ba12e284",