diff --git a/.github/workflows/release-smoke.yml b/.github/workflows/release-smoke.yml index e0aae6d1a7..40fa175346 100644 --- a/.github/workflows/release-smoke.yml +++ b/.github/workflows/release-smoke.yml @@ -23,6 +23,11 @@ on: required: false default: release-smoke type: string + qualification_source_sha: + description: Optional immutable source SHA for provider-free native package qualification + required: false + default: "" + type: string workflow_call: inputs: paperclip_version: @@ -36,6 +41,13 @@ on: required: false default: release-smoke type: string + qualification_source_sha: + required: false + default: "" + type: string + +permissions: + contents: read jobs: # The Docker smoke below can never exercise the background-service leg of @@ -45,6 +57,7 @@ jobs: # on the runner VM's systemd and require the installed service to end up # serving. smoke_service: + if: inputs.qualification_source_sha == '' runs-on: ubuntu-latest timeout-minutes: 30 @@ -98,6 +111,7 @@ jobs: retention-days: 14 smoke: + if: inputs.qualification_source_sha == '' runs-on: ubuntu-latest timeout-minutes: 45 @@ -198,3 +212,162 @@ jobs: - name: Stop Docker smoke container if: always() run: docker rm -f "$SMOKE_CONTAINER_NAME" >/dev/null 2>&1 || true + + qualification_daemons: + if: inputs.qualification_source_sha != '' + uses: ./.github/workflows/release-verify.yml + with: + ref: ${{ inputs.qualification_source_sha }} + runner_binaries_only: true + + qualification_pack: + if: inputs.qualification_source_sha != '' + needs: qualification_daemons + name: Assemble and verify public Linux packages + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ inputs.qualification_source_sha }} + persist-credentials: false + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 + with: + version: 9.15.4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: ${{ needs.qualification_daemons.outputs.runner_artifact_prefix }}-* + path: ${{ runner.temp }}/release-runner-inputs + merge-multiple: true + - name: Bind the immutable Linux producer source + env: + SOURCE_SHA: ${{ inputs.qualification_source_sha }} + run: | + set -euo pipefail + [[ "$SOURCE_SHA" =~ ^[a-f0-9]{40}$ ]] || exit 1 + [[ "$(git rev-parse HEAD)" == "$SOURCE_SHA" ]] || exit 1 + - name: Build normal workspace outputs and public CLI + run: | + pnpm install --no-frozen-lockfile + pnpm build + bash scripts/build-npm.sh --skip-checks --skip-typecheck + mv cli/package.dev.json cli/package.json + - name: Assemble exact-source daemons before server packing + env: + SOURCE_SHA: ${{ inputs.qualification_source_sha }} + run: | + node packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs \ + assemble-daemons "$RUNNER_TEMP/release-runner-inputs" "$SOURCE_SHA" + cp -R packages/paperclip-runner/dist/bin/. server/dist/vendor/paperclip-runner/bin/ + - name: Verify clean public installation and retain the same package bytes + env: + PAPERCLIP_RUNNER_PUBLIC_PACKS_DIR: ${{ runner.temp }}/public-runner-packs + run: node scripts/verify-grok-npm-install.mjs + - name: Locate completed package assembly + id: pack_identity + if: ${{ !cancelled() }} + run: | + if [[ -s "$RUNNER_TEMP/public-runner-packs/manifest.json" ]]; then + echo "available=true" >> "$GITHUB_OUTPUT" + fi + - name: Retain completed package assembly even after a later Linux failure + if: ${{ !cancelled() && steps.pack_identity.outputs.available == 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ${{ inputs.artifact_name }}-public-packages + path: ${{ runner.temp }}/public-runner-packs/ + if-no-files-found: error + retention-days: 14 + + qualification_native: + if: ${{ !cancelled() && inputs.qualification_source_sha != '' && needs.qualification_daemons.result == 'success' }} + needs: [qualification_daemons, qualification_pack] + name: Clean installed package (${{ matrix.target }}) + runs-on: ${{ matrix.host }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.qualification_daemons.outputs.native_consumer_matrix) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ inputs.qualification_source_sha }} + persist-credentials: false + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 + with: + version: 9.15.4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ${{ inputs.artifact_name }}-public-packages + path: ${{ runner.temp }}/public-runner-packs + - name: Install only the existing verifier tooling + run: pnpm install --ignore-scripts --no-frozen-lockfile + - name: Checkout immutable ARM qualification tooling + if: ${{ matrix.target == 'linux-arm64' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: .qualification-tooling + sparse-checkout: scripts/prepare-linux-arm64-qualification.sh + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Prepare Linux ARM sandbox prerequisites + if: ${{ matrix.target == 'linux-arm64' }} + timeout-minutes: 5 + env: + SOURCE_SHA: ${{ inputs.qualification_source_sha }} + TOOLING_SHA: ${{ job.workflow_sha }} + TOOLING_REPOSITORY: ${{ job.workflow_repository }} + TOOLING_WORKFLOW_REF: ${{ job.workflow_ref }} + TOOLING_WORKFLOW_PATH: ${{ job.workflow_file_path }} + run: | + set -euo pipefail + [[ "$SOURCE_SHA" =~ ^[a-f0-9]{40}$ ]] || exit 1 + [[ "$TOOLING_SHA" =~ ^[a-f0-9]{40}$ ]] || exit 1 + [[ "$(git rev-parse HEAD)" == "$SOURCE_SHA" ]] || exit 1 + [[ "$(git -C .qualification-tooling rev-parse HEAD)" == "$TOOLING_SHA" ]] || exit 1 + node --input-type=module <<'NODE' + import { createHash } from 'node:crypto'; + import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { join } from 'node:path'; + const directory = join(process.env.RUNNER_TEMP, 'paperclip-arm-bwrap-prerequisite'); + mkdirSync(directory, { recursive: true }); + const helper = '.qualification-tooling/scripts/prepare-linux-arm64-qualification.sh'; + writeFileSync(join(directory, 'qualification-identity.json'), JSON.stringify({ + productSourceSha: process.env.SOURCE_SHA, + toolingSourceSha: process.env.TOOLING_SHA, + toolingRepository: process.env.TOOLING_REPOSITORY, + toolingWorkflowRef: process.env.TOOLING_WORKFLOW_REF, + toolingWorkflowPath: process.env.TOOLING_WORKFLOW_PATH, + helperSha256: createHash('sha256').update(readFileSync(helper)).digest('hex'), + }, null, 2) + '\n'); + NODE + bash .qualification-tooling/scripts/prepare-linux-arm64-qualification.sh + - name: Verify host identity and actual clean npm consumer + env: + SOURCE_SHA: ${{ inputs.qualification_source_sha }} + TARGET: ${{ matrix.target }} + run: | + set -euo pipefail + [[ "$(git rev-parse HEAD)" == "$SOURCE_SHA" ]] || exit 1 + [[ "$(node -p 'process.platform + "-" + process.arch')" == "$TARGET" ]] || exit 1 + mode=--mac-consumer + if [[ "$TARGET" == linux-arm64 ]]; then mode=--linux-arm64-consumer; fi + node scripts/verify-grok-npm-install.mjs "$mode" \ + "$RUNNER_TEMP/public-runner-packs" "$SOURCE_SHA" "$RUNNER_TEMP/native-public-install.json" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: ${{ inputs.artifact_name }}-${{ matrix.target }} + path: | + ${{ runner.temp }}/native-public-install.json + ${{ runner.temp }}/paperclip-arm-bwrap-prerequisite/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 986968201d..a4632dfb44 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -7,18 +7,30 @@ on: description: Commit SHA, branch, or tag to verify required: true type: string + runner_binaries_only: + description: Build exact-source release daemons without publishing or provider calls + required: false + default: false + type: boolean + outputs: + runner_artifact_prefix: + value: ${{ jobs.runner_binaries.outputs.artifact_prefix }} + native_consumer_matrix: + value: ${{ jobs.runner_binary_targets.outputs.native_consumers }} # Caller-provided refs may name unmerged PR code. AWS is eligible only when # the caller runs on canonical master and verifies that event's exact SHA. # The organization group also restricts these workflow files to master. jobs: runner_chaos_evals: + if: ${{ !inputs.runner_binaries_only }} name: Pre-release Runner chaos evals uses: ./.github/workflows/runner-chaos-evals.yml with: ref: ${{ inputs.ref }} typecheck: + if: ${{ !inputs.runner_binaries_only }} name: Typecheck runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 @@ -97,6 +109,7 @@ jobs: run: pnpm -r typecheck general_tests: + if: ${{ !inputs.runner_binaries_only }} name: General tests (${{ matrix.group_label }}) runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 @@ -208,6 +221,7 @@ jobs: fi serialized_tests: + if: ${{ !inputs.runner_binaries_only }} name: Serialized tests (${{ matrix.shard_label }}) runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 @@ -257,6 +271,7 @@ jobs: run: pnpm test:run:serialized -- --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }} runner_workflow_evals: + if: ${{ !inputs.runner_binaries_only }} name: Runner workflow eval scorer contract runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 10 @@ -287,6 +302,7 @@ jobs: run: pnpm test:runner-workflow-evals verify_paperclip_runner: + if: ${{ !inputs.runner_binaries_only }} name: Verify Paperclip Runner (${{ matrix.lane }}) runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 @@ -470,6 +486,7 @@ jobs: key: runner-release-crate-v1-${{ steps.runner_release_crate_key.outputs.scope }}-${{ hashFiles('packages/paperclip-runner/rust-toolchain.toml', 'packages/paperclip-runner/runner/Cargo.toml', 'packages/paperclip-runner/runner/Cargo.lock', 'packages/paperclip-runner/runner/crates/**', 'packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/test-fixtures/**', 'packages/paperclip-runner/test/fixtures/**', 'tests/runner-e2e/fixtures/**') }} build: + if: ${{ !inputs.runner_binaries_only }} name: Build runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 @@ -498,3 +515,181 @@ jobs: - name: Build run: pnpm build + + runner_binary_targets: + if: inputs.runner_binaries_only + name: Read exact-source daemon targets + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + matrix: ${{ steps.targets.outputs.matrix }} + native_consumers: ${{ steps.targets.outputs.native_consumers }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Preserve the immutable source's platform contract + id: targets + env: + SOURCE_SHA: ${{ inputs.ref }} + run: | + set -euo pipefail + [[ "$SOURCE_SHA" =~ ^[a-f0-9]{40}$ ]] || exit 1 + [[ "$(git rev-parse HEAD)" == "$SOURCE_SHA" ]] || exit 1 + node --input-type=module <<'NODE' + import assert from 'node:assert/strict'; + import { execFileSync } from 'node:child_process'; + import { appendFileSync } from 'node:fs'; + const source = process.env.SOURCE_SHA; + const readBlob = file => { + const entries = execFileSync('git', ['ls-tree', '-z', source, '--', file], { + encoding: 'utf8', maxBuffer: 4096, + }).split('\0').filter(Boolean); + if (!entries.length) return undefined; + assert.equal(entries.length, 1, 'Ambiguous exact-source target contract'); + assert.ok(entries[0].startsWith('100644 blob ') && entries[0].endsWith('\t' + file), 'Target contract must be a regular source file'); + return execFileSync('git', ['show', source + ':' + file], { encoding: 'utf8', maxBuffer: 64 * 1024 }); + }; + const data = readBlob('packages/paperclip-runner/release-runner-targets.json'); + let declared; + if (data !== undefined) { + const contract = JSON.parse(data); + assert.ok(contract && contract.schema === 'paperclip.runner.release-targets.v1' + && Object.keys(contract).sort().join() === 'schema,targets', 'Unknown exact-source daemon platform contract'); + declared = contract.targets; + } else { + const legacy = readBlob('packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs'); + const declarations = [...(legacy ?? '').matchAll(/^[ \t]*export const RELEASE_RUNNER_TARGETS = (?:Object\.freeze\()?(\[[^\r\n]*\])\)?;[ \t]*$/gm)]; + assert.equal(declarations.length, 1, 'Unknown exact-source daemon platform contract'); + declared = JSON.parse(declarations[0][1]); + } + const historical = ['darwin-arm64', 'darwin-x64', 'linux-x64']; + const current = [...historical, 'linux-arm64']; + assert.ok(Array.isArray(declared), 'Unknown exact-source daemon platform contract'); + const matches = expected => JSON.stringify([...declared].sort()) === JSON.stringify([...expected].sort()); + assert.ok(matches(historical) || matches(current), 'Unknown exact-source daemon platform contract'); + const targets = matches(current) ? current : historical; + const hosts = { 'linux-x64': targets.length === current.length ? 'ubuntu-24.04' : 'ubuntu-latest', 'linux-arm64': 'ubuntu-24.04-arm', 'darwin-arm64': 'macos-15', 'darwin-x64': 'macos-15-intel' }; + const include = targets.map(target => ({ target, host: hosts[target] })); + appendFileSync(process.env.GITHUB_OUTPUT, `matrix=${JSON.stringify({ include })}\nnative_consumers=${JSON.stringify({ include: include.filter(row => row.target !== 'linux-x64') })}\n`); + NODE + + runner_binaries: + if: inputs.runner_binaries_only + needs: runner_binary_targets + name: Release daemon (${{ matrix.target }}) + runs-on: ${{ matrix.host }} + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.runner_binary_targets.outputs.matrix) }} + outputs: + artifact_prefix: ${{ steps.identity.outputs.artifact_prefix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 + with: + version: 9.15.4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Bind immutable producer source and target + id: identity + env: + SOURCE_SHA: ${{ inputs.ref }} + TARGET: ${{ matrix.target }} + run: | + set -euo pipefail + [[ "$SOURCE_SHA" =~ ^[a-f0-9]{40}$ ]] || exit 1 + [[ "$(git rev-parse HEAD)" == "$SOURCE_SHA" ]] || exit 1 + [[ "$(node -p 'process.platform + "-" + process.arch')" == "$TARGET" ]] || exit 1 + echo "artifact_prefix=runner-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + - name: Pin native Cargo workspace and toolchain + id: native_rust_workspace + working-directory: packages/paperclip-runner + run: | + set -euo pipefail + pinned="$HOME/paperclip-runner-rust" + if [[ -e "$pinned" && ! -L "$pinned" ]]; then + echo '::error::Native Cargo cache path must not replace a real directory' + exit 1 + fi + rm -f "$pinned" + ln -s "$GITHUB_WORKSPACE/packages/paperclip-runner/runner" "$pinned" + echo "path=$pinned" >> "$GITHUB_OUTPUT" + rustup show + toolchain="$(rustup show active-toolchain | awk '{print $1}')" + echo "RUSTUP_TOOLCHAIN=$toolchain" >> "$GITHUB_ENV" + extra_toolchains="$(rustup toolchain list | awk '{print $1}' | grep -vx "$toolchain" || true)" + if [[ -n "$extra_toolchains" ]]; then + echo "$extra_toolchains" | xargs -n1 rustup toolchain uninstall \ + || echo '::notice title=Native Rust cache::could not remove an extra toolchain; the cache key may not match' + fi + rustup toolchain list + - name: Cache native Rust dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 + with: + workspaces: ${{ steps.native_rust_workspace.outputs.path }} -> target + shared-key: release-native-v2-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }} + cache-workspace-crates: false + cache-bin: false + cache-on-failure: false + save-if: ${{ github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha }} + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + - name: Build TypeScript contract and host daemon concurrently + run: | + node --input-type=module <<'NODE' + import { mkdirSync, readFileSync } from 'node:fs'; + import { join } from 'node:path'; + import { startPrefetch, stopPrefetches } from './scripts/prefetch-child.mjs'; + const root = process.cwd(), logs = join(process.env.RUNNER_TEMP, 'native-build-preparation'); + mkdirSync(logs, { recursive: true }); + const builds = []; + let failure; + try { + for (const [name, args, cwd] of [ + ['typescript', ['--filter', '@paperclipai/paperclip-runner', 'build:typescript'], root], + ['native', ['build:binary'], join(root, 'packages/paperclip-runner')], + ]) { + builds.push(startPrefetch({ name, cmd: 'pnpm', args, cwd, env: process.env, logPath: join(logs, name + '.log') })); + } + await Promise.all(builds.map(build => build.done)); + } catch (error) { failure = error; } + finally { + await stopPrefetches(builds); + const results = await Promise.allSettled(builds.map(build => build.done)); + builds.forEach((build, index) => { + console.log('::group::' + build.name + ' build'); + process.stdout.write(readFileSync(join(logs, build.name + '.log'), 'utf8')); + console.log('::endgroup::'); + console.log(JSON.stringify({ build: build.name, status: results[index].status, + exitCode: build.child.exitCode, signal: build.child.signalCode, durationMs: build.durationMs })); + }); + } + if (failure) throw failure; + NODE + - name: Record actual daemon metadata and immutable bytes + env: + SOURCE_SHA: ${{ inputs.ref }} + TARGET: ${{ matrix.target }} + run: | + node packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs record "$SOURCE_SHA" "$TARGET" \ + packages/paperclip-runner/dist/bin/paperclip-runnerd "$RUNNER_TEMP/runner-release/$TARGET" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ${{ steps.identity.outputs.artifact_prefix }}-${{ matrix.target }} + path: ${{ runner.temp }}/runner-release/ + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2c51dbc74..19d2853b4f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -340,9 +340,52 @@ jobs: SOURCE_SHA: ${{ github.sha }} run: node scripts/cloud-source-verification.mjs "$SOURCE_SHA" + release_runner_source: + needs: [verify_canary, select_nightly, select_beta, preflight_stable] + if: >- + !cancelled() && ( + (github.event_name == 'push' && needs.verify_canary.result == 'success') || + (needs.select_nightly.result == 'success' && needs.select_nightly.outputs.proceed == 'true') || + needs.select_beta.result == 'success' || needs.preflight_stable.result == 'success') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + sha: ${{ steps.source.outputs.sha }} + required: ${{ steps.source.outputs.required }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ needs.preflight_stable.outputs.sha || needs.select_beta.outputs.sha || needs.select_nightly.outputs.sha || github.sha }} + persist-credentials: false + - name: Read the exact source packaging contract + id: source + env: + SOURCE_SHA: ${{ needs.preflight_stable.outputs.sha || needs.select_beta.outputs.sha || needs.select_nightly.outputs.sha || github.sha }} + run: | + set -euo pipefail + [[ "$SOURCE_SHA" =~ ^[a-f0-9]{40}$ ]] || exit 1 + [[ "$(git rev-parse HEAD)" == "$SOURCE_SHA" ]] || exit 1 + echo "sha=$SOURCE_SHA" >> "$GITHUB_OUTPUT" + source_script="$(git show "$SOURCE_SHA:scripts/release.sh")" || exit 1 + if printf '%s\n' "$source_script" | grep -F 'assemble-daemons "$PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR"' > /dev/null; then + echo "required=true" >> "$GITHUB_OUTPUT" + else + echo "required=false" >> "$GITHUB_OUTPUT" + fi + + release_runner_binaries: + needs: release_runner_source + if: needs.release_runner_source.outputs.required == 'true' + uses: ./.github/workflows/release-verify.yml + with: + ref: ${{ needs.release_runner_source.outputs.sha }} + runner_binaries_only: true + publish_canary: - if: github.event_name == 'push' - needs: verify_canary + if: ${{ !cancelled() && github.event_name == 'push' && needs.verify_canary.result == 'success' && needs.release_runner_source.result == 'success' && (needs.release_runner_source.outputs.required == 'false' || needs.release_runner_binaries.result == 'success') }} + needs: [verify_canary, release_runner_source, release_runner_binaries] runs-on: ubuntu-latest timeout-minutes: 150 environment: npm-canary @@ -380,6 +423,17 @@ jobs: git checkout -B master "$SOURCE_SHA" test "$(git rev-parse HEAD)" = "$SOURCE_SHA" + - name: Download exact-source release daemon receipts + if: needs.release_runner_source.outputs.required == 'true' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: ${{ needs.release_runner_binaries.outputs.runner_artifact_prefix }}-* + path: ${{ runner.temp }}/release-runner-inputs + merge-multiple: true + - name: Bind daemon inputs for release assembly + if: needs.release_runner_source.outputs.required == 'true' + run: echo "PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR=$RUNNER_TEMP/release-runner-inputs" >> "$GITHUB_ENV" + - name: Setup pnpm uses: pnpm/action-setup@v6 with: @@ -633,12 +687,14 @@ jobs: artifact_name: nightly-release-smoke publish_nightly: - needs: [select_nightly, smoke_nightly] + needs: [select_nightly, smoke_nightly, release_runner_source, release_runner_binaries] # Publish when smoke passed, or when smoke was deliberately skipped by a # dry-run dispatch (the publish itself is a dry-run in that case). if: >- !cancelled() && needs.select_nightly.outputs.proceed == 'true' && + needs.release_runner_source.result == 'success' && + (needs.release_runner_source.outputs.required == 'false' || needs.release_runner_binaries.result == 'success') && (needs.smoke_nightly.result == 'success' || (needs.smoke_nightly.result == 'skipped' && github.event_name == 'workflow_dispatch' && inputs.dry_run)) runs-on: ubuntu-latest @@ -664,6 +720,17 @@ jobs: ref: ${{ needs.select_nightly.outputs.sha }} fetch-depth: 0 + - name: Download exact-source release daemon receipts + if: needs.release_runner_source.outputs.required == 'true' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: ${{ needs.release_runner_binaries.outputs.runner_artifact_prefix }}-* + path: ${{ runner.temp }}/release-runner-inputs + merge-multiple: true + - name: Bind daemon inputs for release assembly + if: needs.release_runner_source.outputs.required == 'true' + run: echo "PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR=$RUNNER_TEMP/release-runner-inputs" >> "$GITHUB_ENV" + - name: Setup pnpm uses: pnpm/action-setup@v6 with: @@ -901,10 +968,12 @@ jobs: ref: ${{ needs.select_beta.outputs.sha }} publish_beta: - needs: [select_beta, verify_beta_candidate] + needs: [select_beta, verify_beta_candidate, release_runner_source, release_runner_binaries] if: >- !cancelled() && needs.select_beta.result == 'success' && + needs.release_runner_source.result == 'success' && + (needs.release_runner_source.outputs.required == 'false' || needs.release_runner_binaries.result == 'success') && (needs.verify_beta_candidate.result == 'success' || (needs.verify_beta_candidate.result == 'skipped' && needs.select_beta.outputs.mode == 'promote')) runs-on: ubuntu-latest @@ -930,6 +999,17 @@ jobs: ref: ${{ needs.select_beta.outputs.sha }} fetch-depth: 0 + - name: Download exact-source release daemon receipts + if: needs.release_runner_source.outputs.required == 'true' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: ${{ needs.release_runner_binaries.outputs.runner_artifact_prefix }}-* + path: ${{ runner.temp }}/release-runner-inputs + merge-multiple: true + - name: Bind daemon inputs for release assembly + if: needs.release_runner_source.outputs.required == 'true' + run: echo "PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR=$RUNNER_TEMP/release-runner-inputs" >> "$GITHUB_ENV" + - name: Setup pnpm uses: pnpm/action-setup@v6 with: @@ -1286,8 +1366,8 @@ jobs: ref: ${{ needs.preflight_stable.outputs.sha }} preview_stable: - if: github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' && inputs.dry_run - needs: [preflight_stable, verify_stable] + if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' && inputs.dry_run && needs.preflight_stable.result == 'success' && needs.verify_stable.result == 'success' && needs.release_runner_source.result == 'success' && (needs.release_runner_source.outputs.required == 'false' || needs.release_runner_binaries.result == 'success') }} + needs: [preflight_stable, verify_stable, release_runner_source, release_runner_binaries] runs-on: ubuntu-latest timeout-minutes: 45 permissions: @@ -1300,6 +1380,17 @@ jobs: fetch-depth: 0 ref: ${{ needs.preflight_stable.outputs.sha }} + - name: Download exact-source release daemon receipts + if: needs.release_runner_source.outputs.required == 'true' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: ${{ needs.release_runner_binaries.outputs.runner_artifact_prefix }}-* + path: ${{ runner.temp }}/release-runner-inputs + merge-multiple: true + - name: Bind daemon inputs for release assembly + if: needs.release_runner_source.outputs.required == 'true' + run: echo "PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR=$RUNNER_TEMP/release-runner-inputs" >> "$GITHUB_ENV" + - name: Setup pnpm uses: pnpm/action-setup@v6 with: @@ -1328,8 +1419,8 @@ jobs: ./scripts/release.sh "${args[@]}" publish_stable: - if: github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' && !inputs.dry_run - needs: [preflight_stable, verify_stable] + if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' && !inputs.dry_run && needs.preflight_stable.result == 'success' && needs.verify_stable.result == 'success' && needs.release_runner_source.result == 'success' && (needs.release_runner_source.outputs.required == 'false' || needs.release_runner_binaries.result == 'success') }} + needs: [preflight_stable, verify_stable, release_runner_source, release_runner_binaries] runs-on: ubuntu-latest timeout-minutes: 150 environment: npm-stable @@ -1347,6 +1438,17 @@ jobs: fetch-depth: 0 ref: ${{ needs.preflight_stable.outputs.sha }} + - name: Download exact-source release daemon receipts + if: needs.release_runner_source.outputs.required == 'true' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: ${{ needs.release_runner_binaries.outputs.runner_artifact_prefix }}-* + path: ${{ runner.temp }}/release-runner-inputs + merge-multiple: true + - name: Bind daemon inputs for release assembly + if: needs.release_runner_source.outputs.required == 'true' + run: echo "PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR=$RUNNER_TEMP/release-runner-inputs" >> "$GITHUB_ENV" + - name: Setup pnpm uses: pnpm/action-setup@v6 with: diff --git a/doc/plans/2026-10-06-default-agent-runner.md b/doc/plans/2026-10-06-default-agent-runner.md index d28e7960ea..925dd7cca1 100644 --- a/doc/plans/2026-10-06-default-agent-runner.md +++ b/doc/plans/2026-10-06-default-agent-runner.md @@ -1,6 +1,395 @@ # Runner defaults: staged delivery -Updated 2026-10-08. +Updated 2026-10-13 UTC. + +## Fresh cloud onboarding remains unqualified + +The source audit found two ordinary-use gaps in the dependent release. Codex +onboarding resolves Auto from saved environment facts before setup testing. +The normal setup path does not perform the saved environment platform probe. +An immutable OCI environment without prior facts therefore creates a Legacy +agent. Existing managed E2E fixtures explicitly probe first; those fixtures do +not prove fresh onboarding. A private UI correction is being tested against +the existing onboarding suite. It uses the existing permission-gated saved +probe on explicit Connect, refreshes company-scoped discovery, then tests the +selected runtime. Creation and discovery still allocate nothing. Probe errors +must stay actionable, and permissions must not broaden. + +Cloud master `23531d729756ab0461a53f9358beb8df5c1781ed` distributes named +Daytona snapshots. Its recorded runtime image digest identifies the app copied +into that snapshot, not the final boot image. Current source does not bind a +fresh company's qualification to a verified boot identity. The pinned official +Daytona SDK accepts snapshot IDs, but does not guarantee a boot-verified digest. +A read-only lookup of the existing snapshot was rejected before execution by +automatic approval review because direct credential authorization was missing. +The exact limited authorization is now requested. No key was read or request +sent. Source audits and CLI version probes do not close live cloud acceptance. + +These are release blockers for the Codex-default and actual Cloud distribution +stages, not new runtime changes in this prerequisite. No source is merged or +production-ready. The original budget holds, local preview, existing runners, +remote-build boundary and no-production-deployment boundary remain intact. + +## Held-authorization fixture correction + +The ordinary reconciliation candidate `dae808fe02694d13d322c535944910813b21f6d6` +has fresh Greptile 5/5, all four resolved threads, passing isolated native +compilation and the required browser aggregate. Its runner shard passes both +previously corrected cases, but fails a different `authorize_before_bootstrap` +cell: the same implicit one-second polling deadline expires before the fixture +installs its release callback. The shard records 1,672 passes, one failure and +three skips. The exact failure and gate receipt remain: +`/private/tmp/ci-final-dae808-ci-review-failure-receipt.json`. + +The existing fixture now signals actual entry after installing the release +callback in the exact before-bootstrap authorization hook. Race that signal +against the original recovery request under the unchanged five-second bound. +All authority, stage, denied-work, command queue, process, provider-state, +durable-byte and cleanup assertions remain. On canonical Node 24, controlled +1,250 ms scheduling delay reproduces the old undefined-callback assertion and +passes after correction. An injected early recovery error surfaces unchanged. +The original hosted cause remains inconclusive; this proves fixture weakness +and preserves the next diagnostic. These are historical-mock-binary controls, +not final installed runtime qualification. Matching-source independent review +is clear. Receipts: +`/private/tmp/ci-final-dae808-held-authorization-signal-receipt.json` and +`/private/tmp/pr2-dae808-held-authorization-independent-review.json`. + +The already-running package campaign +https://github.com/paperclipai/paperclip/actions/runs/38342104102 +is immutable to literal `dae808` throughout: daemon, assembly and installed +consumer checkouts use its explicit 40-character SHA, while the tooling checkout +uses the captured workflow SHA. Publishing this test-only correction cannot +change that run's source. Let it finish without rerun or cancellation, keep its +receipts literal and overlap fresh current-head CI with its remaining steps. +No new package campaign starts until the existing one is terminal. + +Both worktrees and root's test-drive data stay preserved. Its loopback health +is OK at literal `fe1ecf1c34`, recovery ready, zero active runs; a sandbox-blocked +read was corrected with the authorized read-only loopback path, without restart +or build. Root HEAD stays `1a0ca662`. Codex and Claude reconciliation/acceptance, +normal code-owner approval, private Cloud composition and production baseline +remain pending. No merge/deployment/default promotion, paid provider call, +new paid host or local native/Docker build occurred. + +## Final CI fixture corrections and current master + +Published prerequisite `16a53254c495f3e49a28cc911f5dcd60c41779fc` +has fresh Greptile 5/5 and all four review threads resolved. Full CI records +52 successful checks, four conditional skips and two failures: the runner +Vitest first shard and its aggregate. The actual failing shard passes 1,671 +cases and skips three; its two failures remain recorded in +`/private/tmp/pr2-16a532-runner-vitest-failure.log`. + +The durable-resume case exceeded its outer five-second test envelope. Reuse +Claude owner's existing correction from `c60bf4b6`, changing only that case's +outer limit to 15 seconds. On canonical Node 24, controlled 3,800 ms fake +latency reproduces the old outer timeout, while corrected and ordinary cases +pass. Internal 500 ms/5 second turn deadlines, distinct attempts, same thread, +effect deduplication and the complete usage assertions remain unchanged. +Receipt: `/private/tmp/pr2-16a532-durable-resume-correction-receipt.json`. + +The warm-recovery callback fixture polled callback installation with Vitest's +implicit one-second wait, despite its existing five-second request bound. +Replace polling with actual callback-entry notification raced against request +settlement under that same bound. Controlled 1,250 ms scheduling delay fails +the original assertion and passes the correction; an early recovery error now +surfaces unchanged. All fence, snapshot, retry and side-effect assertions remain. +The original hosted request outcome was hidden, so its underlying cause remains +inconclusive until fresh hosted CI. These controls use recorded historical mock +binaries and do not qualify a final native package. Receipt: +`/private/tmp/ci-final-16a532-warm-callback-signal-receipt.json`. +Independent matching-source review of both test-only changes is clear: +`/private/tmp/pr2-16a532-fixture-corrections-independent-review.json`. + +Current master `e168512f7bdbb6af6ea01954c6d7ed80b91e903e` merges cleanly. +Runner/surfaces/server/E2E no-emit checks pass on the composed source. Maps +remain aligned at 35 areas/171 IDs and token gates pass. Master changed three +activity/inbox/recovery recipes without synchronizing 25 catalog fingerprints. +Independent inspection verifies those entries are exact inherited master drift; +all 92 declared non-map story/component/wrapper/fixture inputs are unchanged. +Only their guarded fingerprints and a source-only review note are synchronized. +Existing 24 unreviewed/one current classifications, IDs, fixtures and INDEX stay +unchanged. The catalog check passes 223 files/2,210 stories. No new live or +visual coverage is claimed for the inherited backend requirements. + +The exact `16a532` remote package campaign is +https://github.com/paperclipai/paperclip/actions/runs/38338806076. +The campaign completed successfully. All four native producers, public Linux +package assembly and the Apple Silicon, Intel Mac and Linux ARM64 installed +consumers pass. The earlier Intel timeout remains a separate failed receipt; +this success does not establish its cause or a diagnostic-code fix. +This remains literal source evidence after the current master/test changes. +No queued job is rerun or obsolete package result relabeled. + +Next action: preserve the completed package result, publish this ordinary master +reconciliation with both scoped fixture corrections, require fresh full CI and +review, and qualify the final package source. The prerequisite remains draft, +auto-merge off, and actual code-owner review is pending. Codex stays `1a0ca662` +until actual prerequisite merge and source reconciliation; the Claude owner +retains its branches and separate $50 allowance. Final immutable managed QA, +actual private Cloud distribution, warm/cold signup and backward-image or +forward-recovery evidence remain required. Production Admin sign-in is pending. +Nothing is merged, deployed or production-ready. The idle root test-drive and +its data are preserved. Paid holds remain $248.179254104/$250; no provider turn, +new paid host, local native/Docker build, rollout or default promotion occurred. + +## Master reconciliation and installed-package checkpoint + +The published prerequisite `64c074fb8da1b31f8d6042c4fef48dc4164e759b` +passes 54 current-head checks with four conditional skips and fresh Greptile 5/5. +All four review threads are resolved. The actual hosted transport suite passes +246 tests and startup trust passes 22; this closes the earlier local fake-binary +limitation at that literal source. Full workspace checks, Linux DB controls, +Rust suites, isolated native compilation and all eight browser shards pass. +Receipt: `/private/tmp/pr2-64c074-full-ci-review-0144-receipt.json`. + +The [existing final package campaign](https://github.com/paperclipai/paperclip/actions/runs/38332921463) +completed on literal `64c074`. All four native producers, public Linux package +assembly, Apple Silicon and Linux ARM64 installed consumers pass. Intel Mac +fails during initial `npm install --ignore-scripts --omit=dev`, with `ETIMEDOUT` +after the existing 10-minute bound. It never reaches lifecycle or native runtime +assertions. The log establishes an installation timeout, not its network/host +cause and not a runner failure. Intel installed execution remains unqualified. +Keep the failed receipt; do not relax assertions or relabel the successful +platforms as a complete campaign. No automatic retry was dispatched. + +Master then advanced to `f329b24ee12496e710ad17377ec7716e0eee22ef`, adding +separately merged durable task workspaces. It conflicts only in the test-home +helper/test and the Storybook review record. Both independent additive resolution +proposals are preserved. The cleanup keeps guarded read-only permission recovery +and its unrelated-ENOTEMPTY negative, adopts master's POSIX error assertion, +and retains master's new isolated-home identity/symlink protections byte-for-byte. +Both source-only Storybook review appendices remain; the catalog merges without +new story/fixture edits or changed review classifications. The task-workspace +migration and runtime changes come from master, not this PR's feature scope; +final image/migrator and backward-image proof must include them. + +The composed worktree passes six existing home controls, with zero failures or +skips, plus runner/surfaces/server/E2E TypeScript checks, token gates, both map +checks (35 areas/171 IDs) and Storybook structure (223 files/2,210 stories). +Only scratch-local TypeScript preparation ran; root's idle test-drive and data +stay unchanged. The contribution check has no below-1,000-to-above-1,000 code +crossing. The two whitespace lines already in master's unrelated privacy test +are not modified; the PR diff against master is clean. The review remains 63 +files. These composed checks have their working-tree source identity and are not +claimed as execution of the old published commit. Receipts are +`/private/tmp/pr2-f329-uncommitted-home-controls-receipt.json` and +`/private/tmp/pr2-f329-composed-source-checks-receipt.json`. + +The existing Mac installer now retains bounded npm timing/progress diagnostics +in its already-uploaded receipt path before cleanup. Only allowlisted metadata +is retained; argv, URLs, environment, raw stderr and exception text are omitted. +At most two 16 KiB file tails and 32 entries per tail are read. Symlink exclusions +and the original exception identity remain. Failure is explicitly FAILED, +unqualified and runtime-not-attempted. The existing 10-minute deadline and all +package, lock, payload, lifecycle and startup oracles are unchanged. Three narrow +cases extend the existing suite; all 24 pass on canonical Node 24, and independent +matching-source review is clear. This adds diagnostics, not an installation +success or explanation of the old timeout. Receipts: +`/private/tmp/pr2-mac-initial-install-diagnostics-repair-receipt.json`, +`/private/tmp/ci-final-mac-install-diagnostics-independent-review.json` and +`/private/tmp/pr2-f329-install-diagnostics-node24.log`. + +Next action: publish the ordinary master-reconciliation commit, require fresh +full CI/review and exact-source package qualification, then normal code-owner +review and merge. Prior `64c074` CI/install receipts remain literal after this +source change. The Claude owner is waiting for the exact canonical parent before +its next composed campaign; keep all its Claude fixes and separate $50 allowance. +Codex remains `1a0ca662` until actual prerequisite merge and reconciliation. +Final immutable OCI QA onboarding/task/follow-up, private Cloud composition, +warm/cold signup and backward-image or forward-recovery proof remain required. +Production Fleet Admin sign-in and code-owner approval are still pending. +No slice is merged or production-ready. Paid holds remain $248.179254104/$250; +no new paid/provider action, local native/Docker build or rollout took place. + +## Production readiness: owned startup correction + +The human requests production readiness; preparation and normal sequential +merges remain authorized. Production deployment and provisioning-default +promotion remain separate actions. Nothing in this checkpoint is merged or +deployed, and the stack is not production-ready. + +Runtime prerequisite behavior is `ce23716503a2c9158b5a7ab74ab61571ff4ee7a3` +on `codex/runner-native-platform-prerequisites`, in +[#16028](https://github.com/paperclipai/paperclip/pull/16028). Final qualification +reproduced close returning before credential preparation drained, followed by +registration and launch. Both direct close and real driver cancellation failed +on literal `5a9ce11`. The initial shared repair from Claude commit `463ebb` is +reused, with independently identified duplicate-admission, shutdown-readiness and +resume ordering corrections. All five source findings are closed in matching +source review. Absolute cleanup deadlines, primary cancellation errors, +unauthenticated-process protection and existing agents' runner choices remain. + +The existing suites pass 32 credential-free controls across three files; both +canonical runner TypeScript no-emit checks pass. No new suite, local Rust/Docker +build, provider call or credential-content read was used. Actual daemon +readiness/recovery controls remain **hosted-CI required**: local worktrees lack +the existing fake-Codex/fake-harness executables. The failed local native attempt +never reached the readiness assertion and is not qualification. Receipts are +`/private/tmp/pr2-463ebb-owned-startup-review-closure-receipt.json` and +`/private/tmp/ci-final-owned-startup-five-gap-closure-review.json`. + +Normal automatic merge is disabled and the prerequisite is draft while this +correction receives fresh full CI, package qualification and 5/5 review. +Earlier green checks at `5a9ce11` remain historical. Actual code-owner approval +is still required; no protection is bypassed. The updated runs/adapters map pair +records source-only scope, bringing the prerequisite review to 63 files after the required Storybook +catalog review below. + +Codex remains `1a0ca66242121e39d88455aa2b52a07acc4a07d9` in +[#15683](https://github.com/paperclipai/paperclip/pull/15683), based on the previous +prerequisite. The Claude owner retains its branches, cached guided-setup fix, +verification and separate $50 allowance. Coordinate reconciliation of the common +repair before rewriting those branches; don't duplicate it in a leaf slice. + +The idle [local preview](http://127.0.0.1:3125/COD/agents/all) still serves literal +behavior `fe1ecf1c34`, recovery ready and zero active runs. Its four saved +legacy/native configurations and two historical native records remain preserved; +it has not been refreshed to this new runtime. Final local/managed acceptance +must use the tested source, independent file checks and follow-up continuity. + +The prepared private execution checklist reuses existing evals, installation, +Cloud composition and campaign paths. Required holds are final source/image/ +migrator, the actual verified-master private distribution, pinned QA onboarding, +warm/cold signup, production baseline and previous-image compatibility. Production +Fleet Admin sign-in is pending. Current paid holds remain $248.179254104 of $250; +unknown/cleanup holds stay reserved, and the unstarted managed envelope is $2.50. +No new image, paid resource, rollout or default promotion was dispatched. + +Master reconciliation uses `4b958ecbd8f33a7782fe6f29ec825f80f636b833`. +The new contribution rules are satisfied: 157 added comment annotations were +removed from 35 files, with identical executable JavaScript/TypeScript emission, +parsed workflow values, 77 action pins and shell commands. No code file crosses +from below 1,000 lines to above that limit. Final focused checks pass 248 tests +(107 workflow/package, 54 runner, 42 server, 45 E2E support). Both runner no-emit +checks and the canonical server/E2E no-emit checks pass with isolated matching +workspace dependencies. Only JavaScript/TypeScript preparation ran locally; +root preview links and data stayed unchanged. The earlier linked-source failures +remain recorded and are superseded only by the new canonical check receipt. + +The existing Storybook catalog has 21 source-review fingerprint updates because +its declared runs/adapters map input changed. All affected stories, subjects, +wrappers and fixtures remain byte-identical to master. Review classifications +stay unchanged. The catalog check passes 223 files/2,210 stories; this is +structural/source evidence, not visual or live proof. Source receipts are +`/private/tmp/pr2-9f59-comments-cleanup-receipt.json`, +`/private/tmp/pr2-storybook-owned-final-receipt.json` and +`/private/tmp/pr2-frozen-canonical-noemit-receipt.json`. + +Next action: publish this reviewed prerequisite correction, require fresh hosted +gates, reconcile Codex and Claude with its actual source, then resume the existing +remote package/immutable preview and dedicated staging qualification. Merge ready +slices normally in the agreed order; actual production readiness requires the +private distribution and rollback/forward-recovery evidence afterward. + +## Final review checkpoint: shared fixes and sandbox qualification + +The user authorizes normal sequential merges after the applicable gates pass. +Runtime prerequisite [#16028](https://github.com/paperclipai/paperclip/pull/16028) +and Codex defaults [#15683](https://github.com/paperclipai/paperclip/pull/15683) +remain draft. Their pre-fix sources are `00ac509b0a` and `8de43be4cd`. +The shared SSH argv and verified cold-resume corrections are already composed. +Six argv controls, fourteen recovery controls, and actual SSH subprocess checks +pass. The separate Claude owner reports real ARM and Linux x64 task/follow-up +pairs across controller restarts on its literal installed-package overlays; +those results do not qualify an unmodified final Codex package. + +The Mac test-drive serves `8de43be4cd` at port 3125. A read-only restart comparison +preserves four full legacy/native execution configurations and two historical +native runs with the deprecated native flag false. Zero provider turns and +zero local native/Docker builds were used. The canonical server no-emit passes +on the composed root and actual prerequisite worktree. + +Both shared fixes are implemented and frozen. The actual workflow selector +rejects candidate execution/output injection in real Git fixtures. Its full +existing release-workflow suite passes 70 tests; existing target assembly passes +7 tests. Registration passes 15 focused controls and canonical runner no-emit. +An independent scoped review confirms bounded teardown and retained private +state on unconfirmed release. The first broader local workflow attempt failed +only because mock loopback listeners were sandbox-blocked; its loopback-enabled +rerun passes. No daemon/provider execution was used by these controls. + +The generic sandbox qualification support passes 30 existing probe and route +controls plus the canonical server no-emit command. It accepts only OCI digest +images, checks the returned cleanup receipt, and reports conditional-update races +without claiming a record. Mutable or name-only configurations retain existing +connection behavior and receive no native compatibility credit. The default +consumer and explicit-permission/import controls are being composed next. + +Fresh review found four source defects: candidate-code execution during CI runner +selection, unbounded registration teardown, a lost explicit Codex permission +setting on automatic Legacy selection, and missing sandbox platform/libc facts. +Existing subagents own narrow fixes in parallel. Use inert target data and +workflow-owned public labels; preserve primary failures and private state on +unconfirmed cleanup; preserve explicit permissions; and use server-measured +sandbox qualification bound to its effective image/config and company scope. +The existing explicit environment-probe/setup flow owns qualification. Saving +and discovery must not provision resources or accept client-authored evidence. +Unknown targets retain automatic Legacy with an actionable explicit-native error. +No new product API or database migration is planned. + +Preview runs [38303434211](https://github.com/paperclipai/paperclip/actions/runs/38303434211), +[38304889308](https://github.com/paperclipai/paperclip/actions/runs/38304889308), and +[38306885592](https://github.com/paperclipai/paperclip/actions/runs/38306885592) +were cancelled as shared source defects changed their candidates. They are +excluded from final qualification. Preserve historical evidence and all cost +and cleanup holds. The next action is to finish the four fixes, run affected +controls, obtain fresh exact-source checks/review, then build the final immutable +preview and matching DB/shared pair remotely for the pinned QA tenant. +Managed Codex retains its $2.50 setup/task envelope and $2 cleanup within the +original $250 ceiling. Claude retains its separate $50 ceiling. No merge or +production deployment has occurred. + + +## Latest checkpoint: runtime prerequisite and authorized merges + +The user authorizes preparing and merging ready PRs in dependency order. +Production deployment and provisioning-default promotion remain separate actions. +The Codex-default candidate is preserved at +`bb7591dfd0c4c8569e2a0b4303ff06de8ca3e9e2`; its running test-drive remains at +[3125](http://127.0.0.1:3125/COD/agents/all). No existing agent changes runner. + +Its qualification work grew the review to 135 files. Extract the independent +runtime, installed-package and hosted-build changes onto +`codex/runner-native-platform-prerequisites`, based on master +`7d9b93aee2b79b2f035c5838741ce0b94168ec83`. This prerequisite contains 39 source +files and this record. Creation policy and experimental gates remain identical +to that master. README activation text and eight existing default-selection +controls stay in the dependent [Codex PR #15683](https://github.com/paperclipai/paperclip/pull/15683), +whose expected review is 98 files. Preserve all work and both branches. + +The isolated extraction passes 276 existing focused controls. Runner no-emit +passes. The standard server no-emit initially finds a linked dependency-output +skew: the new master's ACP constants are absent from the old checkout's compiled +adapter-utils. A source-aware diagnostic passes; canonical verification remains +pending until its matching JavaScript dependency is prepared. Full hosted CI, +native compilation and fresh review are still required for the extracted head. +Do not count the diagnostic as a passing standard command. + +Retained qualification keeps literal identities: four producers and Linux x64 +and both Mac consumers pass at `a200495`; installed ARM native startup passes +with scoped stock Ubuntu prerequisites in +[38291535914](https://github.com/paperclipai/paperclip/actions/runs/38291535914). +Those artifacts are not relabeled as the extracted source. The Linux baseline +is GNU libc 2.39+, on x64/AArch64. Windows and Alpine remain deferred. +Cargo caches and overlapping JavaScript/native preparation use existing hosted +workflows. No local Docker/Rust builds or bundled Codex payloads are introduced. + +Merge order is runtime prerequisite, Codex defaults, then the existing Claude +[protocol #15994](https://github.com/paperclipai/paperclip/pull/15994), +[default #15993](https://github.com/paperclipai/paperclip/pull/15993), and +[platform #15997](https://github.com/paperclipai/paperclip/pull/15997) PRs. +The authorized Claude chat owns its verification and fixes; this task coordinates +normal squash merges after applicable gates. The older Grok Cloud PR is an +independent lane, with its own missing snapshot-cleanup access and live proof. + +Next action: open the extracted prerequisite, compose the preserved default +branch on it, require current-source CI/review, and remotely prepare the exact +Codex staging image and DB/shared pair. The existing QA tenant remains pinned to +old `2dbe43a4`. Managed Codex retains its $2.50 setup/task envelope and $2 cleanup; +historical unknown holds remain within the original $250 ceiling. Claude has +its separately authorized $50 ceiling. Missing live proof remains unresolved. ## Outcome and ownership @@ -94,3 +483,24 @@ Next action: commit and push the narrowed candidate, rerun its checks and fresh review, then update the PR and existing evidence report. Start the Codex-default slice after the user chooses to proceed. Merging and deployment remain outside this task. + +### Occupied credential fixture follow-up — 2026-10-12 + +Current-head `021df207` Runner Vitest 2 failed during fixture preparation with +`EADDRINUSE`, before credential staging. Reused the shared correction from +`50e7554069a31af72069e7632e9ae6151019f449`: prepare a fresh home with two +deliberately occupied candidates, preserve the foreign listener, and retain the +actual staging denial and unchanged auth-file assertion. The three-line code +annotation was removed to follow CONTRIBUTING; its explanation lives here. + +Only fixture preparation retries, up to eight fresh homes. It releases only its +owned partial or unselected listeners, then runs credential staging once. A bind +race during that production call remains a visible failure and receives no +retry. No production credential or permission behavior changed. + +A controlled occupied-port baseline reproduces the old setup failure. All 42 +existing credential controls pass on Node 24 with isolated synthetic data and +zero provider calls or native builds. Comment-only removal preserves the tested +executable structure; the suite was not repeated. Evidence is recorded in +`/private/tmp/ci-final-021df2-shared-credential-quorum-receipt.json` and its +comment-closure receipt. Fresh published-head CI and review remain required. diff --git a/maps/agent-access-map/execution-environments.md b/maps/agent-access-map/execution-environments.md index 74860af970..5c2b258316 100644 --- a/maps/agent-access-map/execution-environments.md +++ b/maps/agent-access-map/execution-environments.md @@ -10,6 +10,12 @@ Each capability needs operation-level review before its entire scope can be qualified. No live agent execution is claimed. Follow the [map contract](../README.md) for format, permission checks, evidence and maintenance. +Scoped immutable-platform source review: 2026-10-12 at +`d16eccf109b524778ceb896f79ccb3881bc61a9b`, limited to saved sandbox probing, +server-owned qualification metadata and read/invalidation behavior. The original +baseline commit and review depth remain for unrelated operations; no live agent +or provider qualification is claimed by this update. + ## `configuration` Review: `baseline`. @@ -34,6 +40,18 @@ own policy and access review. Gaps: API exists; an agent cannot use instance environment management as an ordinary company actor. +Scoped source review: [environment routes](../../server/src/routes/environments.ts) +require board organization access for reads and local-implicit board or instance +administrator for management. Create/PATCH reject client-supplied platform-proof +metadata; [service updates](../../server/src/services/environments.ts) discard +supplied proof, preserve current server proof atomically for metadata-only edits, +and invalidate it when the target configuration or driver changes. Public row +responses hide the private record. This is source-only at the scoped commit. +Verify forged metadata rejection, concurrent probe/update preservation, target +invalidation and restricted-reader redaction; actual agent-denial proof remains +not run. Company/team packages do not import or export environment rows, so those +packages do not transfer platform qualification. + ## `capabilities` Review: `baseline`. @@ -73,6 +91,15 @@ Agent access: `restricted` — Board/admin probes / CLI environment. Policy: Environment management is instance administrator/board; permitted assignment uses agent/project policy. +Both probe endpoints require a board actor that is local-implicit or an instance +administrator; ordinary agent identity and company membership alone are denied. +The saved sandbox probe also resolves company/environment and secret access +before allocating a fresh lease. Its measurement is company-bound to an immutable +image digest, effective provider configuration and active custom-image identity; +only confirmed cleanup permits recording it. A draft probe does not record saved +qualification. Reading stored qualification does not lease/provision a target and +does not grant an agent permission to probe or manage that target. + Evidence: `source-only` — [implementation](../../server/src/routes/environments.ts#L353) at the reviewed commit above. @@ -82,6 +109,24 @@ own policy and access review. Gaps: Reachability probes are not granted by an agent's right to execute assigned work. +Scoped Evidence: `source-only` — [probe authorization](../../server/src/routes/environments.ts), +[immutable binding, measurement and read](../../server/src/services/environment-probe.ts), +[private metadata writes](../../server/src/services/environments.ts) and +[custom-image binding](../../server/src/services/environment-custom-image-runtime.ts) +at the scoped commit. A connection-only success or a fixture cannot certify +native compatibility, account authentication or agent access. + +Scoped Verification: `not-run` — on one authorized owned target, compare pinned +image proof with mutable/snapshot and changed-config controls; require independent +cleanup evidence before recording proof. Assert agent/non-admin board denial, +secret-context denial where the actor lacks access and forged-proof rejection. +Verify API reads hide the private record and metadata-only updates cannot overwrite it. Then use +[runtime validation](./runs-adapters.md#validation) for account/model/native startup. + +There is no new delegated agent probe binding or client proof-write API. Any +human-action request/result handoff requires its own admitted path and verification; +an accepted interaction does not turn the caller into a board administrator. + ## `assignment` Review: `baseline`. diff --git a/maps/agent-access-map/runs-adapters.md b/maps/agent-access-map/runs-adapters.md index bd2f509c54..93c0adc6bd 100644 --- a/maps/agent-access-map/runs-adapters.md +++ b/maps/agent-access-map/runs-adapters.md @@ -9,6 +9,11 @@ Each capability needs operation-level review before its entire scope can be qualified. No live agent execution is claimed. Follow the [map contract](../README.md) for format, permission checks, evidence and maintenance. +Scoped native cancellation source review: 2026-10-12 at +`ce23716503a2c9158b5a7ab74ab61571ff4ee7a3`, limited to owned startup and +shutdown/readiness ordering. The original baseline and review depth remain for +unrelated operations; no live provider or agent authorization is claimed. + ## `availability` Review: `baseline`. @@ -109,3 +114,16 @@ own policy and access review. Gaps: Execution context reset is not a permitted generic native API mutation or an ordinary agent management path. + +Scoped Evidence: `source-only` — the +[native transport](../../packages/paperclip-runner/src/live/runnerd-codex-transport.ts) +joins owned startup before certifying cleanup. Rejected duplicates cannot replace +that owner or mutate recovery state. Unconfirmed preparation or registration +retains recovery state; late registration may release only its own route and +cannot launch after close. Shutdown interrupts readiness without granting +authority over an unauthenticated adopted process. + +Scoped Verification: `not-run` — verify allowed/denied access using the actual +actor independently from the credential-free ownership controls. This correction +adds no account-setup, cleanup-override or session-reset authority. The real native +readiness/recovery fixtures and installed provider acceptance remain separate gates. diff --git a/maps/feature-map/execution-environments.md b/maps/feature-map/execution-environments.md index bb1fa10d78..dda710a1d8 100644 --- a/maps/feature-map/execution-environments.md +++ b/maps/feature-map/execution-environments.md @@ -38,6 +38,33 @@ Automated: [environment UI](../../ui/src/pages/CompanyEnvironments.test.tsx), [d Manual: Save a disposable target, reload it, and run its available probe. Verify both success and missing-credential/reachability errors identify the target. Confirm unavailable providers are not offered as working run targets. +For an immutable sandbox image, pin the OCI image with `@sha256` and explicitly +probe the saved environment in the intended company. The existing connection +probe allocates a fresh lease and can measure its OS, architecture and libc. +Qualification is recorded only when the selected image/configuration matches +the live lease and cleanup is confirmed. Inspect `platformQualificationRecorded` +and its diagnostic in the probe result: connection success alone does not mean +the image has native runtime qualification. + +The measured record is server-owned, company-bound and tied to the effective +configuration and active custom-image identity. Mutable tags, provider defaults +and name-only snapshots do not qualify. Changing the target or image invalidates +the record; normal metadata edits cannot forge or overwrite it, and API reads +omit the private metadata. Reading this evidence for discovery or creation does +not allocate a sandbox. The draft `probe-config` checks are not a saved-image +qualification operation. See the [runtime contract](../../packages/paperclip-runner/README.md) +and [adapter validation recipe](./runs-adapters.md#adapter-setup) for native +binary/provider readiness after platform qualification. + +Verification recipe: probe a pinned owned image, record its company/config/image +identity and cleanup result, then change the image or target and verify its old +qualification is not reused. A mutable-image connection can succeed while +qualification remains absent. Assert ordinary agent and non-admin board probe +requests are denied, and environment create/PATCH payloads cannot supply +qualification metadata. +This recipe and its [probe controls](../../server/src/__tests__/environment-probe.test.ts) +are not claims that a live provider or agent-access journey ran for this map update. + The local row has no UI Test button. Its API/CLI probe reports host/cwd availability only; it does not validate an adapter binary, workspace, or task execution. Remote-driver probes and a real bounded run need their own evidence. @@ -51,5 +78,7 @@ Manual: Run a bounded command in the intended target, inspect its output and wor ## Gotchas - A probe is narrower evidence than a complete task with tools, files, and resumption. +- Platform qualification does not authenticate a model account or qualify every + harness; the selected native runtime still needs its own setup and useful-task proof. - The local option may appear only when editing an existing local environment; availability follows current UI/provider capabilities. - Workspace service reachability and environment reachability are separate checks. diff --git a/maps/feature-map/runs-adapters.md b/maps/feature-map/runs-adapters.md index bec86d34f2..89c1c6c7d9 100644 --- a/maps/feature-map/runs-adapters.md +++ b/maps/feature-map/runs-adapters.md @@ -48,3 +48,12 @@ Manual: Start a disposable task, open its running transcript, and follow it to a - The run log is instance database data; it is not the opt-out first-party Telemetry system or operator-configured OpenTelemetry tracing. - Authentication/model discovery does not establish tool, image, steering, or resumption capability parity. - Use [recovery](./recovery.md) for interrupted runs and [execution environments](./execution-environments.md) for local/SSH/sandbox differences. + +Native setup retirement joins owned startup before cleanup can be confirmed. +[Transport ownership controls](../../packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts) +hold preparation or registration without launching a provider, then check bounded +close, rejected duplicate requests, no later launch and retention of unconfirmed +state. Readiness observes shutdown; resumed startup cannot reactivate ingress +after close. These are lifecycle controls, separate from installed provider proof. +The real daemon readiness and recovery controls require their existing native +fixtures in hosted CI; passing credential-free controls does not qualify them. diff --git a/maps/storybook-map/REVIEW.md b/maps/storybook-map/REVIEW.md index 1c2c8c6264..7697eb6c38 100644 --- a/maps/storybook-map/REVIEW.md +++ b/maps/storybook-map/REVIEW.md @@ -159,6 +159,33 @@ Use `maintain-storybook` to correct drift and `review-storybook` to record separ source, render, interaction, visual-comparison, and production evidence. Both skills consume the feature map; neither changes its upstream contract. +## Native runtime prerequisite: scoped source follow-up — 2026-10-12 + +Source comparison: `9f59ff36cf2da963709ca19c06ab005510c835a6` against actual +master `4b958ecbd8f33a7782fe6f29ec825f80f636b833`. The reviewed +`maps/feature-map/runs-adapters.md` bytes have SHA-256 +`ca5aff86608b7d7987dc0ee814475a7b93e3faff1f99ce090429d2e932b7e16b`. + +Twenty-one entries in Runs, harnesses, and model accounts and Task creation and +lifecycle had review-fingerprint drift. Their 117 unique declared sources have +one changed input: the runs-adapters feature recipe. All 21 story files and the +other 116 declared production subjects, wrappers and fixture sources are +unchanged from master. The recipe adds bounded native startup and registration +retirement controls. It separates those provider-free controls from real daemon +and installed-provider proof; it changes no represented UI or fixture contract. + +Refreshed only those 21 reviewed fingerprints. No story, fixture, export, stable +ID, title, capability reference or product/implementation/data classification +changed. Existing mixed and unreviewed parity limitations remain. No fixture +correction is needed for this scoped prerequisite; this does not establish full +state parity for the affected suites. + +This follow-up is source inspection only. Rendering, play assertions, visual +comparisons and production provider acceptance were not run. Native lifecycle +behavior remains with the linked transport controls and their existing hosted +CI fixtures. Catalog validation proves structure and reviewed dependency hashes, +not that these live journeys passed. + ## Task workspace reconciliation — 2026-10-12 Scope: the 26 declared-source entries affected by task-workspace work at @@ -213,3 +240,41 @@ Reviewed the same twelve workspace-recipe-dependent entries after extending the ## Default-source recipe follow-up — 2026-10-13 Reviewed the twelve unchanged workspace recipe subjects and fixtures after correcting implicit default selection. Task creation and controller classification now share configured-policy, primary, creation-time and ID precedence. Explicit choices and retained denial markers still suppress controller hooks. Project strategy fixtures show configuration, while runtime status and file/browser/artifact/service fixtures do not execute source scripts or prove this backend ordering. Refreshed only their reviewed recipe fingerprints and retained all classifications and stable IDs. No browser render, play, screenshot, Storybook build or live runtime acceptance was performed. + + +## Inherited activity, inbox and recovery recipes — 2026-10-12 + +Source comparison: master `e168512f7bdbb6af6ea01954c6d7ed80b91e903e` +against previous master `f329b24ee12496e710ad17377ec7716e0eee22ef`, +while reconciling the native-runtime prerequisite. All 25 affected catalog +entries are identical to e168 master; their stale fingerprints were inherited. + +The 101 unique declared inputs have three changed feature recipes. The remaining 98 inputs, including all 25 story files, are unchanged +between those masters and the merged worktree. They include 92 non-map +story, subject, wrapper and fixture sources and six unchanged map files. The reviewed recipe +bytes have these SHA-256 identities: + +- `maps/feature-map/activity.md`: `3867cc392c031cc5ca9ec61bfc4211d685da37e39fb36e9b152497b7cf970caf`. +- `maps/feature-map/inbox-search.md`: `8ff5431ddfe54c0c288eecffb785f864d8bfeb38be50c66615999ffbbfba6863`. +- `maps/feature-map/recovery.md`: `46fee5940f23a6d1483697e0b07817a78d993d4748b51121d2e113dd2a96173b`. + +The activity addition names issue-scoped activity authorization and sandbox +bridge controls; company audit/display fixtures do not establish live sandbox +agent access. The inbox recipe excludes automatic historical read cleanup from +user participation while retaining explicit decisions; static inbox/decision +examples do not execute that query or the newly required user/company journey. +The recovery recipe retains native workspace-owner holds across task completion +and repeated reads until exact operator retirement; recovery display fixtures +do not establish ownership, copyback denial, action identity or live operator +recovery. These backend acceptance requirements remain separate gaps. + +The proposed maintenance changes only those 25 reviewed dependency fingerprints. +Retain the existing 24 unreviewed and one current classification, simulation +boundaries, story exports, titles and stable IDs. No fixture or story correction +is indicated by this declared-source comparison. This does not promote full +state parity or coverage for the added backend requirements. + +Source inspection only: no browser rendering, play, visual baseline, provider +run or live production acceptance was performed for this reconciliation. The +existing catalog checker must run on the actual merged tree after synchronization; +its result verifies structure and declared hashes, not these live journeys. diff --git a/maps/storybook-map/catalog.json b/maps/storybook-map/catalog.json index c376bdce63..1395cb94ed 100644 --- a/maps/storybook-map/catalog.json +++ b/maps/storybook-map/catalog.json @@ -246,7 +246,7 @@ "ui/src/pages/tools/AuditTab.tsx", "ui/storybook/stories/activity/activity-feed.stories.tsx" ], - "fingerprint": "e38af3b7963caf0cc3b13f7681da370a579f97ad78a6064559fbb18aeb73b4ed", + "fingerprint": "42f1ebbd11a7b47dd507f43c287c46a9191b8ce323b9c15d83a6d8ec5521eeee", "stableId": "tools-activity-audit-feed", "sidebarTitle": "Oversight and operation/Dashboards and audit trails/Tools/Activity (audit feed)" }, @@ -363,7 +363,7 @@ "ui/storybook/fixtures/onboardingEnvironment.ts", "ui/storybook/stories/runs-adapters/agent-connection-reuse.stories.tsx" ], - "fingerprint": "4dfc350a48d8f378eabf1b7785d1b867541121fa7715da5fd9812b28a0288d9b", + "fingerprint": "7b2571f03ffa3a559a9c228f78c87302d346183599b667c1102d131b4f33593a", "stableId": "onboarding-saved-connections", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/Onboarding/Saved connections" }, @@ -728,7 +728,7 @@ "ui/storybook/prototypes/AiConnectionUsageReview.tsx", "ui/storybook/stories/runs-adapters/ai-connection-usage.stories.tsx" ], - "fingerprint": "0b03669a6abba327c01222793a41d898958d50a483ed1a2e842c195484cb33ec", + "fingerprint": "293a8407e94d525533e5beb8920d78e4b47b2326568460bfe02bb3bdd9f81eaf", "stableId": "ai-connections-usage-limits", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Usage limits" }, @@ -765,7 +765,7 @@ "ui/storybook/prototypes/AiTaskConnectionReview.tsx", "ui/storybook/stories/runs-adapters/ai-connections.stories.tsx" ], - "fingerprint": "2f32110235d513dbfb6086126180d03107bc2167bc472da6280dfeb64629343a", + "fingerprint": "9a5d741ffbff64ab6e14e03130a1b7a53403d6e69ee48108ba1358d0093facf2", "stableId": "ai-connections-review", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Review" }, @@ -808,7 +808,7 @@ "ui/src/pages/apps/app-detail/types.ts", "ui/storybook/stories/activity/app-activity-lifecycle.stories.tsx" ], - "fingerprint": "1642f6a6692ccd3ac058d1841f388607b3379c683711bbe558aad3cc79ffc077", + "fingerprint": "6c813a16621d93fe9f817b1b01d451a35b4b552727976168e938409d100b26e9", "stableId": "apps-app-detail-·-activity-lifecycle", "sidebarTitle": "Oversight and operation/Dashboards and audit trails/Apps/App detail · Activity (lifecycle)" }, @@ -1170,7 +1170,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/tasks/assigned-backlog-safeguards.stories.tsx" ], - "fingerprint": "3a3610f557ca0b31528e7df0a2faf0885132ba6c947380588bc63b144c14833e", + "fingerprint": "f6d94acdd29ba5ca746a6fcba87a44f4384288c02d736aa496a4d30bd94a922a", "stableId": "paperclip-assigned-backlog-safeguards", "sidebarTitle": "Tasks and conversations/Task creation and lifecycle/Paperclip/Assigned Backlog Safeguards" }, @@ -1194,7 +1194,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/inbox-search/blocked-inbox.stories.tsx" ], - "fingerprint": "d7406369152782fe626f9be02cecde54a7abc58753fc9e5a5b0d167f2a5df3f2", + "fingerprint": "b6f5f4b7c00e48da7320bef20e5904527f83665e5fdac4fbb44c866e15547502", "stableId": "product-inbox-blocked-tab", "sidebarTitle": "Tasks and conversations/Inbox, decisions, and search/Product/Inbox/Blocked tab" }, @@ -1374,7 +1374,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/tasks/chat-comments.stories.tsx" ], - "fingerprint": "aae493759f1217f0805fc6a577c8c1444d95ddbc3abc08cafe7288a543020657", + "fingerprint": "78ef704288c2c9ab63641e16b0e5fbe9d70bd81ec6dadacb1a31658388249c6d", "stableId": "product-chat-comments", "sidebarTitle": "Tasks and conversations/Task creation and lifecycle/Product/Chat & Comments" }, @@ -1476,7 +1476,7 @@ "ui/src/components/CloudAccessGate.tsx", "ui/storybook/stories/recovery/cloud-access-recovery.stories.tsx" ], - "fingerprint": "ce4f356d3f8296a2f6e3c6da8bb6b3a57866a0ad9c9e74850a76ff7fd9484b54", + "fingerprint": "bbec224b127ec6898cf95dfae922e6747483c0d0d4ec55ff7d1560c0c4a5c041", "stableId": "app-connection-recovery", "sidebarTitle": "Projects and execution/Recovery/App/Connection recovery" }, @@ -1500,7 +1500,7 @@ "ui/storybook/prototypes/completed-activity/CompletedActivityPreview.tsx", "ui/storybook/stories/tasks/completed-activity.stories.tsx" ], - "fingerprint": "76af14f51a89a98ed64685cee95edb29133f31bf6f3e8356c31a3b7c7b1d00b2", + "fingerprint": "7c91ff66b892ee3b50680e65cab8c83ec56851a75504ff5e3e6db4270f7b730e", "stableId": "tasks-completed-activity-preview", "sidebarTitle": "Tasks and conversations/Task creation and lifecycle/Tasks/Completed activity preview" }, @@ -1550,7 +1550,7 @@ "ui/storybook/prototypes/composer-model-picker/fixtures.ts", "ui/storybook/stories/tasks/composer-mobile-assignee.stories.tsx" ], - "fingerprint": "1288567427591ad546f49303ae7922bfd2b02c01635528cc4ae5470b926b723d", + "fingerprint": "b4d5689a5fd6a891e06d98d698f2556832969ac67c56f2e2a8d9da26b0510616", "stableId": "composer-mobile-assignee-picker", "sidebarTitle": "Tasks and conversations/Task creation and lifecycle/Composer/Mobile assignee picker" }, @@ -1578,7 +1578,7 @@ "ui/storybook/prototypes/composer-model-picker/fixtures.ts", "ui/storybook/stories/tasks/composer-model-picker.stories.tsx" ], - "fingerprint": "ecd36c7bcc1b331cc83aa50fc15bc18842e6fdb813f21889b30966c803204fcd", + "fingerprint": "f1ddb769d225b27ea4275da372027f7bc6f11ed079a8e7b4a3b7409744707b1f", "stableId": "composer-model-and-effort-picker", "sidebarTitle": "Tasks and conversations/Task creation and lifecycle/Composer/Model and effort picker" }, @@ -1679,7 +1679,7 @@ "ui/storybook/prototypes/ConnectionPoolConnectorHost.tsx", "ui/storybook/stories/runs-adapters/connection-pool-connector.stories.tsx" ], - "fingerprint": "045ed080d3c5ebecf92131e090085065ca2753ac7eadd66e3386b88542ff543c", + "fingerprint": "f6208d2a6b919cbd511baefdb5e45f79eb2c2d851e6035ea9778a13fc827544c", "stableId": "connectors-pool-host", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/Connectors/Pool host" }, @@ -1705,7 +1705,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/runs-adapters/connection-pools.stories.tsx" ], - "fingerprint": "dd72a70a6dc0173d82c92dc9aedaba6b5e0414d2befb8a8d4ee38c5e3cce8494", + "fingerprint": "b614f0bcfb914ae817cbfba36d8dd80091068ad6e6a630ec239a7d9f7b3e05c1", "stableId": "ai-connections-connection-pools", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Connection pools" }, @@ -1763,7 +1763,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/tasks/control-plane-surfaces.stories.tsx" ], - "fingerprint": "121469a91e60dbb4e085512d86e3bd8a24b1590469d0f1321ea5e1a869df7da0", + "fingerprint": "55b7a763e93f099aa6085dc7de65474f619aebd1ec9d114ee9c70fd0d2cf42c6", "stableId": "product-control-plane-surfaces", "sidebarTitle": "Tasks and conversations/Task creation and lifecycle/Product/Control Plane Surfaces" }, @@ -1807,7 +1807,7 @@ "ui/src/components/task-chat/TaskChatMarker.tsx", "ui/storybook/stories/secrets/credential-access/01-notice.stories.tsx" ], - "fingerprint": "d4d4b6098d76fa827e6529a6f071948f1f4e6a15a337da4a85ffe9f0234d8565", + "fingerprint": "2eba045db9127e5cd8d8bf7c9664e4598a5c84be117ae6159ce7171b24f2005d", "stableId": "task-credential-access-01-notice", "sidebarTitle": "Apps, channels, and extensions/Secrets and proposals/Task credential access/01 Notice" }, @@ -1833,7 +1833,7 @@ "ui/storybook/stories/secrets/credential-access/02-task-page.stories.tsx", "ui/storybook/support/credential-access/CredentialTaskPage.tsx" ], - "fingerprint": "b7f5b355919f9e11c7d079955113f365d3a41e956fd6cc6746e4c3fdddae644e", + "fingerprint": "aadd6c8f82db53b2e910829deb409344199353cb4956313424ce8d1415bead85", "stableId": "task-credential-access-02-task-page", "sidebarTitle": "Apps, channels, and extensions/Secrets and proposals/Task credential access/02 Task page" }, @@ -1886,7 +1886,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/activity/data-viz-misc.stories.tsx" ], - "fingerprint": "5cc019e34fd9a2adae706143c0af4fd78334a2ffefac62eb82d7ec7f64ee0ea3", + "fingerprint": "2861c5c594bbc3c30682958dd3727488f776e411368bcfdcbde39c14cb64cef6", "stableId": "product-data-visualization-misc", "sidebarTitle": "Oversight and operation/Dashboards and audit trails/Product/Data Visualization & Misc" }, @@ -1906,7 +1906,7 @@ "ui/src/components/DecisionCard.tsx", "ui/storybook/stories/inbox-search/decision-card.stories.tsx" ], - "fingerprint": "151c678aa5f7befc369d2def8df019b626c865f1c5b7bf91543dd4454395cb66", + "fingerprint": "220a4f037c1dd4a456ac7df75d6ee7fd0d844ff7db3fd30ca1da2fff9477cbc8", "stableId": "decisions-decisioncard", "sidebarTitle": "Tasks and conversations/Inbox, decisions, and search/Decisions/DecisionCard" }, @@ -1999,7 +1999,7 @@ "ui/src/pages/WhatNeedsMe.tsx", "ui/storybook/stories/inbox-search/decisions-desk.stories.tsx" ], - "fingerprint": "614be5300540b7c3c8942a8a043e005d0f92e0a021819ade6ad291526b923566", + "fingerprint": "1e49e931cddd7ef77de70f25fc4ee991d3de9d933670ae4412c5d17fad6db0aa", "stableId": "pages-decisions-desk", "sidebarTitle": "Tasks and conversations/Inbox, decisions, and search/Pages/Decisions Desk" }, @@ -2121,7 +2121,7 @@ "ui/src/components/DotRunnerConnection.tsx", "ui/storybook/stories/runs-adapters/dot-runner-connection.stories.tsx" ], - "fingerprint": "a5fbe31bc2b35136830bfd8fb1cf10e1b35cd7cc6995f108ceca5a6773a4db73", + "fingerprint": "65219fd330a54350d2c91a485039d2a8dc6886cce3dbde64bce2b514d2bdcc95", "stableId": "assistant-connections-dot-runner", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/Assistant connections/Dot Runner" }, @@ -2184,7 +2184,7 @@ "ui/src/components/ExecutionBlockerNotice.tsx", "ui/storybook/stories/recovery/execution-blocker-notice.stories.tsx" ], - "fingerprint": "19fa3d95095bbcd67c5306d6ad3c8c00ae526bab9d8cbc49e69ca36c15a873c2", + "fingerprint": "bee366ba1266e6692c8f0ee027c449fdda6b51762369ba431871a5af5ee018c6", "stableId": "task-chat-recovery-notice", "sidebarTitle": "Projects and execution/Recovery/Task chat/Recovery notice" }, @@ -2208,7 +2208,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/recovery/execution-recovery.stories.tsx" ], - "fingerprint": "003b5d7b2f8d0baea06402776157854f6b55d06bfd4d6223300413c284a22fd2", + "fingerprint": "6f8b245b169d95046bdb52513dfaf4937a96f807b72470c198863471f59c8149", "stableId": "tasks-execution-recovery", "sidebarTitle": "Projects and execution/Recovery/Tasks/Execution recovery" }, @@ -2820,7 +2820,7 @@ "ui/src/components/task-chat/TaskChatBlockerLinks.tsx", "ui/storybook/stories/recovery/issue-blocked-notice.stories.tsx" ], - "fingerprint": "4eb78b4e772567d657339435dc650e9b59c2347792cf764c7b376d7f631fe4de", + "fingerprint": "6a84d7f60318d33a1835a668873095aaad1c215a83bbbb4735823fdcb3291140", "stableId": "product-issue-blocked-notice", "sidebarTitle": "Projects and execution/Recovery/Product/Issue/Blocked notice" }, @@ -2942,7 +2942,7 @@ "ui/src/components/AdapterLoginChrome.tsx", "ui/storybook/stories/runs-adapters/local-ai-subscription.stories.tsx" ], - "fingerprint": "79d32a9424c87080ebaa86216c23e3c36fcb1d3c8f4051a109a201e4e3adf1c6", + "fingerprint": "7ec5fe9129699ed14dd55c4e3849c602100a1a934bad44b6fd6d31af4e8a0458", "stableId": "ai-connections-provider-routing-07-subscription-sign-in", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Provider routing/07 Subscription sign-in" }, @@ -3047,7 +3047,7 @@ "ui/src/components/IssueScheduledRetryCard.tsx", "ui/storybook/stories/recovery/model-capacity-retry.stories.tsx" ], - "fingerprint": "4f7421c0267fee71b9de5e38916ab86c66a2c69caa950730751c031a86503b4f", + "fingerprint": "5e08f681f3f96cb7be32e316d3f4066e5239dcf0f4393cb9ec09d599734f201e", "stableId": "tasks-model-capacity-retry", "sidebarTitle": "Projects and execution/Recovery/Tasks/Model capacity retry" }, @@ -4031,7 +4031,7 @@ "ui/storybook/prototypes/provider-routing/story-support.ts", "ui/storybook/stories/runs-adapters/provider-routing-00-overview.stories.tsx" ], - "fingerprint": "9f1c436cf8ac4a409ca597d95abf4a975ab5a58be08c37edbcd44547bea04d55", + "fingerprint": "58ea208bbfd6bbc068ff89a49aa2c71d55f244710d84f8d8574511bcc3593257", "stableId": "ai-connections-provider-routing-00-overview", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Provider routing/00 Overview" }, @@ -4081,7 +4081,7 @@ "ui/storybook/prototypes/provider-routing/story-support.ts", "ui/storybook/stories/runs-adapters/provider-routing-01-onboarding.stories.tsx" ], - "fingerprint": "17ed706f27cb9282f6918dd855160aeb6a63c003d7891c6fa6a336cd7cec1f8b", + "fingerprint": "f39e0ecde64690f67d56519efc0b7b45f790f21b7836057f6735fffaff3d3ef5", "stableId": "ai-connections-provider-routing-01-onboarding", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Provider routing/01 Onboarding" }, @@ -4112,7 +4112,7 @@ "ui/storybook/prototypes/provider-routing/story-support.ts", "ui/storybook/stories/runs-adapters/provider-routing-02-connect.stories.tsx" ], - "fingerprint": "25d750d1d9ab3e1adf71bc1273d577e7b59939f6db1b96a55f59e6da3bea8b8e", + "fingerprint": "728d7d6f61446c644f7d1402b683ade30dbef0c893e409a5fa9526e247e1c179", "stableId": "ai-connections-provider-routing-02-connect", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Provider routing/02 Connect" }, @@ -4162,7 +4162,7 @@ "ui/storybook/prototypes/provider-routing/story-support.ts", "ui/storybook/stories/runs-adapters/provider-routing-03-agent.stories.tsx" ], - "fingerprint": "dd04df0958753c6b4e3fbf97b118e1d69543c8e213970f2cfeffebdc5a7ecc91", + "fingerprint": "5d0c5dcd46b3814727322016a2bb4b58695f1435a9bd015c67898b549aac40fb", "stableId": "ai-connections-provider-routing-03-agent", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Provider routing/03 Agent" }, @@ -4195,7 +4195,7 @@ "ui/storybook/prototypes/provider-routing/story-support.ts", "ui/storybook/stories/runs-adapters/provider-routing-04-manage.stories.tsx" ], - "fingerprint": "f549ac215f9307cb77deefce243283d673b497901c996f5be67eeff177420024", + "fingerprint": "34331e78cd92bccd47f5e2c7b83609e137846117e137615b4bd29bda065524e4", "stableId": "ai-connections-provider-routing-04-manage", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Provider routing/04 Manage" }, @@ -4220,7 +4220,7 @@ "ui/storybook/prototypes/provider-routing/story-support.ts", "ui/storybook/stories/runs-adapters/provider-routing-05-recovery.stories.tsx" ], - "fingerprint": "5cfce358ed547b3eba402b1cdd5c58bac7ea71359c4edc7674c0c7e4de72e51e", + "fingerprint": "fe9596ebe4de1f2e315d6e024c636f56e0b253b3af5d3a9c4cd6a3015309e7e7", "stableId": "ai-connections-provider-routing-05-recovery", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Provider routing/05 Recovery" }, @@ -4251,7 +4251,7 @@ "ui/storybook/prototypes/provider-routing/shared.tsx", "ui/storybook/stories/runs-adapters/provider-routing-06-production.stories.tsx" ], - "fingerprint": "405597333685a02824aa162927f2e0c544647ec7079adbc74c61031e8258c8e6", + "fingerprint": "ab0fe3f233b6f8dafdb6a946447535366ba687b9ccf8fafd2b877aa4e7cac230", "stableId": "ai-connections-provider-routing-06-production-components", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/AI Connections/Provider routing/06 Production components" }, @@ -4384,7 +4384,7 @@ "ui/storybook/prototypes/recovery-notice/RecoveryNoticePreview.tsx", "ui/storybook/stories/recovery/recovery-notice-preview.stories.tsx" ], - "fingerprint": "4e880f02947c96e751ecca7a0f662106874ac5b58f254915592ecdec699298de", + "fingerprint": "cfd7345545112ea9ecc13035ebd187451117de5ed010f439cfb8c6962461fb3f", "stableId": "design-previews-recovery-notice", "sidebarTitle": "Projects and execution/Recovery/Design previews/Recovery notice" }, @@ -4616,7 +4616,7 @@ "ui/storybook/prototypes/runner-activity/RunnerActivityPreview.tsx", "ui/storybook/stories/tasks/runner-activity.stories.tsx" ], - "fingerprint": "fc9539ac75fd89906fcd403ca1e5792b21440b898d0662317fbe28d7192e725d", + "fingerprint": "f39f36b1276c741530f6da95724d0e29fbc43c3bdd87b75640047bbc9f753810", "stableId": "tasks-runner-activity-preview", "sidebarTitle": "Tasks and conversations/Task creation and lifecycle/Tasks/Runner activity preview" }, @@ -4638,7 +4638,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/recovery/scheduled-retry.stories.tsx" ], - "fingerprint": "b07bdc368ce48d184b5cd99156ed23c03251ac9ecd90de47812db351261cc5c0", + "fingerprint": "43ef1917079251d558b3f52e5af750dfb69a54d3326d31494dbf037baeefe193", "stableId": "product-issue-scheduled-retry-surfaces", "sidebarTitle": "Projects and execution/Recovery/Product/Issue Scheduled retry surfaces" }, @@ -4665,7 +4665,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/inbox-search/search.stories.tsx" ], - "fingerprint": "dfd04cb52d810ed016fae1508dea6fa7baeb8a71676e605a4eff269d844957b6", + "fingerprint": "4ebfe41c086ce2e0ae674263dfa50239a0770e5e6f98c3c24f54955f6cbdd3d2", "stableId": "product-search-command-k", "sidebarTitle": "Tasks and conversations/Inbox, decisions, and search/Product/Search & Command K" }, @@ -4729,7 +4729,7 @@ "ui/src/components/task-chat/RunnerGoalWidget.tsx", "ui/storybook/stories/runs-adapters/session-goals.stories.tsx" ], - "fingerprint": "1d6e7f30007a296443b0670142cb13694e0e955e6f97687f306f69155e766593", + "fingerprint": "91775a33b452fed0055f7e607384f3f95801a8bafbdf369cf149f8ebcc3943a8", "stableId": "composer-session-goals", "sidebarTitle": "Agents and reusable capabilities/Runs, harnesses, and model accounts/Composer/Session goals" }, @@ -4950,7 +4950,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/recovery/source-issue-recovery.stories.tsx" ], - "fingerprint": "ad00308c561a44cfb60a7b359ac88b48ad5ee2f13d4b36af99552c29c0a3df67", + "fingerprint": "1b6e44d4fb4ab7e2e63fb8816aa31fbe3267e10647f238ed82e6591a82d2e918", "stableId": "paperclip-source-issue-recovery", "sidebarTitle": "Projects and execution/Recovery/Paperclip/Source Issue Recovery" }, @@ -4972,7 +4972,7 @@ "ui/src/components/SourceResolvedFoldCallout.tsx", "ui/storybook/stories/recovery/source-resolved-fold.stories.tsx" ], - "fingerprint": "f6a8c8477c45cbf1c98a0e0a8618373800183aa49a189b00efac62c42fb0b92e", + "fingerprint": "3af3daff618b8c278bcbf10ba3e5d081d87240c680758ccb3078eccdf9283122", "stableId": "paperclip-source-resolved-fold", "sidebarTitle": "Projects and execution/Recovery/Paperclip/Source-resolved Fold" }, @@ -5049,7 +5049,7 @@ "ui/storybook/fixtures/paperclipData.ts", "ui/storybook/stories/tasks/successful-run-handoff.stories.tsx" ], - "fingerprint": "dee060e56c932b5c076cde939cc8cad36b46452fd3fd85118e1eb6664deb47ad", + "fingerprint": "dd47ab0240dd12f6bdeddeaf155be7afc472e728f69e77579adaf7aeec0eb588", "stableId": "paperclip-successful-run-handoff", "sidebarTitle": "Tasks and conversations/Task creation and lifecycle/Paperclip/Successful Run Handoff" }, @@ -5120,7 +5120,7 @@ "ui/src/components/task-chat/task-chat-model.ts", "ui/storybook/stories/tasks/task-chat-chain-of-thought.stories.tsx" ], - "fingerprint": "c9641a4c8ac8ab68a6d1ad7b9a730f5c3b4f894d5a60262ba2ef196d1c1da7d2", + "fingerprint": "2c926951a7911a3ca79ceae50578e581f389e7cffe68bce15e862ae95a429644", "stableId": "tasks-task-chat-review-fixes", "sidebarTitle": "Tasks and conversations/Task creation and lifecycle/Tasks/Task chat review fixes" }, @@ -5463,7 +5463,7 @@ "ui/src/components/ToastViewport.tsx", "ui/storybook/stories/inbox-search/what-needs-me.stories.tsx" ], - "fingerprint": "b84aef9b566d476455d6b441adfb301d92f68148a409eff29f5850c047d04baf", + "fingerprint": "d8b838185489f6b5009e52d9c2ac3a1fea3b2718211a1ddc393c404a0335a466", "stableId": "pages-decisions", "sidebarTitle": "Tasks and conversations/Inbox, decisions, and search/Pages/Decisions" }, @@ -5486,7 +5486,7 @@ "ui/storybook/fixtures/workTimeline.sample.json", "ui/storybook/stories/activity/work-timeline.stories.tsx" ], - "fingerprint": "ef4f6622a54c742869916b1bded0fd410aaf1faa3bf9e5ff37182eac7ac6aa50", + "fingerprint": "4f6e9f556073d4bc6a812f633bcff32cb84c4c02488746b37797501684ef3899", "stableId": "pages-work-timeline", "sidebarTitle": "Oversight and operation/Dashboards and audit trails/Pages/Work Timeline" }, @@ -5507,7 +5507,7 @@ "ui/storybook/stories/recovery/workspace-branch-recovery/01-component.stories.tsx", "ui/storybook/support/workspace-branch-recovery/fixtures.ts" ], - "fingerprint": "411a3a9332d5348327cb4c04e508895f9f1989d6da27fe8d1d73ebd34be87a82", + "fingerprint": "77f708cc660f6829cadaffa6139930a102109976c41f7e24e2f662ed687d2412", "stableId": "workspace-branch-recovery-01-repair-card", "sidebarTitle": "Projects and execution/Recovery/Workspace branch recovery/01 Repair card" }, @@ -5532,7 +5532,7 @@ "ui/storybook/stories/recovery/workspace-branch-recovery/02-task-journey.stories.tsx", "ui/storybook/support/workspace-branch-recovery/fixtures.ts" ], - "fingerprint": "85db6aeb522937db01d8e307eab632d16f887abe88301c5d2f3c2f797fcb7cec", + "fingerprint": "d34d8092f10377b2280f9511205b5eae76b5dbfd9cde002669eefc51aa5d16d6", "stableId": "workspace-branch-recovery-02-task-journey", "sidebarTitle": "Projects and execution/Recovery/Workspace branch recovery/02 Task journey" }, diff --git a/packages/adapter-utils/src/ssh-command-managed.test.ts b/packages/adapter-utils/src/ssh-command-managed.test.ts new file mode 100644 index 0000000000..57ed8b5d61 --- /dev/null +++ b/packages/adapter-utils/src/ssh-command-managed.test.ts @@ -0,0 +1,70 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + execFile: vi.fn((file: string, args: string[], options: object, callback: Parameters[3]) => { + if (file !== "ssh") throw new Error(`Unexpected subprocess ${file}`); + return actual.execFile("sh", ["-c", args.at(-1)!], options, callback); + }), + spawn: vi.fn((file: string, args: string[], options: Parameters[2]) => { + if (file !== "ssh") throw new Error(`Unexpected subprocess ${file}`); + return actual.spawn("sh", ["-c", args.at(-1)!], options); + }), + }; +}); + +import { createSshCommandManagedRuntimeRunner } from "./ssh.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); +}); + +async function runner() { + const cwd = await mkdtemp(path.join(os.tmpdir(), "ssh-managed-argv-")); + roots.push(cwd); + return createSshCommandManagedRuntimeRunner({ + spec: { + host: "fixture.invalid", username: "fixture", port: 22, + remoteCwd: cwd, remoteWorkspacePath: cwd, + privateKey: null, knownHosts: null, strictHostKeyChecking: true, + }, + }); +} + +describe("SSH command-managed argv", () => { + it("rejects environment keys that could be interpreted as env options", async () => { + await expect((await runner()).execute({ command: "true", env: { "-u": "PATH" } })) + .rejects.toThrow("Invalid SSH environment variable key"); + }); + + it.each([[false, false], [true, false], [false, true], [true, true]])( + "preserves shell positional arguments with environment=%s and stdin=%s", + async (withEnvironment, withStdin) => { + const value = "literal 'quote' $(printf unsafe) `printf unsafe`"; + const result = await (await runner()).execute({ + command: "sh", + args: ["-c", 'set -eu; printf "%s\\n" "$0" "$1" "$2" "${SENTINEL_TEXT-unset}"', "named-probe", "two words", value], + ...(withEnvironment ? { env: { SENTINEL_TEXT: value } } : {}), + ...(withStdin ? { stdin: "" } : {}), + timeoutMs: 5000, + }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(["named-probe", "two words", value, withEnvironment ? value : "unset"].join("\n") + "\n"); + }, + ); + + it("applies environment to a non-shell command", async () => { + const result = await (await runner()).execute({ + command: "printenv", args: ["SENTINEL_TEXT"], + env: { SENTINEL_TEXT: "two words" }, timeoutMs: 5000, + }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe("two words\n"); + }); +}); diff --git a/packages/adapter-utils/src/ssh.ts b/packages/adapter-utils/src/ssh.ts index ddb07d6ac7..47a9c0b82b 100644 --- a/packages/adapter-utils/src/ssh.ts +++ b/packages/adapter-utils/src/ssh.ts @@ -60,17 +60,15 @@ export function createSshCommandManagedRuntimeRunner(input: { const cwd = commandInput.cwd?.trim() || defaultCwd; const envEntries = Object.entries(commandInput.env ?? {}) .filter((entry): entry is [string, string] => typeof entry[1] === "string"); + for (const [key] of envEntries) { + if (!isValidShellEnvKey(key)) { + throw new Error(`Invalid SSH environment variable key: ${key}`); + } + } const envPrefix = envEntries.length > 0 - ? `env ${envEntries.map(([key, value]) => `${key}=${shellQuote(value)}`).join(" ")} ` + ? `env ${envEntries.map(([key, value]) => shellQuote(`${key}=${value}`)).join(" ")} ` : ""; - const exportPrefix = envEntries.length > 0 - ? envEntries.map(([key, value]) => `export ${key}=${shellQuote(value)};`).join(" ") + " " - : ""; - const commandScript = command === "sh" || command === "bash" - ? (args[0] === "-c" || args[0] === "-lc") && typeof args[1] === "string" - ? `${exportPrefix}${args[1]}` - : `${envPrefix}exec ${[shellQuote(command), ...args.map((arg) => shellQuote(arg))].join(" ")}` - : `${envPrefix}exec ${[shellQuote(command), ...args.map((arg) => shellQuote(arg))].join(" ")}`; + const commandScript = `exec ${envPrefix}${[shellQuote(command), ...args.map((arg) => shellQuote(arg))].join(" ")}`; const remoteCommand = `cd ${shellQuote(cwd)} && ${commandScript}`; try { diff --git a/packages/adapters/codex-local/src/ui/build-config.test.ts b/packages/adapters/codex-local/src/ui/build-config.test.ts index 82db065ae3..019f635a35 100644 --- a/packages/adapters/codex-local/src/ui/build-config.test.ts +++ b/packages/adapters/codex-local/src/ui/build-config.test.ts @@ -118,6 +118,11 @@ describe("buildCodexLocalConfig", () => { }); describe("buildPaperclipRunnerConfig", () => { + it.each(["", "max"])("preserves or overrides saved native Codex reasoning effort (%s)", thinkingEffort => { + const values = makeValues({ thinkingEffort, adapterSchemaValues: { provider: "codex", modelReasoningEffort: "high" } }); + expect(buildPaperclipRunnerConfig(values).modelReasoningEffort).toBe(thinkingEffort || "high"); + expect(buildPaperclipRunnerConfig({ ...values, adapterSchemaValues: { provider: "acpx", acpxAgent: "claude", modelReasoningEffort: "high" } })).not.toHaveProperty("modelReasoningEffort"); + }); it.each([undefined, false, true])("preserves Dot attachment consent in create/import forms: %s", value => { const config = buildPaperclipRunnerConfig(makeValues({ adapterType: "paperclip_runner", adapterSchemaValues: { provider: "openai_dot", dotAttachmentAccess: value } })); expect(config.dotAttachmentAccess).toBe(value === true); @@ -142,6 +147,7 @@ describe("buildPaperclipRunnerConfig", () => { codexPermissionMode: "never", lifecycleMode: "per_turn", model: "gpt-5.4", + modelReasoningEffort: "high", timeoutSec: 0, graceSec: 15, }); @@ -149,7 +155,6 @@ describe("buildPaperclipRunnerConfig", () => { "engine", "agentCommand", "stateDir", - "modelReasoningEffort", "search", "fastMode", "dangerouslyBypassApprovalsAndSandbox", diff --git a/packages/adapters/codex-local/src/ui/build-config.ts b/packages/adapters/codex-local/src/ui/build-config.ts index 3a6853b927..51c6a4bf3d 100644 --- a/packages/adapters/codex-local/src/ui/build-config.ts +++ b/packages/adapters/codex-local/src/ui/build-config.ts @@ -76,6 +76,7 @@ export function buildCodexLocalConfig(v: CreateConfigValues): Record { const config = buildCodexLocalConfig(v); const schemaValues = normalizeLegacyRunnerProvider({ ...(v.adapterSchemaValues ?? {}) }); + const codexReasoningEffort = config.modelReasoningEffort ?? schemaValues.modelReasoningEffort ?? schemaValues.reasoningEffort; for (const unsupportedKey of [ "engine", "agentCommand", @@ -232,7 +233,8 @@ export function buildPaperclipRunnerConfig(v: CreateConfigValues): Record Result<(), Box> { } "initialized" => {} "thread/start" => { + if let Some(delay) = argument(&args, "--thread-start-delay-ms") { + thread::sleep(Duration::from_millis(delay.parse()?)); + } if require_external_sandbox && (message.pointer("/params/sandbox") != Some(&json!("danger-full-access")) || message.pointer("/params/permissions").is_some()) @@ -1076,7 +1079,8 @@ fn run() -> Result<(), Box> { } send(json!({ "id": id, - "result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}} + "result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}, + "model": argument(&args, "--returned-model").map(Value::String).unwrap_or_else(|| message.pointer("/params/model").filter(|model| model.is_string()).cloned().unwrap_or(json!("gpt-test")))} }))?; if agent_created_goal { send(json!({ diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 61432b9dbe..cf211a6c97 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -367,6 +367,8 @@ pub struct CodexProviderConfig { pub approval_policy: String, #[serde(default)] pub externally_sandboxed: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub read_only: bool, // Older persisted configurations deliberately retain the provider default. #[serde(default, skip_serializing_if = "Option::is_none")] pub include_skill_instructions: Option, @@ -669,6 +671,7 @@ pub struct CodexProvider { next_request_id: u64, thread_id: String, provider_session_id: Option, + observed_model: Option, active_provider_turn_id: Option, pending_messages: VecDeque, deferred_ambiguous_messages: VecDeque, @@ -831,6 +834,10 @@ fn codex_permission_profile(provider: &str, external_sandbox: bool) -> &'static } } +fn is_false(value: &bool) -> bool { + !*value +} + impl CodexProvider { pub fn start( config: &CodexProviderConfig, @@ -904,11 +911,15 @@ impl CodexProvider { )); } let authorized_tools = authorized_tools.into_iter().collect::>(); - let permission_profile = codex_permission_profile( - &config.provider, - config.externally_sandboxed - || std::env::var("PAPERCLIP_RUNNER_EXTERNAL_SANDBOX").as_deref() == Ok("1"), - ); + let permission_profile = if config.read_only { + "paperclip-runner-workspace-read-only" + } else { + codex_permission_profile( + &config.provider, + config.externally_sandboxed + || std::env::var("PAPERCLIP_RUNNER_EXTERNAL_SANDBOX").as_deref() == Ok("1"), + ) + }; let (dynamic_tools, authorized_tool_ids) = codex_dynamic_tools(authorized_tools.iter().cloned())?; let common_environment_keys = [ @@ -1003,6 +1014,7 @@ impl CodexProvider { next_request_id: 1, thread_id: String::new(), provider_session_id: None, + observed_model: None, active_provider_turn_id: None, pending_messages: VecDeque::new(), deferred_ambiguous_messages: VecDeque::new(), @@ -1133,6 +1145,12 @@ impl CodexProvider { .and_then(Value::as_str) .filter(|value| !value.is_empty()) .map(str::to_owned); + provider.observed_model = opened + .get("model") + .or_else(|| opened.pointer("/thread/model")) + .and_then(Value::as_str) + .filter(|value| !value.is_empty() && value.len() <= 240) + .map(str::to_owned); if resume_thread_id.is_some() { stage = ProviderStartupStage::ThreadRead; @@ -1175,6 +1193,10 @@ impl CodexProvider { self.provider_session_id.as_deref() } + pub fn observed_model(&self) -> Option<&str> { + self.observed_model.as_deref() + } + pub(crate) fn take_provider_trace_frame_id(&mut self) -> Option { self.last_trace_frame_id.take() } @@ -1685,6 +1707,16 @@ impl CodexProvider { message: &str, cwd: &str, options: CodexTurnOptions<'_>, + ) -> Result { + self.start_turn_with_options_and_effort(message, cwd, options, None) + } + + pub(crate) fn start_turn_with_options_and_effort( + &mut self, + message: &str, + cwd: &str, + options: CodexTurnOptions<'_>, + effort: Option<&Value>, ) -> Result { let CodexTurnOptions { skills, @@ -1705,6 +1737,17 @@ impl CodexProvider { for skill in skills { skill.validate()?; } + let effort = effort.filter(|value| !value.is_null()); + if effort.is_some_and(|value| { + !matches!( + value.as_str(), + Some("minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "ultra") + ) + }) { + return Err(LocalRunnerError::invalid( + "unsupported native Codex reasoning effort", + )); + } if self.quarantined { return Err(LocalRunnerError::invalid( "Codex provider is quarantined after unsafe recovered work", @@ -1745,6 +1788,9 @@ impl CodexProvider { let turn_params_object = turn_params .as_object_mut() .expect("Codex turn parameters are an object"); + if let Some(effort) = effort { + turn_params_object.insert("effort".to_owned(), effort.clone()); + } if let Some(mode) = reasoning_mode { turn_params_object.insert("reasoningMode".to_owned(), json!(mode)); } @@ -4094,6 +4140,7 @@ done instructions: "Test only.".to_owned(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }; @@ -4481,6 +4528,7 @@ done instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }; @@ -4601,6 +4649,7 @@ done instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 329c3dc7bf..e6c2c977db 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -2608,7 +2608,14 @@ impl CodexCommandExecutor { .as_ref() .and_then(|state| state.thread_id.as_ref()) .is_some(); - let (thread_id, provider_session_id, process_id, active_provider_turn_id, goal_probe) = { + let ( + thread_id, + provider_session_id, + process_id, + active_provider_turn_id, + goal_probe, + observed_model, + ) = { let provider = self.ensure_provider()?; ( provider.thread_id().to_owned(), @@ -2616,6 +2623,7 @@ impl CodexCommandExecutor { provider.process_id(), provider.active_provider_turn_id().map(str::to_owned), provider.get_goal(), + provider.observed_model().map(str::to_owned), ) }; let (goal_capability, goal) = match goal_probe { @@ -2703,6 +2711,7 @@ impl CodexCommandExecutor { "providerSessionId": thread_id, "providerAccountSessionId": provider_session_id, "processId": process_id, + "providerDescriptor": { "driver": driver, "providerVersion": provider_version, "effectiveModel": observed_model }, }), ), ( @@ -3003,13 +3012,14 @@ impl CodexCommandExecutor { rejected_accepted_turn, ) = { let provider = self.ensure_provider()?; - let result = provider.start_turn_with_options( + let result = provider.start_turn_with_options_and_effort( text, &cwd, CodexTurnOptions { skills: &skills, reasoning_mode, }, + payload.get("effort"), ); ( result, @@ -4998,6 +5008,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -5379,6 +5390,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -5470,6 +5482,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -5523,6 +5536,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -5566,6 +5580,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -5658,6 +5673,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -5725,6 +5741,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -5775,6 +5792,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -5899,6 +5917,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -6011,6 +6030,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -6068,6 +6088,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -6210,6 +6231,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -6251,6 +6273,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -6289,6 +6312,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, @@ -6366,6 +6390,7 @@ mod tests { instructions: String::new(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, }, diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index b4bc2a316b..e586105d51 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -65,6 +65,7 @@ fn provider_config(directory: &Path, switches: &[&str]) -> CodexProviderConfig { instructions: "Stay inside the test workspace.".to_owned(), approval_policy: "never".to_owned(), externally_sandboxed: false, + read_only: false, include_skill_instructions: None, conversation_mode: None, } diff --git a/packages/paperclip-runner/scripts/release-provider-pack.test.mjs b/packages/paperclip-runner/scripts/release-provider-pack.test.mjs index 33d6de5874..171edd46f0 100644 --- a/packages/paperclip-runner/scripts/release-provider-pack.test.mjs +++ b/packages/paperclip-runner/scripts/release-provider-pack.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { spawnSync } from "node:child_process"; -import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -9,6 +9,7 @@ import test from "node:test"; import profiles from "../acpx-profiles.json" with { type: "json" }; import distributions from "../cursor-distributions.json" with { type: "json" }; import { verifyReleaseProviderPack } from "./release-provider-pack.mjs"; +import { existingReleaseProviderPack } from "./stage-release-runner-binaries.mjs"; const revision = "a".repeat(40); const digest = "sha256:" + "b".repeat(64); @@ -47,18 +48,18 @@ test("rejects omitted Cursor, conflicting legacy inventory, stale sources and fo }); test("release assembly rejects rehashed stale or missing packs before staging any files", () => { - const root = mkdtempSync(join(tmpdir(), "release-provider-pack-")); + const root = realpathSync(mkdtempSync(join(tmpdir(), "release-provider-pack-"))); const source = dirname(dirname(fileURLToPath(import.meta.url))); try { // Run the actual assembler in an isolated layout so it cannot overwrite built artifacts. - for (const file of ["scripts/stage-release-runner-binaries.mjs", "scripts/release-provider-pack.mjs", "src/live/runner-binary.ts", "acpx-profiles.json", "cursor-distributions.json"]) { + for (const file of ["scripts/stage-release-runner-binaries.mjs", "scripts/release-provider-pack.mjs", "src/live/runner-binary.ts", "acpx-profiles.json", "cursor-distributions.json", "release-runner-targets.json"]) { mkdirSync(dirname(join(root, file)), { recursive: true }); cpSync(join(source, file), join(root, file)); } const platforms = {}; - for (const target of ["darwin-arm64", "darwin-x64", "linux-x64"]) { + for (const target of ["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"]) { const bytes = Buffer.alloc(64); - if (target === "linux-x64") { - bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); bytes.writeUInt16LE(62, 18); + if (target.startsWith("linux-")) { + bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); bytes.writeUInt16LE(target === "linux-x64" ? 62 : 183, 18); } else { bytes.writeUInt32LE(0xfeedfacf, 0); bytes.writeUInt32LE(target === "darwin-arm64" ? 0x0100000c : 0x01000007, 4); } @@ -77,3 +78,19 @@ test("release assembly rejects rehashed stale or missing packs before staging an } } finally { rmSync(root, { recursive: true, force: true }); } }); + + +test("daemon-only assembly retains an existing same-source Linux provider-pack identity", () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), "release-provider-retained-"))); + try { + assert.equal(existingReleaseProviderPack(root, revision), undefined); + const file = join(root, "dist/remote-provider-packs/linux-x64/provider-pack.json"); + mkdirSync(dirname(file), { recursive: true }); + const pack = seal(fixture()); writeFileSync(file, JSON.stringify(pack)); + assert.deepEqual(existingReleaseProviderPack(root, revision), pack); + assert.throws(() => existingReleaseProviderPack(root, "c".repeat(40)), /release source/); + const wrongTarget = fixture(); wrongTarget.target = { platform: "darwin", architecture: "arm64" }; + writeFileSync(file, JSON.stringify(seal(wrongTarget))); + assert.throws(() => existingReleaseProviderPack(root, revision), /Linux x64 target/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs b/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs index f0e505363a..5f82bd937f 100644 --- a/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs +++ b/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs @@ -1,55 +1,139 @@ -// Assemble the three independently built release daemons before npm packing. // Usage: node packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs MANIFEST.json // Manifest: {sourceRevision, platforms: {"darwin-arm64": {path, sha256}, ...}, remoteProviderPack: {path, sha256}}. import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; -import { chmodSync, copyFileSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { dirname, isAbsolute, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { runnerBinaryTarget } from "../src/live/runner-binary.ts"; import { verifyReleaseProviderPack } from "./release-provider-pack.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const required = ["darwin-arm64", "darwin-x64", "linux-x64"]; -if (process.argv.length !== 3) throw Error("Usage: stage-release-runner-binaries.mjs MANIFEST.json"); -const manifest = JSON.parse(readFileSync(process.argv[2], "utf8")); -if (!/^[a-f0-9]{40}$/.test(manifest.sourceRevision) || !manifest.platforms || Object.keys(manifest.platforms).sort().join() !== [...required].sort().join()) throw Error("Release daemon manifest must bind one source and all three supported targets"); -const verified = required.map(target => { - const artifact = manifest.platforms[target]; - if (!artifact || !isAbsolute(artifact.path) || !/^sha256:[a-f0-9]{64}$/.test(artifact.sha256)) throw Error(`Invalid ${target} release artifact`); - const bytes = readFileSync(artifact.path), sha256 = "sha256:" + createHash("sha256").update(bytes).digest("hex"); - if (sha256 !== artifact.sha256 || runnerBinaryTarget(bytes) !== target) throw Error(`Release daemon ${target} digest or architecture mismatch`); - return { target, source: artifact.path, sha256 }; -}); -if (!manifest.remoteProviderPack) throw Error("Release assembly requires the matching Linux remote provider pack"); -const entry = manifest.remoteProviderPack; -if (!isAbsolute(entry.path) || !/^sha256:[a-f0-9]{64}$/.test(entry.sha256)) throw Error("Invalid remote provider-pack release artifact"); -const packBytes = readFileSync(entry.path); -if (packBytes.length > 512 * 1024 || "sha256:" + createHash("sha256").update(packBytes).digest("hex") !== entry.sha256) throw Error("Remote provider-pack release manifest digest mismatch"); -const packIdentity = verifyReleaseProviderPack(JSON.parse(packBytes), manifest.sourceRevision); -const platforms = {}; -for (const artifact of verified) { - const relative = `${artifact.target}/paperclip-runnerd`, destination = join(root, "dist", "bin", relative); - mkdirSync(dirname(destination), { recursive: true }); copyFileSync(artifact.source, destination); chmodSync(destination, 0o755); - // Copied Mach-O inodes need a fresh ad-hoc signature on macOS. Record the - // resulting packaged bytes, as well as the verified build artifact identity. - if (process.platform === "darwin" && artifact.target.startsWith("darwin-")) { - execFileSync("codesign", ["--force", "--sign", "-", destination], { stdio: "pipe" }); +export function releaseRunnerTargets(contract) { + const supported = ["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"]; + if (!contract || contract.schema !== "paperclip.runner.release-targets.v1" + || Object.keys(contract).sort().join() !== "schema,targets" + || !Array.isArray(contract.targets) + || JSON.stringify([...contract.targets].sort()) !== JSON.stringify([...supported].sort())) { + throw Error("Unknown release daemon target contract"); } - const sha256 = "sha256:" + createHash("sha256").update(readFileSync(destination)).digest("hex"); - platforms[artifact.target] = { path: relative, sha256, sourceSha256: artifact.sha256 }; + return Object.freeze([...contract.targets]); } -{ - const destination = join(root, "dist", "remote-provider-packs", "linux-x64", "provider-pack.json"); - mkdirSync(dirname(destination), { recursive: true }); - // Replace the inode so repeat assembly can preserve a read-only manifest. - const staged = `${destination}.staging-${process.pid}`; - try { - writeFileSync(staged, packBytes, { mode: 0o444, flag: "wx" }); - renameSync(staged, destination); - } finally { - rmSync(staged, { force: true }); +export const RELEASE_RUNNER_TARGETS = releaseRunnerTargets(JSON.parse( + readFileSync(new URL("../release-runner-targets.json", import.meta.url), "utf8"), +)); +const required = RELEASE_RUNNER_TARGETS; +const digest = bytes => "sha256:" + createHash("sha256").update(bytes).digest("hex"); +function regularFile(path, maximumSize) { + const stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size < 1 || stat.size > maximumSize + || realpathSync(path) !== resolve(path)) throw Error("Release artifact must be a bounded canonical regular file"); + return readFileSync(path); +} + +async function metadataContract() { + const { PAPERCLIP_RUNNER_BUILD_METADATA } = await import("../dist/evals/build-metadata.js"); + const { parsePaperclipRunnerdBuildMetadata } = await import("../dist/evals/runnerd-artifact.js"); + return metadata => verifyReleaseRunnerMetadata(parsePaperclipRunnerdBuildMetadata(metadata), PAPERCLIP_RUNNER_BUILD_METADATA); +} +export function verifyReleaseRunnerMetadata(metadata, expected) { + if (metadata.binaryContractVersion !== expected.contracts.runnerdArtifact + || metadata.nativeExecutionVersion !== expected.contracts.nativeExecution + || metadata.harnessDriverVersion !== expected.contracts.harnessDriver + || metadata.packageName !== expected.package.name || metadata.packageVersion !== expected.package.version + || metadata.prp?.name !== expected.prp.name || metadata.prp.minimumVersion > expected.prp.maximumVersion + || metadata.prp.maximumVersion < expected.prp.minimumVersion) throw Error("Release daemon build metadata is incompatible with the production contract"); + return metadata; +} + +export function releaseDaemonManifest(artifactDirectory, sourceRevision, verifyMetadata) { + if (!/^[a-f0-9]{40}$/.test(sourceRevision)) throw Error("Release daemons require an immutable source revision"); + if (readdirSync(artifactDirectory).sort().join() !== [...required].sort().join()) throw Error("Release daemons require exactly the four supported target directories"); + const platforms = {}; + for (const target of required) { + const directory = join(resolve(artifactDirectory), target); + if (readdirSync(directory).sort().join() !== "paperclip-runnerd,receipt.json") throw Error("Release target must contain only its daemon and receipt"); + const receipt = JSON.parse(regularFile(join(directory, "receipt.json"), 64 * 1024)); + const path = join(directory, "paperclip-runnerd"), bytes = regularFile(path, 256 * 1024 * 1024); + if (receipt.schema !== "paperclip.runner.release-artifact.v1" || receipt.sourceRevision !== sourceRevision + || receipt.target !== target || receipt.sha256 !== digest(bytes) || runnerBinaryTarget(bytes) !== target) { + throw Error(`Release daemon ${target} source, digest or architecture mismatch`); + } + verifyMetadata(receipt.buildMetadata); + platforms[target] = { path, sha256: receipt.sha256, buildMetadata: receipt.buildMetadata }; } + return { sourceRevision, platforms }; } -writeFileSync(join(root, "dist", "bin", "release-manifest.json"), JSON.stringify({schema:"paperclip.runner.release-binaries.v1",sourceRevision:manifest.sourceRevision,platforms, - remoteProviderPack: { target: "linux-x64", digest: packIdentity.digest, sourceRevision: packIdentity.payload.runnerSourceRevision }},null,2)+"\n"); -console.log("Staged verified release daemons for " + required.join(", ")); + +export function existingReleaseProviderPack(packageRoot, sourceRevision) { + const file = join(packageRoot, "dist/remote-provider-packs/linux-x64/provider-pack.json"); + if (!existsSync(file)) return undefined; + return verifyReleaseProviderPack(JSON.parse(regularFile(file, 512 * 1024)), sourceRevision); +} + +async function main() { + if (process.argv[2] === "record") { + const [, , , sourceRevision, target, source, output] = process.argv; + if (process.argv.length !== 7 || !/^[a-f0-9]{40}$/.test(sourceRevision) || !required.includes(target) + || target !== `${process.platform}-${process.arch}`) throw Error("Record requires the exact source and actual producer host target"); + const bytes = regularFile(resolve(source), 256 * 1024 * 1024); + if (runnerBinaryTarget(bytes) !== target) throw Error("Built daemon architecture does not match its producer"); + const buildMetadata = JSON.parse(execFileSync(resolve(source), ["--build-metadata"], { + timeout: 15_000, maxBuffer: 64 * 1024, encoding: "utf8", + })); + (await metadataContract())(buildMetadata); + mkdirSync(output, { recursive: true }); + copyFileSync(source, join(output, "paperclip-runnerd")); + writeFileSync(join(output, "receipt.json"), JSON.stringify({ schema: "paperclip.runner.release-artifact.v1", + sourceRevision, target, sha256: digest(bytes), buildMetadata }, null, 2) + "\n"); + return; + } + const daemonOnly = process.argv[2] === "assemble-daemons"; + if (daemonOnly ? process.argv.length !== 5 : process.argv.length !== 3) throw Error("Usage: stage-release-runner-binaries.mjs MANIFEST.json | assemble-daemons ARTIFACT_DIRECTORY SOURCE_SHA"); + const manifest = daemonOnly + ? releaseDaemonManifest(process.argv[3], process.argv[4], await metadataContract()) + : JSON.parse(regularFile(resolve(process.argv[2]), 512 * 1024)); + if (!/^[a-f0-9]{40}$/.test(manifest.sourceRevision) || !manifest.platforms || Object.keys(manifest.platforms).sort().join() !== [...required].sort().join()) throw Error("Release daemon manifest must bind one source and all four supported targets"); + const verified = required.map(target => { + const artifact = manifest.platforms[target]; + if (!artifact || !isAbsolute(artifact.path) || !/^sha256:[a-f0-9]{64}$/.test(artifact.sha256)) throw Error(`Invalid ${target} release artifact`); + const bytes = regularFile(artifact.path, 256 * 1024 * 1024), sha256 = "sha256:" + createHash("sha256").update(bytes).digest("hex"); + if (sha256 !== artifact.sha256 || runnerBinaryTarget(bytes) !== target) throw Error(`Release daemon ${target} digest or architecture mismatch`); + return { target, source: artifact.path, sha256 }; + }); + if (!manifest.remoteProviderPack && !daemonOnly) throw Error("Release assembly requires the matching Linux remote provider pack"); + let packBytes, packIdentity; + if (manifest.remoteProviderPack) { + const entry = manifest.remoteProviderPack; + if (!isAbsolute(entry.path) || !/^sha256:[a-f0-9]{64}$/.test(entry.sha256)) throw Error("Invalid remote provider-pack release artifact"); + packBytes = regularFile(entry.path, 512 * 1024); + if (packBytes.length > 512 * 1024 || "sha256:" + createHash("sha256").update(packBytes).digest("hex") !== entry.sha256) throw Error("Remote provider-pack release manifest digest mismatch"); + packIdentity = verifyReleaseProviderPack(JSON.parse(packBytes), manifest.sourceRevision); + } else if (daemonOnly) { + packIdentity = existingReleaseProviderPack(root, manifest.sourceRevision); + } + const platforms = {}; + for (const artifact of verified) { + const relative = `${artifact.target}/paperclip-runnerd`, destination = join(root, "dist", "bin", relative); + mkdirSync(dirname(destination), { recursive: true }); copyFileSync(artifact.source, destination); chmodSync(destination, 0o755); + if (process.platform === "darwin" && artifact.target.startsWith("darwin-")) { + execFileSync("codesign", ["--force", "--sign", "-", destination], { stdio: "pipe", timeout: 30_000 }); + } + const sha256 = "sha256:" + createHash("sha256").update(readFileSync(destination)).digest("hex"); + platforms[artifact.target] = { path: relative, sha256, sourceSha256: artifact.sha256 }; + } + if (packBytes) { + const destination = join(root, "dist", "remote-provider-packs", "linux-x64", "provider-pack.json"); + mkdirSync(dirname(destination), { recursive: true }); + const staged = `${destination}.staging-${process.pid}`; + try { + writeFileSync(staged, packBytes, { mode: 0o444, flag: "wx" }); + renameSync(staged, destination); + } finally { + rmSync(staged, { force: true }); + } + } + writeFileSync(join(root, "dist", "bin", "release-manifest.json"), JSON.stringify({schema:"paperclip.runner.release-binaries.v1",sourceRevision:manifest.sourceRevision,platforms, + ...(packIdentity ? { remoteProviderPack: { target: "linux-x64", digest: packIdentity.digest, sourceRevision: packIdentity.payload.runnerSourceRevision } } : {})},null,2)+"\n"); + console.log("Staged verified release daemons for " + required.join(", ")); +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main(); diff --git a/packages/paperclip-runner/scripts/stage-runner-binary.mjs b/packages/paperclip-runner/scripts/stage-runner-binary.mjs index 48643ea708..6b415d58db 100644 --- a/packages/paperclip-runner/scripts/stage-runner-binary.mjs +++ b/packages/paperclip-runner/scripts/stage-runner-binary.mjs @@ -28,11 +28,14 @@ export async function stageRunnerBinary(source, destination) { } } +export async function stageBuiltRunnerBinary(packageRoot) { + const executable = process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd"; + const source = path.join(packageRoot, "runner", "target", "release", executable); + const binaryRoot = path.join(packageRoot, "dist", "bin"); + await stageRunnerBinary(source, path.join(binaryRoot, executable)); +} + if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); - const executable = process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd"; - await stageRunnerBinary( - path.join(packageRoot, "runner", "target", "release", executable), - path.join(packageRoot, "dist", "bin", executable), - ); + await stageBuiltRunnerBinary(packageRoot); } diff --git a/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs b/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs index 81696b8ba6..ddffc53e4c 100644 --- a/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs +++ b/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs @@ -1,11 +1,87 @@ import assert from "node:assert/strict"; -import { spawn } from "node:child_process"; +import { execFileSync, spawn } from "node:child_process"; import { once } from "node:events"; -import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { copyFile, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, symlink, utimes, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; -import { stageRunnerBinary } from "./stage-runner-binary.mjs"; +import { stageBuiltRunnerBinary, stageRunnerBinary } from "./stage-runner-binary.mjs"; +import { RELEASE_RUNNER_TARGETS, releaseRunnerTargets, releaseDaemonManifest, verifyReleaseRunnerMetadata } from "./stage-release-runner-binaries.mjs"; +import { paperclipRunnerBinaryNeedsBuild, resolveNativeRunnerDevBinary } from "../../../scripts/dev-runner-native-binary.mjs"; +import { resolvePackagedRunnerBinary } from "../src/live/runner-binary.ts"; + +const contract = { package: { name: "@paperclipai/paperclip-runner", version: "0.0.0" }, + contracts: { runnerdArtifact: 2, nativeExecution: 1, harnessDriver: 1 }, prp: { name: "paperclip.runner", minimumVersion: 1, maximumVersion: 3 } }; +const metadata = { binaryContractVersion: 2, nativeExecutionVersion: 1, harnessDriverVersion: 1, + packageName: "@paperclipai/paperclip-runner", packageVersion: "0.0.0", prp: { name: "paperclip.runner", minimumVersion: 1, maximumVersion: 2 } }; + +test("release assembly uses the inert four-target contract and rejects unexpected selection data", async () => { + const contract = JSON.parse(await readFile(new URL("../release-runner-targets.json", import.meta.url), "utf8")); + assert.deepEqual(RELEASE_RUNNER_TARGETS, ["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"]); + assert.deepEqual(releaseRunnerTargets(contract), RELEASE_RUNNER_TARGETS); + assert.ok(Object.isFrozen(RELEASE_RUNNER_TARGETS)); + for (const invalid of [null, {}, { ...contract, schema: "other" }, { ...contract, host: "trusted-fleet" }, + { ...contract, targets: contract.targets.slice(0, 3) }, { ...contract, targets: [...contract.targets, "linux-arm64"] }, + { ...contract, targets: [...contract.targets.slice(0, 3), "win32-x64"] }, { ...contract, targets: "linux-arm64" }]) { + assert.throws(() => releaseRunnerTargets(invalid), /Unknown release daemon target contract/); + } +}); + +test("release assembly binds all four actual targets, source, hashes and compatible metadata", async () => { + const root = await realpath(await mkdtemp(path.join(tmpdir(), "runner-release-inputs-"))), source = "a".repeat(40); + const verify = value => verifyReleaseRunnerMetadata(value, contract); + const receiptPath = target => path.join(root, target, "receipt.json"); + try { + for (const target of RELEASE_RUNNER_TARGETS) { + await mkdir(path.join(root, target)); + const bytes = Buffer.alloc(64); + if (target.startsWith("linux-")) { + bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); bytes.writeUInt16LE(target === "linux-x64" ? 62 : 183, 18); + } else { bytes.writeUInt32LE(0xfeedfacf); bytes.writeUInt32LE(target === "darwin-x64" ? 0x01000007 : 0x0100000c, 4); } + await writeFile(path.join(root, target, "paperclip-runnerd"), bytes); + await writeFile(receiptPath(target), JSON.stringify({ schema: "paperclip.runner.release-artifact.v1", sourceRevision: source, + target, sha256: "sha256:" + createHash("sha256").update(bytes).digest("hex"), buildMetadata: metadata })); + } + const actual = releaseDaemonManifest(root, source, verify); + assert.deepEqual(Object.keys(actual.platforms), RELEASE_RUNNER_TARGETS); + assert.equal(actual.remoteProviderPack, undefined, "Daemon-only data does not qualify a provider pack"); + assert.throws(() => releaseDaemonManifest(root, "b".repeat(40), verify), /source, digest or architecture/); + const original = await readFile(receiptPath("darwin-arm64"), "utf8"), receipt = JSON.parse(original); + for (const delta of [{ target: "darwin-x64" }, { sha256: "sha256:" + "0".repeat(64) }, { sourceRevision: "b".repeat(40) }, + { buildMetadata: { ...metadata, binaryContractVersion: 99 } }]) { + await writeFile(receiptPath("darwin-arm64"), JSON.stringify({ ...receipt, ...delta })); + assert.throws(() => releaseDaemonManifest(root, source, verify)); + } + await writeFile(receiptPath("darwin-arm64"), original); + const binary = path.join(root, "darwin-arm64/paperclip-runnerd"), bytes = await readFile(binary); + await writeFile(binary, await readFile(path.join(root, "darwin-x64/paperclip-runnerd"))); + assert.throws(() => releaseDaemonManifest(root, source, verify), /digest or architecture/); + await rm(binary); await symlink(path.join(root, "darwin-x64/paperclip-runnerd"), binary); + assert.throws(() => releaseDaemonManifest(root, source, verify), /regular file/); + await rm(binary); await writeFile(binary, bytes); + const armBinary = path.join(root, "linux-arm64/paperclip-runnerd"), armBytes = await readFile(armBinary); + const armReceipt = JSON.parse(await readFile(receiptPath("linux-arm64"), "utf8")); + const wrongArm = await readFile(path.join(root, "linux-x64/paperclip-runnerd")); + await writeFile(armBinary, wrongArm); + await writeFile(receiptPath("linux-arm64"), JSON.stringify({ ...armReceipt, sha256: "sha256:" + createHash("sha256").update(wrongArm).digest("hex") })); + assert.throws(() => releaseDaemonManifest(root, source, verify), /architecture mismatch/); + await writeFile(armBinary, armBytes); await writeFile(receiptPath("linux-arm64"), JSON.stringify(armReceipt)); + await mkdir(path.join(root, "win32-x64")); + assert.throws(() => releaseDaemonManifest(root, source, verify), /exactly the four/); + await rm(path.join(root, "win32-x64"), { recursive: true }); + await rm(receiptPath("linux-arm64")); + assert.throws(() => releaseDaemonManifest(root, source, verify), /only its daemon and receipt/); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("release metadata rejects incompatible native, harness and protocol contracts without semver heuristics", () => { + assert.equal(verifyReleaseRunnerMetadata(metadata, contract), metadata); + for (const delta of [{ nativeExecutionVersion: 9 }, { harnessDriverVersion: 9 }, { packageName: "untrusted" }, + { prp: { ...metadata.prp, name: "other" } }, { prp: { ...metadata.prp, minimumVersion: 4, maximumVersion: 5 } }]) { + assert.throws(() => verifyReleaseRunnerMetadata({ ...metadata, ...delta }, contract), /incompatible/); + } +}); async function withOwnedChildren(root, action) { const owned = []; @@ -53,6 +129,80 @@ test("atomically publishes while the previous executable is running", { skip: pr }); }); +test("a source rebuild selects fresh dev bytes while preserving release binaries and their manifest", { skip: process.platform === "win32", timeout: 15_000 }, async () => { + const root = await mkdtemp(path.join(tmpdir(), "runnerd-stage-rebuild-")); + const packageRoot = path.join(root, "packages", "paperclip-runner"); + const binaryRoot = path.join(packageRoot, "dist", "bin"); + const hostTarget = path.join(binaryRoot, `${process.platform}-${process.arch}`, "paperclip-runnerd"); + const source = path.join(packageRoot, "runner", "crates", "runner-core", "src", "main.rs"); + const built = path.join(packageRoot, "runner", "target", "release", "paperclip-runnerd"); + await withOwnedChildren(root, async start => { + const copyHostFixture = async (input, output) => { + await copyFile(input, output); + if (process.platform === "darwin") { + const slices = execFileSync("/usr/bin/lipo", ["-archs", input], { encoding: "utf8" }).trim().split(/\s+/); + const architecture = slices.find(value => process.arch === "arm64" ? value.startsWith("arm64") : value === "x86_64"); + assert.ok(architecture); + if (slices.length > 1) execFileSync("/usr/bin/lipo", [input, "-thin", architecture, "-output", output]); + } + }; + await mkdir(path.dirname(source), { recursive: true }); + await writeFile(source, "fn main() {}\n"); + await mkdir(path.dirname(built), { recursive: true }); + await copyHostFixture("/bin/echo", built); + const previousBuild = path.join(root, "previous-build"); + await copyHostFixture("/bin/sleep", previousBuild); + await stageRunnerBinary(previousBuild, hostTarget); + await stageRunnerBinary(built, path.join(binaryRoot, "paperclip-runnerd")); + const platforms = {}, releaseBytes = new Map(); + for (const target of RELEASE_RUNNER_TARGETS) { + const artifact = path.join(binaryRoot, target, "paperclip-runnerd"); + if (artifact !== hostTarget) { + const header = Buffer.alloc(64); + if (target.startsWith("linux-")) { + header.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); header.writeUInt16LE(target === "linux-x64" ? 62 : 183, 18); + } else { header.writeUInt32LE(0xfeedfacf); header.writeUInt32LE(target === "darwin-x64" ? 0x01000007 : 0x0100000c, 4); } + await mkdir(path.dirname(artifact), { recursive: true }); + await writeFile(artifact, header); + } + const bytes = await readFile(artifact); + releaseBytes.set(artifact, bytes); + platforms[target] = { path: `${target}/paperclip-runnerd`, sha256: "sha256:" + createHash("sha256").update(bytes).digest("hex") }; + } + const oldTime = new Date(Date.now() - 10_000); + await utimes(hostTarget, oldTime, oldTime); + await utimes(path.join(binaryRoot, "paperclip-runnerd"), oldTime, oldTime); + const oldArtifact = await readFile(hostTarget); + const oldIdentity = await stat(hostTarget); + const manifestPath = path.join(binaryRoot, "release-manifest.json"); + const manifest = JSON.stringify({ schema: "paperclip.runner.release-binaries.v1", sourceRevision: "a".repeat(40), platforms }); + await writeFile(manifestPath, manifest); + const old = start(hostTarget, ["30"]); + await old.spawned; + const requirement = { repoRoot: root, nativeRunnerRequired: true }; + assert.equal(paperclipRunnerBinaryNeedsBuild(requirement), true); + + await stageBuiltRunnerBinary(packageRoot); + + const selected = resolveNativeRunnerDevBinary(requirement); + assert.deepEqual(selected, { binary: path.join(binaryRoot, "paperclip-runnerd"), sourceBuild: true }); + const published = await stat(selected.binary); + assert.equal(published.mode & 0o777, 0o755); + assert.notDeepEqual(await readFile(selected.binary), oldArtifact); + assert.equal((await stat(hostTarget)).ino, oldIdentity.ino); + assert.deepEqual(await readFile(hostTarget), oldArtifact); + assert.equal(await readFile(manifestPath, "utf8"), manifest); + for (const [artifact, bytes] of releaseBytes) assert.deepEqual(await readFile(artifact), bytes); + assert.equal(resolvePackagedRunnerBinary(path.join(packageRoot, "dist")), hostTarget); + assert.equal(paperclipRunnerBinaryNeedsBuild(requirement), false); + assert.equal(old.child.exitCode, null); + assert.equal(old.child.signalCode, null); + const next = start(selected.binary, ["rebuilt"]); + await next.spawned; await next.closed; + assert.equal(next.child.exitCode, 0); + }); +}); + test("a second spawn error still retires the live old child and removes its fixture", { skip: process.platform === "win32", timeout: 15_000 }, async () => { const root = await mkdtemp(path.join(tmpdir(), "runnerd-stage-spawn-failure-")); let old; diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts index 8a18b20012..f1ac121755 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts @@ -245,13 +245,30 @@ describe("managed Codex credentials", () => { } }); + it("prepares two occupied candidates without disturbing a foreign collision", async () => { + const foreign = await silentPrimaryQuorumFixture(); + const nextFixture = vi.fn().mockResolvedValueOnce(foreign.fixture).mockImplementation(credentialFixture); + let prepared: Awaited> | undefined; + try { + prepared = await silentPrimaryQuorumFixture(nextFixture, "quorum"); + expect(nextFixture.mock.calls.length).toBeGreaterThanOrEqual(2); + expect(nextFixture.mock.calls.length).toBeLessThanOrEqual(8); + expect(prepared.fixture.home).not.toBe(foreign.fixture.home); + await expect(listenSilently(credentialLeasePorts(foreign.fixture.home)[0])).rejects.toMatchObject({ code: "EADDRINUSE" }); + const destination = join(prepared.fixture.home, "auth.json"); + await writeFile(destination, '{"sentinel":true}', { mode: 0o600 }); + await expect(stageManagedCodexCredential({ agentHomeDirectory: prepared.fixture.home, environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}' } })).rejects.toThrow("already has an active lease"); + await expect(readFile(destination, "utf8")).resolves.toBe('{"sentinel":true}'); + } finally { + await prepared?.occupied.close(); + await foreign.occupied.close(); + } + }); + it("fails before auth mutation when two quorum candidates are occupied", async () => { - const fixture = await credentialFixture(); + const { fixture, occupied } = await silentPrimaryQuorumFixture(credentialFixture, "quorum"); const destination = join(fixture.home, "auth.json"); await writeFile(destination, '{"sentinel":true}', { mode: 0o600 }); - const ports = credentialLeasePorts(await realpath(fixture.home)); - const first = await listenSilently(ports[0]); - const second = await listenSilently(ports[1]); try { await expect( stageManagedCodexCredential({ @@ -265,7 +282,7 @@ describe("managed Codex credentials", () => { '{"sentinel":true}', ); } finally { - await Promise.all([first.close(), second.close()]); + await occupied.close(); } }); @@ -1654,7 +1671,7 @@ async function credentialFixture(): Promise<{ root: string; home: string }> { async function silentPrimaryQuorumFixture( nextFixture = credentialFixture, - occupiedIndex: 0 | 1 = 0, + occupiedIndex: 0 | 1 | "quorum" = 0, ) { for (let attempt = 0; attempt < 8; attempt += 1) { const fixture = await nextFixture(); @@ -1674,22 +1691,27 @@ async function silentPrimaryQuorumFixture( continue; throw error; } - // Only fixture preparation may retry. Release the two free candidates - // immediately before the caller stages once. A foreign bind racing this - // handoff remains a visible production-call failure, never a hidden retry. + const occupiedIndices = occupiedIndex === "quorum" ? [0, 1] : [occupiedIndex]; + const occupied = { + close: async () => { + const closed = await Promise.allSettled(owned.filter((_, index) => occupiedIndices.includes(index)).map(listener => listener.close())); + const failure = closed.find(result => result.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; + }, + }; const released = await Promise.allSettled( owned - .filter((_, index) => index !== occupiedIndex) + .filter((_, index) => !occupiedIndices.includes(index)) .map((listener) => listener.close()), ); const releaseFailure = released.find( (result) => result.status === "rejected", ); if (releaseFailure?.status === "rejected") { - await owned[occupiedIndex]!.close(); + await occupied.close(); throw releaseFailure.reason; } - return { fixture, occupied: owned[occupiedIndex]! }; + return { fixture, occupied }; } throw new Error("Silent-primary credential fixture reservation exhausted."); } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-startup-trust.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-startup-trust.test.ts index aad5b5c737..e8ef76d4d7 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-startup-trust.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-startup-trust.test.ts @@ -5,16 +5,64 @@ import { rmSync, writeFileSync, realpathSync, + statSync, symlinkSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { execFileSync } from "node:child_process"; -import { describe, it, expect } from "vitest"; +import { beforeEach, afterEach, describe, it, expect, vi } from "vitest"; import { parse } from "smol-toml"; import { trustCodexStartupRoot } from "./codex-startup-trust.js"; -describe("isolated Codex startup trust", () => { +const parserStyle = vi.hoisted(() => ({ nullPrototypeTables: false })); +vi.mock("smol-toml", async (importOriginal) => { + const actual = await importOriginal(); + const setTablePrototypes = (value: unknown): void => { + if (value === null || typeof value !== "object" || value instanceof Date) return; + if (Array.isArray(value)) { + value.forEach(setTablePrototypes); + return; + } + Object.values(value).forEach(setTablePrototypes); + Object.setPrototypeOf(value, parserStyle.nullPrototypeTables ? null : Object.prototype); + }; + return { + ...actual, + parse: (...args: Parameters) => { + const result = actual.parse(...args); + setTablePrototypes(result); + return result; + }, + }; +}); + +describe.each([ + { name: "ordinary tables", nullPrototypeTables: false }, + { name: "null-prototype tables", nullPrototypeTables: true }, +])("isolated Codex startup trust with $name", ({ nullPrototypeTables }) => { + beforeEach(() => { parserStyle.nullPrototypeTables = nullPrototypeTables; }); + afterEach(() => { parserStyle.nullPrototypeTables = false; }); + + it.each(["missing", "empty"])("initializes %s config in an isolated home", (kind) => { + const temp = realpathSync(mkdtempSync(join(tmpdir(), "codex-trust-empty-"))); + try { + const cwd = join(temp, "project"); + const home = join(temp, "home"); + mkdirSync(cwd); + if (kind === "empty") { + mkdirSync(home); + writeFileSync(join(home, "config.toml"), ""); + } + trustCodexStartupRoot(home, cwd); + const path = join(home, "config.toml"); + const updated = readFileSync(path, "utf8"); + expect(parse(updated)).toEqual({ projects: { [cwd]: { trust_level: "trusted" } } }); + expect(statSync(path).mode & 0o777).toBe(0o600); + trustCodexStartupRoot(home, cwd); + expect(readFileSync(path, "utf8")).toBe(updated); + } finally { rmSync(temp, { recursive: true, force: true }); } + }); it.each(["missing", "table", "empty-table", "inline", "empty-inline", "inline-projects"])( "preserves managed comments and unrelated content with %s project configuration", (kind) => { const temp = realpathSync(mkdtempSync(join(tmpdir(), "codex-trust-comments-"))); @@ -60,15 +108,17 @@ describe("isolated Codex startup trust", () => { mkdirSync(home); writeFileSync( join(home, "config.toml"), - 'model = "test"\n[mcp_servers.fixture]\nurl = "http://localhost/example"\n', + 'model = "test"\nstarted = 2026-10-12T12:00:00Z\nlimits = [inf, nan, 9007199254740991]\n[mcp_servers.fixture]\nurl = "http://localhost/example"\n', ); + const original = readFileSync(join(home, "config.toml"), "utf8"); + const unrelated = parse(original); trustCodexStartupRoot(home, cwd); trustCodexStartupRoot(home, cwd); expect(parse(readFileSync(join(home, "config.toml"), "utf8"))).toEqual({ - model: "test", - mcp_servers: { fixture: { url: "http://localhost/example" } }, + ...unrelated, projects: { [cwd]: { trust_level: "trusted" } }, }); + expect(readFileSync(join(home, "config.toml"), "utf8")).toContain(original); expect(readFileSync(join(home, "config.toml"), "utf8")).not.toContain( `[projects."${temp}"]`, ); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-startup-trust.ts b/packages/paperclip-runner/src/drivers/codex/codex-startup-trust.ts index d85f002552..fb748ae169 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-startup-trust.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-startup-trust.ts @@ -92,24 +92,23 @@ export function trustCodexStartupRoot(codexHome: string, cwd: string): void { const path = join(codexHome, "config.toml"); const source = existsSync(path) ? readFileSync(path, "utf8") : ""; const config = parse(source); - const projects = config.projects ?? {}; + const projects = config.projects ?? parse(""); if ( typeof projects !== "object" || Array.isArray(projects) || projects instanceof Date ) throw new Error("codex_startup_trust_invalid_projects"); - const project = projects[root] ?? {}; + const project = projects[root] ?? parse(""); if ( typeof project !== "object" || Array.isArray(project) || project instanceof Date ) throw new Error("codex_startup_trust_invalid_project"); - config.projects = { - ...projects, - [root]: { ...project, trust_level: "trusted" }, - }; + project.trust_level = "trusted"; + projects[root] = project; + config.projects = projects; const updated = editTrust(source, root, config); if (updated === source) return; const temporary = resolve(codexHome, `config.toml.${randomUUID()}.tmp`); diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index fe7e4c7097..32103f88d9 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -46,6 +46,7 @@ export type { DurableRecoveryIdentity } from "./control-plane/prp-transport-type export * from "./drivers/codex/app-server-transport.js"; export * from "./drivers/codex/codex-app-server-driver.js"; export { resolveCodexCommand, resolvePinnedCodexCommand } from "./drivers/codex/codex-command.js"; +export { runnerBinaryTarget } from "./live/runner-binary.js"; export * from "./drivers/opencode/opencode-server-driver.js"; export * from "./drivers/opencode/mcp-bridge.js"; export * from "./drivers/acpx/qualified-profiles.js"; @@ -59,6 +60,7 @@ export * from "./drivers/acpx/sidecar-protocol.js"; export * from "./drivers/runner-tool-bridge.js"; export { createRunnerdCodexTransport, + probeNativeRunnerEnvironment, runnerCodexDynamicToolsFit, defaultCapabilityRunnerdBinary, readRunnerdArtifactBinding, diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 27fdc73202..9292aa316a 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -1871,7 +1871,7 @@ describe("Capability live runnerd and Codex session", () => { reasoningTokens: 5, costNanodollars: 2_500, }); - }); + }, 15_000); it("does not raise an unhandled rejection when interrupt() outlasts the turn timeout during reconcileActiveTurn", async () => { const state = providerState(); diff --git a/packages/paperclip-runner/src/live/runner-binary.test.ts b/packages/paperclip-runner/src/live/runner-binary.test.ts index 914f443998..6f27822fc6 100644 --- a/packages/paperclip-runner/src/live/runner-binary.test.ts +++ b/packages/paperclip-runner/src/live/runner-binary.test.ts @@ -17,9 +17,9 @@ function binary(root: string, target: string | null, bytes: Buffer) { const file = join(folder, "paperclip-runnerd"); writeFileSync(file, bytes); return file; } describe("packaged native daemon target", () => { - it.each(["darwin-arm64", "darwin-x64", "linux-x64"])("selects %s from a universal package rather than the build-host legacy binary", target => { + it.each(["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"])("selects %s from a universal package rather than the build-host legacy binary", target => { const root = fixture(); binary(root, null, header("darwin-arm64")); - for (const t of ["darwin-arm64", "darwin-x64", "linux-x64"]) binary(root, t, header(t)); + for (const t of ["darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"]) binary(root, t, header(t)); const [platform, architecture] = target.split("-"); expect(runnerBinaryTarget(header(target))).toBe(target); expect(resolvePackagedRunnerBinary(join(root, "dist"), platform as NodeJS.Platform, architecture as typeof process.arch)).toBe(join(root, "dist", "bin", target, "paperclip-runnerd")); @@ -32,6 +32,16 @@ describe("packaged native daemon target", () => { const root = fixture(); binary(root, null, header("linux-x64")); binary(root, "linux-x64", header("darwin-arm64")); expect(() => resolvePackagedRunnerBinary(join(root, "dist"), "linux", "x64")).toThrow("runner_runtime_incompatible"); }); + it.each([["linux-x64", "arm64"], ["linux-arm64", "x64"]] as const)("selects remote bytes beside a %s controller for linux-%s", (controller, remoteArch) => { + const root = fixture(); + binary(root, null, header(controller)); + const remote = binary(root, `linux-${remoteArch}`, header(`linux-${remoteArch}`)); + expect(resolvePackagedRunnerBinary(join(root, "dist"), "linux", remoteArch)).toBe(remote); + }); + it.each([["linux-x64", "arm64"], ["linux-arm64", "x64"]] as const)("rejects a missing remote target instead of falling back to the %s controller for linux-%s", (controller, remoteArch) => { + const root = fixture(); binary(root, null, header(controller)); + expect(() => resolvePackagedRunnerBinary(join(root, "dist"), "linux", remoteArch)).toThrow("runner_runtime_incompatible"); + }); it("retains a matching local legacy build and permits the source-only debug fallback", () => { const root = fixture(); expect(resolvePackagedRunnerBinary(join(root, "dist"), "linux", "x64")).toBeNull(); const legacy = binary(root, null, header("linux-x64")); expect(resolvePackagedRunnerBinary(join(root, "dist"), "linux", "x64")).toBe(legacy); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index a64c51c26e..b44f6b7c36 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -69,9 +69,11 @@ import { codexSemanticToolSpecs, } from "../drivers/codex/codex-app-server-driver.js"; import { releaseMaterializedNativeRuntimeSkills } from "../drivers/runtime-context-materializer.js"; +import * as runtimeMaterializer from "../drivers/runtime-context-materializer.js"; import { RUNNERD_CANONICAL_ITEM } from "../drivers/codex/codex-driver-values.js"; import { + probeNativeRunnerEnvironment, parseAcpxTurnControlCapabilities, authorizedToolSetForProvider, createCapabilityRunnerdCodexTransport, @@ -394,6 +396,8 @@ it("publishes spawned runner ownership before waiting for provider startup", asy }); const opening = bundle.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: codexSemanticToolSpecs() }); try { + await expect(bundle.transport.request("thread/read", { threadId: "rejected-duplicate" })) + .rejects.toThrow("PRP provider thread is already started"); await vi.waitFor(() => expect(onSpawn).toHaveBeenCalledOnce()); expect(onSpawn).toHaveBeenCalledWith({ pid: expect.any(Number), processGroupId: expect.any(Number), startedAt: expect.any(String) }); expect(bundle.transport.processInfo?.().pid).toBeGreaterThan(0); @@ -401,13 +405,17 @@ it("publishes spawned runner ownership before waiting for provider startup", asy releaseOwnership(); await opening; expect(activate).toHaveBeenCalledOnce(); + const turnStarted = expectTurnStarted(bundle.transport.notifications()[Symbol.asyncIterator]()); + void turnStarted.catch(() => undefined); + await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Verify fixture turn delivery." }] }); + await turnStarted; } finally { releaseOwnership(); await opening.catch(() => undefined); await bundle.transport.close(); await rm(stateDirectory, { recursive: true, force: true }); } -}); +}, 30_000); it("launches runnerd with its production durable outbox limits", () => { expect(runnerdLaunchProfileInternals.maxOutboxBytes).toBe(16 * 1024 * 1024); @@ -2175,6 +2183,113 @@ it.each(["installed", "explicit"] as const)("records the selected %s Codex comma } }); +it("does not invoke deferred startup preparation after immediate close", async () => { + const prepare = vi.spyOn(runtimeMaterializer, "prepareIsolatedCodexHome").mockResolvedValue(undefined); + const registration = vi.fn(async () => ({ connectUrl: "ws://127.0.0.1:1", release: () => undefined })); + const launch = vi.fn(() => { throw new Error("unexpected launch"); }); + const { transport } = createCapabilityRunnerdCodexTransport({ + runnerBinary: resolve(import.meta.dirname, "../../package.json"), codexCommand: "/unused-fixture-command", sourceCodexHome: "", + controlPlaneRegistration: registration, runnerProcessLauncher: launch, + }); + const opening = transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [] }); + void opening.catch(() => undefined); + try { + await transport.close(); + await expect(opening).rejects.toThrow(/closed/); + expect(prepare).not.toHaveBeenCalled(); + expect(registration).not.toHaveBeenCalled(); + expect(launch).not.toHaveBeenCalled(); + } finally { + await opening.catch(() => undefined); + await transport.close().catch(() => undefined); + prepare.mockRestore(); + } +}); + +it("owns blocked Codex home preparation through driver abort without later registration or launch", async () => { + const root = await mkdtemp(join(tmpdir(), "runnerd-blocked-home-preparation-")); + const runnerBinary = join(root, "runnerd"); + const command = join(root, "codex"); + await writeFile(runnerBinary, "unexecuted fixture artifact", { mode: 0o700 }); + await writeFile(command, "#!/bin/sh\nexit 99\n", { mode: 0o700 }); + let releasePreparation!: () => void; + const preparationReleased = new Promise(resolve => { releasePreparation = resolve; }); + let markPreparationStarted!: () => void; + const preparationStarted = new Promise(resolve => { markPreparationStarted = resolve; }); + let closeSettled = false; + let preparationContinuedAfterClose = false; + const prepare = vi.spyOn(runtimeMaterializer, "prepareIsolatedCodexHome").mockImplementation(async input => { + markPreparationStarted(); + await preparationReleased; + preparationContinuedAfterClose = closeSettled; + await mkdir(input.codexHome, { mode: 0o700 }); + await writeFile(join(input.codexHome, "preparation-marker"), "fixture preparation drained\n", { mode: 0o600 }); + }); + const registration = vi.fn(async () => ({ + connection: { mode: "connect" as const, connectUrl: "ws://127.0.0.1:1/fixture-never-connected" }, + release: () => undefined, + })); + const launch = vi.fn(() => { throw new Error("fixture_stop_before_runner_launch"); }); + const { transport } = createCapabilityRunnerdCodexTransport({ + provider: "codex", runnerBinary, codexCommand: command, stateDirectory: root, + sourceCodexHome: "", environment: {}, controlPlaneRegistration: registration, runnerProcessLauncher: launch, + }); + const request = transport.request.bind(transport); + let bootstrapOperation: Promise> | undefined; + const requestSpy = vi.spyOn(transport, "request").mockImplementation((method, params) => { + const operation = request(method, params); + if (method === "thread/start") bootstrapOperation = operation; + return operation; + }); + const closeTransport = transport.close.bind(transport); + let closeOperation: Promise | undefined; + const closeSpy = vi.spyOn(transport, "close").mockImplementation(reason => { + closeOperation ??= closeTransport(reason).then(() => { closeSettled = true; }); + return closeOperation; + }); + const controller = new AbortController(); + const cancelled = new Error("fixture Codex bootstrap cancelled"); + const driver = new CodexAppServerDriver({ + taskEnvelope: { + schema: "paperclip.skillless_task.v1", objective: "Credential-free bootstrap ownership fixture.", + completionContract: { revision: "blocked-preparation-v1", criteria: [] }, + constraints: [], expectedResultSchema: "paperclip.run_result.v1", + }, + conversationMode: "direct", environment: {}, transportFactory: () => transport, + }); + let openingSettled = false; + const opening = driver.openSession({ runId: "fixture-blocked-home", normalizedSessionId: "fixture-blocked-home", workingDirectory: root, signal: controller.signal }) + .then(value => { openingSettled = true; return { value }; }, error => { openingSettled = true; return { error }; }); + try { + await preparationStarted; + await expect(transport.request("thread/start", { cwd: root, dynamicTools: [] })) + .rejects.toThrow("PRP provider thread is already started"); + controller.abort(cancelled); + await new Promise(resolve => setImmediate(resolve)); + expect.soft(closeOperation).toBeDefined(); + expect.soft(closeSettled).toBe(false); + expect.soft(openingSettled).toBe(false); + releasePreparation(); + const outcome = await opening; + await closeOperation; + await bootstrapOperation?.catch(() => undefined); + expect.soft(preparationContinuedAfterClose).toBe(false); + expect.soft(registration.mock.calls.length).toBe(0); + expect.soft(launch.mock.calls.length).toBe(0); + expect.soft(outcome).toHaveProperty("error"); + expect.soft(outcome).toHaveProperty("error", cancelled); + } finally { + releasePreparation(); + await opening; + await bootstrapOperation?.catch(() => undefined); + await transport.close().catch(() => undefined); + requestSpy.mockRestore(); + closeSpy.mockRestore(); + prepare.mockRestore(); + await rm(root, { recursive: true, force: true }); + } +}); + it("rejects remote Codex without a guest executable before resolving controller dependencies", async () => { const root = await mkdtemp(join(tmpdir(), "paperclip-runner-remote-codex-command-")); const runnerBinary = join(root, "runnerd"); @@ -2715,6 +2830,89 @@ const fakeCodex = resolve( "../../runner/target/debug/fake-codex-app-server", ); +it.each(["stuck", "late-release", "late-release-error", "stuck-release", "cooperative-release", "launch-failure"] as const)( + "bounds setup registration teardown and retains unconfirmed state (%s)", + async mode => { + const root = await mkdtemp(join(tmpdir(), "runnerd-setup-registration-")); + const runtimeDirectory = join(root, "native-state"); + await mkdir(runtimeDirectory, { mode: 0o700 }); + let registrationStarted!: () => void; + const started = new Promise(resolve => { registrationStarted = resolve; }); + let returnRegistration!: (registration: { connection: { mode: "listen"; listenAddress: string; listenPort: number; listenPath: string }; release: () => Promise }) => void; + const registration = new Promise[0]>(resolve => { returnRegistration = resolve; }); + let finishRelease!: () => void; + const releasing = new Promise(resolve => { finishRelease = resolve; }); + const releaseError = new Error("setup registration release failed"); + const release = vi.fn(async () => { + if (mode === "stuck-release") await releasing; + if (mode === "late-release-error") throw releaseError; + }); + const registered = { + connection: { mode: "listen" as const, listenAddress: "127.0.0.1", listenPort: 43127, listenPath: "/setup-probe" }, + release, + }; + let registrationSignal: AbortSignal | undefined; + const cleanup = vi.fn(async () => { await rm(runtimeDirectory, { recursive: true, force: true }); }); + const launch = vi.fn(() => { throw new Error("owned launcher failed before execution"); }); + let settled = false; + vi.useFakeTimers(); + const result = probeNativeRunnerEnvironment({ + runtimeDirectory, provider: "codex", model: "gpt-test", timeoutMs: 1_000, + environment: { PATH: "", OPENAI_API_KEY: "inert-test-key" }, + transportOptions: { + runnerBinary: resolve(import.meta.dirname, "../../package.json"), + codexCommand: resolve(import.meta.dirname, "../../package.json"), + sourceCodexHome: "", runnerProcessLauncher: launch, + controlPlaneRegistration: async (_authority, _identity, signal) => { + registrationSignal = signal; + if (mode === "cooperative-release") signal?.addEventListener("abort", () => returnRegistration(registered), { once: true }); + registrationStarted(); + return registration; + }, + }, + onCleanupConfirmed: cleanup, + }).then(value => { settled = true; return value; }, (error: unknown) => { settled = true; return error; }); + try { + await started; + if (mode === "launch-failure") returnRegistration(registered); + await vi.advanceTimersByTimeAsync(1_000); + if (mode === "late-release" || mode === "late-release-error" || mode === "stuck-release") returnRegistration(registered); + await vi.advanceTimersByTimeAsync(10_000); + if (mode === "late-release" || mode === "cooperative-release" || mode === "launch-failure") await result; + expect(settled).toBe(true); + expect(registrationSignal?.aborted).toBe(true); + const failure = await result; + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain(mode === "launch-failure" ? "owned launcher failed before execution" : "Native provider hello probe timed out."); + if (mode === "stuck" || mode === "stuck-release" || mode === "late-release-error") { + expect(failure).toBeInstanceOf(AggregateError); + expect((failure as AggregateError).cause).toMatchObject({ message: "Native provider hello probe timed out." }); + expect(cleanup).not.toHaveBeenCalled(); + expect((await stat(runtimeDirectory)).isDirectory()).toBe(true); + if (mode === "late-release-error") expect((failure as AggregateError).errors).toContain(releaseError); + returnRegistration(registered); + finishRelease(); + await vi.advanceTimersByTimeAsync(0); + expect(release).toHaveBeenCalledOnce(); + expect(cleanup).not.toHaveBeenCalled(); + } else { + expect(release).toHaveBeenCalledOnce(); + expect(cleanup).toHaveBeenCalledOnce(); + await expect(stat(runtimeDirectory)).rejects.toThrow(); + } + expect(launch).toHaveBeenCalledTimes(mode === "launch-failure" ? 1 : 0); + expect(vi.getTimerCount()).toBe(0); + } finally { + returnRegistration(registered); + finishRelease(); + await vi.advanceTimersByTimeAsync(0); + await result; + vi.useRealTimers(); + await rm(root, { recursive: true, force: true }); + } + }, +); + function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { return [ "--state-file", @@ -2723,6 +2921,249 @@ function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { ]; } +it("joins owned credential preparation before close returns and never launches after close", async () => { + const root = await mkdtemp(join(tmpdir(), "runnerd-owned-startup-")); + const artifact = join(root, "unexecuted-runner-artifact"); + await writeFile(artifact, "fixture artifact bytes; never executed"); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + let entered!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const preparation = vi.spyOn(runtimeMaterializer, "prepareIsolatedCodexHome").mockImplementation(async ({ codexHome }) => { + entered(); + await gate; + await mkdir(codexHome, { recursive: true }); + await writeFile(join(codexHome, "owned-write.txt"), "completed owned preparation"); + }); + const registration = vi.fn(async () => ({ connectUrl: "ws://127.0.0.1:1", release: async () => {} })); + const launch = vi.fn(() => { throw new Error("unexecuted late launch"); }); + const bundle = createCapabilityRunnerdCodexTransport({ stateDirectory: root, runnerBinary: artifact, codexCommand: artifact, provider: "codex", closeGraceMs: 1000, + controlPlaneRegistration: registration, runnerProcessLauncher: launch }); + let opening: Promise | undefined; + let closing: Promise | undefined; + try { + opening = bundle.transport.request("thread/start", { cwd: root, model: "gpt-test", dynamicTools: [] }); + void opening.catch(() => {}); + await started; + let closeReturned = false; + closing = bundle.transport.close("owned preparation test"); + void closing.then(() => { closeReturned = true; }, () => {}); + await new Promise(resolve => setTimeout(resolve, 25)); + expect(closeReturned).toBe(false); + release(); + await expect(opening).rejects.toThrow(/closed/); + await closing; + expect(await readFile(join(root, "codex-home", "owned-write.txt"), "utf8")).toBe("completed owned preparation"); + expect(registration).not.toHaveBeenCalled(); + expect(launch).not.toHaveBeenCalled(); + } finally { + release(); + await Promise.allSettled([opening, closing]); + preparation.mockRestore(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("retains owned state when credential preparation outlives bounded close", async () => { + const root = await mkdtemp(join(tmpdir(), "runnerd-retained-startup-")); + const artifact = join(root, "unexecuted-artifact"); + await writeFile(artifact, "never executed"); + let release!: () => void, entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const preparation = vi.spyOn(runtimeMaterializer, "prepareIsolatedCodexHome").mockImplementation(async () => { entered(); await gate; }); + const registration = vi.fn(async () => ({ connectUrl: "ws://127.0.0.1:1", release: async () => {} })); + const launch = vi.fn(() => { throw new Error("unexpected launch"); }); + const bundle = createCapabilityRunnerdCodexTransport({ stateDirectory: root, runnerBinary: artifact, codexCommand: artifact, provider: "codex", closeGraceMs: 100, + controlPlaneRegistration: registration, runnerProcessLauncher: launch }); + const opening = bundle.transport.request("thread/start", { cwd: root, model: "gpt-test", dynamicTools: [] }); + void opening.catch(() => {}); + try { + await started; + await expect(bundle.transport.close("bounded preparation")).rejects.toThrow("startup cleanup is unconfirmed"); + expect((await stat(root)).isDirectory()).toBe(true); + release(); + await expect(opening).rejects.toThrow(/closed/); + expect(registration).not.toHaveBeenCalled(); + expect(launch).not.toHaveBeenCalled(); + } finally { + release(); await opening.catch(() => {}); preparation.mockRestore(); + await rm(root, { recursive: true, force: true }); + } +}); + +it.each(["settled", "late", "late-release-error"] as const)("retires owned startup registration without post-close launch (%s)", async mode => { + const root = await mkdtemp(join(tmpdir(), "runnerd-startup-registration-")); + const artifact = join(root, "unexecuted-artifact"); + await writeFile(artifact, "never executed"); + let release!: () => void, entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const registered = new Promise(resolve => { entered = resolve; }); + const preparation = vi.spyOn(runtimeMaterializer, "prepareIsolatedCodexHome").mockResolvedValue(undefined); + const retire = vi.fn(async () => { if (mode === "late-release-error") throw new Error("owned registration release failed"); }); + const registration = vi.fn(async () => { entered(); await gate; return { connectUrl: "ws://127.0.0.1:1", release: retire }; }); + const launch = vi.fn(() => { throw new Error("unexpected launch"); }); + const bundle = createCapabilityRunnerdCodexTransport({ stateDirectory: root, runnerBinary: artifact, codexCommand: artifact, provider: "codex", closeGraceMs: mode === "settled" ? 1000 : 100, + controlPlaneRegistration: registration, runnerProcessLauncher: launch }); + const opening = bundle.transport.request("thread/start", { cwd: root, model: "gpt-test", dynamicTools: [] }); + void opening.catch(() => {}); + let closing: Promise | undefined; + try { + await registered; + let closeReturned = false; + closing = bundle.transport.close("owned registration"); + void closing.then(() => { closeReturned = true; }, () => {}); + if (mode === "settled") { + await new Promise(resolve => setTimeout(resolve, 25)); + expect(closeReturned).toBe(false); + } else { + await expect(closing).rejects.toThrow("startup cleanup is unconfirmed"); + expect((await stat(root)).isDirectory()).toBe(true); + } + release(); + await expect(opening).rejects.toThrow(mode === "late-release-error" ? "owned registration release failed" : /closed/); + if (mode === "settled") await closing; + expect(retire).toHaveBeenCalledTimes(1); + expect(launch).not.toHaveBeenCalled(); + } finally { + release(); await Promise.allSettled([opening, closing]); preparation.mockRestore(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("does not certify probe cleanup while owned credential preparation is unconfirmed", async () => { + const root = await mkdtemp(join(tmpdir(), "runnerd-probe-owned-preparation-")); + const artifact = join(root, "unexecuted-artifact"); + await writeFile(artifact, "never executed"); + let release!: () => void, entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const preparation = vi.spyOn(runtimeMaterializer, "prepareIsolatedCodexHome").mockImplementation(async () => { entered(); await gate; }); + const cleanup = vi.fn(async () => {}); + const launch = vi.fn(() => { throw new Error("unexpected launch"); }); + const probe = probeNativeRunnerEnvironment({ runtimeDirectory: root, provider: "codex", model: "gpt-test", timeoutMs: 100, + environment: { OPENAI_API_KEY: "fixture" }, onCleanupConfirmed: cleanup, + transportOptions: { runnerBinary: artifact, codexCommand: artifact, runnerProcessLauncher: launch, + controlPlaneRegistration: async () => ({ connectUrl: "ws://127.0.0.1:1", release: async () => {} }) }, + }); + const result = probe.catch(error => error); + try { + await started; + const error = await result; + expect(cleanup).not.toHaveBeenCalled(); + expect(error).toBeInstanceOf(AggregateError); + expect(error.message).toContain("hello probe timed out"); + expect(error.message).toContain("startup cleanup is unconfirmed"); + expect((await stat(root)).isDirectory()).toBe(true); + release(); + await new Promise(resolve => setTimeout(resolve, 25)); + expect(launch).not.toHaveBeenCalled(); + } finally { + release(); await result; preparation.mockRestore(); + await new Promise(resolve => setTimeout(resolve, 25)); + await rm(root, { recursive: true, force: true }); + } +}, 15_000); + +it.each(["complete", "api-key", "default-model", "mismatched-model", "model-refresh-error", "model-cleanup-error", "missing-model", "missing-runtime", "missing-completion", "late-bootstrap", "remote-complete", "remote-mismatched-model", "late-remote-preparation", "late-remote-release-error"] as const)("native Codex setup requires its daemon/app-server hello (%s)", async mode => { + const root = await mkdtemp(join(tmpdir(), "runnerd-native-setup-")); + const runtimeDirectory = join(root, "native-state"); + await mkdir(runtimeDirectory, { mode: 0o700 }); + const calls = join(root, "calls.jsonl"); + const sourceHome = join(root, "bound-account"); + await mkdir(sourceHome, { mode: 0o700 }); + await writeFile(join(sourceHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: mode === "api-key" ? "unrelated-login" : "selected-account" }), { mode: 0o600 }); + if (mode === "api-key") await writeFile(join(sourceHome, "config.toml"), 'model = "ambient-model"\n[mcp_servers.ambient]\ncommand = "untrusted-ambient-command"\n', { mode: 0o600 }); + const remote = mode.startsWith("remote-") || mode.startsWith("late-remote-"); + const remoteRoot = join(root, "assigned-remote-filesystem"); + const remoteFilesystem = join(remoteRoot, "filesystem"); + const remoteWorkspace = join(remoteFilesystem, "workspace"); + let registrationReleased = false; + let refreshed = false; + let cleanupConfirmed = false; + try { + const probe = probeNativeRunnerEnvironment({ + runtimeDirectory, provider: "codex", model: mode === "default-model" ? null : "gpt-6.1-sol", reasoningEffort: "low", + environment: { PATH: process.env.PATH, OPENAI_API_KEY: mode === "api-key" ? "selected-account" : "", HOME: "/unrelated/home" }, + timeoutMs: mode.startsWith("late-remote-") ? 800 : mode === "late-bootstrap" ? 100 : mode === "missing-completion" ? 1_000 : 10_000, + ...(remote ? { workingDirectory: remoteWorkspace } : {}), + transportOptions: { + ...(remote ? { + runnerFilesystemRoot: remoteFilesystem, runnerStateDirectory: join(remoteRoot, "runner"), + readRunnerState: async () => JSON.parse(await readFile(join(remoteRoot, "runner", "runner-state.json"), "utf8")), + controlPlaneRegistration: async authority => { + await mkdir(remoteWorkspace, { recursive: true, mode: 0o700 }); + const remoteHome = join(remoteFilesystem, "codex-home"); + await mkdir(remoteHome, { mode: 0o700 }); + for (const name of ["auth.json", "config.toml"] as const) await cp(join(runtimeDirectory, "codex-home", name), join(remoteHome, name)); + if (mode.startsWith("late-remote-")) await new Promise(resolve => setTimeout(resolve, 1000)); + await authority.start(); + return { connectUrl: authority.connectUrl, release: async () => { registrationReleased = true; await authority.stop(); if (mode === "late-remote-release-error") throw new Error("PRP registration release failed"); } }; + }, + } : {}), + runnerBinary: mode === "missing-runtime" ? join(root, "missing-runner") : resolve("runner/target/debug/paperclip-runnerd"), + codexCommand: fakeCodex, codexArgs: fakeCodexArgs(root, "--request-log", calls, + ...(mode === "missing-completion" ? ["--hold-turn"] : []), + ...(mode === "mismatched-model" || mode === "remote-mismatched-model" || mode === "model-refresh-error" || mode === "model-cleanup-error" ? ["--returned-model", "another-model"] : []), + ...(mode === "missing-model" ? ["--returned-model", ""] : []), + ...(mode === "late-bootstrap" ? ["--thread-start-delay-ms", "1000"] : [])), + sourceCodexHome: sourceHome, + }, + onCodexCredentialRefresh: async filename => { + expect(filename).toBe(join(remote ? remoteFilesystem : runtimeDirectory, "codex-home", "auth.json")); + expect(JSON.parse(await readFile(filename, "utf8"))).toEqual({ OPENAI_API_KEY: "selected-account" }); + if (mode === "api-key") { + const config = await readFile(join(runtimeDirectory, "codex-home", "config.toml"), "utf8"); + expect(config).not.toContain("ambient-model"); expect(config).not.toContain("untrusted-ambient-command"); + expect((await stat(filename)).mode & 0o777).toBe(0o600); + } + refreshed = true; + if (mode === "model-refresh-error") throw new Error("credential refresh failed after the native model error"); + }, + onCleanupConfirmed: async () => { + cleanupConfirmed = true; + expect(refreshed || mode === "missing-runtime" || mode === "late-bootstrap" || mode.startsWith("late-remote-")).toBe(true); + if (mode === "model-cleanup-error") throw new Error("owned temporary state cleanup failed"); + await rm(runtimeDirectory, { recursive: true, force: true }); + }, + }); + if (mode === "complete" || mode === "api-key" || mode === "default-model" || mode === "remote-complete") { + const observedModel = mode === "default-model" ? "gpt-test" : "gpt-6.1-sol"; + await expect(probe).resolves.toEqual({ provider: "codex", providerDriver: "codex_app_server", effectiveModel: observedModel, helloProbePassed: true }); + expect(refreshed).toBe(true); + if (remote) expect(registrationReleased).toBe(true); + expect(cleanupConfirmed).toBe(true); + await expect(stat(runtimeDirectory)).rejects.toThrow(); + const requests = (await readFile(calls, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + expect(requests.find(value => value.method === "thread/start").params).toMatchObject({ ...(remote ? { cwd: remoteWorkspace } : {}), permissions: "paperclip-runner-workspace-read-only", dynamicTools: [] }); + expect(requests.find(value => value.method === "turn/start").params).toMatchObject({ effort: "low", permissions: "paperclip-runner-workspace-read-only" }); + } else { + await expect(probe).rejects.toThrow(mode === "missing-runtime" ? /ENOENT|unavailable|cleanup/i : mode === "mismatched-model" || mode === "remote-mismatched-model" || mode === "model-refresh-error" || mode === "model-cleanup-error" || mode === "missing-model" ? /observed model|mismatched provider model/i : /timed out|cleanup|suspension/i); + if (mode === "late-remote-release-error") { + expect(registrationReleased).toBe(true); expect(cleanupConfirmed).toBe(false); + expect((await stat(runtimeDirectory)).isDirectory()).toBe(true); + } + if (mode === "model-refresh-error") { + expect(refreshed).toBe(true); + expect(cleanupConfirmed).toBe(false); + expect((await stat(runtimeDirectory)).isDirectory()).toBe(true); + } + if (mode === "model-cleanup-error") { + expect(cleanupConfirmed).toBe(true); + expect((await stat(runtimeDirectory)).isDirectory()).toBe(true); + } + if (mode === "mismatched-model" || mode === "remote-mismatched-model" || mode === "missing-model") { + expect(cleanupConfirmed).toBe(true); + await expect(stat(runtimeDirectory)).rejects.toThrow(); + } + if (mode === "mismatched-model" || mode === "remote-mismatched-model" || mode === "model-refresh-error" || mode === "model-cleanup-error" || mode === "missing-model" || mode === "late-bootstrap" || mode.startsWith("late-remote-")) { + const requests = (await readFile(calls, "utf8").catch(() => "")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line)); + expect(requests.some(value => value.method === "turn/start")).toBe(false); + } + } + } finally { await rm(root, { recursive: true, force: true }); } +}, 20_000); + function assignedRuntimeContext( skillRoot: string, instructionRoot: string, @@ -5452,6 +5893,10 @@ it.each([ let recoveryFenceActive = true; const completionSnapshotIds: string[] = []; let rejectHeldCompletion: ((error: Error) => void) | undefined; + let markHeldCompletionEntered!: () => void; + const heldCompletionEntered = new Promise((resolve) => { + markHeldCompletionEntered = resolve; + }); let snapshotObserverSpy: ReturnType | undefined; const routeServer = createServer((_request, response) => response.writeHead(404).end(), @@ -5988,6 +6433,10 @@ it.each([ recoveryFault === "none" ? null : await fingerprintTree(stateDirectory); const authorizationStages: string[] = []; let releaseHeldAuthorization: (() => void) | undefined; + let markHeldAuthorizationEntered!: () => void; + const heldAuthorizationEntered = new Promise((resolve) => { + markHeldAuthorizationEntered = resolve; + }); const authorizationFault = recoveryFault.startsWith("authorize_"); const failedAuthorizationStage = recoveryFault.slice("authorize_".length); const beforeProviderBytes = await readFile( @@ -6030,6 +6479,7 @@ it.each([ ) { return new Promise((_resolve, reject) => { rejectHeldCompletion = reject; + markHeldCompletionEntered(); }); } recoveryFenceActive = false; @@ -6046,6 +6496,7 @@ it.each([ ) { await new Promise((resolveGate) => { releaseHeldAuthorization = resolveGate; + markHeldAuthorizationEntered(); }); } if (stage === failedAuthorizationStage) @@ -6078,9 +6529,13 @@ it.each([ authorizationFault && failedAuthorizationStage === "before_bootstrap" ) { - await vi.waitFor(() => - expect(releaseHeldAuthorization).toBeTypeOf("function"), - ); + await within(Promise.race([ + heldAuthorizationEntered, + recoveryRequest.then(() => { + throw new Error("warm restart authorization callback was not entered"); + }), + ])); + expect(releaseHeldAuthorization).toBeTypeOf("function"); const queuedBefore = cores[1]!.store.state.commands.map( (command) => command.commandId, ); @@ -6185,9 +6640,13 @@ it.each([ const firstRead = within(resumed.transport.request("thread/read", {})); void firstRead.catch(() => undefined); if (ordinaryFollowup === "callback_async_failure") { - await vi.waitFor(() => - expect(rejectHeldCompletion).toBeTypeOf("function"), - ); + await within(Promise.race([ + heldCompletionEntered, + firstRead.then(() => { + throw new Error("warm restart completion callback was not entered"); + }), + ])); + expect(rejectHeldCompletion).toBeTypeOf("function"); try { expect(recoveryFenceActive).toBe(true); const queuedBefore = cores[1]!.store.state.commands.map( @@ -7677,6 +8136,41 @@ it("keeps an adopted runner alive when the detached controller finalizer closes" it("adopts an opening session without a checkpoint instead of bootstrapping a duplicate provider", () => verifyLiveRunnerAdoption(false, false, false, false, true), 30_000); +it("cancels startup readiness before releasing the route that settles its ready callback", async () => { + const root = await mkdtemp(join(tmpdir(), "runnerd-ready-release-cycle-")); + let releaseReady!: () => void; + const readyReleased = new Promise(resolve => { releaseReady = resolve; }); + let markReadyStarted!: () => void; + const readyStarted = new Promise(resolve => { markReadyStarted = resolve; }); + const release = vi.fn(async () => { releaseReady(); }); + let core!: DurablePrpControlPlane; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(root), stateDirectory: root, sourceCodexHome: "", environment: {}, closeGraceMs: 2_000, + controlPlaneRegistration: async authority => { + core = authority; + await authority.start(); + return { ready: () => { markReadyStarted(); return readyReleased; }, release }; + }, + }); + const opening = bundle.transport.request("thread/start", { cwd: root, dynamicTools: [] }); + void opening.catch(() => undefined); + try { + await readyStarted; + await vi.waitFor(() => expect(core.store.state.commands.find(command => command.type === "session.open")?.status).toBe("completed")); + await expect(bundle.transport.close("cancel pending ingress authentication")).resolves.toBeUndefined(); + await expect(opening).rejects.toThrow(/closed/); + expect(release).toHaveBeenCalledOnce(); + expect(bundle.evidence().runnerExited).toBe(true); + expect(bundle.evidence().diagnostics.join("\n")).not.toContain("startup cleanup is unconfirmed"); + } finally { + releaseReady(); + await opening.catch(() => undefined); + await bundle.transport.close().catch(() => undefined); + await rm(root, { recursive: true, force: true }); + } +}, 30_000); + it("surfaces a runner exit while provider-ingress readiness is still pending", async () => { const neverReady = new Promise(() => undefined); const bundle = createCapabilityRunnerdCodexTransport({ diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 4d7be0fbe6..75a7543ff7 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -39,6 +39,7 @@ import type { } from "../drivers/codex/app-server-transport.js"; import { createSanitizedCodexEnvironment } from "../drivers/codex/app-server-transport.js"; import { + CodexAppServerDriver, codexSemanticToolSpecs, createIsolatedCodexAppServerArgs, } from "../drivers/codex/codex-app-server-driver.js"; @@ -50,6 +51,7 @@ import type { NativeRunIdentity, NativeTurnControlCapabilities } from "../contra import type { PrpEvent } from "../protocol/replay-contract.js"; import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; import type { + HarnessSession, HarnessRuntimeRequestResolution, PersistedHarnessProviderIdentity, } from "../contracts/harness-driver.js"; @@ -1333,6 +1335,7 @@ export interface CapabilityRunnerdCodexTransportOptions { controlPlaneRegistration?: ( authority: DurablePrpControlPlane, identity?: DurableRecoveryIdentity, + signal?: AbortSignal, ) => Promise<{ connectUrl?: string; connection?: RunnerProcessConnection; @@ -3548,6 +3551,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #deferredTurnStartEvents: DurableRecoveryCommittedEvent[] = []; #recoveryTurnBindingPending = false; #threadId = ""; + #observedProviderModel: string | undefined; #sessionId: string | null = null; #providerIdentity: Record | null = null; #turnControls: NativeTurnControlCapabilities = { steering: false, queuedFollowUp: false }; @@ -3593,7 +3597,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #authorizedTools: Record | null = null; #runAttachTemplate: Record | null = null; #closed = false; + readonly #closedSignal: Promise; + #resolveClosedSignal!: () => void; #closePromise: Promise | null = null; + #startupOperation: Promise | null = null; + #startupDrainTimedOut = false; #controllerDetachedForRestart = false; #failure: Error | null = null; readonly #failureSignal: Promise; @@ -3630,6 +3638,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#failureSignal = new Promise((_resolve, reject) => { this.#rejectFailureSignal = reject; }); + this.#closedSignal = new Promise(resolve => { + this.#resolveClosedSignal = resolve; + }); // Failure is also observed by request/notification paths. Register an // internal handler so a process exit after the owner has closed the // session cannot become an unhandled process-level rejection. @@ -3706,7 +3717,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { throw new Error("native_runner_warm_transition_completion_pending"); } if (method === "initialize") return { user: {} }; - if (method === "thread/start") return this.#start(params); + if (method === "thread/start") return this.#trackStartup(() => this.#start(params)); if (method === "collaborationMode/list") { // runnerd negotiates the real Codex preset or the provider-proxy-owned // planning contract during session.open. This transport-level mask @@ -3805,8 +3816,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } if (method === "thread/read") { if (this.#core === null) { - this.#recoveryTurnBindingPending = true; - await this.#resume(); + await this.#trackStartup(() => { + this.#recoveryTurnBindingPending = true; + return this.#resume(); + }); } // Ask the authenticated runner for its live provider snapshot rather // than reading its filesystem. This both supports remote process owners @@ -4420,6 +4433,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (this.#closed) return; this.#controllerDetachedForRestart = true; this.#closed = true; + this.#resolveClosedSignal(); this.#turnStartAdmission?.resolve(false); if (this.#pump !== null) clearInterval(this.#pump); this.#pump = null; @@ -4446,8 +4460,41 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); } + #trackStartup(start: () => Promise): Promise { + if (this.#startupOperation !== null) { + return Promise.reject(new Error("PRP provider thread is already started")); + } + const operation = Promise.resolve().then(() => { + this.#assertStartupOpen(); + return start(); + }); + this.#startupOperation = operation; + return operation; + } + + #assertStartupOpen(): void { + if (this.#closed) throw new Error("runnerd transport closed during provider startup"); + } + async #closeOnce(): Promise { this.#closed = true; + this.#resolveClosedSignal(); + const { preparationDeadline, closeDeadline } = runnerCloseDeadlines(Date.now(), runnerCloseGraceMs(this.options)); + let startupConfirmed = true; + if (this.#startupOperation !== null) { + let timer: ReturnType | undefined; + try { + await Promise.race([ + this.#startupOperation.catch(() => undefined), + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error("Native startup drain timed out")), Math.max(0, preparationDeadline - Date.now())); + }), + ]); + } catch { + startupConfirmed = false; + this.#startupDrainTimedOut = true; + } finally { clearTimeout(timer); } + } this.#turnStartAdmission?.resolve(false); const adoptedRunner = this.options.adoptExistingRunner; // `settled` is a durable-state assertion, not merely the absence of a @@ -4472,10 +4519,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // A terminal provider frame can become visible one control loop before // its durable provider suffix is ACKed. Drain it before suspension so a // fresh run authority never inherits the prior run's pending events. - const { preparationDeadline, closeDeadline } = runnerCloseDeadlines( - Date.now(), - runnerCloseGraceMs(this.options), - ); + if (!(await this.#runnerHasExited())) { // Let an already-admitted tool result reach its original provider // before turn.stop can retire that tool-call identity. This shares the @@ -4602,7 +4646,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // an incomplete or identity-conflicting state remains fail-closed. const finalProviderState = this.#providerDrainState(); const runnerSettled = - runnerSuspended && + startupConfirmed && runnerSuspended && providerDrained && this.#core?.semanticToolResultsSettled() === true && (finalProviderState === null || @@ -4654,6 +4698,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } throw new NativeSessionCloseUnrecoverableError(settlement); } + if (!startupConfirmed) { + throw new Error("Native runner startup cleanup is unconfirmed; recovery state retained."); + } if (this.#ownsRoot && !adoptedRunner) { rmSync(this.#root, { recursive: true, force: true }); } @@ -4747,6 +4794,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { nativeMcp: nativeMcpLaunchBinding(this.options.environment), }); } + this.#assertStartupOpen(); const opencodeProxyPath = this.options.opencodeProxyPath ?? (provider === "opencode" && !this.options.runnerFilesystemRoot @@ -5005,6 +5053,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { externallySandboxed: provider === "codex" && this.options.externallySandboxed === true, + ...(provider === "codex" && params.permissions === "paperclip-runner-workspace-read-only" ? { readOnly: true } : {}), instructions: provider === "codex" ? withCodexCollaborationRuntimeInstructions( @@ -5045,6 +5094,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#controlPlaneCheckpoint = registration.checkpoint ?? null; this.#controlPlaneRelease = registration.release; } + if (this.#closed && this.#startupDrainTimedOut) { + const release = this.#controlPlaneRelease; + this.#controlPlaneRelease = null; + await release?.(); + } + this.#assertStartupOpen(); const admissionDeadline = this.#coldAdmissionDeadline(); const handle = spawnRunner({ connection: registration?.connection ?? { @@ -5087,7 +5142,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#handle = handle; this.#watchRunner(handle); await this.#publishSpawnedProcess(handle); + this.#assertStartupOpen(); await registration?.activate?.(); + this.#assertStartupOpen(); if (registration?.failure) { void registration.failure.catch((error: unknown) => { this.#failTransport( @@ -5096,6 +5153,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }); } await this.#awaitRegistrationReady(registration?.ready); + this.#assertStartupOpen(); this.#evidence.runnerPid = handle.child.pid ?? null; this.#evidence.runnerProcessGroupId = handle.processGroupId ?? null; this.#publish(); @@ -5112,6 +5170,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const provider = this.options.provider ?? "codex"; const acpxAgent = this.options.acpxAgent ?? "codex"; return { + ...(provider === "codex" ? { model: this.#observedProviderModel } : {}), thread: { id: this.#threadId, sessionId: this.#sessionId, @@ -5549,6 +5608,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { >; let oldTransitionRegistration: RecoveryRegistration | null = null; let newTransitionRegistration: RecoveryRegistration | null = null; + this.#assertStartupOpen(); if (warmRecovery && this.options.controlPlaneRegistration) { try { oldTransitionRegistration = await this.options.controlPlaneRegistration( @@ -5567,6 +5627,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { "native_runner_warm_transition_registered_endpoint_mismatch", ); } + this.#assertStartupOpen(); newTransitionRegistration = await this.options.controlPlaneRegistration( core, warmRecovery.proof.receipt.newIdentity, @@ -5621,6 +5682,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } : registration.release; } + if (this.#closed && this.#startupDrainTimedOut) { + const release = this.#controlPlaneRelease; + this.#controlPlaneRelease = null; + await release?.(); + } + this.#assertStartupOpen(); // Route and immutable endpoint admission precedes every launch-material // write. A refused transition leaves its retained home/context untouched. if (runtimeContext !== null) { @@ -5636,6 +5703,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await releaseMaterializedNativeRuntimeSkills( resolve(localCodexHome, "skills"), ); + this.#assertStartupOpen(); await prepareIsolatedCodexHome({ context: sourceRuntimeContext, codexHome: localCodexHome, @@ -5652,6 +5720,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (warmRecovery) { await this.options.authorizeWarmTransitionRecovery?.("before_bootstrap"); } + this.#assertStartupOpen(); const bootstrapTicket = adoptedRunner ? null : warmRecovery @@ -5667,6 +5736,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await this.options.authorizeWarmTransitionRecovery?.("before_spawn"); } // A replacement executor restores its cold provider before acknowledging recovery. + this.#assertStartupOpen(); const admissionDeadline = this.#coldAdmissionDeadline(adoptedRunner === undefined); const handle = adoptedRunner ? null @@ -5712,10 +5782,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#watchRunner(handle); await this.#publishSpawnedProcess(handle); } + this.#assertStartupOpen(); if (oldTransitionRegistration && newTransitionRegistration) { await oldTransitionRegistration.activate?.(); + this.#assertStartupOpen(); await newTransitionRegistration.activate?.(); } else await registration?.activate?.(); + this.#assertStartupOpen(); if (registration?.failure) { void registration.failure.catch((error: unknown) => { this.#failTransport( @@ -5724,6 +5797,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }); } if (!adoptedRunner) await this.#awaitRegistrationReady(registration?.ready); + this.#assertStartupOpen(); if (adoptedRunner) { this.#evidence.runnerPid = adoptedRunner.pid; this.#evidence.runnerProcessGroupId = adoptedRunner.processGroupId; @@ -5740,6 +5814,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { !recoveryIdentityMatches(core.store.state.identity, desiredIdentity) || core.store.state.warmTransition !== undefined ) { + this.#assertStartupOpen(); if (Date.now() >= deadline || (await this.#runnerHasExited())) { throw new Error("native_runner_warm_transition_recovery_pending"); } @@ -5754,11 +5829,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#deferredTurnStartEvents = []; if (oldTransitionRegistration && newTransitionRegistration) { await oldTransitionRegistration.release(); + this.#assertStartupOpen(); this.#controlPlaneRelease = newTransitionRegistration.release; this.#controlPlaneCheckpoint = newTransitionRegistration.checkpoint ?? null; } } + this.#assertStartupOpen(); this.#pump = setInterval(() => this.#pumpEventsSafely(), 5); if (adoptedRunner) { await this.#awaitAdoptedRunnerConnection( @@ -5949,6 +6026,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { "turn.start", { text: message, + ...((this.options.provider ?? "codex") === "codex" && params.effort !== undefined ? { effort: params.effort } : {}), ...(reasoningMode === undefined ? {} : { reasoningMode }), ...(skills.length ? { skills } : {}), turnId: pendingTurnId, @@ -6579,6 +6657,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const started = record(record(event.envelope.payload).payload); const runtimeIdentity = record(started.runtimeIdentity); const descriptor = record(started.providerDescriptor); + if (typeof descriptor.effectiveModel === "string" && descriptor.effectiveModel.trim()) this.#observedProviderModel = descriptor.effectiveModel; const { processId: pid, threadId, @@ -6794,8 +6873,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await awaitAdoptedRunnerAuthentication({ activeConnectionCount: () => core.activeRunnerConnectionCount(), isAlive: () => adoptedRunner.isAlive(), - throwIfFailed: () => this.#throwIfFailed(), - failure: this.#failureSignal, + throwIfFailed: () => { this.#assertStartupOpen(); this.#throwIfFailed(); }, + failure: Promise.race([ + this.#failureSignal, + this.#closedSignal.then(() => { throw new Error("runnerd transport closed during provider startup"); }), + ]), ready, timeoutMs: this.options.runnerReconnectGraceMs ?? 30_000, }); @@ -6804,6 +6886,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#failTransport(failure); throw failure; } + this.#assertStartupOpen(); this.#adoptedRunnerAuthenticated = true; this.#diagnostic( `adopted runner ${adoptedRunner.pid} authenticated to its durable PRP authority`, @@ -7008,11 +7091,16 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { async #awaitRegistrationReady( ready: (() => Promise) | undefined, ): Promise { + this.#assertStartupOpen(); if (ready === undefined) { this.#throwIfFailed(); return; } - await Promise.race([ready(), this.#failureSignal]); + await Promise.race([ + ready(), this.#failureSignal, + this.#closedSignal.then(() => { this.#assertStartupOpen(); }), + ]); + this.#assertStartupOpen(); this.#throwIfFailed(); } @@ -7029,13 +7117,16 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } } -export function defaultCapabilityRunnerdBinary(): string { +export function defaultCapabilityRunnerdBinary(platform = process.platform, architecture = process.arch): string { // Standalone builds use dist/live; the server vendors that compiled tree // directly under vendor/paperclip-runner. Resolve beside our own live module. const outputRoot = resolve(fileURLToPath(new URL("../", import.meta.url))); - const staged = resolvePackagedRunnerBinary(outputRoot) - ?? resolvePackagedRunnerBinary(resolve(packageRoot, "dist")); + const staged = resolvePackagedRunnerBinary(outputRoot, platform, architecture) + ?? resolvePackagedRunnerBinary(resolve(packageRoot, "dist"), platform, architecture); if (staged) return staged; + if (platform !== process.platform || architecture !== process.arch) { + throw new Error(`runner_remote_artifact_unavailable: install the Paperclip release containing the ${platform}-${architecture} daemon or configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH`); + } return resolve( packageRoot, `runner/target/debug/paperclip-runnerd${executableSuffix}`, @@ -7057,6 +7148,194 @@ export function createCapabilityRunnerdCodexTransport( export const createRunnerdCodexTransport = createCapabilityRunnerdCodexTransport; +export async function probeNativeRunnerEnvironment(options: { + runtimeDirectory: string; + provider: "codex"; + model: string | null; + reasoningEffort?: string; + environment: NodeJS.ProcessEnv; + timeoutMs?: number; + workingDirectory?: string; + transportOptions?: CapabilityRunnerdCodexTransportOptions; + onCodexCredentialRefresh?: (filename: string) => Promise; + onCleanupConfirmed?: () => Promise; +}): Promise<{ + provider: "codex"; + providerDriver: string; + effectiveModel: string; + helloProbePassed: true; +}> { + const environment = { ...options.environment }; + const remoteFilesystemRoot = options.transportOptions?.runnerFilesystemRoot; + if (options.workingDirectory !== undefined && !remoteFilesystemRoot) throw new Error("Native setup remote working directory requires a remote runner filesystem."); + const workingDirectory = options.workingDirectory ?? resolve(options.runtimeDirectory, "workspace"); + if (!remoteFilesystemRoot) mkdirSync(workingDirectory, { mode: 0o700 }); + const controller = new AbortController(); + let registrationTeardownError: unknown; + let registrationOperation: Promise | undefined; + let registrationCleanupConfirmed = true; + const sourceCodexHome = options.transportOptions?.sourceCodexHome ?? resolveSourceCodexHome(environment); + delete environment.PAPERCLIP_RUNNER_EXTERNAL_SANDBOX; + environment.HOME = resolve(remoteFilesystemRoot ?? options.runtimeDirectory, "probe-home"); + environment.CODEX_HOME = resolve(remoteFilesystemRoot ?? options.runtimeDirectory, "codex-home"); + environment.PAPERCLIP_WORKSPACE_CWD = workingDirectory; + const bundle = createRunnerdCodexTransport({ + ...options.transportOptions, + ...(options.transportOptions?.controlPlaneRegistration ? { controlPlaneRegistration: (authority, identity) => { + const operation = (async () => { + controller.signal.throwIfAborted(); + registrationCleanupConfirmed = false; + const registration = await options.transportOptions!.controlPlaneRegistration!(authority, identity, controller.signal); + let releaseOperation: Promise | undefined; + const release = () => releaseOperation ??= Promise.resolve() + .then(() => registration.release()) + .then(() => { registrationCleanupConfirmed = true; }, error => { + registrationTeardownError = error; + throw error; + }); + if (controller.signal.aborted) { + await release(); + controller.signal.throwIfAborted(); + } + return { ...registration, release }; + })(); + registrationOperation = operation; + return operation; + } } : {}), + ...(options.transportOptions?.runnerProcessLauncher ? { runnerProcessLauncher: spec => { + controller.signal.throwIfAborted(); + return options.transportOptions!.runnerProcessLauncher!(spec); + } } : {}), + provider: options.provider, + sourceCodexHome: sourceCodexHome ?? "", + stateDirectory: options.runtimeDirectory, + environment, + externallySandboxed: false, + closeGraceMs: 10_000, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + }); + const driver = new CodexAppServerDriver({ + taskEnvelope: { + schema: "paperclip.skillless_task.v1", objective: "Verify the selected native account and model.", + completionContract: { revision: "environment-probe-v1", criteria: [] }, + constraints: [], expectedResultSchema: "paperclip.run_result.v1", + }, + ...(options.model ? { model: options.model } : {}), + ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } : {}), + conversationMode: "direct", + requestedCollaborationMode: "plan", + approvalPolicy: "never", + includeCollaborationModeInstructions: false, + dynamicTools: [], + environment, + transportFactory: () => bundle.transport, + requireProviderSessionIdentity: true, + ...(remoteFilesystemRoot ? { workingDirectoryAuthority: "remote_runner" as const } : {}), + }); + let rejectAborted!: (error: unknown) => void; + const aborted = new Promise((_resolve, reject) => { rejectAborted = reject; }); + const timer = setTimeout(() => { + const error = new Error("Native provider hello probe timed out."); + controller.abort(error); + rejectAborted(error); + }, Math.max(1, Math.min(options.timeoutMs ?? 45_000, 120_000))); + let session: HarnessSession | undefined; + let completed = false; + let effectiveModel: string | undefined; + let failure: unknown; + const operation = (async () => { + session = await driver.openSession({ runId: "environment-probe", normalizedSessionId: "environment-probe", + workingDirectory, signal: controller.signal }); + controller.signal.throwIfAborted(); + const ids = session.ids(); + const expectedDriver = "codex_app_server"; + if (!ids.driverSessionId || !ids.providerSessionId || bundle.evidence().providerDriver !== expectedDriver) { + throw new Error(`The native environment probe returned no matching provider identity (${bundle.evidence().providerDriver ?? "missing driver"}).`); + } + const events = session.events()[Symbol.asyncIterator](); + const first = await events.next(); + const context = record(first.value?.payload.context); + if (first.value?.eventType !== "session.started" || !context.model || record(record(context.sandbox).permissionProfile).id !== "paperclip-runner-workspace-read-only") { + throw new Error("The native environment probe returned no observed model or read-only runtime identity."); + } + { + effectiveModel = typeof context.model === "string" && context.model.trim() && context.model !== "unknown" ? context.model : undefined; + if (!effectiveModel || (options.model !== null && effectiveModel !== options.model)) throw new Error("The selected native runtime returned a missing or mismatched provider model."); + } + controller.signal.throwIfAborted(); + const { turnId } = await session.startTurn({ message: { role: "user", text: "Respond only with hello. Do not use tools or inspect files." } }); + let responseObserved = false; + while (true) { + const next = await events.next(); + if (next.done) break; + const event = next.value; + if (event.turnId !== turnId) continue; + if (event.eventType === "item.completed" && event.payload.kind === "agentMessage" + && typeof event.payload.text === "string" && event.payload.text.trim()) { + responseObserved = true; + } + if (event.eventType === "turn.completed") { + if (!responseObserved || !effectiveModel) throw new Error("The native provider hello probe returned no response or observed model."); + completed = true; + return; + } + if (event.eventType === "turn.failed" || event.eventType === "turn.interrupted" || event.eventType === "turn.cancelled") { + const detail = record(event.payload.error).message; + throw new Error(typeof detail === "string" ? detail : "The native provider hello probe failed."); + } + } + throw new Error("The native provider hello probe ended without a successful turn."); + })(); + try { + await Promise.race([operation, aborted]); + } catch (error) { + failure = error; + } finally { + clearTimeout(timer); + controller.abort(failure ?? new Error("Native provider hello probe complete.")); + const teardownErrors: unknown[] = []; + const teardownDeadline = Date.now() + 10_000; + const awaitTeardown = async (pending: Promise, detail: string): Promise => { + let timeout: ReturnType | undefined; + try { + await Promise.race([ + pending, + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error(`Native setup probe cleanup timed out while ${detail}; recovery state retained.`)), Math.max(0, teardownDeadline - Date.now())); + }), + ]); + } finally { clearTimeout(timeout); } + }; + try { + await awaitTeardown(session ? session.close({ reason: "environment probe complete" }) : bundle.transport.close("environment probe complete"), "closing the selected runtime"); + if (bundle.evidence().runnerPid !== null && !bundle.evidence().runnerExited) throw new Error("Native setup probe cleanup is incomplete."); + } catch (error) { teardownErrors.push(error); } + try { await awaitTeardown(operation.catch(() => undefined), "draining native setup"); } + catch (error) { teardownErrors.push(error); } + if (registrationOperation) { + try { await awaitTeardown(registrationOperation.catch(() => undefined), "draining authority registration"); } + catch (error) { teardownErrors.push(error); } + } + if (registrationTeardownError !== undefined && !teardownErrors.includes(registrationTeardownError)) teardownErrors.push(registrationTeardownError); + if (!registrationCleanupConfirmed) teardownErrors.push(new Error("Native setup probe authority release is unconfirmed; recovery state retained.")); + if (bundle.evidence().runnerPid !== null && bundle.evidence().runnerExited && options.provider === "codex" && options.onCodexCredentialRefresh) { + try { await options.onCodexCredentialRefresh(resolve(remoteFilesystemRoot ?? options.runtimeDirectory, "codex-home", "auth.json")); } + catch (error) { teardownErrors.push(error); } + } + if (!teardownErrors.length && options.onCleanupConfirmed) { + try { await options.onCleanupConfirmed(); } + catch (error) { teardownErrors.push(error); } + } + if (teardownErrors.length) { + const errors = [...(failure === undefined ? [] : [failure]), ...teardownErrors]; + throw new AggregateError(errors, errors.map(error => error instanceof Error ? error.message : "Native probe cleanup failed.").join("; "), { cause: failure ?? teardownErrors[0] }); + } + } + if (failure !== undefined) throw failure; + if (!completed) throw new Error("The native provider hello probe did not complete."); + return Object.freeze({ provider: options.provider, providerDriver: bundle.evidence().providerDriver!, effectiveModel: effectiveModel!, helloProbePassed: true }); +} + /** The transport provider identifies the biller, independently of model family. */ function openedThreadModelProvider( provider: CapabilityRunnerdCodexTransportOptions["provider"], diff --git a/scripts/__tests__/grok-public-install-sandbox.test.mjs b/scripts/__tests__/grok-public-install-sandbox.test.mjs index 71f6971d9b..8c43c72a13 100644 --- a/scripts/__tests__/grok-public-install-sandbox.test.mjs +++ b/scripts/__tests__/grok-public-install-sandbox.test.mjs @@ -1,16 +1,131 @@ import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; import { fileURLToPath } from 'node:url'; import { isWorkspacePackage, parseWorkspaceEntries } from '../build-standalone-public-packages.mjs'; -import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertInstalledCodexPlatformProvenance, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource } from '../grok-public-install-sandbox.mjs'; +import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertInstalledCodexPlatformProvenance, assertInstalledNativeConsumerContinuity, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource, installedDaemonProbeSource, macNodeHeadersDirectory, macPublicInstallLifecycleProfile, macInstalledStartupProbeSource, runMacInitialInstallWithDiagnostics } from '../grok-public-install-sandbox.mjs'; +import { verifyPiPackagedDaemon } from '../pi-public-install-probe.mjs'; +import { runnerBinaryTarget } from '../../packages/paperclip-runner/src/live/runner-binary.ts'; const paths = { assets: '/private/staging/assets', consumer: '/private/staging/consumer', cache: '/private/staging/cache', uid: 1001, gid: 1001 }; const values = (args, flag) => args.flatMap((value, index) => value === flag ? [args[index + 1]] : []); +test('Mac initial install failures retain bounded npm progress without secret or environment values', () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'mac-install-diagnostics-test-'))); + try { + const cacheDirectory = join(root, 'cache'), logs = join(cacheDirectory, '_logs'), receiptPath = join(root, 'failure.json'); + mkdirSync(logs, { recursive: true }); + writeFileSync(join(logs, '0000-debug-0.log'), '0 timing idealTree:init Completed in 987654321ms\n'); + const debug = Array.from({ length: 1000 }, (_, index) => `${index} timing idealTree:node_modules/private-secret Completed in ${index}ms`).join('\n'); + writeFileSync(join(logs, '0010-debug-0.log'), debug + '\n1001 verbose reify PASSWORD=991122ms\n' + + '1002 http fetch GET 200 https://user:password@registry.npmjs.org/pkg?token=query-secret 312ms (cache miss)\n' + + '1003 verbose cli /private/secret/path --token=argv-secret\n1004 silly fetch manifest Authorization=Bearer-header-secret\n'); + writeFileSync(join(logs, '0020-timing.json'), JSON.stringify({ timers: { 'idealTree:userRequests': 92967, 'reify:unpack': 76, + 'ENV_SECRET': 112233, 'reify:private-secret': 17 }, environment: { PASSWORD: 'environment-secret' } })); + const original = Object.assign(new Error('password=exception-secret'), { code: 'ETIMEDOUT', errno: -60, status: 1, signal: 'SIGTERM', + stdout: 'stdout-secret', stderr: 'stderr-secret', spawnargs: ['--token=argument-secret'] }); + const options = { sourceRevision: 'a'.repeat(40), platform: 'darwin', architecture: 'x64', cacheDirectory, receiptPath }; + assert.throws(() => runMacInitialInstallWithDiagnostics(() => { throw original; }, options), error => error === original); + const text = readFileSync(receiptPath, 'utf8'), receipt = JSON.parse(text); + assert.equal(receipt.sourceRevision, options.sourceRevision); assert.equal(receipt.platform, 'darwin'); assert.equal(receipt.architecture, 'x64'); + assert.equal(receipt.outcome, 'FAILED'); assert.equal(receipt.qualified, false); assert.equal(receipt.runtimeNotAttempted, true); + assert.equal(receipt.phase, 'initial-npm-install'); assert.equal(receipt.providerCalls, 0); assert.equal(receipt.timeoutMilliseconds, 600000); + assert.ok(receipt.elapsedMilliseconds >= 0); + assert.deepEqual(receipt.error, { code: 'ETIMEDOUT', status: 1, signal: 'SIGTERM', errno: -60 }); + assert.equal(receipt.npmDiagnosticTail.length, 2); + assert.ok(receipt.npmDiagnosticTail.every(tail => tail.entries.length <= 32)); + assert.equal(receipt.npmDiagnosticTail[0].truncated, true); + assert.ok(receipt.npmDiagnosticTail[1].entries.some(entry => entry.phase === 'idealTree:userRequests' && entry.durationMilliseconds === 92967)); + assert.ok(receipt.npmDiagnosticTail[0].entries.some(entry => entry.method === 'GET' && entry.status === 200 && entry.durationMilliseconds === 312)); + assert.ok(Buffer.byteLength(text) < 10000); + assert.doesNotMatch(text, /secret|PASSWORD|Authorization|https:|private|991122|987654321|112233/i); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('Mac install diagnostics never follow outside log symlinks or replace the primary failure', () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'mac-install-diagnostics-boundary-'))); + try { + const cacheDirectory = join(root, 'cache'), logs = join(cacheDirectory, '_logs'), receiptPath = join(root, 'failure.json'); + mkdirSync(logs, { recursive: true }); + writeFileSync(join(root, 'outside.log'), '1 timing idealTree:userRequests Completed in 987654321ms\n'); + symlinkSync(join(root, 'outside.log'), join(logs, '0001-debug-0.log')); + const original = Object.assign(new Error('primary'), { code: 'unsafe-secret', status: null, signal: 'unsafe-secret' }); + const options = { sourceRevision: 'b'.repeat(40), platform: 'darwin', architecture: 'arm64', cacheDirectory, receiptPath }; + assert.throws(() => runMacInitialInstallWithDiagnostics(() => { throw original; }, options), error => error === original); + const receipt = JSON.parse(readFileSync(receiptPath, 'utf8')); + assert.deepEqual(receipt.npmDiagnosticTail, []); assert.equal(receipt.error.code, null); assert.equal(receipt.error.signal, null); + rmSync(receiptPath); mkdirSync(receiptPath); + assert.throws(() => runMacInitialInstallWithDiagnostics(() => { throw original; }, options), error => error === original); + rmSync(receiptPath, { recursive: true }); rmSync(logs, { recursive: true }); symlinkSync(root, logs); + assert.throws(() => runMacInitialInstallWithDiagnostics(() => { throw original; }, options), error => error === original); + assert.deepEqual(JSON.parse(readFileSync(receiptPath, 'utf8')).npmDiagnosticTail, []); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('successful Mac installs do not create failed receipts or change the bounded install contract', () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'mac-install-diagnostics-success-'))); + try { + const receiptPath = join(root, 'receipt.json'); + assert.equal(runMacInitialInstallWithDiagnostics(() => 'installed', { sourceRevision: 'c'.repeat(40), platform: 'darwin', + architecture: 'x64', cacheDirectory: join(root, 'cache'), receiptPath }), 'installed'); + assert.equal(existsSync(receiptPath), false); + const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + assert.match(source, /timeout: 10 \* 60_000/); + assert.match(source, /runMacInitialInstallWithDiagnostics\(\(\) => run\(process.execPath, \[npmCli, 'install', '--ignore-scripts', '--omit=dev', '--timing'/); + assert.ok(source.indexOf('runMacInitialInstallWithDiagnostics(()') < source.indexOf("assert.equal(existsSync(sentinel), false)")); + assert.match(source, /finally \{ rmSync\(root, \{ recursive: true, force: true \}\); \}/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('native host startup preserves the consumer lock and both executable byte identities', () => { + const before = { lockSha256: 'a'.repeat(64), runnerSha256: 'b'.repeat(64), codexCommandSha256: 'c'.repeat(64), codexNativeExecutableSha256: 'e'.repeat(64) }; + assert.doesNotThrow(() => assertInstalledNativeConsumerContinuity(before, { ...before })); + for (const field of Object.keys(before)) { + assert.throws(() => assertInstalledNativeConsumerContinuity(before, { ...before, [field]: 'd'.repeat(64) }), new RegExp(field)); + assert.throws(() => assertInstalledNativeConsumerContinuity({ ...before, [field]: '' }, { ...before, [field]: '' }), new RegExp(field)); + } + const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + const arm = source.split('async function verifyLinuxArmConsumer')[1].split("if (process.argv[2] === '--linux-arm64-consumer')")[0]; + assert.match(arm, /const hostEnv = \{ PATH: .*HOME: home, TMPDIR: hostTmp, LC_ALL: 'C', TZ: 'UTC' \}/); + assert.match(arm, /execFileSync\(process.execPath, \[file\], \{ cwd: consumer, env: hostEnv/); + assert.ok(arm.indexOf('isolated(GROK_PUBLIC_INSTALL_LIFECYCLE)') < arm.indexOf("hostProbe('host-startup-probe.mjs'")); + assert.ok(arm.indexOf('const before =') < arm.indexOf("hostProbe('host-startup-probe.mjs'")); + assert.ok(arm.indexOf("hostProbe('host-startup-probe.mjs'") < arm.indexOf('const after =')); + assert.match(arm, /assertInstalledNativeConsumerContinuity\(before, after\)/); + assert.doesNotMatch(arm, /externallySandboxed|externalSandbox|sudo|sysctl|--privileged|seccomp=unconfined|unshare/); +}); + +test('Mac offline lifecycle uses only the matching executing Node SDK', () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'mac-node-headers-test-'))); + try { + const prefix = join(root, 'sdk'), executable = join(prefix, 'bin/node'), include = join(prefix, 'include/node'); + mkdirSync(join(prefix, 'bin'), { recursive: true }); mkdirSync(include, { recursive: true }); + writeFileSync(executable, 'fixture Node executable'); + writeFileSync(join(include, 'node.h'), '#include "node_version.h"\n'); + const header = '#define NODE_MAJOR_VERSION 24\n#define NODE_MINOR_VERSION 20\n#define NODE_PATCH_VERSION 0\n'; + writeFileSync(join(include, 'node_version.h'), header); + assert.equal(macNodeHeadersDirectory(executable, '24.20.0'), prefix); + assert.throws(() => macNodeHeadersDirectory(executable, '24.19.0'), /must match the executing Node/); + assert.throws(() => macNodeHeadersDirectory(executable, '24.20.0-custom'), /exact release version/); + writeFileSync(join(include, 'node_version.h'), header + '#define NODE_PATCH_VERSION 0\n'); + assert.throws(() => macNodeHeadersDirectory(executable, '24.20.0'), /declare one patch version/); + writeFileSync(join(include, 'node_version.h'), header); + rmSync(join(include, 'node.h')); + assert.throws(() => macNodeHeadersDirectory(executable, '24.20.0'), /ENOENT/); + writeFileSync(join(root, 'outside-node.h'), '#include "node_version.h"\n'); + symlinkSync(join(root, 'outside-node.h'), join(include, 'node.h')); + assert.throws(() => macNodeHeadersDirectory(executable, '24.20.0'), /inside the executing Node prefix/); + const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + assert.ok(source.includes('macNodeHeadersDirectory(process.execPath, process.versions.node)')); + assert.ok(source.includes('consumer, { ...env, npm_config_nodedir: nodeHeaders }'), 'SDK is bound only to the deferred lifecycle environment'); + assert.equal(source.split('npm_config_nodedir:').length, 2); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + test('lifecycle execution has no network, host credentials, checkout, or elevated privileges', () => { const args = grokConsumerDockerArgs({ ...paths, command: GROK_PUBLIC_INSTALL_LIFECYCLE }); assert.deepEqual(values(args, '--network'), ['none']); @@ -85,7 +200,8 @@ test('verification never elevates PR-controlled provisioning or cleanup on the h test('the image and Grok downloads start before staging and join without weakening the npm checks', () => { const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); - const position = fragment => { const index = source.indexOf(fragment); assert.notEqual(index, -1, `verifier must contain ${fragment}`); return index; }; + const linuxSource = source.slice(source.indexOf("assert.equal(process.platform, 'linux'")); + const position = fragment => { const index = linuxSource.indexOf(fragment); assert.notEqual(index, -1, `verifier must contain ${fragment}`); return index; }; // Exactly the pinned consumer image is pulled, for the platform the probes run on. const imagePull = position("prefetch('image', 'docker', ['pull', '--platform', 'linux/amd64', GROK_PUBLIC_INSTALL_IMAGE])"); // Grok is provisioned into the prefetch directory, never straight into `native/grok`. @@ -109,7 +225,7 @@ test('the image and Grok downloads start before staging and join without weakeni // cleanup waits for them before removing the directory they write into. assert.ok(source.includes("import { startPrefetch, stopPrefetches } from './prefetch-child.mjs'")); assert.ok(position('await stopPrefetches(prefetches)') < position('rmSync(root, { recursive: true, force: true })')); - assert.equal(source.match(/rmSync\(root/g).length, 1); + assert.equal(linuxSource.match(/rmSync\(root/g).length, 1); }); test('the bundled-dependency installs run together while the other packages are copied, and pack in order', () => { @@ -137,7 +253,8 @@ test('the bundled-dependency installs run together while the other packages are const checks = position("for (const tarball of tarballs) assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball])"); const copied = position('for (const tarball of tarballs) {'); assert.ok(bundledSources < start && start < registered && registered < otherSources && otherSources < join && join < packs && packs < reordered && reordered < checks && checks < copied); - assert.equal(source.match(/run\('npm', \['pack'/g).length, 1, 'only the lifecycle sentinel is packed in-process'); + const packagingLane = source.slice(position("} else {\nassert.equal(process.platform, 'linux'")); + assert.equal(packagingLane.match(/run\('npm', \['pack'/g).length, 1, 'only the lifecycle sentinel is packed in-process in the packaging lane'); assert.ok(source.includes("destination: join(root, `pack-${index}`)"), 'each pack writes into its own directory'); // The timing line reports the split staging phase, each install, and the // pack step with what the packs would have cost one after another. @@ -201,6 +318,14 @@ test('the installed Codex probe exercises the public export and rejects incomple const platformPath = `node_modules/@openai/codex-${target}`; const platform = join(consumer, platformPath); mkdirSync(platform, { recursive: true }); const nativeVendor = join(platform, 'vendor'); mkdirSync(nativeVendor); + const triple = { 'linux-x64': 'x86_64-unknown-linux-musl', 'linux-arm64': 'aarch64-unknown-linux-musl', + 'darwin-x64': 'x86_64-apple-darwin', 'darwin-arm64': 'aarch64-apple-darwin', 'win32-x64': 'x86_64-pc-windows-msvc', 'win32-arm64': 'aarch64-pc-windows-msvc' }[target]; + const nativeBytes = Buffer.alloc(64); + if (process.platform === 'darwin') { nativeBytes.writeUInt32LE(0xfeedfacf); nativeBytes.writeUInt32LE(process.arch === 'arm64' ? 0x0100000c : 0x01000007, 4); } + else if (process.platform === 'win32') { nativeBytes.set([0x4d, 0x5a]); nativeBytes.writeUInt32LE(40, 60); nativeBytes.writeUInt32LE(0x00004550, 40); nativeBytes.writeUInt16LE(process.arch === 'arm64' ? 0xaa64 : 0x8664, 44); } + else { nativeBytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); nativeBytes.writeUInt16LE(process.arch === 'arm64' ? 183 : 62, 18); } + const nativeFile = join(nativeVendor, triple, 'bin', process.platform === 'win32' ? 'codex.exe' : 'codex'); + mkdirSync(join(nativeVendor, triple, 'bin'), { recursive: true }); writeFileSync(nativeFile, nativeBytes, { mode: 0o755 }); const manifest = { name: '@openai/codex', version: `0.160.0-${target}`, os: [process.platform], cpu: [process.arch] }; writeFileSync(join(platform, 'package.json'), JSON.stringify(manifest)); const entry = { name: manifest.name, version: manifest.version, @@ -209,6 +334,8 @@ test('the installed Codex probe exercises the public export and rejects incomple writeFileSync(join(consumer, 'package-lock.json'), JSON.stringify(lock)); const server = join(consumer, 'node_modules/@paperclipai/server'); const vendor = join(server, 'dist/vendor/paperclip-runner'), drivers = join(vendor, 'drivers/acpx'); mkdirSync(drivers, { recursive: true }); + mkdirSync(join(vendor, 'live')); + writeFileSync(join(vendor, 'live/runner-binary.js'), `export const runnerBinaryTarget = ${runnerBinaryTarget.toString()};`); writeFileSync(join(server, 'package.json'), JSON.stringify({ name: '@paperclipai/server', type: 'module' })); const integrity = join(drivers, 'installation-integrity.js'); const security = join(vendor, 'drivers/codex/codex-security-config.js'); mkdirSync(join(vendor, 'drivers/codex')); @@ -233,6 +360,24 @@ test('the installed Codex probe exercises the public export and rejects incomple assert.equal(JSON.parse(result.stdout).pinnedCodexCommandVerified, true); assert.equal(JSON.parse(result.stdout).codexPlatformPackageSource, 'official npm'); assert.equal(JSON.parse(result.stdout).codexConsumerPlatform, target); + assert.equal(JSON.parse(result.stdout).codexCommandSha256, createHash('sha256').update(readFileSync(command)).digest('hex')); + assert.equal(JSON.parse(result.stdout).codexNativeExecutableSha256, createHash('sha256').update(nativeBytes).digest('hex')); + const original = JSON.parse(result.stdout); + nativeBytes[63] = 1; writeFileSync(nativeFile, nativeBytes); + result = run(); assert.equal(result.status, 0, result.stderr); + const changed = JSON.parse(result.stdout); + assert.equal(changed.codexCommandSha256, original.codexCommandSha256, 'native mutation does not change the npm launcher'); + assert.notEqual(changed.codexNativeExecutableSha256, original.codexNativeExecutableSha256); + assert.throws(() => assertInstalledNativeConsumerContinuity({ lockSha256: 'a'.repeat(64), runnerSha256: 'b'.repeat(64), ...original }, + { lockSha256: 'a'.repeat(64), runnerSha256: 'b'.repeat(64), ...changed }), /codexNativeExecutableSha256/); + nativeBytes[63] = 0; writeFileSync(nativeFile, nativeBytes); + rmSync(nativeFile); + result = run(); assert.notEqual(result.status, 0); assert.match(result.stderr, /one native executable/); + const foreignNative = join(root, 'foreign-codex'); writeFileSync(foreignNative, nativeBytes, { mode: 0o755 }); + symlinkSync(foreignNative, nativeFile); + result = run(); assert.notEqual(result.status, 0); assert.match(result.stderr, /bounded regular file in its selected platform package/); + rmSync(nativeFile); writeFileSync(nativeFile, nativeBytes, { mode: 0o755 }); + result = run(); assert.equal(result.status, 0, result.stderr); assert.equal(JSON.parse(result.stdout).codexQualifiedInstallationVerified, true); assert.equal(JSON.parse(result.stdout).codexCommandLeaseVerified, true); assert.equal(JSON.parse(result.stdout).codexNativeSandboxResourcesVerified, true); @@ -253,6 +398,18 @@ test('the installed Codex probe exercises the public export and rejects incomple writeFileSync(legacyCommand, '#!/bin/sh\nexit 0\n', { mode: 0o755 }); writeFileSync(index, `export const resolvePinnedCodexCommand = () => ${JSON.stringify(legacyCommand)};`); result = run(); assert.notEqual(result.status, 0); assert.match(result.stderr, /Selected Codex platform version must match the qualified pin/); + mkdirSync(join(legacy, 'vendor')); + const legacyNative = join(legacy, 'vendor', triple, 'codex', process.platform === 'win32' ? 'codex.exe' : 'codex'); + mkdirSync(join(legacy, 'vendor', triple, 'codex'), { recursive: true }); writeFileSync(legacyNative, nativeBytes, { mode: 0o755 }); + writeFileSync(legacyCommand, '#!/bin/sh\nprintf "codex-cli 0.156.1\\n"\n', { mode: 0o755 }); + writeFileSync(security, securitySource.replaceAll(JSON.stringify(nativeVendor), JSON.stringify(join(legacy, 'vendor')))); + writeFileSync(probe, installedCodexProbeSource(index, consumer, undefined, { nativeCodex: true })); + result = run(); assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).installedCodexVersion, 'codex-cli 0.156.1'); + assert.equal(JSON.parse(result.stdout).codexQualifiedInstallationVerified, undefined); + writeFileSync(probe, installedCodexProbeSource(index, consumer, '0.160.0')); + writeFileSync(security, securitySource); + writeFileSync(index, `import { writeFileSync } from 'node:fs'; export const resolvePinnedCodexCommand = () => { writeFileSync(${JSON.stringify(join(platform, 'package.json'))}, ${JSON.stringify(JSON.stringify({ ...manifest, altered: true }))}); return ${JSON.stringify(command)}; };`); @@ -308,6 +465,7 @@ test('actual tar member checks allow Codex JavaScript but reject nested platform assert.doesNotThrow(() => assertNoBundledCodexPayloads(javascript)); for (const payload of [ 'package/node_modules/@openai/codex-linux-x64/package.json', + 'package/node_modules/@openai/codex-linux-arm64/package.json', 'package/node_modules/bridge/node_modules/@openai/codex-darwin-arm64/package.json', 'package/node_modules/@openai/codex-win32-x64/package.json', 'package/node_modules/@openai/codex/vendor/x86_64-unknown-linux-musl/codex/codex', @@ -320,24 +478,129 @@ test('actual tar member checks allow Codex JavaScript but reject nested platform }); test('consumer provenance rejects bundled, foreign, inconsistent and non-official Codex payloads', () => { - const path = 'node_modules/@openai/codex-linux-x64'; - const manifest = { name: '@openai/codex', version: '0.160.0-linux-x64', os: ['linux'], cpu: ['x64'] }; + for (const target of ['linux-x64', 'linux-arm64']) { + const path = `node_modules/@openai/codex-${target}`; + const manifest = { name: '@openai/codex', version: `0.160.0-${target}`, os: ['linux'], cpu: [target.slice('linux-'.length)] }; const entry = { version: manifest.version, resolved: `https://registry.npmjs.org/@openai/codex/-/codex-${manifest.version}.tgz`, integrity: `sha512-${Buffer.alloc(64, 1).toString('base64')}` }; - const check = (value = entry, installed = [{ path, manifest }]) => assertInstalledCodexPlatformProvenance({ packages: { [path]: value } }, installed, 'linux-x64'); + const check = (value = entry, installed = [{ path, manifest }]) => assertInstalledCodexPlatformProvenance({ packages: { [path]: value } }, installed, target); assert.equal(check().codexConsumerHostOnly, true); - const legacyManifest = { ...manifest, version: '0.156.1-linux-x64' }; + const legacyManifest = { ...manifest, version: `0.156.1-${target}` }; assert.equal(check({ ...entry, version: legacyManifest.version, - resolved: 'https://registry.npmjs.org/@openai/codex/-/codex-0.156.1-linux-x64.tgz' }, [{ path, manifest: legacyManifest }]).codexConsumerHostOnly, true); - assert.throws(() => check(entry, [{ path, manifest: { ...manifest, version: '^0.160.0-linux-x64' } }]), /exact host version/); + resolved: `https://registry.npmjs.org/@openai/codex/-/codex-0.156.1-${target}.tgz` }, [{ path, manifest: legacyManifest }]).codexConsumerHostOnly, true); + assert.throws(() => check(entry, [{ path, manifest: { ...manifest, version: `^0.160.0-${target}` } }]), /exact host version/); for (const change of [ - { inBundle: true }, { link: true }, { version: '0.156.1-linux-x64' }, - { resolved: 'https://example.com/@openai/codex/-/codex-0.160.0-linux-x64.tgz' }, - { resolved: 'https://registry.npmjs.org/@openai/codex/-/codex-0.156.1-linux-x64.tgz' }, - { resolved: 'https://user@registry.npmjs.org/@openai/codex/-/codex-0.160.0-linux-x64.tgz' }, + { inBundle: true }, { link: true }, { version: `0.156.1-${target}` }, + { resolved: `https://example.com/@openai/codex/-/codex-0.160.0-${target}.tgz` }, + { resolved: `https://registry.npmjs.org/@openai/codex/-/codex-0.156.1-${target}.tgz` }, + { resolved: `https://user@registry.npmjs.org/@openai/codex/-/codex-0.160.0-${target}.tgz` }, { integrity: undefined }, { integrity: 'sha512-AA==' }, ]) assert.throws(() => check({ ...entry, ...change })); assert.throws(() => check(entry, []), /must install its host/); assert.throws(() => check(entry, [{ path: 'node_modules/@openai/codex-darwin-arm64', manifest }]), /only the host/); assert.throws(() => check(entry, [{ path, manifest: { ...manifest, name: 'untrusted' } }]), /identity/); + } +}); + + +test('hosted Mac lifecycle denies OS networking and writes outside owned scratch', () => { + const profile = macPublicInstallLifecycleProfile('/Users/runner/work/_temp/owned-check'); + assert.match(profile, /\(deny network\*\)/); + assert.match(profile, /\(deny file-write\*/); + assert.match(profile, /subpath "\/Users\/runner\/work\/_temp\/owned-check"/); + assert.throws(() => macPublicInstallLifecycleProfile('relative'), /absolute/); + assert.throws(() => macPublicInstallLifecycleProfile('/tmp/owned\0bad'), /absolute/); + const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + assert.match(source, /process.env.GITHUB_ACTIONS, 'true'/); + assert.match(source, /networkProbe, 'allowed'/); + assert.match(source, /networkProbe, 'denied'/); + assert.match(source, /error.code,'EPERM'/); + assert.match(source, /'rebuild', '--offline', '--ignore-scripts=false', '--dangerously-allow-all-scripts'/); + assert.match(source, /readFileSync\(sentinel, 'utf8'\), 'ok'/); + assert.match(source, /readFileSync\(join\(consumer, 'package-lock.json'\), 'utf8'\), lock/); + assert.match(source, /finally \{ rmSync\(root, \{ recursive: true, force: true \}\); \}/); +}); + +test('installed daemon proof uses the production host resolver and complete source-bound manifest', () => { + const source = installedDaemonProbeSource('/consumer/server/dist/vendor/paperclip-runner', '/consumer', 'a'.repeat(40)); + assert.match(source, /defaultCapabilityRunnerdBinary\(\)/); + assert.match(source, /Object.keys\(manifest.platforms\).sort\(\), targets.sort\(\)/); + assert.match(source, /manifest.sourceRevision, "a{40}"/); + assert.match(source, /runnerBinaryTarget\(bytes\), target/); + assert.match(source, /createHash\('sha256'\)/); + assert.match(source, /metadata.binaryContractVersion, expected.contracts.runnerdArtifact/); + assert.match(source, /'linux-arm64'/); + assert.doesNotMatch(source, /PAPERCLIP_RUNNERD_BINARY|runner\/target\/debug|execFileSync\('paperclip-runnerd'/); +}); + +test('native ARM clean consumers keep the existing container and installed-runtime boundaries', () => { + const args = grokConsumerDockerArgs({ ...paths, platform: 'linux/arm64', command: GROK_PUBLIC_INSTALL_LIFECYCLE }); + assert.deepEqual(values(args, '--platform'), ['linux/arm64']); + assert.deepEqual(values(args, '--network'), ['none']); + assert.deepEqual(values(args, '--user'), ['1001:1001']); + assert.ok(args.includes('--read-only')); + assert.deepEqual(values(args, '--cap-drop'), ['ALL']); + assert.deepEqual(values(args, '--security-opt'), ['no-new-privileges']); + assert.ok(values(args, '--mount').some(value => value.endsWith('dst=/packages,readonly'))); + assert.throws(() => grokConsumerDockerArgs({ ...paths, platform: 'linux/arm/v7', command: [] }), /supported native Linux platform/); + const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + const arm = source.split('async function verifyLinuxArmConsumer(')[1].split("if (process.argv[2] === '--linux-arm64-consumer')")[0]; + assert.match(arm, /process.platform \+ '-' \+ process.arch, 'linux-arm64'/); + assert.match(arm, /const platform = 'linux\/arm64'/); + assert.match(arm, /isolated\(GROK_PUBLIC_INSTALL_LIFECYCLE\)/); + assert.match(arm, /installedCodexProbeSource\(installed \+ '\/index.js', '\/consumer', undefined, \{ nativeCodex: true \}\)/); + assert.match(arm, /tests\/runner-e2e\/installed-cli.ts/); + assert.match(arm, /process.versions.node.split\('\.'\)\[0\], '24'/); + assert.match(arm, /macInstalledStartupProbeSource\(consumer, hostServer, daemon.sha256, join\(assets, 'installed-cli.ts'\)\)/); + assert.match(arm, /startupEnvironment: 'native host, same installed graph, isolated empty homes'/); + assert.doesNotMatch(arm, /stage-runner|runner\/target|PAPERCLIP_RUNNER_BINARY|prerequisite:/); +}); + + +test('Mac startup wiring normalizes only validated manifest digests and uses server-scoped tooling', () => { + const source = macInstalledStartupProbeSource('/owned/consumer', '/owned/consumer/server', 'sha256:' + 'a'.repeat(64), '/source/tests/runner-e2e/installed-cli.ts'); + assert.match(source, /expectedRunnerSha256:"a{64}"/); + assert.doesNotMatch(source, /expectedRunnerSha256:"sha256:/); + for (const digest of ['a'.repeat(64), 'sha256:bad', 'sha512:' + 'a'.repeat(64), 'sha256:' + 'A'.repeat(64)]) { + assert.throws(() => macInstalledStartupProbeSource('/owned/consumer', '/owned/consumer/server', digest, '/source/helper.ts'), /verified SHA-256/); + } + const verifier = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + assert.match(verifier, /\['--import', 'tsx', startupFile\], join\(repo, 'server'\)/); + const closedEnvironment = verifier.match(/const env = \{ PATH: `\$\{tools\}[\s\S]*?\};/)[0]; + assert.doesNotMatch(closedEnvironment, /NODE_PATH|NODE_OPTIONS|\.\.\.process.env/); +}); + +test('Pi admission verifies assembled host daemon identity and preserves the flat source-package control', async () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'pi-installed-daemon-'))), server = join(root, 'server'); + const output = join(server, 'dist/vendor/paperclip-runner'), target = 'linux-x64', source = 'a'.repeat(40); + const binary = join(output, 'bin', target, 'paperclip-runnerd'), manifestPath = join(output, 'bin/release-manifest.json'); + const bytes = Buffer.alloc(64); bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); bytes.writeUInt16LE(62, 18); + const sha256 = value => `sha256:${createHash('sha256').update(value).digest('hex')}`; + const manifest = { schema: 'paperclip.runner.release-binaries.v1', sourceRevision: source, + platforms: { [target]: { path: `${target}/paperclip-runnerd`, sha256: sha256(bytes) } } }; + const check = () => verifyPiPackagedDaemon(server, binary, target, runnerBinaryTarget); + try { + mkdirSync(join(output, 'bin', target), { recursive: true }); + writeFileSync(binary, bytes); writeFileSync(join(server, 'dist/build-info.json'), JSON.stringify({ commit: source })); + writeFileSync(manifestPath, JSON.stringify(manifest)); + await check(); + for (const changed of [null, { ...manifest, schema: 'untrusted' }, { ...manifest, sourceRevision: 'b'.repeat(40) }, + { ...manifest, platforms: { [target]: { ...manifest.platforms[target], path: '../outside' } } }, + { ...manifest, platforms: { [target]: { ...manifest.platforms[target], sha256: 'sha256:' + '0'.repeat(64) } } }]) { + writeFileSync(manifestPath, JSON.stringify(changed)); await assert.rejects(check()); + } + const wrongHeader = Buffer.from(bytes); wrongHeader.writeUInt16LE(183, 18); + writeFileSync(binary, wrongHeader); + writeFileSync(manifestPath, JSON.stringify({ ...manifest, platforms: { [target]: { ...manifest.platforms[target], sha256: sha256(wrongHeader) } } })); + await assert.rejects(check(), /linux-arm64/); + writeFileSync(binary, bytes); writeFileSync(manifestPath, JSON.stringify(manifest)); + const outside = join(root, 'outside'); writeFileSync(outside, bytes); + rmSync(binary); symlinkSync(outside, binary); await assert.rejects(check()); + rmSync(binary); writeFileSync(binary, bytes); rmSync(manifestPath); + const flat = join(output, 'bin/paperclip-runnerd'); writeFileSync(flat, bytes); + await verifyPiPackagedDaemon(server, flat, target, runnerBinaryTarget); + await assert.rejects(check(), /strictly equal/); + writeFileSync(manifestPath, 'null'); + await assert.rejects(verifyPiPackagedDaemon(server, flat, target, runnerBinaryTarget), /manifest must be an object/); + } finally { rmSync(root, { recursive: true, force: true }); } }); diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 9d00a15b79..c6db590248 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -1,8 +1,10 @@ import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; +import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; import test from "node:test"; import { createServer } from "node:http"; @@ -81,14 +83,14 @@ test("chaos verification isolates callers that verify the same source commit", ( test("canary reuses exact-source proof while stable keeps full verification", () => { const releaseWorkflow = readWorkflow("release.yml"); - const canary = releaseWorkflow.split(" verify_canary:\n")[1].split("\n publish_canary:")[0]; + const canary = releaseWorkflow.match(/ verify_canary:\n[\s\S]*?(?=\n [a-z_]+:)/)[0]; assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/); assert.match(canary, /actions: read/); assert.match(canary, /ref: \$\{\{ github\.sha \}\}/); assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/); assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/); assert.doesNotMatch(canary, /release-verify\.yml|continue-on-error|always\(\)/); - assert.match(releaseWorkflow, /publish_canary:\n\s+if: github\.event_name == 'push'\n\s+needs: verify_canary/); + assert.match(releaseWorkflow, /publish_canary:\n\s+if: .*github\.event_name == 'push'.*\n\s+needs: \[verify_canary, release_runner_source, release_runner_binaries\]/); // The stable lane is gated on the stable channel since the nightly lane // was added; a `needs:` line (for example a preflight job) may sit between // the gate and the delegation. @@ -742,6 +744,454 @@ test("direct protocol concurrency override only lowers the configured ceiling", } }); +test("release daemon production uses exact-source native hosts and no publisher or provider permissions", () => { + const workflow = readWorkflow("release-verify.yml"); + const producer = workflow.split(" runner_binaries:\n")[1]; + const targets = workflow.split(" runner_binary_targets:\n")[1].split(" runner_binaries:\n")[0]; + for (const [target, host] of [["linux-arm64", "ubuntu-24.04-arm"], ["darwin-arm64", "macos-15"], ["darwin-x64", "macos-15-intel"]]) { + assert.ok(targets.includes(`'${target}': '${host}'`)); + } + assert.match(producer, /matrix: \$\{\{ fromJSON\(needs.runner_binary_targets.outputs.matrix\) \}\}/); + assert.match(targets, /Unknown exact-source daemon platform contract/); + assert.match(targets, /release-runner-targets\.json/); + assert.match(targets, /execFileSync\('git', \['show', source \+ ':' \+ file\]/); + assert.doesNotMatch(targets, /import .*from '\.\/|await import\(|\beval\(|new Function/); + assert.doesNotMatch(targets, /secrets\.|id-token: write|contents: write|npm publish|environment:/); + assert.match(producer, /persist-credentials: false/); + assert.match(producer, /contents: read/); + assert.doesNotMatch(producer, /secrets\.|id-token: write|contents: write|npm publish|environment:/); + assert.match(producer, /\[\[ "\$SOURCE_SHA" =~ \^\[a-f0-9\]\{40\}\$ \]\]/); + assert.match(producer, /git rev-parse HEAD/); + assert.match(producer, /process.platform \+ "-" \+ process.arch/); + assert.match(producer, /\['native', \['build:binary'\]/); + const runner = JSON.parse(readFileSync(path.join(repoRoot, 'packages/paperclip-runner/package.json'), 'utf8')); + assert.match(runner.scripts['build:binary'], /cargo build --release --manifest-path runner\/Cargo.toml --locked/); + assert.match(runner.scripts['build:binary'], /&& node scripts\/stage-runner-binary.mjs$/); + assert.match(producer, /stage-release-runner-binaries.mjs record/); + for (const job of ["runner_chaos_evals", "typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"]) { + assert.ok(workflow.includes(`${job}:\n if: \${{ !inputs.runner_binaries_only }}`)); + } + const release = readWorkflow("release.yml"); + for (const job of ["publish_canary", "publish_nightly", "publish_beta", "preview_stable", "publish_stable"]) { + const block = release.split(` ${job}:\n`)[1].split(/\n [a-z_]+:\n/)[0]; + assert.match(block, /needs: .*release_runner_binaries/); + assert.match(block, /pattern: \$\{\{ needs.release_runner_binaries.outputs.runner_artifact_prefix \}\}-\*/); + assert.match(block, /PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR/); + } + const script = readFileSync(path.join(repoRoot, 'scripts/release.sh'), 'utf8'); + assert.ok(script.indexOf('assemble-daemons "$PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR"') < script.indexOf('scripts/build-standalone-public-packages.mjs')); + assert.match(script, /cp -R .*paperclip-runner\/dist\/bin\/\." .*server\/dist\/vendor\/paperclip-runner\/bin\//); + assert.match(script, /elif \[ "\$dry_run" = false \]; then[\s\S]*?release_fail "Public releases require/); +}); + +function nativeProducer() { + return readWorkflow('release-verify.yml').split(' runner_binaries:\n')[1]; +} + +test('native dependency caches are target-bound and unmerged master callers cannot save them', () => { + const producer = nativeProducer(); + const cache = producer.split(' - name: Cache native Rust dependencies\n')[1].split('\n - ')[0]; + assert.match(cache, /Swatinem\/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6/); + assert.match(cache, /workspaces: \$\{\{ steps.native_rust_workspace.outputs.path \}\} -> target/); + assert.match(cache, /shared-key: release-native-v2-\$\{\{ matrix.host \}\}-\$\{\{ matrix.target \}\}-\$\{\{ hashFiles\('packages\/paperclip-runner\/protocol\/\*\*', 'packages\/paperclip-runner\/src\/\*\*', 'packages\/paperclip-runner\/\*.json'\) \}\}/); + for (const option of ['cache-workspace-crates', 'cache-bin', 'cache-on-failure']) { + assert.match(cache, new RegExp(option + ': false')); + } + const expression = cache.match(/save-if: \$\{\{ (.+) \}\}/)?.[1]; + assert.ok(expression, 'cache writes need a separate source trust gate'); + const canSave = new Function('github', 'inputs', `return (${expression});`); + const github = { repository: 'paperclipai/paperclip', repository_id: '1170821064', event_name: 'push', ref: 'refs/heads/master', sha: 'a'.repeat(40) }; + assert.equal(canSave(github, { ref: github.sha }), true); + for (const [overrides, input] of [ + [{}, 'b'.repeat(40)], [{}, ''], + [{ event_name: 'workflow_dispatch' }, github.sha], + [{ event_name: 'workflow_dispatch' }, 'b'.repeat(40)], + [{ event_name: 'pull_request', ref: 'refs/pull/123/merge' }, github.sha], + [{ ref: 'refs/heads/topic' }, github.sha], + [{ repository: 'fork/paperclip' }, github.sha], + [{ repository_id: '1' }, github.sha], + ]) assert.equal(canSave({ ...github, ...overrides }, { ref: input }), false); + assert.match(producer, /pinned="\$HOME\/paperclip-runner-rust"/); + assert.match(producer, /ln -s "\$GITHUB_WORKSPACE\/packages\/paperclip-runner\/runner" "\$pinned"/); + assert.match(producer, /echo "RUSTUP_TOOLCHAIN=\$toolchain" >> "\$GITHUB_ENV"/); + assert.doesNotMatch(producer, /cache-hit|continue-on-error/); + assert.ok(producer.indexOf('Bind immutable producer source and target') < producer.indexOf('Cache native Rust dependencies')); + assert.ok(producer.indexOf('Build TypeScript contract and host daemon concurrently') < producer.indexOf('Record actual daemon metadata and immutable bytes')); +}); + +function nativePreparationScript() { + const step = nativeProducer().split(' - name: Build TypeScript contract and host daemon concurrently\n')[1].split('\n - ')[0]; + const script = step.match(/node --input-type=module <<'NODE'\n([\s\S]*?)\n NODE/)?.[1]; + assert.ok(script); + return script.replace(/^ {10}/gm, ''); +} + +const fixtureProcessGone = pid => { + try { process.kill(pid, 0); } catch (error) { return error.code === 'ESRCH'; } + try { return /^\d+ \(.*\) Z /.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch { return false; } +}; +const waitForFixture = async (predicate, label) => { + for (let attempt = 0; attempt < 200; attempt++) { + if (predicate()) return; + await new Promise(resolve => setTimeout(resolve, 25)); + } + assert.fail(label); +}; + +async function withNativePreparationFixture(mode, check) { + const root = realpathSync(mkdtempSync(path.join(tmpdir(), 'native-preparation-'))); + let controller; + try { + for (const directory of ['scripts', 'tools', 'packages/paperclip-runner', 'logs']) { + mkdirSync(path.join(root, directory), { recursive: true }); + } + for (const helper of ['prefetch-child.mjs', 'prefetch-supervisor.mjs']) { + copyFileSync(path.join(repoRoot, 'scripts', helper), path.join(root, 'scripts', helper)); + } + writeFileSync(path.join(root, 'prepare.mjs'), nativePreparationScript()); + writeFileSync(path.join(root, 'tools/pnpm'), `#!${process.execPath} +const { spawn } = require('node:child_process'); +const { existsSync, writeFileSync } = require('node:fs'); +const { join } = require('node:path'); +const root = process.env.FIXTURE_ROOT; +const name = process.argv.includes('build:binary') ? 'native' : 'typescript'; +writeFileSync(join(root, name + '.started'), JSON.stringify({ pid: process.pid, cwd: process.cwd(), args: process.argv.slice(2) })); +const wait = setInterval(() => { + if (!existsSync(join(root, 'native.started')) || !existsSync(join(root, 'typescript.started'))) return; + clearInterval(wait); + if (process.env.FIXTURE_MODE === 'success') { + setTimeout(() => { writeFileSync(join(root, name + '.finished'), 'joined'); console.log(name + ' complete'); }, 150); + return; + } + const child = spawn(process.execPath, ['-e', \ + "const fs=require('node:fs'); fs.writeFileSync(process.argv[1], String(process.pid)); setInterval(() => fs.writeFileSync(process.argv[2], String(Date.now())), 25)",\ + join(root, name + '.descendant'), join(root, name + '.writing')], { stdio: 'inherit' }); + child.on('exit', () => process.exit(0)); + const barrier = setInterval(() => { + if (!existsSync(join(root, 'native.descendant')) || !existsSync(join(root, 'typescript.descendant'))) return; + clearInterval(barrier); + if (process.env.FIXTURE_MODE === name) { console.error(name + ' failed'); process.exit(23); } + }, 10); +}, 10); +`, { mode: 0o755 }); + controller = spawn(process.execPath, ['prepare.mjs'], { cwd: root, env: { + ...process.env, PATH: `${path.join(root, 'tools')}:${process.env.PATH}`, RUNNER_TEMP: path.join(root, 'logs'), FIXTURE_ROOT: root, FIXTURE_MODE: mode, + }, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = '', stderr = ''; + controller.stdout.on('data', chunk => { stdout += chunk; }); + controller.stderr.on('data', chunk => { stderr += chunk; }); + const completed = new Promise((resolve, reject) => { + controller.once('error', reject); + controller.once('close', (status, signal) => resolve({ status, signal, stdout, stderr })); + }); + await check({ root, controller, completed }); + } finally { + if (controller?.exitCode === null && controller.signalCode === null) controller.kill('SIGKILL'); + for (const name of ['native', 'typescript']) { + for (const filename of [name + '.started', name + '.descendant']) { + const file = path.join(root, filename); + if (!existsSync(file)) continue; + const data = readFileSync(file, 'utf8'); + const pid = filename.endsWith('.started') ? JSON.parse(data).pid : Number(data); + if (!fixtureProcessGone(pid)) { + try { process.kill(pid, 'SIGKILL'); } catch (error) { if (error.code !== 'ESRCH') throw error; } + } + } + } + rmSync(root, { recursive: true, force: true }); + } +} + +test('native preparation overlaps independent outputs and joins both before recording', async () => { + await withNativePreparationFixture('success', async ({ root, completed }) => { + const result = await completed; + assert.equal(result.status, 0, result.stderr); + for (const name of ['native', 'typescript']) assert.equal(readFileSync(path.join(root, name + '.finished'), 'utf8'), 'joined'); + assert.equal(JSON.parse(readFileSync(path.join(root, 'native.started'), 'utf8')).cwd, path.join(root, 'packages/paperclip-runner')); + assert.deepEqual(JSON.parse(readFileSync(path.join(root, 'native.started'), 'utf8')).args, ['build:binary']); + assert.deepEqual(JSON.parse(readFileSync(path.join(root, 'typescript.started'), 'utf8')).args, ['--filter', '@paperclipai/paperclip-runner', 'build:typescript']); + const results = result.stdout.split('\n').filter(line => line.startsWith('{')).map(line => JSON.parse(line)); + assert.deepEqual(results.map(item => [item.build, item.status, item.exitCode]), [['typescript', 'fulfilled', 0], ['native', 'fulfilled', 0]]); + }); +}); + +for (const name of ['typescript', 'native']) { + test(`${name} preparation failure fails the join and retires both owned process trees`, async () => { + await withNativePreparationFixture(name, async ({ root, completed }) => { + const result = await completed; + assert.equal(result.status, 1, result.stderr); + assert.match(result.stderr, new RegExp(name + ' prefetch failed \\(exit 23\\)')); + const results = result.stdout.split('\n').filter(line => line.startsWith('{')).map(line => JSON.parse(line)); + assert.equal(results.length, 2); + assert.ok(results.every(item => item.status === 'rejected')); + for (const build of ['native', 'typescript']) { + const pid = Number(readFileSync(path.join(root, build + '.descendant'), 'utf8')); + await waitForFixture(() => fixtureProcessGone(pid), build + ' descendant outlived failed join'); + } + }); + }); +} + +test('canceling native preparation retires both supervisors and their descendants', { skip: process.platform === 'win32' }, async () => { + await withNativePreparationFixture('cancel', async ({ root, controller, completed }) => { + await waitForFixture(() => ['native', 'typescript'].every(name => existsSync(path.join(root, name + '.descendant'))), 'both native build trees must start'); + controller.kill('SIGTERM'); + const result = await completed; + assert.equal(result.signal, 'SIGTERM'); + for (const name of ['native', 'typescript']) { + const pid = Number(readFileSync(path.join(root, name + '.descendant'), 'utf8')); + await waitForFixture(() => fixtureProcessGone(pid), name + ' descendant outlived cancellation'); + } + }); +}); + +test("premerge native qualification consumes the same Linux-packed npm bytes without publishing", () => { + const smoke = readWorkflow("release-smoke.yml"); + assert.match(smoke, /smoke_service:\n\s+if: inputs.qualification_source_sha == ''/); + assert.match(smoke, /smoke:\n\s+if: inputs.qualification_source_sha == ''/); + const qualification = smoke.split(" qualification_daemons:\n")[1]; + assert.doesNotMatch(qualification, /secrets\.|npm publish|id-token: write|contents: write|continue-on-error/); + assert.match(qualification, /runner_binaries_only: true/); + assert.match(qualification, /PAPERCLIP_RUNNER_PUBLIC_PACKS_DIR/); + assert.match(qualification, /if: \$\{\{ !cancelled\(\) && steps.pack_identity.outputs.available == 'true' \}\}/); + assert.match(qualification, /name: \$\{\{ inputs.artifact_name \}\}-public-packages/); + assert.match(qualification, /--mac-consumer/); + assert.match(qualification, /--linux-arm64-consumer/); + assert.match(qualification, /fromJSON\(needs.qualification_daemons.outputs.native_consumer_matrix\)/); + assert.match(qualification, /pnpm install --ignore-scripts --no-frozen-lockfile/); +}); + +test("native ARM qualification prepares its scoped distro sandbox before the existing consumer", () => { + const smoke = readWorkflow("release-smoke.yml"); + const native = smoke.split(" qualification_native:\n")[1]; + const preparation = native.split(" - name: Prepare Linux ARM sandbox prerequisites\n")[1].split("\n - ")[0]; + assert.match(preparation, /if: \$\{\{ matrix\.target == 'linux-arm64' \}\}/); + assert.match(preparation, /timeout-minutes: 5/); + assert.match(preparation, /bash \.qualification-tooling\/scripts\/prepare-linux-arm64-qualification\.sh/); + const tooling = native.split(" - name: Checkout immutable ARM qualification tooling\n")[1].split("\n - ")[0]; + assert.match(tooling, /if: \$\{\{ matrix\.target == 'linux-arm64' \}\}/); + assert.match(tooling, /repository: \$\{\{ job\.workflow_repository \}\}/); + assert.match(tooling, /ref: \$\{\{ job\.workflow_sha \}\}/); + assert.match(tooling, /sparse-checkout: scripts\/prepare-linux-arm64-qualification\.sh/); + assert.match(tooling, /sparse-checkout-cone-mode: false/); + assert.match(tooling, /persist-credentials: false/); + assert.doesNotMatch(tooling + preparation, /github\.workflow_sha|github\.sha|ref: (?:master|main)|ref: \$\{\{ inputs\.qualification_source_sha/); + assert.ok(native.indexOf("Checkout immutable ARM qualification tooling") < native.indexOf("Prepare Linux ARM sandbox prerequisites")); + assert.ok(native.indexOf("Prepare Linux ARM sandbox prerequisites") < native.indexOf("Verify host identity and actual clean npm consumer")); + assert.match(native, /path: \|\n\s+\$\{\{ runner\.temp \}\}\/native-public-install\.json\n\s+\$\{\{ runner\.temp \}\}\/paperclip-arm-bwrap-prerequisite\//); + assert.doesNotMatch(smoke.split(" qualification_daemons:\n")[0], /prepare-linux-arm64-qualification/, + "ordinary service and Docker smoke keep their existing setup"); +}); + +test("ARM qualification binds separate product and reusable-workflow revisions even when the product lacks the helper", { skip: process.platform === "win32" }, () => { + const native = readWorkflow("release-smoke.yml").split(" qualification_native:\n")[1]; + const step = native.split(" - name: Prepare Linux ARM sandbox prerequisites\n")[1].split("\n - ")[0]; + const script = step.split(" run: |\n")[1].split("\n").map(line => line.startsWith(" ") ? line.slice(10) : line).join("\n"); + const root = mkdtempSync(path.join(tmpdir(), "arm-separate-tooling-")); + try { + const tools = path.join(root, "tools"); mkdirSync(tools); + const tooling = path.join(root, ".qualification-tooling", "scripts"); mkdirSync(tooling, { recursive: true }); + const helper = 'printf "%s\\n" "$TOOLING_SHA" > "$FIXTURE_PREPARED"\n'; + writeFileSync(path.join(tooling, "prepare-linux-arm64-qualification.sh"), helper); + writeFileSync(path.join(tools, "git"), '#!/bin/bash\nif [[ "$1" == -C ]]; then printf "%s\\n" "$FIXTURE_TOOLING_HEAD"; else printf "%s\\n" "$FIXTURE_PRODUCT_HEAD"; fi\n', { mode: 0o755 }); + const productSha = "a".repeat(40), toolingSha = "b".repeat(40); + for (const [productHead, toolingHead, expectedStatus] of [[productSha, toolingSha, 0], ["c".repeat(40), toolingSha, 1], [productSha, "c".repeat(40), 1], [productSha, "invalid", 1]]) { + const report = path.join(root, "report"); mkdirSync(report); + const prepared = path.join(root, "prepared"); + const result = spawnSync("/bin/bash", ["-c", script], { cwd: root, encoding: "utf8", env: { + PATH: `${tools}:${path.dirname(process.execPath)}:/usr/bin:/bin`, RUNNER_TEMP: report, + SOURCE_SHA: productSha, TOOLING_SHA: toolingSha, TOOLING_REPOSITORY: "paperclipai/paperclip", + TOOLING_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/release-smoke.yml@refs/heads/qualification", + TOOLING_WORKFLOW_PATH: ".github/workflows/release-smoke.yml", + FIXTURE_PRODUCT_HEAD: productHead, FIXTURE_TOOLING_HEAD: toolingHead, FIXTURE_PREPARED: prepared, + GITHUB_REPOSITORY: "different/caller", GITHUB_SHA: "d".repeat(40), + } }); + assert.equal(result.status, expectedStatus, result.stderr); + assert.equal(existsSync(path.join(root, "scripts", "prepare-linux-arm64-qualification.sh")), false); + assert.equal(existsSync(prepared), expectedStatus === 0); + if (expectedStatus === 0) { + assert.equal(readFileSync(prepared, "utf8").trim(), toolingSha); + const identity = JSON.parse(readFileSync(path.join(report, "paperclip-arm-bwrap-prerequisite", "qualification-identity.json"), "utf8")); + assert.equal(identity.productSourceSha, productSha); + assert.equal(identity.toolingSourceSha, toolingSha); + assert.equal(identity.toolingRepository, "paperclipai/paperclip"); + assert.equal(identity.toolingWorkflowPath, ".github/workflows/release-smoke.yml"); + assert.equal(identity.helperSha256, createHash("sha256").update(helper).digest("hex")); + } + rmSync(report, { recursive: true, force: true }); + rmSync(prepared, { force: true }); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("Linux ARM prerequisite setup rejects other hosts before invoking tools and preserves scoped security", { skip: process.platform === "win32" }, () => { + const helper = path.join(repoRoot, "scripts/prepare-linux-arm64-qualification.sh"); + const source = readFileSync(helper, "utf8"); + const root = mkdtempSync(path.join(tmpdir(), "arm-prerequisite-guard-")); + try { + const tools = path.join(root, "tools"); mkdirSync(tools); + const sentinel = path.join(root, "invoked"); + for (const command of ["dpkg", "uname", "mkdir", "mktemp", "sysctl", "sudo"]) { + writeFileSync(path.join(tools, command), '#!/bin/bash\nprintf "%s\\n" "$0" >> "$PREREQUISITE_SENTINEL"\nexit 85\n', { mode: 0o755 }); + } + for (const [actions, runnerOs] of [["", "Linux"], ["false", "Linux"], ["true", "macOS"], ["true", "Windows"]]) { + const result = spawnSync("/bin/bash", [helper], { encoding: "utf8", env: { + PATH: tools, GITHUB_ACTIONS: actions, RUNNER_OS: runnerOs, RUNNER_TEMP: root, PREREQUISITE_SENTINEL: sentinel, + } }); + assert.equal(result.status, 1); + assert.match(result.stderr, /requires a GitHub Actions Linux runner/); + assert.equal(existsSync(sentinel), false, "host guard must precede every external tool"); + assert.equal(existsSync(path.join(root, "paperclip-arm-bwrap-prerequisite")), false); + } + assert.ok(source.indexOf('"${GITHUB_ACTIONS:-}" != true') < source.indexOf(". /etc/os-release")); + assert.match(source, /-o Acquire::AllowInsecureRepositories=false/); + assert.match(source, /download "\$package_name=\$package_version"/); + assert.match(source, /printf '%s %s\\n' "\$package_sha256" "\$package_file" \| sha256sum -c -/); + assert.match(source, /dpkg-deb --fsys-tarfile "\$package_file"/); + assert.match(source, /tar -xO \.\/usr\/bin\/bwrap/); + assert.match(source, /11d39094f044f0cda0febb3ad517b830301da6b2ce929664af09ee9e4dd264f9/); + assert.ok(source.indexOf('test ! -L "$target"') < source.indexOf('sudo install -o root -g root -m 0644 "$stock_profile" "$target"')); + assert.match(source, /timeout 15s sudo "\$parser" -r "\$target"/); + for (const setting of ["userns", "apparmor", "clone"]) assert.ok(source.includes(`test "$after_${setting}" = "$before_${setting}"`)); + const aptCommands = source.replace(/\\\n\s*/g, " ").split("\n").filter(line => /\bapt-get\b/.test(line)); + assert.equal(aptCommands.length, 2, "only signed-index update and download may invoke APT"); + assert.match(aptCommands[0], /\bupdate\b/); + assert.match(aptCommands[1], /\bdownload\b/); + for (const command of aptCommands) assert.doesNotMatch(command, /\b(?:install|upgrade|remove|purge)\b/); + assert.doesNotMatch(source, /apt-get[^\n]* install|\bdpkg(?:-deb)?\s+(?:-i|--install|--unpack)|sysctl\s+(?:-w|--system)|\baa-disable\b|\bsetcap\b|systemctl|--no-sandbox/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("exact-source daemon matrices read inert data and never execute preview runner selection", () => { + const workflow = readWorkflow("release-verify.yml"); + const step = workflow.split(' - name: Preserve the immutable source\'s platform contract')[1].split('\n runner_binaries:')[0]; + const script = step.split(' run: |\n')[1].split('\n').map(line => line.startsWith(' ') ? line.slice(10) : line).join('\n'); + const root = mkdtempSync(path.join(tmpdir(), 'release-target-contract-')); + try { + const environment = { + PATH: `${path.dirname(process.execPath)}:/usr/bin:/bin`, HOME: root, + GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', + }; + const git = args => { + const result = spawnSync('git', args, { cwd: root, env: environment, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return result.stdout.trim(); + }; + const template = path.join(root, 'empty-template'); mkdirSync(template); + git(['init', '-q', '--template=' + template]); + const source = path.join(root, 'packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs'); + const dataFile = path.join(root, 'packages/paperclip-runner/release-runner-targets.json'); + mkdirSync(path.dirname(source), { recursive: true }); + const output = path.join(root, 'outputs'), marker = path.join(root, 'candidate-executed'); + const historical = ['darwin-arm64', 'darwin-x64', 'linux-x64']; + const current = [...historical, 'linux-arm64']; + const contract = targets => JSON.stringify({ schema: 'paperclip.runner.release-targets.v1', targets }); + const malicious = `import { writeFileSync } from 'node:fs';\nwriteFileSync(${JSON.stringify(marker)}, 'executed');\nwriteFileSync(process.env.GITHUB_OUTPUT, 'matrix={"include":[{"target":"linux-x64","host":"runs-on/fleet=paperclip-post-merge-x64/env=public-ci"}]}\\n');\nprocess.exit(0);\n`; + const pin = (module, data) => { + rmSync(dataFile, { force: true }); + writeFileSync(source, module); + if (data !== undefined) writeFileSync(dataFile, data); + git(['add', 'packages']); + git(['-c', 'user.name=Release fixture', '-c', 'user.email=fixture@example.invalid', '-c', 'commit.gpgsign=false', + 'commit', '-qm', 'Fixture', '--allow-empty']); + return git(['rev-parse', 'HEAD']); + }; + const run = sha => { + writeFileSync(output, ''); + const result = spawnSync('bash', ['-euo', 'pipefail', '-c', script], { cwd: root, encoding: 'utf8', env: { + ...environment, SOURCE_SHA: sha, GITHUB_OUTPUT: output, + } }); + assert.equal(existsSync(marker), false, 'Candidate code must never execute while selecting runners'); + return result; + }; + const assertMatrix = targets => { + const data = Object.fromEntries(readFileSync(output, 'utf8').trim().split('\n').map(line => { + const index = line.indexOf('='); return [line.slice(0, index), JSON.parse(line.slice(index + 1))]; + })); + assert.deepEqual(data.matrix.include.map(row => row.target), targets); + assert.deepEqual(data.native_consumers.include.map(row => row.target), targets.filter(target => target !== 'linux-x64')); + const hosts = { 'linux-x64': targets.length === 4 ? 'ubuntu-24.04' : 'ubuntu-latest', + 'linux-arm64': 'ubuntu-24.04-arm', 'darwin-arm64': 'macos-15', 'darwin-x64': 'macos-15-intel' }; + for (const row of data.matrix.include) assert.equal(row.host, hosts[row.target], 'Only workflow-owned public hosts may be selected'); + }; + for (const [targets, module, data] of [ + [historical, `export const RELEASE_RUNNER_TARGETS = ${JSON.stringify(historical)};`, undefined], + [current, `${malicious}export const RELEASE_RUNNER_TARGETS = Object.freeze(${JSON.stringify(current)});`, undefined], + [historical, malicious, contract(historical)], + [current, malicious, contract(current)], + ]) { + const result = run(pin(module, data)); + assert.equal(result.status, 0, result.stderr); + assertMatrix(targets); + } + const sha = pin(malicious, contract(current)); + writeFileSync(dataFile, contract(['runs-on/fleet=paperclip-post-merge-x64/env=public-ci'])); + assert.equal(run(sha).status, 0, 'Read the immutable git blob, not a changed checkout file'); + assertMatrix(current); + assert.equal(run('a'.repeat(40)).status, 1, 'Wrong checked-out source must fail before output'); + assert.equal(readFileSync(output, 'utf8'), ''); + for (const data of [ + contract(['linux-arm64']), contract([...current, 'linux-arm64']), contract([...current.slice(0, 3), 'win32-x64']), + JSON.stringify({ schema: 'paperclip.runner.release-targets.v1', targets: current, host: 'trusted-fleet' }), + JSON.stringify({ schema: 'other', targets: current }), contract('linux-arm64'), 'null', '{invalid', ' '.repeat(65 * 1024), + ]) { + const result = run(pin(`export const RELEASE_RUNNER_TARGETS = ${JSON.stringify(historical)};`, data)); + assert.notEqual(result.status, 0, 'Malformed present data must never fall back to a legacy module'); + assert.equal(readFileSync(output, 'utf8'), ''); + } + for (const module of [malicious, + 'export const RELEASE_RUNNER_TARGETS = getTargets();', + `export const RELEASE_RUNNER_TARGETS = ${JSON.stringify(current)};\nexport const RELEASE_RUNNER_TARGETS = ${JSON.stringify(historical)};`, + 'export const RELEASE_RUNNER_TARGETS = ["linux-arm64"];']) { + const result = run(pin(module, undefined)); + assert.notEqual(result.status, 0, 'Unknown legacy declarations must fail closed'); + assert.equal(readFileSync(output, 'utf8'), ''); + } + pin(malicious, undefined); + symlinkSync('scripts/stage-release-runner-binaries.mjs', dataFile); + git(['add', 'packages']); + git(['-c', 'user.name=Release fixture', '-c', 'user.email=fixture@example.invalid', '-c', 'commit.gpgsign=false', 'commit', '-qm', 'Symlink fixture']); + assert.notEqual(run(git(['rev-parse', 'HEAD'])).status, 0, 'Source contract symlinks must not be followed'); + assert.equal(readFileSync(output, 'utf8'), ''); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + + +test("historical promotion sources keep their packaging path while new sources require daemon assembly", () => { + const workflow = readWorkflow("release.yml"); + const selector = workflow.split(' - name: Read the exact source packaging contract')[1].split('\n release_runner_binaries:')[0]; + const script = selector.split(' run: |\n')[1].split('\n').map(line => line.startsWith(' ') ? line.slice(10) : line).join('\n'); + const root = mkdtempSync(path.join(tmpdir(), 'release-source-contract-')); + try { + const tools = path.join(root, 'tools'); mkdirSync(tools); + writeFileSync(path.join(tools, 'git'), '#!/bin/sh\ncase "$1" in\nrev-parse) printf "%s\\n" "$FIXTURE_HEAD";;\nshow) cat "$FIXTURE_SOURCE";;\n*) exit 3;;\nesac\n', { mode: 0o755 }); + const source = path.join(root, 'release.sh'), output = path.join(root, 'outputs'); + for (const [contents, required] of [['pnpm build\n', false], ['assemble-daemons "$PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR" "$SOURCE_SHA"\n', true]]) { + writeFileSync(source, contents); writeFileSync(output, ''); + const result = spawnSync('bash', ['-euo', 'pipefail', '-c', script], { encoding: 'utf8', env: { + PATH: `${tools}:/usr/bin:/bin`, SOURCE_SHA: 'a'.repeat(40), FIXTURE_HEAD: 'a'.repeat(40), FIXTURE_SOURCE: source, GITHUB_OUTPUT: output, + } }); + assert.equal(result.status, 0, result.stderr); + assert.match(readFileSync(output, 'utf8'), new RegExp(`required=${required}`)); + } + const result = spawnSync('bash', ['-euo', 'pipefail', '-c', script], { encoding: 'utf8', env: { + PATH: `${tools}:/usr/bin:/bin`, SOURCE_SHA: 'a'.repeat(40), FIXTURE_HEAD: 'b'.repeat(40), FIXTURE_SOURCE: source, GITHUB_OUTPUT: output, + } }); + assert.equal(result.status, 1, 'Wrong checkout cannot opt out of mandatory assembly'); + const unreadable = spawnSync('bash', ['-euo', 'pipefail', '-c', script], { encoding: 'utf8', env: { + PATH: `${tools}:/usr/bin:/bin`, SOURCE_SHA: 'a'.repeat(40), FIXTURE_HEAD: 'a'.repeat(40), FIXTURE_SOURCE: path.join(root, 'absent'), GITHUB_OUTPUT: output, + } }); + assert.equal(unreadable.status, 1, 'A failed immutable source read must not masquerade as a legacy source'); + + } finally { rmSync(root, { recursive: true, force: true }); } + for (const job of ['publish_canary','publish_nightly','publish_beta','preview_stable','publish_stable']) { + const block = workflow.split(` ${job}:\n`)[1].split(/\n [a-z_]+:\n/)[0]; + assert.match(block, /needs.release_runner_source.result == 'success'/); + assert.match(block, /needs.release_runner_source.outputs.required == 'false' \|\| needs.release_runner_binaries.result == 'success'/); + assert.match(block, /Download exact-source release daemon receipts\n\s+if: needs.release_runner_source.outputs.required == 'true'/); + } +}); + test("canary startup refreshes an incomplete dependency publication, then onboards once", async () => { diff --git a/scripts/dev-runner-native-binary.mjs b/scripts/dev-runner-native-binary.mjs index ecf9cabede..1df5eab04b 100644 --- a/scripts/dev-runner-native-binary.mjs +++ b/scripts/dev-runner-native-binary.mjs @@ -1,5 +1,6 @@ -import { readdirSync, statSync } from "node:fs"; +import { readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; +import { runnerBinaryTarget } from "../packages/paperclip-runner/src/live/runner-binary.ts"; export function resolveNativeRunnerRequirement({ exitCode, stdout }) { if (exitCode !== 0) { @@ -38,20 +39,38 @@ function newestMtimeMs(target) { return newest; } +export function resolveNativeRunnerDevBinary({ + repoRoot, + platform = process.platform, + architecture = process.arch, +}) { + const executable = platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd"; + const packageRoot = path.join(repoRoot, "packages", "paperclip-runner"); + const target = path.join(packageRoot, "dist", "bin", `${platform}-${architecture}`, executable); + const flat = path.join(packageRoot, "dist", "bin", executable); + const targetStat = statSync(target, { throwIfNoEntry: false }); + const flatStat = statSync(flat, { throwIfNoEntry: false }); + if (flatStat?.isFile() && (!targetStat?.isFile() || flatStat.mtimeMs > targetStat.mtimeMs) + && runnerBinaryTarget(readFileSync(flat)) === `${platform}-${architecture}`) { + return { binary: flat, sourceBuild: true }; + } + return targetStat?.isFile() ? { binary: target, sourceBuild: false } : null; +} + export function paperclipRunnerBinaryNeedsBuild({ repoRoot, nativeRunnerRequired, configuredBinary, platform = process.platform, + architecture = process.arch, }) { if (!nativeRunnerRequired) return false; if (configuredBinary?.trim()) return false; - const executable = platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd"; + const selected = resolveNativeRunnerDevBinary({ repoRoot, platform, architecture }); + if (!selected) return true; + const binaryStat = statSync(selected.binary); const packageRoot = path.join(repoRoot, "packages", "paperclip-runner"); - const stagedBinary = path.join(packageRoot, "dist", "bin", executable); - const binaryStat = statSync(stagedBinary, { throwIfNoEntry: false }); - if (!binaryStat?.isFile()) return true; const runnerRoot = path.join(packageRoot, "runner"); const buildInputs = [ diff --git a/scripts/dev-runner.ts b/scripts/dev-runner.ts index 6bd777f333..407a5a3447 100644 --- a/scripts/dev-runner.ts +++ b/scripts/dev-runner.ts @@ -8,6 +8,7 @@ import { stdin, stdout } from "node:process"; import { createCapturedOutputBuffer, parseJsonResponseWithLimit } from "./dev-runner-output.ts"; import { paperclipRunnerBinaryNeedsBuild, + resolveNativeRunnerDevBinary, resolveNativeRunnerRequirement, } from "./dev-runner-native-binary.mjs"; import { applyDevRunnerOptions } from "./dev-runner-options.ts"; @@ -171,6 +172,7 @@ const env: NodeJS.ProcessEnv = { ...process.env, PAPERCLIP_UI_DEV_MIDDLEWARE: explicitUiDevMiddleware ?? (serveBuiltUiForManagedRuntime ? "false" : "true"), }; +const configuredRunnerBinary = env.PAPERCLIP_RUNNER_BINARY; if (mode === "dev") { env.PAPERCLIP_DEV_SERVER_STATUS_FILE = devServerStatusFilePath; @@ -573,28 +575,32 @@ async function buildPaperclipRunner() { process.exit(typescriptResult.code); } + const nativeRunnerRequired = await getNativeRunnerRequired(); if ( - !paperclipRunnerBinaryNeedsBuild({ + paperclipRunnerBinaryNeedsBuild({ repoRoot, - nativeRunnerRequired: await getNativeRunnerRequired(), - configuredBinary: env.PAPERCLIP_RUNNER_BINARY, + nativeRunnerRequired, + configuredBinary: configuredRunnerBinary, }) ) { - return; + console.log("[paperclip] building paperclip runner native binary..."); + const binaryResult = await runPnpm( + ["--filter", "@paperclipai/paperclip-runner", "build:binary"], + { stdio: "inherit" }, + ); + if (binaryResult.signal) { + exitForSignal(binaryResult.signal); + return; + } + if (binaryResult.code !== 0) { + console.error("[paperclip] paperclip runner native binary build failed"); + process.exit(binaryResult.code); + } } - - console.log("[paperclip] building paperclip runner native binary..."); - const binaryResult = await runPnpm( - ["--filter", "@paperclipai/paperclip-runner", "build:binary"], - { stdio: "inherit" }, - ); - if (binaryResult.signal) { - exitForSignal(binaryResult.signal); - return; - } - if (binaryResult.code !== 0) { - console.error("[paperclip] paperclip runner native binary build failed"); - process.exit(binaryResult.code); + if (!configuredRunnerBinary?.trim()) { + const selected = nativeRunnerRequired ? resolveNativeRunnerDevBinary({ repoRoot }) : null; + if (selected?.sourceBuild) env.PAPERCLIP_RUNNER_BINARY = selected.binary; + else delete env.PAPERCLIP_RUNNER_BINARY; } } diff --git a/scripts/grok-public-install-sandbox.mjs b/scripts/grok-public-install-sandbox.mjs index 8ce7663810..1d6279eb83 100644 --- a/scripts/grok-public-install-sandbox.mjs +++ b/scripts/grok-public-install-sandbox.mjs @@ -1,4 +1,6 @@ import assert from 'node:assert/strict'; +import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, readSync, readdirSync, realpathSync, statSync, writeFileSync } from 'node:fs'; +import { basename, dirname, join } from 'node:path'; // Keep public-package lifecycle code off the verification host. Resolve and // cache the public npm graph without scripts, then execute it offline. @@ -9,6 +11,102 @@ export const GROK_PUBLIC_INSTALL_LIFECYCLE = [ 'npm', 'rebuild', '--offline', '--ignore-scripts=false', '--dangerously-allow-all-scripts', ]; +export function runMacInitialInstallWithDiagnostics(install, { sourceRevision, platform, architecture, cacheDirectory, receiptPath }) { + const started = performance.now(); + try { + return install(); + } catch (error) { + try { + const phases = new Set(['npm', 'load', 'config', 'command', 'idealTree', 'reify', 'build', 'extract', 'unpack', 'fetch', 'audit', 'cache', 'arborist', 'metavuln']); + const operations = new Set(['load', 'install', 'init', 'userRequests', 'buildDeps', 'inflate', 'fixDepFlags', 'diffTrees', 'loadTrees', 'retireShallow', 'markDeleted', 'loadBundles', 'unpack', 'unretire', 'build', 'trash', 'save', 'audit', 'manifest', 'failed']); + const safePhase = value => { + const [phase, detail] = value.split(':'); + return phases.has(phase) ? phase + (operations.has(detail) ? ':' + detail : '') : null; + }; + const tails = []; + try { + const logs = join(cacheDirectory, '_logs'); + assert.equal(realpathSync(logs), logs); + for (const name of readdirSync(logs).filter(name => /^[0-9TZ_.:-]+-(?:debug-\d+\.log|timing\.json)$/.test(name)).sort().slice(-2)) { + const file = join(logs, name); + if (!lstatSync(file).isFile()) continue; + const fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = fstatSync(fd); + assert.ok(stat.isFile()); + const size = stat.size; + const bytes = Buffer.alloc(Math.min(size, 16 * 1024)); + const length = readSync(fd, bytes, 0, bytes.length, Math.max(0, size - bytes.length)); + const text = bytes.subarray(0, length).toString('utf8'); + const entries = []; + if (name.endsWith('-timing.json')) { + if (size > bytes.length) continue; + const timing = JSON.parse(text); + for (const [key, duration] of Object.entries(timing.timers ?? {})) { + const phase = safePhase(key); + if (phase && Number.isFinite(duration) && duration >= 0) entries.push({ phase, durationMilliseconds: duration }); + } + } else { + for (const line of text.split('\n')) { + const match = line.match(/^\d+\s+(timing|silly|verbose|http)\s+(\S+)(?:\s+(\S+))?/); + if (!match) continue; + const phase = safePhase(match[2]); + if (!phase) continue; + const entry = { level: match[1], phase }; + if (operations.has(match[3])) entry.operation = match[3]; + const duration = line.match(/^\d+\s+timing\s+\S+\s+Completed in (\d{1,10})ms\s*$/); + if (duration) entry.durationMilliseconds = Number(duration[1]); + if (match[1] === 'http') { + const response = line.match(/^\d+\s+http\s+fetch\s+(GET|POST|PUT|DELETE|HEAD)\s+(\d{3})\s+/); + if (response) { + entry.method = response[1]; entry.status = Number(response[2]); + const fetched = line.match(/\s(\d{1,10})ms(?:\s+\(cache (?:hit|miss|revalidated|updated)\))?\s*$/); + if (fetched) entry.durationMilliseconds = Number(fetched[1]); + } + } + entries.push(entry); + } + } + tails.push({ kind: name.endsWith('-timing.json') ? 'npm-timing' : 'npm-debug', truncated: size > bytes.length || entries.length > 32, entries: entries.slice(-32) }); + } finally { closeSync(fd); } + } + } catch {} + const codes = new Set(['ETIMEDOUT', 'ENOENT', 'EACCES', 'EPERM', 'ENOMEM', 'ENOBUFS', 'EIO', 'EPIPE', 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER']); + const signals = new Set(['SIGTERM', 'SIGKILL', 'SIGABRT', 'SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGSEGV', 'SIGBUS']); + writeFileSync(receiptPath, JSON.stringify({ schema: 'paperclip.runner.mac-public-install-failure.v1', + sourceRevision, platform, architecture, phase: 'initial-npm-install', outcome: 'FAILED', qualified: false, + runtimeNotAttempted: true, providerCalls: 0, elapsedMilliseconds: Math.round(performance.now() - started), timeoutMilliseconds: 600000, + error: { code: codes.has(error?.code) ? error.code : null, status: Number.isInteger(error?.status) ? error.status : null, + signal: signals.has(error?.signal) ? error.signal : null, errno: Number.isInteger(error?.errno) ? error.errno : null }, + npmDiagnosticTail: tails }, null, 2) + '\n'); + } catch {} + throw error; + } +} + +export function macNodeHeadersDirectory(nodeExecutable, nodeVersion) { + assert.match(nodeVersion, /^\d+\.\d+\.\d+$/, 'Executing Node must have an exact release version'); + const executable = realpathSync(nodeExecutable); + assert.equal(basename(executable), 'node', 'Node SDK must belong to the executing Node binary'); + assert.ok(statSync(executable).isFile(), 'Executing Node must be a regular file'); + const prefix = realpathSync(join(dirname(executable), '..')); + const include = join(prefix, 'include/node'); + for (const name of ['node.h', 'node_version.h']) { + const header = join(include, name); + assert.equal(realpathSync(header), header, 'Node SDK headers must remain inside the executing Node prefix'); + assert.ok(statSync(header).isFile(), 'Node SDK headers must be regular files'); + } + assert.match(readFileSync(join(include, 'node.h'), 'utf8'), /#\s*include\s+"node_version\.h"/, 'Node SDK must include its version header'); + const versionHeader = readFileSync(join(include, 'node_version.h'), 'utf8'); + const versions = ['MAJOR', 'MINOR', 'PATCH'].map(part => { + const matches = [...versionHeader.matchAll(new RegExp('^#\\s*define\\s+NODE_' + part + '_VERSION\\s+(\\d+)\\s*$', 'gm'))]; + assert.equal(matches.length, 1, 'Node SDK must declare one ' + part.toLowerCase() + ' version'); + return matches[0][1]; + }); + assert.equal(versions.join('.'), nodeVersion, 'Node SDK version must match the executing Node'); + return prefix; +} + // Inspect the actual npm archive, independently of the staging helper. export function assertNoBundledCodexPayloads(entries) { assert.ok(Array.isArray(entries) && entries.length > 0, 'Packed package must have tar members'); @@ -60,10 +158,11 @@ export function assertInstalledCodexPlatformProvenance(lock, installed, target) // Execute this in the existing offline public consumer, never in the checkout. // The public index and returned command must both come from the installed graph. -export function installedCodexProbeSource(indexPath, consumerRoot, expectedVersion) { +export function installedCodexProbeSource(indexPath, consumerRoot, expectedVersion, { nativeCodex = false } = {}) { return ` import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; + import { createHash } from 'node:crypto'; import { createRequire } from 'node:module'; import { accessSync, constants, existsSync, lstatSync, readFileSync, readdirSync, realpathSync, statSync } from 'node:fs'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; @@ -118,11 +217,11 @@ export function installedCodexProbeSource(indexPath, consumerRoot, expectedVersi assert.ok(selectedPlatform, 'Codex wrapper must resolve an official host platform package from its installed graph'); const selectedMetadata = JSON.parse(readFileSync(selectedPlatformManifest, 'utf8')); assert.deepEqual(selectedMetadata, selectedPlatform.manifest, 'Selected Codex platform metadata must match its scanned provenance'); - assert.equal(selectedMetadata.version, ${JSON.stringify(expectedVersion)} + '-' + process.platform + '-' + process.arch, - 'Selected Codex platform version must match the qualified pin'); + ${nativeCodex ? '' : `assert.equal(selectedMetadata.version, ${JSON.stringify(expectedVersion)} + '-' + process.platform + '-' + process.arch, + 'Selected Codex platform version must match the qualified pin');`} const version = execFileSync(command, ['--version'], { timeout: 30_000, maxBuffer: 128 * 1024, encoding: 'utf8', - env: { PATH: '/usr/local/bin:/usr/bin:/bin', HOME: '/tmp', NODE_PATH: '' } }).trim(); - assert.equal(version, 'codex-cli ' + ${JSON.stringify(expectedVersion)}, 'Installed Codex version must match the qualified pin'); + env: { PATH: process.env.PATH, HOME: process.env.HOME, NODE_PATH: '' } }).trim(); + ${nativeCodex ? "assert.match(version, /^codex-cli /, 'Installed official Codex command must execute');" : `assert.equal(version, 'codex-cli ' + ${JSON.stringify(expectedVersion)}, 'Installed Codex version must match the qualified pin');`} const vendor = dirname(index); const integrityPath = join(vendor, 'drivers/acpx/installation-integrity.js'); const profilesPath = join(vendor, 'drivers/acpx/qualified-profiles.js'); @@ -132,26 +231,112 @@ export function installedCodexProbeSource(indexPath, consumerRoot, expectedVersi const platformRoot = realpathSync(dirname(selectedPlatformManifest)); const nativeVendor = realpathSync(join(platformRoot, 'vendor')); assert.ok(contained(nativeVendor) && nativeVendor.startsWith(platformRoot + sep), 'Codex vendor resources must remain inside their installed platform package'); + const triple = { 'linux-x64': 'x86_64-unknown-linux-musl', 'linux-arm64': 'aarch64-unknown-linux-musl', + 'darwin-x64': 'x86_64-apple-darwin', 'darwin-arm64': 'aarch64-apple-darwin', + 'win32-x64': 'x86_64-pc-windows-msvc', 'win32-arm64': 'aarch64-pc-windows-msvc' }[process.platform + '-' + process.arch]; + assert.ok(triple, 'Codex native payload requires a known host target'); + const nativeName = process.platform === 'win32' ? 'codex.exe' : 'codex'; + const nativeCandidates = ['bin', 'codex'].map(directory => join(nativeVendor, triple, directory, nativeName)).filter(existsSync); + assert.equal(nativeCandidates.length, 1, 'Selected official Codex platform must contain one native executable'); + const nativeExecutable = realpathSync(nativeCandidates[0]); + const nativeStat = lstatSync(nativeCandidates[0]); + assert.ok(nativeExecutable.startsWith(nativeVendor + sep) && nativeStat.isFile() && !nativeStat.isSymbolicLink() + && nativeStat.size >= 20 && nativeStat.size <= 512 * 1024 * 1024, 'Codex native executable must be a bounded regular file in its selected platform package'); + accessSync(nativeExecutable, constants.X_OK); + const nativeBytes = readFileSync(nativeExecutable); + const { runnerBinaryTarget } = await import(pathToFileURL(join(vendor, 'live/runner-binary.js')).href); + assert.equal(runnerBinaryTarget(nativeBytes), process.platform + '-' + process.arch, 'Codex native executable must match the actual host architecture'); const sandboxSource = { HOME: '/private-provider-home', PATH: '/usr/local/bin:/usr/bin:/bin' }; const readRoots = codexExecutableReadOnlyRoots(sandboxSource, command); assert.ok(readRoots.includes(nativeVendor), 'Selected native Codex vendor directory must be readable inside its sandbox'); assert.ok(!readRoots.includes(platformRoot) && !readRoots.includes(root) && !readRoots.includes(sandboxSource.HOME), 'Codex sandbox must not grant enclosing npm or credential-home access'); const sandboxArgs = createIsolatedCodexAppServerArgs(sandboxSource, readRoots).join('\\n'); assert.ok(sandboxArgs.includes(JSON.stringify(nativeVendor) + '="read"'), 'Codex sandbox configuration must retain the selected native executable resources'); - const { createAcpxPackageJsonResolver, verifyQualifiedAcpxInstallation } = await import(pathToFileURL(integrityPath).href); + ${nativeCodex ? '' : `const { createAcpxPackageJsonResolver, verifyQualifiedAcpxInstallation } = await import(pathToFileURL(integrityPath).href); const { resolveQualifiedAcpxProfile } = await import(pathToFileURL(profilesPath).href); const serverRoot = resolve(vendor, '../../..'); assert.ok(contained(serverRoot), 'Codex package authority must be the installed server'); const installation = await verifyQualifiedAcpxInstallation(resolveQualifiedAcpxProfile('codex', 'gpt-5.4'), createAcpxPackageJsonResolver(serverRoot, join(serverRoot, 'package.json'))); const lease = await installation.openCommand(); - await lease.close(); - console.log(JSON.stringify({ pinnedCodexCommandVerified: true, pinnedCodexVersion: ${JSON.stringify(expectedVersion)}, - ...provenance, codexSelectedPlatformPath: selectedPlatform.path, codexSelectedPlatformVersion: selectedMetadata.version, codexNativeSandboxResourcesVerified: true, codexQualifiedInstallationVerified: true, codexCommandLeaseVerified: true, providerCalls: 0 })); + await lease.close();`} + console.log(JSON.stringify({ pinnedCodexCommandVerified: true, installedCodexVersion: version, + codexCommandSha256: createHash('sha256').update(readFileSync(command)).digest('hex'), + codexNativeExecutableSha256: createHash('sha256').update(nativeBytes).digest('hex'), + codexNativeExecutablePath: relative(root, nativeExecutable).split(sep).join('/'), + ...provenance, codexSelectedPlatformPath: selectedPlatform.path, codexSelectedPlatformVersion: selectedMetadata.version, + codexNativeSandboxResourcesVerified: true, + ${nativeCodex ? '' : `pinnedCodexVersion: ${JSON.stringify(expectedVersion)}, codexQualifiedInstallationVerified: true, codexCommandLeaseVerified: true,`} + providerCalls: 0 })); `; } -export function grokConsumerDockerArgs({ assets, consumer, cache, command, uid, gid, download = false, prerequisite, temporarySizeMiB = 256, temporaryExecutable = false }) { +export function assertInstalledNativeConsumerContinuity(before, after) { + for (const field of ['lockSha256', 'runnerSha256', 'codexCommandSha256', 'codexNativeExecutableSha256']) { + if (!/^[a-f0-9]{64}$/.test(before[field]) || before[field] !== after[field]) { + throw new Error(`Installed native host startup changed ${field}`); + } + } +} + +export function installedDaemonProbeSource(outputRoot, consumerRoot, sourceRevision) { + return ` + import assert from 'node:assert/strict'; + import { createHash } from 'node:crypto'; + import { execFileSync } from 'node:child_process'; + import { readFileSync, realpathSync } from 'node:fs'; + import { isAbsolute, join, relative, sep } from 'node:path'; + import { pathToFileURL } from 'node:url'; + const root = realpathSync(${JSON.stringify(consumerRoot)}), output = realpathSync(${JSON.stringify(outputRoot)}); + const inside = path => { const value = relative(root, realpathSync(path)); return value !== '' && value !== '..' && !value.startsWith('..' + sep) && !isAbsolute(value); }; + assert.ok(inside(output)); + const { defaultCapabilityRunnerdBinary } = await import(pathToFileURL(join(output, 'live/runnerd-codex-transport.js')).href); + const { runnerBinaryTarget } = await import(pathToFileURL(join(output, 'live/runner-binary.js')).href); + const { parsePaperclipRunnerdBuildMetadata } = await import(pathToFileURL(join(output, 'evals/runnerd-artifact.js')).href); + const { PAPERCLIP_RUNNER_BUILD_METADATA: expected } = await import(pathToFileURL(join(output, 'evals/build-metadata.js')).href); + const manifest = JSON.parse(readFileSync(join(output, 'bin/release-manifest.json'), 'utf8')); + const targets = ['darwin-arm64', 'darwin-x64', 'linux-x64', 'linux-arm64']; + assert.equal(manifest.schema, 'paperclip.runner.release-binaries.v1'); + assert.equal(manifest.sourceRevision, ${JSON.stringify(sourceRevision)}); + assert.deepEqual(Object.keys(manifest.platforms).sort(), targets.sort()); + for (const target of targets) { + const record = manifest.platforms[target]; + assert.equal(record.path, target + '/paperclip-runnerd'); + const file = join(output, 'bin', record.path); assert.ok(inside(file)); + const bytes = readFileSync(file); + assert.equal(runnerBinaryTarget(bytes), target); + assert.equal('sha256:' + createHash('sha256').update(bytes).digest('hex'), record.sha256); + } + const target = process.platform + '-' + process.arch, selected = defaultCapabilityRunnerdBinary(); + assert.ok(inside(selected)); + assert.equal(realpathSync(selected), realpathSync(join(output, 'bin', manifest.platforms[target].path)), 'Production must select the packaged host daemon'); + const metadata = parsePaperclipRunnerdBuildMetadata(JSON.parse(execFileSync(selected, ['--build-metadata'], { + timeout: 15000, maxBuffer: 64 * 1024, encoding: 'utf8', env: { PATH: process.env.PATH, HOME: process.env.HOME, NODE_PATH: '' }, + }))); + assert.equal(metadata.binaryContractVersion, expected.contracts.runnerdArtifact); + assert.equal(metadata.nativeExecutionVersion, expected.contracts.nativeExecution); + assert.equal(metadata.harnessDriverVersion, expected.contracts.harnessDriver); + assert.equal(metadata.prp.name, expected.prp.name); + assert.ok(metadata.prp.minimumVersion <= expected.prp.maximumVersion && metadata.prp.maximumVersion >= expected.prp.minimumVersion); + console.log(JSON.stringify({ installedRunnerDaemonVerified: true, target, sourceRevision: manifest.sourceRevision, + sha256: manifest.platforms[target].sha256, buildMetadata: metadata, providerCalls: 0 })); + `; +} + +export function macPublicInstallLifecycleProfile(ownedRoot) { + if (!ownedRoot.startsWith('/') || ownedRoot.includes('\0')) throw new Error('Mac lifecycle requires an absolute owned root'); + return `(version 1)\n(allow default)\n(deny network*)\n(deny file-write* (require-all (require-not (subpath ${JSON.stringify(ownedRoot)})) (require-not (literal "/dev/null"))))\n`; +} + +export function macInstalledStartupProbeSource(consumerRoot, serverRoot, runnerDigest, helperPath) { + if (!/^sha256:[a-f0-9]{64}$/.test(runnerDigest)) throw new Error('Installed Mac startup requires the verified SHA-256 manifest identity'); + return `import { probeInstalledCodexStartup } from ${JSON.stringify(helperPath)}; + console.log(JSON.stringify(await probeInstalledCodexStartup({consumerRoot:${JSON.stringify(consumerRoot)}, + serverRoot:${JSON.stringify(serverRoot)},expectedRunnerSha256:${JSON.stringify(runnerDigest.slice(7))}})));`; +} + +export function grokConsumerDockerArgs({ assets, consumer, cache, command, uid, gid, download = false, prerequisite, temporarySizeMiB = 256, temporaryExecutable = false, platform = 'linux/amd64' }) { + if (!['linux/amd64', 'linux/arm64'].includes(platform)) throw new Error('Public-install verification requires a supported native Linux platform'); if (!Number.isSafeInteger(uid) || uid <= 0 || !Number.isSafeInteger(gid) || gid <= 0) { throw new Error('Public-install verification requires an unprivileged host user'); } @@ -162,7 +347,7 @@ export function grokConsumerDockerArgs({ assets, consumer, cache, command, uid, throw new Error('Executable runtime snapshots require an offline verification sandbox'); } return [ - 'run', '--rm', '--platform', 'linux/amd64', + 'run', '--rm', '--platform', platform, '--user', `${uid}:${gid}`, '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--pids-limit', '256', '--memory', '3g', diff --git a/scripts/pi-public-install-probe.mjs b/scripts/pi-public-install-probe.mjs index 663754fcbc..22f547d64a 100644 --- a/scripts/pi-public-install-probe.mjs +++ b/scripts/pi-public-install-probe.mjs @@ -2,10 +2,42 @@ // Run in a credential-free public npm consumer after `paperclipai runtime setup pi`. // Use the server's normal packaged daemon resolver; never submit a prompt. import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; import { mkdir, mkdtemp, readFile, realpath, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export async function verifyPiPackagedDaemon(server, daemon, target, runnerBinaryTarget) { + assert.match(target, /^(?:linux|darwin)-(?:x64|arm64)$/); + const output = join(server, 'dist/vendor/paperclip-runner'), manifestPath = join(output, 'bin/release-manifest.json'); + let manifest; + try { manifest = JSON.parse(await readFile(manifestPath, 'utf8')); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + let expectedPath = join(output, 'bin/paperclip-runnerd'); + if (manifest !== undefined) { + assert.ok(manifest && typeof manifest === 'object' && !Array.isArray(manifest), 'Release manifest must be an object'); + assert.equal(await realpath(manifestPath), manifestPath, 'Release manifest must remain inside the installed package'); + assert.equal(manifest.schema, 'paperclip.runner.release-binaries.v1'); + assert.match(manifest.sourceRevision, /^[a-f0-9]{40}$/); + const build = JSON.parse(await readFile(join(server, 'dist/build-info.json'), 'utf8')); + assert.equal(manifest.sourceRevision, build.commit, 'Packaged daemon source must match the installed server build'); + const artifact = manifest.platforms?.[target]; + assert.equal(artifact?.path, `${target}/paperclip-runnerd`, 'Packaged daemon must use the known host target path'); + assert.match(artifact.sha256, /^sha256:[a-f0-9]{64}$/); + expectedPath = join(output, 'bin', artifact.path); + assert.equal(await realpath(daemon), expectedPath, 'Production must select the installed host daemon'); + assert.equal(await realpath(expectedPath), expectedPath, 'Packaged daemon cannot escape through a symlink'); + const bytes = await readFile(expectedPath); + assert.equal(`sha256:${createHash('sha256').update(bytes).digest('hex')}`, artifact.sha256); + assert.equal(runnerBinaryTarget(bytes), target); + } else { + assert.equal(await realpath(daemon), expectedPath); + assert.equal(runnerBinaryTarget(await readFile(expectedPath)), target); + } +} + +async function main() { const [serverDirectory, ...extra] = process.argv.slice(2); assert.ok(serverDirectory && extra.length === 0, 'Usage: pi-public-install-probe.mjs INSTALLED_SERVER_ROOT'); @@ -25,7 +57,8 @@ assert.equal(QUALIFIED_ACPX_PROFILES.pi.commandDigest, 'sha256:e92078bee3c23bec4 assert.equal(Object.hasOwn(QUALIFIED_ACPX_PROFILES.pi, 'qualificationModel'), false); assert.equal(Object.hasOwn(QUALIFIED_ACPX_PROFILES.pi, 'reportedModelId'), false); const daemon = resolvePaperclipRunnerBinary(); -assert.equal(await realpath(daemon), join(server, 'dist/vendor/paperclip-runner/bin/paperclip-runnerd')); +const { runnerBinaryTarget } = await import(pathToFileURL(join(server, 'dist/vendor/paperclip-runner/live/runner-binary.js'))); +await verifyPiPackagedDaemon(server, daemon, `${process.platform}-${process.arch}`, runnerBinaryTarget); const root = await mkdtemp(join(tmpdir(), 'pi-public-install-probe-')); const evidence = []; const started = performance.now(); @@ -69,3 +102,5 @@ try { try { await bundle?.transport.close(); } finally { clearTimeout(watchdog); await rm(root, { recursive: true, force: true }); } } +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main(); diff --git a/scripts/prepare-linux-arm64-qualification.sh b/scripts/prepare-linux-arm64-qualification.sh new file mode 100755 index 0000000000..f1c29f501f --- /dev/null +++ b/scripts/prepare-linux-arm64-qualification.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${GITHUB_ACTIONS:-}" != true || "${RUNNER_OS:-}" != Linux ]]; then + printf 'Linux ARM qualification preparation requires a GitHub Actions Linux runner.\n' >&2 + exit 1 +fi + +. /etc/os-release +test "$ID" = ubuntu +test "$VERSION_ID" = 24.04 +test "$(dpkg --print-architecture)" = arm64 +test "$(uname -m)" = aarch64 +prerequisite_report_dir=${RUNNER_TEMP:?}/paperclip-arm-bwrap-prerequisite +mkdir -p "$prerequisite_report_dir" +prerequisite_download_dir=$(mktemp -d "${RUNNER_TEMP}/paperclip-arm-stock-packages.XXXXXX") +before_userns=$(sysctl -n kernel.apparmor_restrict_unprivileged_userns) +before_apparmor=$(cat /sys/module/apparmor/parameters/enabled) +before_clone=$(sysctl -n kernel.unprivileged_userns_clone) +test "$before_userns" = 1 +test "$before_apparmor" = Y +test "$before_clone" = 1 +printf 'restriction=%s apparmor=%s clone=%s\n' \ + "$before_userns" "$before_apparmor" "$before_clone" | \ + tee "$prerequisite_report_dir/before.txt" + +parser=$(readlink -f "$(command -v apparmor_parser)") +test "$parser" = /usr/sbin/apparmor_parser +test "$(stat -c '%u' "$parser")" = 0 +dpkg-query -W -f='${Package} ${Version} ${Architecture}\n' \ + apparmor libc6 libcap2 libselinux1 | \ + tee "$prerequisite_report_dir/existing-package-versions.txt" +dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libc6)" ge 2.38 +dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libcap2)" ge 1:2.10 +dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libselinux1)" ge 3.1~ + +timeout 120s sudo apt-get \ + -o Acquire::AllowInsecureRepositories=false \ + -o APT::Get::AllowUnauthenticated=false update | \ + tee "$prerequisite_report_dir/apt-update.txt" +apt-cache policy bubblewrap apparmor-profiles | \ + tee "$prerequisite_report_dir/apt-policy.txt" +for package_name in bubblewrap apparmor-profiles; do + package_version=$(apt-cache policy "$package_name" | awk '/Candidate:/ {print $2}') + test -n "$package_version" + test "$package_version" != '(none)' + apt-cache show "$package_name=$package_version" > \ + "$prerequisite_report_dir/$package_name-index.txt" + package_sha256=$(awk '/^SHA256:/ {print $2}' \ + "$prerequisite_report_dir/$package_name-index.txt") + test "${#package_sha256}" = 64 + printf '%s\n' "$package_sha256" | grep -E '^[0-9a-f]{64}$' >/dev/null + ( + cd "$prerequisite_download_dir" + timeout 60s apt-get -o APT::Get::AllowUnauthenticated=false \ + download "$package_name=$package_version" + ) + package_files=("$prerequisite_download_dir"/"$package_name"_*.deb) + test "${#package_files[@]}" = 1 + package_file=${package_files[0]} + test -f "$package_file" + printf '%s %s\n' "$package_sha256" "$package_file" | sha256sum -c - + test "$(dpkg-deb -f "$package_file" Package)" = "$package_name" + test "$(dpkg-deb -f "$package_file" Version)" = "$package_version" + dpkg-deb -f "$package_file" Package Version Architecture | \ + tee "$prerequisite_report_dir/$package_name-downloaded-version.txt" + if test "$package_name" = bubblewrap; then + test "$(dpkg-deb -f "$package_file" Architecture)" = arm64 + dpkg-deb --fsys-tarfile "$package_file" | \ + tar -xO ./usr/bin/bwrap > "$prerequisite_download_dir/bwrap" + else + test "$(dpkg-deb -f "$package_file" Architecture)" = all + dpkg-deb --fsys-tarfile "$package_file" | \ + tar -xO ./usr/share/apparmor/extra-profiles/bwrap-userns-restrict > \ + "$prerequisite_download_dir/bwrap-userns-restrict" + fi +done + +stock_binary=$prerequisite_download_dir/bwrap +stock_profile=$prerequisite_download_dir/bwrap-userns-restrict +printf '11d39094f044f0cda0febb3ad517b830301da6b2ce929664af09ee9e4dd264f9 %s\n' \ + "$stock_profile" | sha256sum -c - +cat "$stock_profile" | tee "$prerequisite_report_dir/stock-profile.txt" +test "$(od -An -tx1 -N4 "$stock_binary" | tr -d ' \n')" = 7f454c46 +readelf -h "$stock_binary" | tee "$prerequisite_report_dir/bwrap-elf-header.txt" +grep -E 'Machine:.*AArch64' "$prerequisite_report_dir/bwrap-elf-header.txt" + +test ! -L /usr/bin/bwrap +if test -e /usr/bin/bwrap; then + test -f /usr/bin/bwrap + cmp "$stock_binary" /usr/bin/bwrap +else + sudo install -o root -g root -m 0755 "$stock_binary" /usr/bin/bwrap +fi +test "$(readlink -f "$(command -v bwrap)")" = /usr/bin/bwrap +test "$(stat -c '%u:%g:%a' /usr/bin/bwrap)" = 0:0:755 +command -v getcap >/dev/null +test -z "$(getcap /usr/bin/bwrap)" +sha256sum "$stock_binary" /usr/bin/bwrap | \ + tee "$prerequisite_report_dir/bwrap-sha256.txt" +timeout 5s /usr/bin/bwrap --version | tee "$prerequisite_report_dir/bwrap-version.txt" +help=$(timeout 5s /usr/bin/bwrap --help) +printf '%s\n' "$help" > "$prerequisite_report_dir/bwrap-help.txt" +printf '%s\n' "$help" | grep -F -- --as-pid-1 >/dev/null +printf '%s\n' "$help" | grep -F -- --perms >/dev/null + +target=/etc/apparmor.d/bwrap-userns-restrict +test ! -L "$target" +test ! -L /etc/apparmor.d/local/bwrap-userns-restrict +test ! -L /etc/apparmor.d/local/unpriv_bwrap +test ! -s /etc/apparmor.d/local/bwrap-userns-restrict +test ! -s /etc/apparmor.d/local/unpriv_bwrap +if test -e "$target"; then + test -f "$target" + cmp "$stock_profile" "$target" +else + sudo install -o root -g root -m 0644 "$stock_profile" "$target" +fi +timeout 15s sudo "$parser" -r "$target" +sha256sum "$stock_profile" "$target" | \ + tee "$prerequisite_report_dir/profile-sha256.txt" +loaded_profiles=$(sudo cat /sys/kernel/security/apparmor/profiles) +for required_profile in 'bwrap (enforce)' 'unpriv_bwrap (enforce)'; do + printf '%s\n' "$loaded_profiles" | grep -Fx "$required_profile" | \ + tee -a "$prerequisite_report_dir/loaded-profiles.txt" +done +after_userns=$(sysctl -n kernel.apparmor_restrict_unprivileged_userns) +after_apparmor=$(cat /sys/module/apparmor/parameters/enabled) +after_clone=$(sysctl -n kernel.unprivileged_userns_clone) +test "$after_userns" = "$before_userns" +test "$after_apparmor" = "$before_apparmor" +test "$after_clone" = "$before_clone" +printf 'restriction=%s apparmor=%s clone=%s\n' \ + "$after_userns" "$after_apparmor" "$after_clone" | \ + tee "$prerequisite_report_dir/after.txt" +timeout 5s /usr/bin/bwrap --die-with-parent --unshare-user --unshare-net \ + --ro-bind / / --cap-drop ALL /bin/true +printf 'stock files materialized without dpkg registration or maintainer scripts; namespace probe passed; no provider processes or turns\n' | \ + tee "$prerequisite_report_dir/namespace-probe.txt" diff --git a/scripts/release.sh b/scripts/release.sh index 7238c4b293..dc9815610b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -284,6 +284,13 @@ release_info "" release_info "==> Step 2/7: Building workspace artifacts..." cd "$REPO_ROOT" pnpm build +if [ -n "${PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR:-}" ]; then + node "$REPO_ROOT/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs" \ + assemble-daemons "$PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR" "$(git rev-parse HEAD)" + cp -R "$REPO_ROOT/packages/paperclip-runner/dist/bin/." "$REPO_ROOT/server/dist/vendor/paperclip-runner/bin/" +elif [ "$dry_run" = false ]; then + release_fail "Public releases require the four hosted runner daemon artifacts; use the supported Release workflow." +fi node "$REPO_ROOT/scripts/build-standalone-public-packages.mjs" bash "$REPO_ROOT/scripts/prepare-server-ui-dist.sh" for pkg_dir in server packages/adapters/claude-local packages/adapters/codex-local; do diff --git a/scripts/verify-grok-npm-install.mjs b/scripts/verify-grok-npm-install.mjs index 524b76b86d..f7af462f8a 100644 --- a/scripts/verify-grok-npm-install.mjs +++ b/scripts/verify-grok-npm-install.mjs @@ -4,17 +4,194 @@ // explicit and separate from npm installation, and is removed in finally. import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; -import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; +import { stripTypeScriptTypes } from 'node:module'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { packConcurrency, packStagedPackages, packTimings } from './pack-staged-packages.mjs'; import { materializePublishManifest } from './prepare-bundled-package.mjs'; -import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource } from './grok-public-install-sandbox.mjs'; +import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertInstalledNativeConsumerContinuity, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource, installedDaemonProbeSource, macNodeHeadersDirectory, macPublicInstallLifecycleProfile, macInstalledStartupProbeSource, runMacInitialInstallWithDiagnostics } from './grok-public-install-sandbox.mjs'; import { startPrefetch, stopPrefetches } from './prefetch-child.mjs'; import { retainRunnerQualificationPackages } from './retain-runner-qualification-packages.mjs'; import { bundledInstallTimings, startBundledInstalls } from './stage-bundled-packages.mjs'; const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +async function verifyMacConsumer(packageDirectory, sourceRevision, receiptPath) { + assert.equal(process.env.GITHUB_ACTIONS, 'true', 'Mac clean consumers run only on disposable hosted CI'); + assert.equal(process.platform, 'darwin'); + assert.ok(['arm64', 'x64'].includes(process.arch)); + assert.match(sourceRevision, /^[a-f0-9]{40}$/); + const packages = realpathSync(packageDirectory); + const manifest = JSON.parse(readFileSync(join(packages, 'manifest.json'), 'utf8')); + assert.equal(manifest.schema, 'paperclip.runner.public-packages/v1'); + assert.equal(manifest.sourceRevision, sourceRevision); + assert.ok(manifest.archives.some(entry => entry.name === '@paperclipai/server')); + assert.ok(manifest.archives.some(entry => entry.name === 'paperclipai')); + assert.equal(new Set(manifest.archives.map(entry => entry.name)).size, manifest.archives.length); + const root = realpathSync(mkdtempSync(join(tmpdir(), 'paperclip-mac-public-install-'))); + try { + const consumer = join(root, 'consumer'), home = join(root, 'home'), tools = join(root, 'tools'); + for (const directory of [consumer, home, tools, join(root, 'tmp'), join(root, 'cache')]) mkdirSync(directory); + const npmCli = realpathSync(execFileSync('/usr/bin/which', ['npm'], { encoding: 'utf8' }).trim()); + symlinkSync(process.execPath, join(tools, 'node')); + const env = { PATH: `${tools}:/usr/bin:/bin:/usr/sbin:/sbin`, HOME: home, TMPDIR: join(root, 'tmp'), + npm_config_cache: join(root, 'cache'), npm_config_audit: 'false', npm_config_fund: 'false' }; + const run = (command, args, cwd = consumer, runEnv = env) => execFileSync(command, args, { cwd, env: runEnv, + timeout: 10 * 60_000, maxBuffer: 32 * 1024 * 1024, encoding: 'utf8' }); + const tarballs = manifest.archives.map(entry => { + assert.match(entry.file, /^[A-Za-z0-9._-]+\.tgz$/); assert.match(entry.sha256, /^[a-f0-9]{64}$/); + const path = join(packages, entry.file); + assert.equal(realpathSync(path), path); + assert.equal(createHash('sha256').update(readFileSync(path)).digest('hex'), entry.sha256); + assertNoBundledCodexPayloads(run('/usr/bin/tar', ['-tzf', path]).trim().split('\n')); + return path; + }); + writeFileSync(join(consumer, 'package.json'), JSON.stringify({ private: true, type: 'module' })); + const sentinelSource = join(root, 'lifecycle-sentinel'); mkdirSync(sentinelSource); + writeFileSync(join(sentinelSource, 'package.json'), JSON.stringify({ name: 'paperclip-verification-lifecycle-sentinel', version: '1.0.0', + scripts: { postinstall: 'node -e "require(\'node:fs\').writeFileSync(\'lifecycle-ran\', \'ok\')"' } })); + run(process.execPath, [npmCli, 'pack', '--ignore-scripts', '--pack-destination', root], sentinelSource); + const sentinelTar = join(root, readdirSync(root).find(name => name.endsWith('.tgz'))); + runMacInitialInstallWithDiagnostics(() => run(process.execPath, [npmCli, 'install', '--ignore-scripts', '--omit=dev', '--timing', ...tarballs, sentinelTar]), + { sourceRevision, platform: process.platform, architecture: process.arch, cacheDirectory: join(root, 'cache'), receiptPath }); + const sentinel = join(consumer, 'node_modules/paperclip-verification-lifecycle-sentinel/lifecycle-ran'); + assert.equal(existsSync(sentinel), false); + const lock = readFileSync(join(consumer, 'package-lock.json'), 'utf8'); + const profile = join(root, 'lifecycle.sb'); writeFileSync(profile, macPublicInstallLifecycleProfile(root)); + const listener = createServer(socket => socket.end()); + await new Promise((resolve, reject) => { listener.once('error', reject); listener.listen(0, '127.0.0.1', resolve); }); + try { + const port = listener.address().port; + const networkProbe = `import assert from 'node:assert/strict'; import net from 'node:net'; + const socket = net.connect({host:'127.0.0.1',port:${port}}); socket.setTimeout(3000,()=>process.exit(2)); + socket.once('connect',()=>{socket.destroy(); ${"process.argv[1] === 'denied' ? process.exit(3) : process.exit(0)"};}); + socket.once('error',error=>{assert.equal(process.argv[1],'denied'); assert.equal(error.code,'EPERM');});`; + run(process.execPath, ['--input-type=module', '-e', networkProbe, 'allowed']); + run('/usr/bin/sandbox-exec', ['-f', profile, process.execPath, '--input-type=module', '-e', networkProbe, 'denied']); + } finally { await new Promise(resolve => listener.close(resolve)); } + const nodeHeaders = macNodeHeadersDirectory(process.execPath, process.versions.node); + run('/usr/bin/sandbox-exec', ['-f', profile, process.execPath, npmCli, 'rebuild', '--offline', '--ignore-scripts=false', '--dangerously-allow-all-scripts', '--foreground-scripts'], + consumer, { ...env, npm_config_nodedir: nodeHeaders }); + assert.equal(readFileSync(sentinel, 'utf8'), 'ok'); + assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), lock); + for (const entry of manifest.archives) { + assert.equal(JSON.parse(readFileSync(join(consumer, 'node_modules', entry.name, 'package.json'))).version, manifest.releaseVersion); + } + const server = join(consumer, 'node_modules/@paperclipai/server'), installed = join(server, 'dist/vendor/paperclip-runner'); + const probe = (name, source) => { const file = join(root, name); writeFileSync(file, source); return JSON.parse(run(process.execPath, [file]).trim()); }; + const daemon = probe('daemon-probe.mjs', installedDaemonProbeSource(installed, consumer, sourceRevision)); + const codex = probe('codex-probe.mjs', installedCodexProbeSource(join(installed, 'index.js'), consumer, undefined, { nativeCodex: true })); + const startupScript = macInstalledStartupProbeSource(consumer, server, daemon.sha256, join(repo, 'tests/runner-e2e/installed-cli.ts')); + const startupFile = join(root, 'startup-probe.mts'); writeFileSync(startupFile, startupScript); + const startup = JSON.parse(run(process.execPath, ['--import', 'tsx', startupFile], join(repo, 'server')).trim()); + const receipt = { schema: 'paperclip.runner.mac-public-install.v1', sourceRevision, platform: process.platform, architecture: process.arch, + cleanNpmInstall: true, packageCount: manifest.archives.length, codexPackedPayloadsOmitted: true, + consumerLockPreserved: true, lifecycleSentinelVerified: true, lifecycleNodeHeadersVerified: true, lifecycleNodeVersion: process.versions.node, + lifecycleNetwork: 'sandbox-exec denied (loopback negative control)', + ...daemon, ...codex, startup, providerCalls: 0 }; + writeFileSync(receiptPath, JSON.stringify(receipt, null, 2) + '\n'); + console.log(JSON.stringify(receipt)); + } finally { rmSync(root, { recursive: true, force: true }); } +} +async function verifyLinuxArmConsumer(packageDirectory, sourceRevision, receiptPath) { + assert.equal(process.env.GITHUB_ACTIONS, 'true', 'Native clean consumers run only on disposable hosted CI'); + assert.equal(process.platform + '-' + process.arch, 'linux-arm64', 'ARM qualification requires a native ARM host'); + assert.equal(process.versions.node.split('.')[0], '24', 'ARM host startup requires the qualified Node24 runtime'); + assert.match(sourceRevision, /^[a-f0-9]{40}$/); + const packages = realpathSync(packageDirectory); + const manifest = JSON.parse(readFileSync(join(packages, 'manifest.json'), 'utf8')); + assert.equal(manifest.schema, 'paperclip.runner.public-packages/v1'); + assert.equal(manifest.sourceRevision, sourceRevision); + assert.ok(manifest.archives.some(entry => entry.name === '@paperclipai/server')); + assert.ok(manifest.archives.some(entry => entry.name === 'paperclipai')); + assert.equal(new Set(manifest.archives.map(entry => entry.name)).size, manifest.archives.length); + const root = realpathSync(mkdtempSync(join(tmpdir(), 'paperclip-arm-public-install-'))); + try { + const assets = join(root, 'assets'), consumer = join(root, 'consumer'), cache = join(root, 'cache'); + for (const directory of [assets, consumer, cache]) mkdirSync(directory); + const run = (command, args, cwd = root) => execFileSync(command, args, { cwd, + env: { PATH: process.env.PATH, HOME: root, TMPDIR: root }, timeout: 10 * 60_000, + maxBuffer: 32 * 1024 * 1024, encoding: 'utf8' }); + const platform = 'linux/arm64'; + run('docker', ['pull', '--platform', platform, GROK_PUBLIC_INSTALL_IMAGE]); + const isolated = (command, options = {}) => run('docker', grokConsumerDockerArgs({ assets, consumer, cache, + uid: process.getuid(), gid: process.getgid(), platform, command, ...options })); + for (const entry of manifest.archives) { + assert.match(entry.file, /^[A-Za-z0-9._-]+\.tgz$/); assert.match(entry.sha256, /^[a-f0-9]{64}$/); + const tarball = join(packages, entry.file); + assert.equal(realpathSync(tarball), tarball); + assert.equal(createHash('sha256').update(readFileSync(tarball)).digest('hex'), entry.sha256); + assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball]).trim().split('\n')); + cpSync(tarball, join(assets, entry.file)); chmodSync(join(assets, entry.file), 0o644); + } + writeFileSync(join(consumer, 'package.json'), JSON.stringify({ private: true, type: 'module' })); + const sentinelSource = join(root, 'lifecycle-sentinel'); mkdirSync(sentinelSource); + writeFileSync(join(sentinelSource, 'package.json'), JSON.stringify({ name: 'paperclip-verification-lifecycle-sentinel', version: '1.0.0', + scripts: { postinstall: 'node -e "require(\'node:fs\').writeFileSync(\'lifecycle-ran\', \'ok\')"' } })); + run('npm', ['pack', '--ignore-scripts', '--pack-destination', assets], sentinelSource); + isolated(['npm', 'install', '--ignore-scripts', '--omit=dev', ...readdirSync(assets).filter(file => file.endsWith('.tgz')).map(file => `/packages/${file}`)], { download: true }); + const sentinel = join(consumer, 'node_modules/paperclip-verification-lifecycle-sentinel/lifecycle-ran'); + assert.equal(existsSync(sentinel), false); + const lock = readFileSync(join(consumer, 'package-lock.json'), 'utf8'); + isolated(GROK_PUBLIC_INSTALL_LIFECYCLE); + assert.equal(readFileSync(sentinel, 'utf8'), 'ok'); + assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), lock); + for (const entry of manifest.archives) { + assert.equal(JSON.parse(readFileSync(join(consumer, 'node_modules', entry.name, 'package.json'))).version, manifest.releaseVersion); + } + const server = '/consumer/node_modules/@paperclipai/server', installed = server + '/dist/vendor/paperclip-runner'; + const probe = (name, source, options = {}) => { writeFileSync(join(assets, name), source); return JSON.parse(isolated(['node', '/packages/' + name], options).trim()); }; + const daemon = probe('daemon-probe.mjs', installedDaemonProbeSource(installed, '/consumer', sourceRevision)); + assert.equal(daemon.target, 'linux-arm64'); + const codex = probe('codex-probe.mjs', installedCodexProbeSource(installed + '/index.js', '/consumer', undefined, { nativeCodex: true })); + cpSync(join(repo, 'tests/runner-e2e/installed-cli.ts'), join(assets, 'installed-cli.ts')); + writeFileSync(join(assets, 'types.js'), stripTypeScriptTypes(readFileSync(join(repo, 'tests/runner-e2e/types.ts'), 'utf8'))); + const home = join(root, 'host-home'), hostTmp = join(root, 'host-tmp'); + for (const directory of [home, hostTmp]) mkdirSync(directory, { mode: 0o700 }); + const hostEnv = { PATH: `${dirname(process.execPath)}:/usr/bin:/bin:/usr/sbin:/sbin`, HOME: home, TMPDIR: hostTmp, LC_ALL: 'C', TZ: 'UTC' }; + const hostProbe = (name, source) => { + const file = join(assets, name); writeFileSync(file, source); + return JSON.parse(execFileSync(process.execPath, [file], { cwd: consumer, env: hostEnv, + timeout: 120_000, maxBuffer: 1024 * 1024, encoding: 'utf8' }).trim()); + }; + const hostServer = join(consumer, 'node_modules/@paperclipai/server'), hostInstalled = join(hostServer, 'dist/vendor/paperclip-runner'); + const hostDaemon = () => hostProbe('host-daemon-probe.mjs', installedDaemonProbeSource(hostInstalled, consumer, sourceRevision)); + const hostCodex = () => hostProbe('host-codex-probe.mjs', installedCodexProbeSource(join(hostInstalled, 'index.js'), consumer, undefined, { nativeCodex: true })); + const digest = value => createHash('sha256').update(value).digest('hex'); + const beforeCodex = hostCodex(); + const before = { lockSha256: digest(lock), runnerSha256: hostDaemon().sha256.slice(7), codexCommandSha256: beforeCodex.codexCommandSha256, codexNativeExecutableSha256: beforeCodex.codexNativeExecutableSha256 }; + assert.equal(before.runnerSha256, daemon.sha256.slice(7), 'Docker and host must select the same packaged daemon bytes'); + assert.equal(before.codexCommandSha256, codex.codexCommandSha256, 'Docker and host must select the same official Codex command bytes'); + assert.equal(before.codexNativeExecutableSha256, codex.codexNativeExecutableSha256, 'Docker and host must select the same official native Codex payload bytes'); + const hostHeader = process.report.getReport().header; + assert.match(hostHeader.glibcVersionRuntime || '', /^\d+\.\d+$/, 'Native host startup must report its actual GNU libc ABI'); + const startup = hostProbe('host-startup-probe.mjs', macInstalledStartupProbeSource(consumer, hostServer, daemon.sha256, join(assets, 'installed-cli.ts'))); + const afterCodex = hostCodex(); + const after = { lockSha256: digest(readFileSync(join(consumer, 'package-lock.json'))), runnerSha256: hostDaemon().sha256.slice(7), codexCommandSha256: afterCodex.codexCommandSha256, codexNativeExecutableSha256: afterCodex.codexNativeExecutableSha256 }; + assertInstalledNativeConsumerContinuity(before, after); + assert.equal(startup.runnerSha256, before.runnerSha256); + assert.equal(startup.codexSha256, before.codexCommandSha256); + const receipt = { schema: 'paperclip.runner.linux-arm64-public-install.v1', sourceRevision, platform: process.platform, architecture: process.arch, + cleanNpmInstall: true, packageCount: manifest.archives.length, codexPackedPayloadsOmitted: true, + consumerImage: GROK_PUBLIC_INSTALL_IMAGE, consumerImagePlatform: platform, + consumerLockPreserved: true, lifecycleSentinelVerified: true, lifecycleNetwork: 'none', + startupEnvironment: 'native host, same installed graph, isolated empty homes', + startupHost: { nodeVersion: process.versions.node, nodeModulesAbi: process.versions.modules, glibcVersionRuntime: hostHeader.glibcVersionRuntime, kernelRelease: hostHeader.osRelease }, + installedIdentityBefore: before, installedIdentityAfter: after, + ...daemon, ...codex, startup, providerCalls: 0 }; + writeFileSync(receiptPath, JSON.stringify(receipt, null, 2) + '\n'); + console.log(JSON.stringify(receipt)); + } finally { rmSync(root, { recursive: true, force: true }); } +} +if (process.argv[2] === '--linux-arm64-consumer') { + assert.equal(process.argv.length, 6, 'Usage: --linux-arm64-consumer PACKAGE_DIRECTORY SOURCE_SHA RECEIPT_PATH'); + await verifyLinuxArmConsumer(process.argv[3], process.argv[4], process.argv[5]); +} else if (process.argv[2] === '--mac-consumer') { + assert.equal(process.argv.length, 6, 'Usage: --mac-consumer PACKAGE_DIRECTORY SOURCE_SHA RECEIPT_PATH'); + await verifyMacConsumer(process.argv[3], process.argv[4], process.argv[5]); +} else { assert.equal(process.platform, 'linux', 'Run this verification on disposable EC2 Linux, not a developer host'); const root = mkdtempSync(join(tmpdir(), 'paperclip-grok-public-install-')); const prerequisite = join(root, 'native/grok'); @@ -146,6 +323,21 @@ try { timing.stage_pack_check_ms = Math.round(performance.now() - checksStarted); timing.stage_pack_ms = Math.round(performance.now() - packStarted); Object.assign(timing, packTimings(packs, concurrency)); + + if (process.env.PAPERCLIP_RUNNER_PUBLIC_PACKS_DIR) { + const output = resolve(process.env.PAPERCLIP_RUNNER_PUBLIC_PACKS_DIR); + assert.equal(JSON.parse(readFileSync(join(repo, 'server/dist/build-info.json'))).commit, sourceRevision); + assert.ok(!existsSync(output), 'Never replace retained public packages'); + mkdirSync(output, { recursive: true }); + const archives = [...needed].map((name, index) => { + const file = basename(tarballs[index]); cpSync(tarballs[index], join(output, file)); + return { name, file, sha256: createHash('sha256').update(readFileSync(tarballs[index])).digest('hex') }; + }); + writeFileSync(join(output, 'manifest.json'), JSON.stringify({ + schema: 'paperclip.runner.public-packages/v1', sourceRevision, releaseVersion, + publicInstallationVerified: false, providerCalls: 0, archives, + }, null, 2) + '\n'); + } const assets = join(root, 'assets'); mkdirSync(assets, { mode: 0o755 }); const consumer = join(root, 'consumer'); mkdirSync(consumer); const cache = join(root, 'cache'); mkdirSync(cache); @@ -259,3 +451,4 @@ try { await stopPrefetches(prefetches); rmSync(root, { recursive: true, force: true }); } +} diff --git a/server/src/__tests__/dev-runner-native-binary.test.ts b/server/src/__tests__/dev-runner-native-binary.test.ts index 705ea8cf76..cb7640ba43 100644 --- a/server/src/__tests__/dev-runner-native-binary.test.ts +++ b/server/src/__tests__/dev-runner-native-binary.test.ts @@ -1,16 +1,22 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { paperclipRunnerBinaryNeedsBuild, + resolveNativeRunnerDevBinary, resolveNativeRunnerRequirement, } from "../../../scripts/dev-runner-native-binary.mjs"; const tempRoots = new Set(); +const initialPlatform = Object.getOwnPropertyDescriptor(process, "platform")!; +const initialArchitecture = Object.getOwnPropertyDescriptor(process, "arch")!; afterEach(() => { + vi.restoreAllMocks(); + Object.defineProperty(process, "platform", initialPlatform); + Object.defineProperty(process, "arch", initialArchitecture); for (const root of tempRoots) { fs.rmSync(root, { recursive: true, force: true }); } @@ -36,10 +42,24 @@ function createRunnerCheckout(): { root: string; source: string; binary: string fs.writeFileSync(path.join(runnerRoot, "Cargo.toml"), "[workspace]\n", "utf8"); fs.writeFileSync(path.join(runnerRoot, "Cargo.lock"), "", "utf8"); fs.writeFileSync(source, "fn main() {}\n", "utf8"); - fs.writeFileSync(binary, "runnerd", "utf8"); + fs.writeFileSync(binary, nativeHeader(process.platform, process.arch)); return { root, source, binary }; } +function nativeHeader(platform: string, architecture: string): Buffer { + const bytes = Buffer.alloc(128); + if (platform === "darwin") { + bytes.writeUInt32LE(0xfeedfacf); + bytes.writeUInt32LE(architecture === "x64" ? 0x01000007 : 0x0100000c, 4); + } else if (platform === "win32") { + bytes.set([0x4d, 0x5a]); bytes.writeUInt32LE(64, 60); + bytes.writeUInt32LE(0x00004550, 64); bytes.writeUInt16LE(architecture === "x64" ? 0x8664 : 0xaa64, 68); + } else { + bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); bytes.writeUInt16LE(architecture === "x64" ? 62 : 183, 18); + } + return bytes; +} + describe("paperclip runner native dev prerequisite", () => { it("uses an explicit status response and fails safe when status is unknown", () => { expect( @@ -110,6 +130,49 @@ describe("paperclip runner native dev prerequisite", () => { ).toBe(false); }); + it.each(["arm64", "x64"])("uses the staged Mac %s target before the flat build output", architecture => { + const checkout = createRunnerCheckout(); + const target = path.join(path.dirname(checkout.binary), `darwin-${architecture}`, "paperclip-runnerd"); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, nativeHeader("darwin", architecture)); + const old = new Date(Date.now() - 2000), current = new Date(Date.now() + 2000); + fs.utimesSync(checkout.binary, old, old); + fs.utimesSync(target, current, current); + const input = { repoRoot: checkout.root, nativeRunnerRequired: true, platform: "darwin", architecture }; + expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(false); + fs.writeFileSync(checkout.binary, nativeHeader("linux", "x64")); + const wrongHostNewer = new Date(Date.now() + 4000); + fs.utimesSync(checkout.binary, wrongHostNewer, wrongHostNewer); + expect(resolveNativeRunnerDevBinary(input)).toEqual({ binary: target, sourceBuild: false }); + expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(false); + fs.rmSync(checkout.binary); + expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(false); + fs.utimesSync(target, old, old); + expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(true); + fs.rmSync(target); + expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(true); + }); + + it.each(["arm64", "x64"])("uses only a newer matching Mac %s source build without changing release selection", architecture => { + const checkout = createRunnerCheckout(); + const target = path.join(path.dirname(checkout.binary), `darwin-${architecture}`, "paperclip-runnerd"); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, nativeHeader("darwin", architecture)); + fs.writeFileSync(checkout.binary, nativeHeader("darwin", architecture)); + const old = new Date(Date.now() - 2000), fresh = new Date(Date.now() + 2000), changed = new Date(Date.now() + 4000); + fs.utimesSync(target, old, old); fs.utimesSync(checkout.binary, fresh, fresh); + const input = { repoRoot: checkout.root, nativeRunnerRequired: true, platform: "darwin", architecture }; + expect(resolveNativeRunnerDevBinary(input)).toEqual({ binary: checkout.binary, sourceBuild: true }); + expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(false); + fs.utimesSync(checkout.source, changed, changed); + expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(true); + fs.writeFileSync(checkout.binary, nativeHeader("darwin", architecture === "x64" ? "arm64" : "x64")); + expect(resolveNativeRunnerDevBinary(input)).toEqual({ binary: target, sourceBuild: false }); + fs.rmSync(target); + expect(resolveNativeRunnerDevBinary(input)).toBeNull(); + expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(true); + }); + it("does not build a workspace binary when an explicit binary is configured", () => { expect( paperclipRunnerBinaryNeedsBuild({ diff --git a/server/src/__tests__/environment-instance-routes.test.ts b/server/src/__tests__/environment-instance-routes.test.ts index 28f24be2fb..51664e01d9 100644 --- a/server/src/__tests__/environment-instance-routes.test.ts +++ b/server/src/__tests__/environment-instance-routes.test.ts @@ -21,6 +21,7 @@ const mockEnvironmentService = vi.hoisted(() => ({ list: vi.fn(), getById: vi.fn(), create: vi.fn(), + update: vi.fn(), })); const mockEnvironmentCustomImageService = vi.hoisted(() => ({ @@ -129,6 +130,26 @@ function createApp(actor: Record) { } describe("environment instance routes", () => { + it.each(["post", "patch"])("rejects client-written sandbox platform evidence on %s", async method => { + mockEnvironmentService.getById.mockResolvedValue(createEnvironment()); + const app = createApp({ type: "board", source: "local_implicit", isInstanceAdmin: true }); + const payload = { ...(method === "post" ? { name: "Fixture", driver: "local", config: {} } : {}), metadata: { paperclipSandboxPlatformProbes: { "company-1": { v: 1 } } } }; + const res = method === "post" ? await request(app).post("/api/companies/company-1/environments").send(payload) + : await request(app).patch("/api/environments/env-1").send(payload); + expect(res.status).toBe(422); + expect(res.body.details?.code ?? res.body.code).toBe("environment_platform_probe_reserved"); + expect(mockEnvironmentService.create).not.toHaveBeenCalled(); + }); + it.each([null, { userNote: "replacement" }])("passes explicit user metadata replacement through without copying an old probe (%j)", async metadata => { + const existing = createEnvironment({ metadata: { userNote: "old", paperclipSandboxPlatformProbes: { "company-1": { v: 1, marker: "stale" } } } }); + mockEnvironmentService.getById.mockResolvedValue(existing); + mockEnvironmentService.update.mockResolvedValue({ ...existing, metadata }); + const app = createApp({ type: "board", source: "local_implicit", isInstanceAdmin: true }); + const res = await request(app).patch("/api/environments/env-1").send({ metadata }); + expect(res.status).toBe(200); + expect(mockEnvironmentService.update).toHaveBeenCalledExactlyOnceWith("env-1", { metadata }, { db: expect.anything() }); + expect(res.body.metadata).toEqual(metadata); + }); beforeEach(() => { mockIssueService.clearExecutionWorkspaceEnvironmentSelection.mockReset(); mockProjectService.clearExecutionWorkspaceEnvironmentSelection.mockReset(); @@ -138,6 +159,7 @@ describe("environment instance routes", () => { mockEnvironmentService.list.mockReset(); mockEnvironmentService.getById.mockReset(); mockEnvironmentService.create.mockReset(); + mockEnvironmentService.update.mockReset(); Object.values(mockEnvironmentCustomImageService).forEach((mock) => mock.mockReset()); mockEnvironmentCustomImageService.getOverview.mockResolvedValue({ activeTemplate: null, diff --git a/server/src/__tests__/environment-probe.test.ts b/server/src/__tests__/environment-probe.test.ts index 9a29883219..71f2c42555 100644 --- a/server/src/__tests__/environment-probe.test.ts +++ b/server/src/__tests__/environment-probe.test.ts @@ -6,12 +6,16 @@ const mockProbePluginSandboxProviderDriver = vi.hoisted(() => vi.fn()); const mockResolvePluginSandboxProviderDriverByKey = vi.hoisted(() => vi.fn()); const mockRuntimeAcquireRunLease = vi.hoisted(() => vi.fn()); const mockRuntimeReleaseRunLease = vi.hoisted(() => vi.fn()); +const mockRuntimeExecute = vi.hoisted(() => vi.fn()); +const mockRecordPlatformProbe = vi.hoisted(() => vi.fn()); const mockEnvironmentRuntimeService = vi.hoisted(() => vi.fn(() => ({ acquireRunLease: mockRuntimeAcquireRunLease, + execute: mockRuntimeExecute, getDriver: vi.fn(() => ({ releaseRunLease: mockRuntimeReleaseRunLease, })), }))); +vi.mock("../services/environments.js", () => ({ environmentService: () => ({ recordSandboxPlatformProbe: mockRecordPlatformProbe }) })); vi.mock("@paperclipai/adapter-utils/ssh", () => ({ ensureSshWorkspaceReady: mockEnsureSshWorkspaceReady, @@ -27,20 +31,120 @@ vi.mock("../services/environment-runtime.js", () => ({ environmentRuntimeService: mockEnvironmentRuntimeService, })); -import { probeEnvironment } from "../services/environment-probe.ts"; +import { probeEnvironment, readSandboxPlatformQualification, SANDBOX_PLATFORM_PROBE_COMMAND } from "../services/environment-probe.ts"; +import * as configRuntime from "../services/environment-config.js"; +import { SANDBOX_PLATFORM_PROBE_METADATA_KEY, resolveActiveEnvironmentCustomImageTemplateForRuntime } from "../services/environment-custom-image-runtime.js"; describe("probeEnvironment", () => { beforeEach(() => { + vi.restoreAllMocks(); + mockRecordPlatformProbe.mockReset().mockResolvedValue(true); + mockRuntimeExecute.mockReset(); mockEnsureSshWorkspaceReady.mockReset(); mockProbePluginEnvironmentDriver.mockReset(); mockProbePluginSandboxProviderDriver.mockReset(); mockResolvePluginSandboxProviderDriverByKey.mockReset(); mockResolvePluginSandboxProviderDriverByKey.mockResolvedValue(null); mockRuntimeAcquireRunLease.mockReset(); - mockRuntimeReleaseRunLease.mockReset(); + mockRuntimeReleaseRunLease.mockReset().mockResolvedValue({ id: "lease", companyId: "company", environmentId: "env", status: "released", cleanupStatus: "success" }); mockEnvironmentRuntimeService.mockClear(); }); + it.each(["2.38", "2.39", "musl"])("records actual immutable-image facts for %s and reads them without another lease", async libc => { + const image = `registry.example.test/runtime@sha256:${"a".repeat(64)}`; + const environment = { id: "env", driver: "sandbox", status: "active", config: { provider: "daytona", image, reuseLease: false }, metadata: null } as any; + vi.spyOn(configRuntime, "resolveEnvironmentDriverConfigForRuntime").mockResolvedValue({ driver: "sandbox", config: environment.config }); + mockRuntimeAcquireRunLease.mockResolvedValue({ lease: { id: "lease", provider: "daytona", metadata: { image, remoteCwd: "/workspace" } } }); + mockRuntimeExecute.mockResolvedValue({ exitCode: 0, stdout: `Linux\naarch64\n${libc === "musl" ? "musl" : `glibc\nglibc ${libc}`}\n` }); + const result = await probeEnvironment({} as any, environment, { companyId: "company", acquireSandboxRuntimeLease: true }); + expect(result.ok).toBe(true); + expect(result.details?.platformQualificationRecorded).toBe(true); + expect(mockRuntimeExecute).toHaveBeenCalledWith(expect.objectContaining({ command: "sh", args: ["-c", SANDBOX_PLATFORM_PROBE_COMMAND], timeoutMs: 10_000, bypassSession: true })); + const proof = mockRecordPlatformProbe.mock.calls[1]![0].proof; + const measured = { platform: "linux", architecture: "arm64", libc: libc === "musl" ? "musl" : "glibc", ...(libc === "musl" ? {} : { glibcVersion: `glibc ${libc}` }) }; + expect(proof.facts).toEqual(measured); + environment.metadata = { [SANDBOX_PLATFORM_PROBE_METADATA_KEY]: { company: proof } }; + expect(await readSandboxPlatformQualification({} as any, environment, "company")).toEqual(measured); + expect(mockRuntimeAcquireRunLease).toHaveBeenCalledTimes(1); + expect(await readSandboxPlatformQualification({} as any, environment, "foreign-company")).toBeNull(); + expect(await readSandboxPlatformQualification({} as any, { ...environment, config: { ...environment.config, image: `registry.example.test/runtime@sha256:${"b".repeat(64)}` } }, "company")).toBeNull(); + vi.mocked(configRuntime.resolveEnvironmentDriverConfigForRuntime).mockImplementation(async (_db, _company, _env, options) => { + options?.onResolvedCustomImageTemplate?.({ id: "changed-template", capturedAt: null, templateKind: "image", bindingField: "image" }); + return { driver: "sandbox", config: environment.config }; + }); + expect(await readSandboxPlatformQualification({} as any, environment, "company")).toBeNull(); + expect(mockRuntimeAcquireRunLease).toHaveBeenCalledTimes(1); + }); + it.each(["measurement", "cleanup", "config-race"])("does not claim qualification after %s failure while retaining connection readiness", async failure => { + const image = `registry.example.test/runtime@sha256:${"a".repeat(64)}`; + const environment = { id: "env", driver: "sandbox", status: "active", config: { provider: "daytona", image }, metadata: null } as any; + vi.spyOn(configRuntime, "resolveEnvironmentDriverConfigForRuntime").mockResolvedValue({ driver: "sandbox", config: environment.config }); + mockRuntimeAcquireRunLease.mockResolvedValue({ lease: { id: "lease", provider: "daytona", metadata: { image } } }); + mockRuntimeExecute.mockResolvedValue({ exitCode: failure === "measurement" ? 1 : 0, stdout: "Linux\nx86_64\nglibc\nglibc 2.39\n" }); + if (failure === "cleanup") mockRuntimeReleaseRunLease.mockRejectedValueOnce(new Error("owned cleanup failed")); + if (failure === "config-race") mockRecordPlatformProbe.mockResolvedValue(false); + const result = await probeEnvironment({} as any, environment, { companyId: "company", acquireSandboxRuntimeLease: true }); + expect(result.ok).toBe(true); + expect(result.details?.platformQualificationRecorded).toBe(false); + expect(mockRecordPlatformProbe.mock.calls[0]![0].proof).toBeNull(); + if (failure !== "config-race") expect(mockRecordPlatformProbe).toHaveBeenCalledTimes(1); + }); + it.each([ + null, + { id: "lease", companyId: "company", environmentId: "env", status: "released", cleanupStatus: "failed" }, + { id: "lease", companyId: "company", environmentId: "env", status: "pending_cleanup", cleanupStatus: "failed" }, + { id: "other-lease", companyId: "company", environmentId: "env", status: "released", cleanupStatus: "success" }, + { id: "lease", companyId: "foreign-company", environmentId: "env", status: "released", cleanupStatus: "success" }, + { id: "lease", companyId: "company", environmentId: "other-environment", status: "released", cleanupStatus: "success" }, + ])("does not record platform evidence without an owned successful cleanup receipt (%j)", async receipt => { + const image = `registry.example.test/runtime@sha256:${"a".repeat(64)}`; + const environment = { id: "env", driver: "sandbox", status: "active", config: { provider: "daytona", image }, metadata: null } as any; + vi.spyOn(configRuntime, "resolveEnvironmentDriverConfigForRuntime").mockResolvedValue({ driver: "sandbox", config: environment.config }); + mockRuntimeAcquireRunLease.mockResolvedValue({ lease: { id: "lease", provider: "daytona", metadata: { image } } }); + mockRuntimeExecute.mockResolvedValue({ exitCode: 0, stdout: "Linux\nx86_64\nglibc\nglibc 2.39\n" }); + mockRuntimeReleaseRunLease.mockResolvedValue(receipt); + const result = await probeEnvironment({} as any, environment, { companyId: "company", acquireSandboxRuntimeLease: true }); + expect(result.ok).toBe(true); + expect(result.details?.platformQualificationRecorded).toBe(false); + expect(mockRecordPlatformProbe).toHaveBeenCalledTimes(1); + expect(mockRecordPlatformProbe.mock.calls[0]![0].proof).toBeNull(); + }); + it("reads an effective custom image without touching its usage timestamp", async () => { + const update = vi.fn(); + const capturedAt = new Date("2026-10-12T00:00:00Z"); + const row = { id: "template", provider: "daytona", templateKind: "image", templateRef: `registry.example.test/runtime@sha256:${"a".repeat(64)}`, capturedAt, metadata: null }; + const db = { select: () => ({ from: () => ({ where: () => ({ orderBy: () => Promise.resolve([row]) }) }) }), update }; + const identity = vi.fn(); + const result = await resolveActiveEnvironmentCustomImageTemplateForRuntime(db as never, { environmentId: "env", baseConfig: { provider: "daytona", reuseLease: false }, runtimeConfig: { provider: "daytona", reuseLease: false }, readOnly: true, onResolvedTemplate: identity }); + expect(result).toMatchObject({ image: row.templateRef }); + expect(identity).toHaveBeenCalledWith({ id: "template", capturedAt: capturedAt.toISOString(), templateKind: "image", bindingField: "image" }); + expect(update).not.toHaveBeenCalled(); + }); + it.each(["provider_template", "unknown"])("does not qualify a named active %s boot binding over a pinned base image", async templateKind => { + const image = `registry.example.test/runtime@sha256:${"a".repeat(64)}`; + const environment = { id: "env", driver: "sandbox", status: "active", config: { provider: "daytona", image }, metadata: null } as any; + vi.spyOn(configRuntime, "resolveEnvironmentDriverConfigForRuntime").mockImplementation(async (_db, _company, _env, options) => { + const bindingField = templateKind === "provider_template" ? "template" : "templateRef"; + options?.onResolvedCustomImageTemplate?.({ id: "active-template", capturedAt: null, templateKind, bindingField }); + return { driver: "sandbox", config: { ...environment.config, [bindingField]: "mutable-template-name" } }; + }); + mockRuntimeAcquireRunLease.mockResolvedValue({ lease: { id: "lease", provider: "daytona", metadata: { image } } }); + const result = await probeEnvironment({} as any, environment, { companyId: "company", acquireSandboxRuntimeLease: true }); + expect(result.ok).toBe(true); + expect(result.details?.platformQualificationRecorded).toBe(false); + expect(mockRuntimeExecute).not.toHaveBeenCalled(); + expect(mockRecordPlatformProbe).toHaveBeenCalledTimes(1); + }); + it("returns false when an atomic probe record loses its config comparison", async () => { + const { environmentService } = await vi.importActual("../services/environments.js"); + const returning = vi.fn().mockResolvedValue([]); + const where = vi.fn(() => ({ returning })); + const db = { update: vi.fn(() => ({ set: vi.fn(() => ({ where })) })) }; + expect(await environmentService(db as never).recordSandboxPlatformProbe({ environmentId: "env", companyId: "company", config: { provider: "daytona" }, proof: { v: 1 } })).toBe(false); + expect(where).toHaveBeenCalledOnce(); + expect(returning).toHaveBeenCalledOnce(); + }); + it("reports local environments as immediately available", async () => { const result = await probeEnvironment({} as any, { id: "env-1", diff --git a/server/src/__tests__/environment-service.test.ts b/server/src/__tests__/environment-service.test.ts index cfa81980bf..50d10441fa 100644 --- a/server/src/__tests__/environment-service.test.ts +++ b/server/src/__tests__/environment-service.test.ts @@ -1,6 +1,8 @@ import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { and, eq } from "drizzle-orm"; +import { and, eq, type SQL } from "drizzle-orm"; +import type { UpdateEnvironment } from "@paperclipai/shared"; +import { SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "../services/environment-custom-image-runtime.js"; import { activityLog, agents, @@ -130,6 +132,63 @@ describeEmbeddedPostgres("environmentService leases", () => { return companyId; } + it("clears user metadata to null when no private platform proof exists", async () => { + const created = await svc.create({ name: "User metadata clear", driver: "sandbox", config: { provider: "daytona" }, metadata: { userNote: "remove" } }); + expect((await svc.update(created.id, { metadata: null }))?.metadata).toBeNull(); + }); + + it.each(["clear", "replace", "unrelated", "status", "config", "driver"] as const)( + "handles %s saves atomically after a concurrent platform probe record", async operation => { + const companyId = await seedCompany(); + const config = { provider: "daytona", image: `registry.example.test/runtime@sha256:${"a".repeat(64)}` }; + const staleProof = { v: 1, marker: "old" }; + const freshProof = { v: 1, marker: "new" }; + const created = await svc.create({ name: "Concurrent metadata", driver: "sandbox", config, + metadata: { userNote: "old", _paperclipManagedArchiveToken: "old-archive-token", + [SANDBOX_PLATFORM_PROBE_METADATA_KEY]: { [companyId]: staleProof } } }); + const patches: Record = { + clear: { metadata: null }, + replace: { metadata: { userNote: "replacement", [SANDBOX_PLATFORM_PROBE_METADATA_KEY]: { [companyId]: staleProof } } }, + unrelated: { description: "unrelated save" }, + status: { status: "active" }, + config: { config: { ...config, image: `registry.example.test/runtime@sha256:${"b".repeat(64)}` } }, + driver: { driver: "ssh", config: { host: "fixture.example.test", username: "fixture" } }, + }; + const delayedWriteDb = { + update(table: typeof environments) { + const prepared = db.update(table); + return { + set(values: Parameters[0]) { + const statement = prepared.set(values); + return { + where(condition: SQL) { + const pending = statement.where(condition); + return { + async returning() { + expect(await svc.recordSandboxPlatformProbe({ + environmentId: created.id, companyId, config, proof: freshProof, + })).toBe(true); + return pending.returning(); + }, + }; + }, + }; + }, + }; + }, + }; + const saved = await svc.update(created.id, patches[operation], { db: delayedWriteDb as never }); + if (operation === "config" || operation === "driver") { + expect(saved?.metadata).not.toHaveProperty(SANDBOX_PLATFORM_PROBE_METADATA_KEY); + } else { + expect(saved?.metadata?.[SANDBOX_PLATFORM_PROBE_METADATA_KEY]).toEqual({ [companyId]: freshProof }); + } + if (operation === "clear") expect(saved?.metadata).toEqual({ [SANDBOX_PLATFORM_PROBE_METADATA_KEY]: { [companyId]: freshProof } }); + else expect(saved?.metadata?.userNote).toBe(operation === "replace" ? "replacement" : "old"); + if (operation !== "unrelated") expect(saved?.metadata).not.toHaveProperty("_paperclipManagedArchiveToken"); + }, + ); + it("acquires and releases a lease for a run", async () => { const { companyId, environmentId, runId } = await seedEnvironment(); diff --git a/server/src/__tests__/remove-test-home.test.ts b/server/src/__tests__/remove-test-home.test.ts index 724f141f7a..3ccd8fcaf1 100644 --- a/server/src/__tests__/remove-test-home.test.ts +++ b/server/src/__tests__/remove-test-home.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { removeTestHome } from "./remove-test-home.js"; @@ -61,6 +61,17 @@ describe("removeTestHome", () => { const home = makeHome(); fs.mkdirSync(path.join(home, "instances", "default"), { recursive: true }); fs.writeFileSync(path.join(home, "instances", "default", "paperclip.json"), "{}\n"); + const error = Object.assign(new Error("unrelated nonempty directory"), { code: "ENOTEMPTY" }); + const removal = vi.spyOn(fs, "rmSync").mockImplementation(() => { throw error; }); + const permissionChange = vi.spyOn(fs, "chmodSync"); + try { + expect(() => removeTestHome(home)).toThrow(error); + expect(removal).toHaveBeenCalledTimes(1); + expect(permissionChange).not.toHaveBeenCalled(); + } finally { + removal.mockRestore(); + permissionChange.mockRestore(); + } removeTestHome(home); expect(fs.existsSync(home)).toBe(false); expect(() => removeTestHome(home)).not.toThrow(); diff --git a/server/src/__tests__/remove-test-home.ts b/server/src/__tests__/remove-test-home.ts index 760c3cc9da..39cbad6087 100644 --- a/server/src/__tests__/remove-test-home.ts +++ b/server/src/__tests__/remove-test-home.ts @@ -19,39 +19,45 @@ export function removeTestHome(home: string): void { try { fs.rmSync(home, options); } catch (error) { - if (!isReadOnlyDirectoryRemovalError(error)) throw error; - makeDirectoriesWritable(home); + const permissionError = isPermissionError(error); + if (!permissionError && (error as NodeJS.ErrnoException | null)?.code !== "ENOTEMPTY") throw error; + const restored = makeDirectoriesWritable(home); + if (!permissionError && !restored) throw error; fs.rmSync(home, options); } } -function isReadOnlyDirectoryRemovalError(error: unknown): boolean { +function isPermissionError(error: unknown): boolean { const code = (error as NodeJS.ErrnoException | null)?.code; - return code === "EACCES" || code === "EPERM" || code === "ENOTEMPTY"; + return code === "EACCES" || code === "EPERM"; } -function makeDirectoriesWritable(directory: string): void { +function makeDirectoriesWritable(directory: string): boolean { let stat: fs.Stats; try { stat = fs.lstatSync(directory); } catch { - return; + return false; } - if (!stat.isDirectory()) return; + if (!stat.isDirectory()) return false; // Owner read, write and search: enough to list and unlink the children. // A symlink is not followed, so nothing outside the home is touched. - try { - fs.chmodSync(directory, stat.mode | 0o700); - } catch { - // A directory that cannot be changed still fails the removal below, loudly. + let restored = false; + if ((stat.mode & 0o700) !== 0o700) { + try { + fs.chmodSync(directory, stat.mode | 0o700); + restored = true; + } catch { + } } let entries: fs.Dirent[]; try { entries = fs.readdirSync(directory, { withFileTypes: true }); } catch { - return; + return restored; } for (const entry of entries) { - if (entry.isDirectory()) makeDirectoriesWritable(path.join(directory, entry.name)); + if (entry.isDirectory() && makeDirectoriesWritable(path.join(directory, entry.name))) restored = true; } + return restored; } diff --git a/server/src/routes/environments.ts b/server/src/routes/environments.ts index 68d51234cb..e0a24a73a1 100644 --- a/server/src/routes/environments.ts +++ b/server/src/routes/environments.ts @@ -57,6 +57,7 @@ import { getConfiguredSecretProvider } from "../secrets/configured-provider.js"; import { assertBoardOrgAccess, getActorInfo } from "./authz.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { environmentService } from "../services/environments.js"; +import { SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "../services/environment-custom-image-runtime.js"; import { environmentRuntimeService } from "../services/environment-runtime.js"; import { executionWorkspaceService } from "../services/execution-workspaces.js"; import { closeWarmNativeSessionsForEnvironment } from "../services/native-runtime/native-session-executor.js"; @@ -311,6 +312,9 @@ async function assertPlatformProvisionedEnvironmentWritable( * Clearing (null/false) is allowed so stale markers can be removed. */ function assertNoClientPlatformProvisionedMarkers(metadata: unknown): void { + if (isPlainRecord(metadata) && Object.prototype.hasOwnProperty.call(metadata, SANDBOX_PLATFORM_PROBE_METADATA_KEY)) { + throw unprocessable("Sandbox platform qualification is recorded only by an explicit environment probe.", { code: "environment_platform_probe_reserved" }); + } if (!isCloudManagedInstance() || !isPlainRecord(metadata)) return; for (const key of PLATFORM_PROVISIONED_MARKER_KEYS) { if (metadata[key] !== undefined && metadata[key] !== null && metadata[key] !== false) { @@ -396,6 +400,10 @@ export function environmentRoutes( envVars?: Record | null; metadata: Record | null; }>(req: Request, environment: T): T { + if (environment.metadata) { + const { [SANDBOX_PLATFORM_PROBE_METADATA_KEY]: _probe, ...metadata } = environment.metadata; + environment = { ...environment, metadata }; + } // Floor: on cloud-managed instances, platform-provisioned rows use the // floored view — credential-shaped config keys are never echoed to any // reader, structural config stays visible, and tenant env vars on the diff --git a/server/src/services/environment-config.ts b/server/src/services/environment-config.ts index d1d1dc184c..002cea6a47 100644 --- a/server/src/services/environment-config.ts +++ b/server/src/services/environment-config.ts @@ -660,6 +660,8 @@ export async function resolveEnvironmentDriverConfigForRuntime( // but must still resolve the active custom image as prepared runtime // configuration and tooling so the test reflects what real agent runs use. applyCustomImageTemplate?: boolean; + readOnlyCustomImageTemplate?: boolean; + onResolvedCustomImageTemplate?: (identity: { id: string; capturedAt: string | null; templateKind: string; bindingField: string }) => void; }, ): Promise { const parsed = parseEnvironmentDriverConfig(environment); @@ -729,6 +731,8 @@ export async function resolveEnvironmentDriverConfigForRuntime( // are excluded when the template's source fingerprint is computed, // so they must be excluded when re-checking it here. secretRefExcludePaths: collectSecretRefPaths(schema), + readOnly: context?.readOnlyCustomImageTemplate, + onResolvedTemplate: context?.onResolvedCustomImageTemplate, }) : runtimeConfig, }; diff --git a/server/src/services/environment-custom-image-runtime.ts b/server/src/services/environment-custom-image-runtime.ts index 996770ef3f..20f596e01e 100644 --- a/server/src/services/environment-custom-image-runtime.ts +++ b/server/src/services/environment-custom-image-runtime.ts @@ -13,6 +13,7 @@ import { readConfigValueAtPath, writeConfigValueAtPath } from "./json-schema-sec type TemplateRow = typeof environmentCustomImageTemplates.$inferSelect; export const ENVIRONMENT_CUSTOM_IMAGE_RUNTIME_CONFIG_BINDING_METADATA_KEY = "runtimeConfigBinding"; +export const SANDBOX_PLATFORM_PROBE_METADATA_KEY = "paperclipSandboxPlatformProbes"; export const ENVIRONMENT_CUSTOM_IMAGE_CONFIG_FINGERPRINT_EXCLUDED_PATHS = [ "timeoutMs", "reuseLease", @@ -557,6 +558,8 @@ export async function resolveActiveEnvironmentCustomImageTemplateForRuntime( runtimeConfig: SandboxEnvironmentConfig; secretRefExcludePaths?: Iterable; now?: Date; + readOnly?: boolean; + onResolvedTemplate?: (identity: { id: string; capturedAt: string | null; templateKind: string; bindingField: string }) => void; }, ): Promise { const row = await db @@ -587,7 +590,9 @@ export async function resolveActiveEnvironmentCustomImageTemplateForRuntime( // match. Runtime-only knobs such as lease reuse, timeouts, and resource hints // are excluded from the fingerprint so those edits do not discard the capture. const now = input.now ?? new Date(); - await db + input.onResolvedTemplate?.({ id: row.id, capturedAt: row.capturedAt?.toISOString() ?? null, + templateKind: row.templateKind ?? "unknown", bindingField: resolveEnvironmentCustomImageRuntimeConfigBinding(row).field }); + if (!input.readOnly) await db .update(environmentCustomImageTemplates) .set({ lastUsedAt: now, updatedAt: now }) .where(eq(environmentCustomImageTemplates.id, active.id)); diff --git a/server/src/services/environment-probe.ts b/server/src/services/environment-probe.ts index ee1bce5d4c..1d6c8ccedc 100644 --- a/server/src/services/environment-probe.ts +++ b/server/src/services/environment-probe.ts @@ -11,6 +11,54 @@ import { isBuiltinSandboxProvider, probeSandboxProvider } from "./sandbox-provid import { probePluginEnvironmentDriver, probePluginSandboxProviderDriver } from "./plugin-environment-driver.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; import { environmentRuntimeService } from "./environment-runtime.js"; +import { environmentService } from "./environments.js"; +import { fingerprintEnvironmentSandboxProviderConfig, SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "./environment-custom-image-runtime.js"; +import type { SandboxEnvironmentConfig } from "@paperclipai/shared"; + +export const SANDBOX_PLATFORM_PROBE_COMMAND = "uname -s; uname -m; if version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$version\" ]; then printf 'glibc\\n%s\\n' \"$version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi"; +type PlatformFacts = { platform: string; architecture: string; libc: "glibc" | "musl" | "unknown"; glibcVersion?: string }; +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function fingerprint(config: Record): string { + return fingerprintEnvironmentSandboxProviderConfig(config as SandboxEnvironmentConfig); +} +function isImmutableImage(value: unknown): value is string { + return typeof value === "string" && /^[^\s@]+@sha256:[a-f0-9]{64}$/.test(value); +} +export function parseSandboxPlatformFacts(output: string): PlatformFacts { + const [os, arch, libc, version] = output.trim().split(/\r?\n/); + return { platform: os === "Linux" ? "linux" : os === "Darwin" ? "darwin" : "unknown", + architecture: arch === "x86_64" ? "x64" : arch === "aarch64" || arch === "arm64" ? "arm64" : "unknown", + libc: libc === "glibc" || libc === "musl" ? libc : "unknown", + ...(libc === "glibc" ? { glibcVersion: version?.trim() ?? "unknown" } : {}) }; +} +async function sandboxBinding(db: Db, environment: Environment, companyId: string) { + const selectedTemplate: { value: { id: string; capturedAt: string | null; templateKind: string; bindingField: string } | null } = { value: null }; + const parsed = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment, { + applyCustomImageTemplate: true, readOnlyCustomImageTemplate: true, + onResolvedCustomImageTemplate: identity => { selectedTemplate.value = identity; }, + }); + if (parsed.driver !== "sandbox") return null; + const image = parsed.config.image; + const template = selectedTemplate.value; + if (!isImmutableImage(image) || ["snapshot", "template", "templateRef"].some(key => record(parsed.config)[key]) + || (template && (template.templateKind !== "image" || template.bindingField !== "image"))) return null; + return { baseConfigFingerprint: fingerprint(environment.config), runtimeConfigFingerprint: fingerprint(parsed.config as unknown as Record), image, template }; +} + +export async function readSandboxPlatformQualification(db: Db, environment: Environment, companyId: string): Promise { + const proof = record(record(environment.metadata?.[SANDBOX_PLATFORM_PROBE_METADATA_KEY])[companyId]); + if (proof.v !== 1 || proof.companyId !== companyId || proof.baseConfigFingerprint !== fingerprint(environment.config)) return null; + const binding = await sandboxBinding(db, environment, companyId); + if (!binding || proof.runtimeConfigFingerprint !== binding.runtimeConfigFingerprint || proof.image !== binding.image + || JSON.stringify(proof.template) !== JSON.stringify(binding.template)) return null; + const facts = record(proof.facts); + if (typeof facts.platform !== "string" || typeof facts.architecture !== "string" + || !["glibc", "musl", "unknown"].includes(String(facts.libc))) return null; + return { platform: facts.platform, architecture: facts.architecture, libc: facts.libc as PlatformFacts["libc"], + ...(typeof facts.glibcVersion === "string" ? { glibcVersion: facts.glibcVersion } : {}) }; +} export async function probeEnvironment( db: Db, @@ -24,6 +72,10 @@ export async function probeEnvironment( } = {}, ): Promise { const resolvedCompanyId = options.companyId ?? null; + const savedSandboxProbe = environment.driver === "sandbox" && options.acquireSandboxRuntimeLease === true && resolvedCompanyId; + const environments = environmentService(db); + if (savedSandboxProbe) await environments.recordSandboxPlatformProbe({ environmentId: environment.id, + companyId: savedSandboxProbe, config: environment.config, proof: null }); const parsed = options.resolvedConfig ?? ( options.acquireSandboxRuntimeLease === true ? parseEnvironmentDriverConfig(environment) @@ -77,6 +129,8 @@ export async function probeEnvironment( }; let leaseRecord: Awaited> | null = null; let releaseStatus: "released" | "failed" = "released"; + let qualification: Record | null = null; + let details: Record | null = null; try { leaseRecord = await runtime.acquireRunLease({ companyId: resolvedCompanyId, @@ -93,19 +147,37 @@ export async function probeEnvironment( const sandboxName = typeof metadata.sandboxName === "string" && metadata.sandboxName.trim().length > 0 ? metadata.sandboxName.trim() : null; + details = { provider, providerLeaseId: leaseRecord.lease.providerLeaseId, leaseId: leaseRecord.lease.id, + leasePolicy: leaseRecord.lease.leasePolicy, metadata, platformQualificationRecorded: false, + platformQualificationDiagnostic: "An immutable OCI image digest is required; this connection probe does not qualify a mutable image or snapshot name." }; + if (isImmutableImage(metadata.image) && !metadata.snapshot) { + try { + const binding = await sandboxBinding(db, environment, resolvedCompanyId); + if (binding && metadata.image === binding.image) { + const platform = await runtime.execute({ environment: probeEnvironmentConfig, lease: leaseRecord.lease, + command: "sh", args: ["-c", SANDBOX_PLATFORM_PROBE_COMMAND], + bypassSession: true, timeoutMs: 10_000, commandTimeoutMs: 10_000 }); + if (platform.exitCode !== 0) throw new Error("platform_probe_failed"); + const facts = parseSandboxPlatformFacts(platform.stdout); + details.platform = facts; + if (facts.platform !== "unknown" && facts.architecture !== "unknown" + && (facts.platform !== "linux" || facts.libc !== "unknown") + && (facts.libc !== "glibc" || /^(?:glibc\s+)?\d+\.\d+(?:\.\d+)?$/.test(facts.glibcVersion ?? ""))) { + qualification = { v: 1, companyId: resolvedCompanyId, ...binding, facts, probedAt: new Date().toISOString() }; + delete details.platformQualificationDiagnostic; + } else details.platformQualificationDiagnostic = "The image's OS, architecture or libc could not be identified. Legacy remains the creation default."; + } else details.platformQualificationDiagnostic = "The live lease does not match the selected immutable image configuration. Probe again."; + } catch { + details.platformQualificationDiagnostic = "Platform qualification could not complete; connection readiness does not establish native compatibility. Check the image and probe again."; + } + } return { ok: true, driver: "sandbox", summary: sandboxName ? `Connected to ${provider} sandbox ${sandboxName}.` : `Connected to ${provider} sandbox environment.`, - details: { - provider, - providerLeaseId: leaseRecord.lease.providerLeaseId, - leaseId: leaseRecord.lease.id, - leasePolicy: leaseRecord.lease.leasePolicy, - metadata, - }, + details, }; } catch (error) { releaseStatus = "failed"; @@ -122,11 +194,25 @@ export async function probeEnvironment( if (leaseRecord) { const driver = runtime.getDriver(environment.driver); try { - await driver?.releaseRunLease({ + const releasedLease = await driver?.releaseRunLease({ environment: probeEnvironmentConfig, lease: leaseRecord.lease, status: releaseStatus, }); + if (qualification && driver && savedSandboxProbe + && releasedLease?.id === leaseRecord.lease.id + && releasedLease.companyId === resolvedCompanyId + && releasedLease.environmentId === environment.id + && releasedLease.status === "released" && releasedLease.cleanupStatus === "success") { + const recorded = await environments.recordSandboxPlatformProbe({ environmentId: environment.id, companyId: savedSandboxProbe, + config: environment.config, proof: qualification }); + if (details) { + details.platformQualificationRecorded = recorded; + if (!recorded) details.platformQualificationDiagnostic = "The environment changed during the probe. Probe its current configuration again."; + } + } else if (qualification && details) { + details.platformQualificationDiagnostic = "The probe lease cleanup could not be verified. Resolve pending cleanup and probe again."; + } } catch (releaseError) { // Cleanup failures must not mask the connection result shown to // the operator, but a leaked sandbox should still be traceable. diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index 4170332cc3..b8922aa61d 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1,5 +1,7 @@ import { randomUUID } from "node:crypto"; -import { and, desc, eq, inArray, isNull, ne, sql } from "drizzle-orm"; + +import { SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "./environment-custom-image-runtime.js"; +import { and, desc, eq, inArray, isNull, ne, sql, type SQL } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, @@ -859,6 +861,21 @@ export function environmentService(db: Db) { }; return { + recordSandboxPlatformProbe: async (input: { + environmentId: string; + companyId: string; + config: Record; + proof: Record | null; + }): Promise => { + const existing = sql`coalesce(${environments.metadata}->${SANDBOX_PLATFORM_PROBE_METADATA_KEY}, '{}'::jsonb)`; + const probes = input.proof === null + ? sql`${existing} - ${input.companyId}` + : sql`${existing} || ${JSON.stringify({ [input.companyId]: input.proof })}::jsonb`; + const rows = await db.update(environments).set({ + metadata: sql`jsonb_set(coalesce(${environments.metadata}, '{}'::jsonb), ARRAY[${SANDBOX_PLATFORM_PROBE_METADATA_KEY}]::text[], ${probes}, true)`, + }).where(and(eq(environments.id, input.environmentId), eq(environments.driver, "sandbox"), eq(environments.config, input.config))).returning({ id: environments.id }); + return rows.length > 0; + }, list: async ( companyIdOrFilters?: string | EnvironmentListFilters, maybeFilters?: EnvironmentListFilters, @@ -1118,20 +1135,28 @@ export function environmentService(db: Db) { if ("envVars" in patch && patch.envVars !== undefined) { values.envVars = (patch.envVars ?? {}) as Record; } + const changesTarget = patch.config !== undefined || patch.driver !== undefined; + let metadataUpdate: SQL | undefined; if (patch.metadata !== undefined) { - values.metadata = withoutManagedEnvironmentArchiveToken(patch.metadata ?? null); - } else if (patch.status !== undefined) { - const existingMetadata = await writeDb - .select({ metadata: environments.metadata }) - .from(environments) - .where(eq(environments.id, id)) - .then((rows) => rows[0]?.metadata ?? null); - values.metadata = withoutManagedEnvironmentArchiveToken(existingMetadata); + const userMetadata = withoutManagedEnvironmentArchiveToken(patch.metadata ?? null); + if (userMetadata) delete userMetadata[SANDBOX_PLATFORM_PROBE_METADATA_KEY]; + if (changesTarget) { + values.metadata = userMetadata; + } else { + const replacement = userMetadata === null ? sql`null::jsonb` : sql`${JSON.stringify(userMetadata)}::jsonb`; + metadataUpdate = sql`case when ${environments.metadata} ? ${SANDBOX_PLATFORM_PROBE_METADATA_KEY} + then coalesce(${replacement}, '{}'::jsonb) || jsonb_build_object( + ${SANDBOX_PLATFORM_PROBE_METADATA_KEY}::text, ${environments.metadata}->${SANDBOX_PLATFORM_PROBE_METADATA_KEY}) + else ${replacement} end`; + } + } else if (changesTarget || patch.status !== undefined) { + metadataUpdate = sql`${environments.metadata} - ${MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_METADATA_KEY}`; + if (changesTarget) metadataUpdate = sql`${metadataUpdate} - ${SANDBOX_PLATFORM_PROBE_METADATA_KEY}`; } const row = await writeDb .update(environments) - .set(values) + .set({ ...values, ...(metadataUpdate ? { metadata: metadataUpdate } : {}) }) .where(eq(environments.id, id)) .returning() .then((rows) => rows[0] ?? null) diff --git a/server/src/services/local-ai-browser-login.test.ts b/server/src/services/local-ai-browser-login.test.ts index 6bfc0d3636..5f02157018 100644 --- a/server/src/services/local-ai-browser-login.test.ts +++ b/server/src/services/local-ai-browser-login.test.ts @@ -2,6 +2,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { mkdtemp, mkdir, readFile, realpath, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { createRequire } from "node:module"; import { startLocalBrowserLogin } from "./local-ai-browser-login.js"; import { resolveCodexCommand } from "../vendor/paperclip-runner/index.js"; @@ -18,6 +20,7 @@ afterEach(async () => { Object.defineProperty(process, "arch", initialArchitecture); process.env.PATH = initialPath; vi.unstubAllEnvs(); + vi.restoreAllMocks(); vi.clearAllMocks(); if (root) await rm(root, { recursive: true, force: true }); root = undefined; @@ -35,14 +38,36 @@ async function fakeCli(name: string, source: string) { return realpath(home); } -function targetPlatform(platform: string, architecture: string) { +function targetPlatform(platform: string, architecture: string, glibc = true, glibcVersionRuntime = "2.39") { Object.defineProperty(process, "platform", { ...initialPlatform, value: platform }); Object.defineProperty(process, "arch", { ...initialArchitecture, value: architecture }); + if (platform === "linux") vi.spyOn(process.report, "getReport").mockReturnValue({ header: glibc ? { glibcVersionRuntime } : {} } as never); +} + +function resolveFixtureCodexCommand(home: string): string { + const source = ` + import { resolveCodexCommand } from ${JSON.stringify(new URL("../../../packages/paperclip-runner/src/drivers/codex/codex-command.ts", import.meta.url).href)}; + try { console.log(JSON.stringify({ command: resolveCodexCommand(${JSON.stringify(path.join(root!, "issuer.js"))}, { PATH: "/missing-codex" }, ${JSON.stringify(home)}) })); } + catch (error) { console.log(JSON.stringify({ error: error.message })); } + `; + const result = JSON.parse(execFileSync(process.execPath, [ + "--no-global-search-paths", "--import", createRequire(import.meta.url).resolve("tsx"), + "--input-type=module", "--eval", source, + ], { cwd: root, env: { PATH: "/usr/bin:/bin", NODE_PATH: "", NODE_OPTIONS: "" }, + encoding: "utf8", timeout: 10000, maxBuffer: 16384 })); + if (typeof result.error === "string") throw new Error(result.error); + if (typeof result.command !== "string") throw new Error("Fixture resolver returned no executable"); + return result.command; } describe.skipIf(process.platform === "win32")("local browser subscription login", () => { - it.each([false, true])("uses the selected Codex command with its credential home and ambient HOME/cwd (legacy Linux ARM64: %s)", async (legacy) => { - if (legacy) targetPlatform("linux", "arm64"); + it.each(["native", "arm32", "musl", "glibc-2.38", "glibc-3.0", "unknown"])("uses the selected Codex command with its credential home and ambient HOME/cwd (%s)", async platform => { + const legacy = platform === "arm32"; + if (platform === "arm32") targetPlatform("linux", "arm"); + if (platform === "musl") targetPlatform("linux", "arm64", false); + if (platform === "glibc-2.38") targetPlatform("linux", "arm64", true, "2.38"); + if (platform === "glibc-3.0") targetPlatform("linux", "arm64", true, "3.0"); + if (platform === "unknown") targetPlatform("linux", "x64", true, "unknown"); const home = await fakeCli("codex", [ '[ "$1 $2" = "login --device-auth" ] || exit 11', '[ "$HOME" = "$CLAUDE_CONFIG_DIR" ] && [ "$HOME" != "$CODEX_HOME" ] && [ "$CODEX_HOME" = "$PAPERCLIP_TEST_LOGIN_CODEX_HOME" ] || exit 12', @@ -60,13 +85,60 @@ describe.skipIf(process.platform === "win32")("local browser subscription login" for (const key of ["OPENAI_API_KEY", "CODEX_API_KEY", "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN"]) vi.stubEnv(key, "unrelated-host-value"); const login = startLocalBrowserLogin("openai", home); - await vi.waitFor(() => expect(login.outcome).toBe("success"), { timeout: 5000 }); + await vi.waitFor(() => expect(login.outcome).toBeDefined(), { timeout: 5000 }); + expect(login.outcome).toBe("success"); if (legacy) expect(resolveCodexCommand).not.toHaveBeenCalled(); else expect(resolveCodexCommand).toHaveBeenCalledOnce(); expect(login.code).toBe("ABCD-EFGHJ"); }); - it.each([["linux", "x64"], ["darwin", "arm64"], ["darwin", "x64"]] as const)("reports an unavailable Codex CLI on %s/%s without revealing private details", async (platform, architecture) => { + it.each(["x64", "arm64"].flatMap(architecture => ["2.38", "musl", "unknown"].map(libc => ({ architecture, libc }))))("uses the installed Codex dependency for login on Linux $architecture/$libc without a global CLI", async ({ architecture, libc }) => { + targetPlatform("linux", architecture, libc !== "musl", libc); + const home = await fakeCli("codex", [ + 'echo owned > "$CODEX_HOME/owned-cli"', + 'printf "1. Open this link in your browser and sign in to your account\\nhttps://auth.openai.com/codex/device\\n2. Enter this one-time code (expires in 15 minutes)\\nABCD-EFGHJ\\n"', + ].join("\n")); + const dependency = path.join(root!, "node_modules/@openai/codex"); + await mkdir(path.join(dependency, "bin"), { recursive: true }); + await writeFile(path.join(dependency, "package.json"), JSON.stringify({ name: "@openai/codex", version: "0.1.0", bin: { codex: "bin/codex" } })); + await writeFile(path.join(dependency, "bin/codex"), await readFile(path.join(root!, "bin/codex")), { mode: 0o700 }); + const command = resolveFixtureCodexCommand(home); + expect(command).toBe(await realpath(path.join(dependency, "bin/codex"))); + vi.mocked(resolveCodexCommand).mockReturnValue(command); + process.env.PATH = "/usr/bin:/bin"; + const login = startLocalBrowserLogin("openai", home); + try { + await vi.waitFor(() => expect(login.outcome).toBeDefined(), { timeout: 5000 }); + expect(login.outcome).toBe("success"); + expect(await readFile(path.join(home, "owned-cli"), "utf8")).toBe("owned\n"); + expect(resolveCodexCommand).toHaveBeenCalledOnce(); + expect(process.report.getReport).not.toHaveBeenCalled(); + } finally { login.abort(); } + }); + + it.each(["2.38", "musl", "unknown"])("reports a real owned-dependency containment failure on Linux/%s without falling back to another CLI", async libc => { + targetPlatform("linux", "arm64", libc !== "musl", libc); + const home = await fakeCli("codex", 'echo invoked > "$CODEX_HOME/unreviewed-cli"'); + const dependency = path.join(root!, "node_modules/@openai/codex"); + await mkdir(dependency, { recursive: true }); + await writeFile(path.join(dependency, "package.json"), JSON.stringify({ name: "@openai/codex", version: "0.1.0", bin: { codex: "../outside-codex" } })); + let resolutionError: unknown; + try { resolveFixtureCodexCommand(home); } catch (error) { resolutionError = error; } + expect(resolutionError).toBeInstanceOf(Error); + expect((resolutionError as Error).message).toContain("executable escapes its package"); + vi.mocked(resolveCodexCommand).mockImplementation(() => { throw resolutionError; }); + const login = startLocalBrowserLogin("openai", home); + try { + await vi.waitFor(() => expect(login.outcome).toBeDefined(), { timeout: 5000 }); + expect(login.outcome).toBe("failure"); + expect(login.error).toMatch(/requires an installed Codex CLI.*Install Codex/); + expect(JSON.stringify(login)).not.toContain(root!); + await expect(readFile(path.join(home, "unreviewed-cli"))).rejects.toMatchObject({ code: "ENOENT" }); + expect(resolveCodexCommand).toHaveBeenCalledOnce(); + } finally { login.abort(); } + }); + + it.each([["linux", "x64"], ["linux", "arm64"], ["darwin", "arm64"], ["darwin", "x64"]] as const)("reports an unavailable Codex CLI on %s/%s without revealing private details", async (platform, architecture) => { targetPlatform(platform, architecture); const home = await fakeCli("codex", 'echo invoked > "$CODEX_HOME/unreviewed-cli"'); const failure = new Error("private-installation-path-or-provider-output"); @@ -79,8 +151,8 @@ describe.skipIf(process.platform === "win32")("local browser subscription login" expect(login.authorizationUrl).toBeUndefined(); }); - it("reports a missing Linux ARM64 Codex CLI without revealing the host PATH", async () => { - targetPlatform("linux", "arm64"); + it("reports a missing Linux ARM32 Codex CLI without revealing the host PATH", async () => { + targetPlatform("linux", "arm"); const home = await fakeCli("codex", "exit 0"); process.env.PATH = "/usr/bin:/bin"; const login = startLocalBrowserLogin("openai", home); diff --git a/server/src/services/local-ai-browser-login.ts b/server/src/services/local-ai-browser-login.ts index ad176b8618..4ae7e26ab1 100644 --- a/server/src/services/local-ai-browser-login.ts +++ b/server/src/services/local-ai-browser-login.ts @@ -56,10 +56,8 @@ export function startLocalBrowserLogin(provider: "anthropic" | "openai", home: s async start(_command: string, onData: (chunk: string) => void): Promise<{ exitCode: number | null }> { let executable = "claude"; if (provider === "openai") { - // Linux ARM64 retains its supported legacy CLI path; it is not a - // qualified native runner target. Other hosts prefer the installed - // dependency, then an existing CLI when the dependency is absent. - const legacyPlatform = process.platform === "linux" && process.arch === "arm64"; + const legacyPlatform = !((process.platform === "linux" || process.platform === "darwin") + && (process.arch === "x64" || process.arch === "arm64")); try { if (legacyPlatform) { await ensureCommandResolvable("codex", home, process.env); diff --git a/server/src/services/native-runtime/native-codex-runner.test.ts b/server/src/services/native-runtime/native-codex-runner.test.ts index 1d1cc71020..f5341c8a73 100644 --- a/server/src/services/native-runtime/native-codex-runner.test.ts +++ b/server/src/services/native-runtime/native-codex-runner.test.ts @@ -9,9 +9,10 @@ import { buildNativeRunnerArguments, buildNativeRunnerPreparePayload, resolvePaperclipRunnerBinary, + resolveRemotePaperclipRunnerBinary, } from "./native-codex-runner.js"; -afterEach(() => vi.restoreAllMocks()); +afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); describe("installed runner selection", () => { it("uses the runner's platform-aware packaged selection", async () => { @@ -45,6 +46,121 @@ describe("installed runner selection", () => { }); }); +describe("selected remote daemon target", () => { + function header(target: string) { + const bytes = Buffer.alloc(64); + if (target.startsWith("darwin")) { + bytes.writeUInt32LE(0xfeedfacf); bytes.writeUInt32LE(target.endsWith("arm64") ? 0x0100000c : 0x01000007, 4); + } else { + bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); bytes.writeUInt16LE(target.endsWith("arm64") ? 183 : 62, 18); + } + return bytes; + } + const remote = (stdout: string, overrides = {}) => ({ execute: vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false, stdout, stderr: "", pid: null, startedAt: null, ...overrides })) }); + it.each([ + ["Linux", "aarch64", "linux", "arm64"], ["Linux", "x86_64", "linux", "x64"], + ["Darwin", "arm64", "darwin", "arm64"], ["Darwin", "x86_64", "darwin", "x64"], + ] as const)("uses the packaged %s/%s daemon independent of the controller", async (os, arch, platform, architecture) => { + const root = await mkdtemp(join(tmpdir(), "paperclip-ssh-daemon-")); + const binary = join(root, "paperclip-runnerd"); + try { + await writeFile(binary, header(`${platform}-${architecture}`)); + const select = vi.spyOn(runner, "defaultCapabilityRunnerdBinary").mockReturnValue(binary); + const target = remote(`${os}\n${arch}\n`); + expect(await resolveRemotePaperclipRunnerBinary({ runner: target, remoteCwd: "/selected-workspace" })).toBe(binary); + expect(select).toHaveBeenCalledExactlyOnceWith(platform, architecture); + expect(target.execute).toHaveBeenCalledExactlyOnceWith({ command: "sh", args: ["-c", "uname -s; uname -m"], cwd: "/selected-workspace", bypassSession: true, timeoutMs: 10_000 }); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it.each(["arm64", "x64"].flatMap(controllerArch => ["arm64", "x64"].map(targetArch => ({ controllerArch, targetArch }))))( + "selects the actual Linux $targetArch sandbox bytes on a Mac $controllerArch controller", async ({ controllerArch, targetArch }) => { + const root = await mkdtemp(join(tmpdir(), "paperclip-sandbox-daemon-")); + const binary = join(root, "paperclip-runnerd"); + try { + vi.stubGlobal("process", { ...process, platform: "darwin", arch: controllerArch }); + await writeFile(binary, header(`linux-${targetArch}`)); + const controllerBinary = join(root, "controller-runnerd"); + await writeFile(controllerBinary, header(`darwin-${controllerArch}`), { mode: 0o700 }); + vi.stubEnv("PAPERCLIP_RUNNER_BINARY", controllerBinary); + const select = vi.spyOn(runner, "defaultCapabilityRunnerdBinary").mockReturnValue(binary); + const target = remote(`Linux\n${targetArch === "arm64" ? "aarch64" : "x86_64"}\n`); + expect(await resolveRemotePaperclipRunnerBinary({ runner: target, remoteCwd: "/sandbox-workspace" })).toBe(binary); + expect(select).toHaveBeenCalledExactlyOnceWith("linux", targetArch); + expect(target.execute).toHaveBeenCalledExactlyOnceWith({ command: "sh", args: ["-c", "uname -s; uname -m"], cwd: "/sandbox-workspace", bypassSession: true, timeoutMs: 10_000 }); + } finally { await rm(root, { recursive: true, force: true }); } + }, + ); + it.each(["linux", "darwin"].flatMap(platform => ["arm64", "x64"].map(architecture => ({ platform, architecture }))))( + "prefers the supervisor's fresh $platform/$architecture override for the same-host target", async ({ platform, architecture }) => { + const root = await mkdtemp(join(tmpdir(), "paperclip-same-host-daemon-")); + try { + vi.stubGlobal("process", { ...process, platform, arch: architecture }); + const fresh = join(root, "fresh-runnerd"), stale = join(root, "stale-packaged-runnerd"); + await writeFile(fresh, Buffer.concat([header(`${platform}-${architecture}`), Buffer.from("fresh")]), { mode: 0o700 }); + await writeFile(stale, Buffer.concat([header(`${platform}-${architecture}`), Buffer.from("stale")]), { mode: 0o700 }); + vi.stubEnv("PAPERCLIP_RUNNER_BINARY", fresh); + const select = vi.spyOn(runner, "defaultCapabilityRunnerdBinary").mockReturnValue(stale); + const target = remote(`${platform === "linux" ? "Linux" : "Darwin"}\n${architecture === "arm64" ? "arm64" : "x86_64"}\n`); + expect(await resolveRemotePaperclipRunnerBinary({ runner: target, remoteCwd: "/workspace" })).toBe(fresh); + expect(select).not.toHaveBeenCalled(); + } finally { await rm(root, { recursive: true, force: true }); } + }, + ); + it.each(["missing", "relative", "malformed", "wrong-architecture"])("retains same-host override policy and target validation (%s)", async scenario => { + const root = await mkdtemp(join(tmpdir(), "paperclip-same-host-daemon-")); + try { + vi.stubGlobal("process", { ...process, platform: "linux", arch: "x64" }); + const override = join(root, "configured-runnerd"), fallback = join(root, "packaged-runnerd"); + if (scenario !== "missing") await writeFile(override, scenario === "malformed" ? Buffer.alloc(64) : header("linux-arm64"), { mode: 0o700 }); + await writeFile(fallback, header("linux-x64"), { mode: 0o700 }); + vi.stubEnv("PAPERCLIP_RUNNER_BINARY", scenario === "relative" ? "relative-runnerd" : override); + vi.spyOn(runner, "defaultCapabilityRunnerdBinary").mockReturnValue(fallback); + const request = resolveRemotePaperclipRunnerBinary({ runner: remote("Linux\nx86_64\n"), remoteCwd: "/workspace" }); + if (scenario === "missing") expect(await request).toBe(fallback); + else await expect(request).rejects.toThrow(scenario === "relative" ? "PAPERCLIP_RUNNER_BINARY must be an absolute path" : "runner_remote_artifact_platform_mismatch"); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it("ignores a relative controller override when the actual target differs", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-cross-host-daemon-")); + try { + vi.stubGlobal("process", { ...process, platform: "darwin", arch: "arm64" }); + vi.stubEnv("PAPERCLIP_RUNNER_BINARY", "relative-controller-runnerd"); + const binary = join(root, "linux-runnerd"); + await writeFile(binary, header("linux-arm64")); + const select = vi.spyOn(runner, "defaultCapabilityRunnerdBinary").mockReturnValue(binary); + expect(await resolveRemotePaperclipRunnerBinary({ runner: remote("Linux\naarch64\n"), remoteCwd: "/workspace" })).toBe(binary); + expect(select).toHaveBeenCalledExactlyOnceWith("linux", "arm64"); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it.each(["matching", "wrong-architecture", "malformed", "missing"])("verifies an explicit controller-owned remote artifact (%s)", async scenario => { + const root = await mkdtemp(join(tmpdir(), "paperclip-ssh-daemon-")); + const binary = join(root, "configured-runnerd"); + try { + vi.stubGlobal("process", { ...process, platform: "linux", arch: "arm64" }); + vi.stubEnv("PAPERCLIP_RUNNER_BINARY", "invalid-relative-controller-override"); + if (scenario !== "missing") await writeFile(binary, scenario === "malformed" ? Buffer.alloc(64) : header(scenario === "matching" ? "linux-arm64" : "linux-x64")); + const select = vi.spyOn(runner, "defaultCapabilityRunnerdBinary"); + const request = resolveRemotePaperclipRunnerBinary({ runner: remote("Linux\naarch64\n"), remoteCwd: "/workspace", configuredPath: binary }); + if (scenario === "matching") expect(await request).toBe(binary); + else await expect(request).rejects.toThrow(scenario === "missing" ? "runner_remote_artifact_unavailable" : "runner_remote_artifact_platform_mismatch"); + expect(select).not.toHaveBeenCalled(); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it("keeps missing package targets actionable without a controller or legacy fallback", async () => { + const select = vi.spyOn(runner, "defaultCapabilityRunnerdBinary").mockImplementation(() => { throw new Error("runner_remote_artifact_unavailable: install the matching release"); }); + await expect(resolveRemotePaperclipRunnerBinary({ runner: remote("Linux\naarch64\n"), remoteCwd: "/workspace" })).rejects.toThrow("runner_remote_artifact_unavailable"); + expect(select).toHaveBeenCalledExactlyOnceWith("linux", "arm64"); + }); + it.each([ + ["Linux\naarch64\n", { exitCode: 1 }], ["Linux\naarch64\n", { timedOut: true }], + ["Linux\nriscv64\n", {}], ["FreeBSD\nx86_64\n", {}], ["", {}], + ])("fails a missing, failed or unsupported platform probe (%s)", async (stdout, overrides) => { + const select = vi.spyOn(runner, "defaultCapabilityRunnerdBinary"); + await expect(resolveRemotePaperclipRunnerBinary({ runner: remote(stdout as string, overrides), remoteCwd: "/workspace" })).rejects.toThrow("runner_remote_artifact_platform_mismatch"); + expect(select).not.toHaveBeenCalled(); + }); +}); + describe("buildNativeRunnerArguments", () => { it("binds every durable identity without exposing the bootstrap ticket", () => { const args = buildNativeRunnerArguments({ diff --git a/server/src/services/native-runtime/native-codex-runner.ts b/server/src/services/native-runtime/native-codex-runner.ts index 14d25ed9a4..f8e22050f2 100644 --- a/server/src/services/native-runtime/native-codex-runner.ts +++ b/server/src/services/native-runtime/native-codex-runner.ts @@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url"; import { and, eq } from "drizzle-orm"; import type { AdapterExecutionResult } from "@paperclipai/adapter-utils"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; import type { Db } from "@paperclipai/db"; import { agentSessionGoalActions, agentTaskSessions } from "@paperclipai/db"; @@ -14,6 +15,7 @@ import { failRunnerGoalAction } from "../runner-goals.js"; import { createPaperclipRunnerAuthorizedToolSet, defaultCapabilityRunnerdBinary, + runnerBinaryTarget, type PaperclipSemanticToolDefinition, } from "../../vendor/paperclip-runner/index.js"; import { runnerPrpCoordinator } from "./runner-prp-coordinator.js"; @@ -206,6 +208,35 @@ export function resolvePaperclipRunnerBinary( ); } +export async function resolveRemotePaperclipRunnerBinary(input: { + runner: CommandManagedRuntimeRunner; + remoteCwd: string; + configuredPath?: string | null; +}): Promise { + const probe = await input.runner.execute({ command: "sh", args: ["-c", "uname -s; uname -m"], + cwd: input.remoteCwd, bypassSession: true, timeoutMs: 10_000 }); + const [os, arch] = probe.stdout.trim().split(/\r?\n/); + const platform = os === "Linux" ? "linux" : os === "Darwin" ? "darwin" : null; + const architecture = arch === "x86_64" ? "x64" : arch === "aarch64" || arch === "arm64" ? "arm64" : null; + if (probe.exitCode !== 0 || probe.timedOut || !platform || !architecture) { + throw new Error("runner_remote_artifact_platform_mismatch: could not identify a supported remote OS and architecture; check the selected execution environment and retry"); + } + const configured = input.configuredPath?.trim(); + if (configured && !isAbsolute(configured)) throw new Error("PAPERCLIP_RUNNER_REMOTE_BINARY_PATH must be an absolute path"); + const controllerOverride = platform === process.platform && architecture === process.arch + ? process.env.PAPERCLIP_RUNNER_BINARY : undefined; + const binary = configured || (controllerOverride + ? resolvePaperclipRunnerBinary(controllerOverride) + : defaultCapabilityRunnerdBinary(platform, architecture)); + let bytes: Buffer; + try { bytes = readFileSync(binary); } + catch (cause) { throw new Error(`runner_remote_artifact_unavailable: install the Paperclip release containing the ${platform}-${architecture} daemon or configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH`, { cause }); } + if (runnerBinaryTarget(bytes) !== `${platform}-${architecture}`) { + throw new Error(`runner_remote_artifact_platform_mismatch: the selected daemon does not match ${platform}-${architecture}; install the matching Paperclip release or configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH`); + } + return binary; +} + export function buildNativeRunnerArguments(input: { connectUrl: string; stateDirectory: string; diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 52b40ff1c7..47cd86bc03 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -206,6 +206,7 @@ const state = vi.hoisted(() => ({ bundledPackManifestPath: null as string | null, bundledRunnerBinary: null as string | null, resolveRunnerBinary: vi.fn(() => "/tmp/paperclip-runnerd"), + resolveRemoteRunnerBinary: vi.fn(async (input: { configuredPath?: string | null }) => input.configuredPath || "/tmp/paperclip-runnerd"), release: null as null | (() => void), })); @@ -300,6 +301,7 @@ vi.mock("../issue-recovery-actions.js", () => ({ vi.mock("./native-codex-runner.js", () => ({ resolvePaperclipRunnerBinary: state.resolveRunnerBinary, + resolveRemotePaperclipRunnerBinary: state.resolveRemoteRunnerBinary, })); import { @@ -373,6 +375,7 @@ beforeEach(() => { state.bundledRunnerBinary = null; state.createAssignedMcpTools.mockReset(); state.resolveRunnerBinary.mockReset().mockReturnValue("/tmp/paperclip-runnerd"); + state.resolveRemoteRunnerBinary.mockReset().mockImplementation(async (input: { configuredPath?: string | null }) => input.configuredPath || "/tmp/paperclip-runnerd"); state.resolveCurrentWakeCommentsBinding.mockReset().mockResolvedValue(null); state.assertCurrentWakeCommentsRead.mockReset().mockResolvedValue(undefined); }); @@ -10485,9 +10488,9 @@ describe("runnerd provider runtime wiring", () => { corrupt: true, accepted: false, }, - ] as const)( - "uses remote prior-run backup when acceptance=$accepted direct=$directLifecycle backup=$backupLifecycle corrupt=$corrupt", - async ({ directLifecycle, backupLifecycle, corrupt, accepted }) => { + ].flatMap(testCase => (["sandbox", "ssh"] as const).map(transport => ({ ...testCase, transport }))))( + "uses $transport prior-run backup when acceptance=$accepted direct=$directLifecycle backup=$backupLifecycle corrupt=$corrupt", + async ({ directLifecycle, backupLifecycle, corrupt, accepted, transport }) => { const priorExecution = scopedNativeExecution("remote-prior-scope"); @@ -10501,7 +10504,10 @@ describe("runnerd provider runtime wiring", () => { const priorRunDb = priorNativeRunDb(priorExecution); const remoteTarget = { kind: "remote" as const, - transport: "sandbox" as const, + transport, + spec: { host: "fixture.invalid", username: "fixture", port: 22, + remoteCwd: "/workspace", remoteWorkspacePath: "/workspace", + privateKey: null, knownHosts: null, strictHostKeyChecking: true }, providerKey: "daytona", leaseId: "environment-lease-remote-prior-scope", remoteCwd: "/home/daytona/paperclip-workspace", @@ -12203,22 +12209,20 @@ describe("runnerd provider runtime wiring", () => { it.each([ ...["current", "preinstalled-exact", "preinstalled-mismatch", "preinstalled-error", "preinstalled-timeout", "stale", "missing", "retained", "retained-mismatch", "retained-error", "retained-timeout", "retained-explicit"] - .map((image) => ({ image, version: "0.160.0", compatible: true })), + .map((image) => ({ image, version: "0.160.0" })), ...["0.149.0", "0.149.1", "0.153.4", "0.156.0", "0.160.1"] - .map((version) => ({ image: "current", version, compatible: true })), + .map((version) => ({ image: "current", version })), ...["0.148.9", "0.161.0", "1.0.0", "0.160.0-alpha.1", "unknown"] - .map((version) => ({ image: "current", version, compatible: false })), - ])("uses shared Codex and the server-owned replacement artifact (image=$image, Codex=$version)", async ({ image, version, compatible }) => { + .map((version) => ({ image: "current", version })), + ])("uses shared Codex and the server-owned replacement artifact (image=$image, Codex=$version)", async ({ image, version }) => { const retained = image.startsWith("retained"); const exactRetained = image === "retained" || image === "retained-explicit"; const needsReplacement = image !== "current" && image !== "preinstalled-exact" && !exactRetained; // The mocked remote executes metadata probes; artifact staging only needs bytes. // Keep this regression independent of a locally compiled Rust runner binary. const controllerArtifact = join(isolatedStateDirectory, "paperclip-runnerd"); - if (needsReplacement || retained || image === "preinstalled-exact") { - await writeFile(controllerArtifact, "fixture runner artifact"); - state.resolveRunnerBinary.mockReturnValueOnce(controllerArtifact); - } + await writeFile(controllerArtifact, "fixture runner artifact"); + state.resolveRemoteRunnerBinary.mockResolvedValue(controllerArtifact); const onLog = vi.fn(async () => undefined); const syncIn = vi.fn(async () => undefined); const remoteExecute = vi.fn( @@ -12300,16 +12304,10 @@ describe("runnerd provider runtime wiring", () => { controlPlaneRegistration: (authority: unknown) => Promise; }; await expect(transport.controlPlaneRegistration({})).rejects.toThrow( - compatible ? "reached-preinstalled-codex-verification" : "runner_remote_provider_artifact_incompatible: supported Codex versions >=0.149.0 <0.161.0", + "reached-preinstalled-codex-verification", ); - if (!compatible) { - expect(syncIn).not.toHaveBeenCalled(); - expect(remoteExecute.mock.calls.some(([call]) => call.command === "npm" || call.args?.[1]?.includes("paperclip_codex_launcher_tmp"))).toBe(false); - expect(onLog).not.toHaveBeenCalledWith("stderr", expect.stringContaining("using compatible Codex")); - return; - } if (version !== "0.160.0") { - expect(onLog).toHaveBeenCalledWith("stderr", expect.stringContaining(`using compatible Codex ${version}`)); + expect(onLog).toHaveBeenCalledWith("stderr", expect.stringContaining(`using installed Codex ${version === "unknown" || version.includes("alpha") ? "unrecognized version" : version}`)); } if (needsReplacement) { expect(transport.runnerBinary).toBe(controllerArtifact); @@ -12355,12 +12353,7 @@ describe("runnerd provider runtime wiring", () => { ].map((scenario) => ({ ...scenario, recover: false })), { version: "0.156.0", model: "gpt-6.1-sol", floor: null, recover: true }, { version: "0.158.0", model: "gpt-6.1-sol", floor: null, recover: true }, - ])("verifies sandbox Codex $version against the prepared $model selection (floor=$floor, recover=$recover)", async ({ version, model, floor, recover }) => { - // A preinstalled Codex inside the compatibility window can still be too - // old for the configured model: the ChatGPT backend rejects every turn - // with "not supported when using Codex with a ChatGPT account". The - // verifier names the stale image instead of letting the run fail as an - // account problem. + ])("lets sandbox Codex $version reach actual model verification ($model, recover=$recover)", async ({ version, model, recover }) => { const preparedModel = recover ? resolvePaperclipRunnerNativeProviderInput({ backend: "codex_app_server", adapterConfig: { provider: "codex", model }, codexCliVersion: version, }).model : model; @@ -12430,32 +12423,33 @@ describe("runnerd provider runtime wiring", () => { ); } await expect(transport.controlPlaneRegistration({})).rejects.toThrow( - floor - ? `runner_remote_provider_artifact_incompatible: ${model} requires Codex ${floor} or newer with ChatGPT sign-in, received ${version} from the sandbox image; promote a sandbox image with Codex 0.160.0 or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC=@openai/codex@0.160.0` - : "reached-preinstalled-codex-verification", + "reached-preinstalled-codex-verification", ); - if (floor) { - // No npm spec is configured, so there is no fallback install; the run - // fails before launch instead of running against the stale CLI. - expect(remoteExecute.mock.calls.some(([call]) => call.command === "npm")).toBe(false); - expect(remoteExecute.mock.calls.some(([call]) => call.args?.[1]?.includes("paperclip_codex_launcher_tmp"))).toBe(false); - } + expect(remoteExecute.mock.calls.some(([call]) => call.command === "npm")).toBe(false); + }); - it("binds a remote launch to the configured controller-owned runner artifact", async () => { + it.each(["ssh", "sandbox"].flatMap(transport => ["explicit", "linux-arm64", "linux-x64", "missing", "mismatch"].map(mode => ({ transport, mode })))) + ("binds a $transport launch to the selected controller-owned runner artifact ($mode)", async ({ transport, mode }) => { const remoteCwd = "/home/daytona/paperclip-workspace"; - const controllerArtifact = "/controller/artifacts/paperclip-runnerd"; + const controllerArtifact = `/controller/artifacts/${mode}/paperclip-runnerd`; + const explicit = mode === "explicit" || mode === "missing" || mode === "mismatch"; + const error = mode === "missing" ? "runner_remote_artifact_unavailable" : mode === "mismatch" ? "runner_remote_artifact_platform_mismatch" : null; + if (error) state.resolveRemoteRunnerBinary.mockRejectedValue(new Error(error)); + else state.resolveRemoteRunnerBinary.mockResolvedValue(controllerArtifact); const remoteExecution = { ...execution, binding: { ...execution.binding, runId: "run-remote-runner-artifact" }, workspace: { ...execution.workspace, cwd: "/host/paperclip-workspace" }, } as NativeExecutionInputV1; - await createRunnerdBackend({ + const backend = createRunnerdBackend({ db: leaseDb(remoteExecution), execution: remoteExecution, runnerInstanceId: "runner", - runnerExecutionTarget: { + runnerExecutionTarget: transport === "sandbox" ? { + kind: "remote", transport: "sandbox", providerKey: "test", remoteCwd, runner: { execute: vi.fn() }, + } : { kind: "remote", transport: "ssh", remoteCwd, @@ -12470,14 +12464,26 @@ describe("runnerd provider runtime wiring", () => { strictHostKeyChecking: true, }, }, - runnerRemoteBinaryPath: controllerArtifact, + ...(explicit ? { runnerRemoteBinaryPath: controllerArtifact } : {}), }); + if (error) { + await expect(backend).rejects.toThrow(error); + expect(state.resolveRemoteRunnerBinary).toHaveBeenCalledWith(expect.objectContaining({ remoteCwd, configuredPath: controllerArtifact })); + expect(state.resolveRunnerBinary).not.toHaveBeenCalled(); + expect(state.createBackend).not.toHaveBeenCalled(); + expect(state.createTransport).not.toHaveBeenCalled(); + return; + } + await backend; + state.createTransport.mockClear(); state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); expect(state.createTransport).toHaveBeenCalledWith( expect.objectContaining({ runnerBinary: controllerArtifact }), ); + expect(state.resolveRemoteRunnerBinary).toHaveBeenCalledWith(expect.objectContaining({ remoteCwd, configuredPath: explicit ? controllerArtifact : undefined })); + expect(state.resolveRunnerBinary).not.toHaveBeenCalled(); }); it.each([ diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 0a31738bd6..ab6d4eb169 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -18,11 +18,7 @@ import { copyBackGrokAuth } from "@paperclipai/adapter-grok-local/server"; import { inferOpenAiCompatibleBiller, type AdapterUsageCheckpoint } from "@paperclipai/adapter-utils"; import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; -import { - isSupportedRemoteCodexVersion, - parseCodexCliVersion, - REMOTE_CODEX_SUPPORTED_RANGE, -} from "./codex-runtime-compatibility.js"; +import { parseCodexCliVersion } from "./codex-runtime-compatibility.js"; import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js"; import { createNativeProviderFailureObservation } from "./native-provider-failure.js"; import { createNativeGitHubAccess, type NativeGitHubAccess } from "./native-github-access.js"; @@ -31,11 +27,6 @@ import { bindManagedNativeCredentialTurn, completeManagedNativeCredentialTurn } import { createLocalNativeQuestionBridge } from "./local-native-question-bridge.js"; import { readVerifiedRemoteWorkspaceFile } from "./remote-deliverable-file.js"; import { copyBackCodexAuth } from "@paperclipai/adapter-codex-local/server"; -import { - codexCliVersionAtLeast, - minimumCodexCliVersionForModel, - normalizeCodexModel, -} from "@paperclipai/adapter-codex-local"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; import { stoppedCodexTurnIsTextOnly } from "./stopped-codex-turn.js"; @@ -181,7 +172,7 @@ import { } from "./status-arbiter.js"; import { conflict, HttpError } from "../../errors.js"; import { redactSensitiveText } from "../../redaction.js"; -import { resolvePaperclipRunnerBinary } from "./native-codex-runner.js"; +import { resolvePaperclipRunnerBinary, resolveRemotePaperclipRunnerBinary } from "./native-codex-runner.js"; import { createNativeRunTrace, isNativeRunRootHistoricalSpan, @@ -7738,9 +7729,7 @@ async function executePaperclipNativeSessionWithinScope( retainedTransition = await migrateRunnerdStateRootForExecution({ db: input.db, execution: input.execution, - allowVerifiedBackup: - input.runnerExecutionTarget?.kind === "remote" && - input.runnerExecutionTarget.transport === "sandbox", + allowVerifiedBackup: input.runnerExecutionTarget?.kind === "remote", // A retained warm runner is deliberately still ready rather than // suspended. Only the exact idle in-process owner may rotate that // prior-run authority; after a hard restart the map is empty and the @@ -10193,7 +10182,7 @@ export function assertRemoteRunnerBuildMetadata( } } -async function stageRemoteRunnerFile(input: { +export async function stageRemoteRunnerFile(input: { target: Extract; runner: CommandManagedRuntimeRunner; sourcePath: string; @@ -10585,7 +10574,7 @@ export async function syncRemoteRunnerDirectoryOut(input: { } } -async function readRemoteRunnerState(input: { +export async function readRemoteRunnerState(input: { runner: CommandManagedRuntimeRunner; stateDirectory: string; }): Promise> { @@ -11207,9 +11196,7 @@ export async function createRunnerdBackend(input: { retainedTransition = await migrateRunnerdStateRootForExecution({ db: input.db, execution: input.execution, - allowVerifiedBackup: - input.runnerExecutionTarget?.kind === "remote" && - input.runnerExecutionTarget.transport === "sandbox", + allowVerifiedBackup: input.runnerExecutionTarget?.kind === "remote", allowRetainedWarmRunner: false, allowLocalRecovery: input.runnerExecutionTarget?.kind !== "remote", onLog: input.onLog, @@ -11560,7 +11547,10 @@ async function createRunnerdBackendWithinSessionClaim( // filesystem; the remote launcher separately owns the sandbox command path. // When an explicit remote artifact is configured, prepareRemoteRunner stages // these exact bytes at remoteBinary before launch. - const controllerRunnerBinary = remoteTarget + const targetMatchedCodexArtifact = remoteTarget !== null && input.execution.provider.kind === "codex"; + const controllerRunnerBinary = targetMatchedCodexArtifact + ? await resolveRemotePaperclipRunnerBinary({ runner: remoteCommandRunner!, remoteCwd: remoteTarget.remoteCwd, configuredPath: input.runnerRemoteBinaryPath }) + : remoteTarget ? input.runnerRemoteBinaryPath?.trim() || remotePiCompanion?.runnerBinary || (useBundledRemoteImageAssets ? bundledRemoteRunnerBinary() : resolvePaperclipRunnerBinary()) : resolvePaperclipRunnerBinary(); const explicitRemoteCodex = input.runnerRemoteCodexPath?.trim() || null; @@ -11651,76 +11641,6 @@ async function createRunnerdBackendWithinSessionClaim( }) : Promise.resolve(); - const verifyRemoteRunner = async ( - requiredMode: "dial_wss" | "listen_ws", - executable = remoteBinary, - ) => { - if (!remoteTarget || !remoteCommandRunner || !executable) return; - const metadataResult = await remoteCommandRunner.execute({ - command: executable, - args: ["--build-metadata"], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 30_000, - }); - if (metadataResult.exitCode !== 0 || metadataResult.timedOut) { - throw new Error("runner_remote_artifact_verification_failed"); - } - let metadata: Record; - try { - metadata = JSON.parse(metadataResult.stdout) as Record; - } catch (error) { - throw new Error("runner_remote_artifact_metadata_invalid", { - cause: error, - }); - } - assertRemoteRunnerBuildMetadata(metadata, requiredMode); - }; - - const reportedCodexVersions = new Set(); - const verifyRemoteCodex = async (executable = remoteCodexBinary) => { - if (!remoteTarget || !remoteCommandRunner || !executable) return; - const versionResult = await remoteCommandRunner.execute({ - command: executable, - args: ["--version"], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 30_000, - }); - if (versionResult.exitCode !== 0 || versionResult.timedOut) { - throw new Error("runner_remote_codex_artifact_verification_failed"); - } - const versionOutput = `${versionResult.stdout}\n${versionResult.stderr}`; - const version = parseCodexCliVersion(versionOutput); - if (!version || !isSupportedRemoteCodexVersion(version)) { - throw new Error( - `runner_remote_provider_artifact_incompatible: supported Codex versions ${REMOTE_CODEX_SUPPORTED_RANGE}, received ${version ?? "an unrecognized or prerelease version"}; install a supported stable Codex release or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC=@openai/codex@${REMOTE_PROVIDER_PACK_PINS.codex}`, - ); - } - // A Codex inside the compatibility window can still be too old for the - // configured model: the ChatGPT backend rejects a model from clients - // below the model's floor on every turn. Fail before launch with the - // exact gap, so a stale sandbox image is not reported as an account - // problem. When a preinstalled Codex fails here and an npm spec is - // configured, the caller falls back to installing the pinned release. - const configuredModel = input.execution.provider.kind === "codex" - ? input.execution.provider.model - : null; - const modelMinimum = minimumCodexCliVersionForModel(configuredModel); - if (modelMinimum && !codexCliVersionAtLeast(version, modelMinimum)) { - throw new Error( - `runner_remote_provider_artifact_incompatible: ${normalizeCodexModel(configuredModel)} requires Codex ${modelMinimum} or newer with ChatGPT sign-in, received ${version} from the sandbox image; promote a sandbox image with Codex ${REMOTE_PROVIDER_PACK_PINS.codex} or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC=@openai/codex@${REMOTE_PROVIDER_PACK_PINS.codex}`, - ); - } - if (version !== REMOTE_PROVIDER_PACK_PINS.codex && !reportedCodexVersions.has(version)) { - reportedCodexVersions.add(version); - await input.onLog?.( - "stderr", - `[paperclip-runner] using compatible Codex ${version} (supported ${REMOTE_CODEX_SUPPORTED_RANGE}; install pin ${REMOTE_PROVIDER_PACK_PINS.codex})\n`, - ); - } - }; - const verifyRemoteProviderPack = async (packRoot: string) => { if (!remoteTarget || !remoteCommandRunner || !expectedProviderPackManifest) return; @@ -11805,41 +11725,20 @@ async function createRunnerdBackendWithinSessionClaim( return parseRemoteExecutableCandidate(result.stdout); }; - // Image policy: keep one latest stable CLI installation shared by native and - // local adapters. Preferred bin entries must point to that same installation; - // never bake an older global CLI alongside a private runner-only version. - const discoverPreinstalledExecutable = async ( - name: "paperclip-runnerd" | "codex", - ) => { - if (!remoteTarget || !remoteCommandRunner) return null; - return discoverRemoteExecutable(remoteCommandRunner, remoteTarget.remoteCwd, name); - }; - - const linkPreinstalledExecutable = async ( - sourcePath: string, - targetPath: string, - ) => { - if (!remoteTarget || !remoteCommandRunner) return; - const escapedSource = sourcePath.replaceAll("'", "'\\''"); - const escapedTarget = targetPath.replaceAll("'", "'\\''"); - const escapedDirectory = posix.dirname(targetPath).replaceAll("'", "'\\''"); - const result = await remoteCommandRunner.execute({ - command: "sh", - args: [ - "-c", - targetPath === remoteCodexBinary - ? buildRemoteCodexLauncherCommand(sourcePath, targetPath) - : `umask 077; mkdir -p '${escapedDirectory}' && ` + - `ln -sfn '${escapedSource}' '${escapedTarget}'`, - ], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 10_000, - }); - if (result.exitCode !== 0 || result.timedOut) { - throw new Error("runner_remote_preinstalled_link_failed"); - } + const remoteNativeArtifacts = remoteTarget && remoteCommandRunner && remoteBinary && remoteRuntimeRoot + ? createRemoteNativeArtifactPreparation({ + target: remoteTarget, runner: remoteCommandRunner, remoteBinary, controllerRunnerBinary, + remoteRuntimeRoot, remoteCodexBinary, runnerRemoteBinaryPath: input.runnerRemoteBinaryPath, + runnerRemoteCodexPath: input.runnerRemoteCodexPath, runnerRemoteCodexNpmSpec: input.runnerRemoteCodexNpmSpec, + model: input.execution.provider.kind === "codex" ? input.execution.provider.model : null, + reuseRetainedRunner: sandboxLeaseAcquisition?.outcome === "resumed", targetPlatformArtifactVerified: remotePiCompanion !== null || useBundledRemoteImageAssets || targetMatchedCodexArtifact, + trace: input.trace, onLog: input.onLog, + }) + : null; + const verifyRemoteRunner = async (mode: "dial_wss" | "listen_ws", executable?: string | null) => { + if (remoteNativeArtifacts) await remoteNativeArtifacts.verifyRemoteRunner(mode, executable ?? undefined); }; + const verifyRemoteCodex = async () => { await remoteNativeArtifacts?.verifyRemoteCodex(); }; const prepareRemoteRunner = async ( requiredMode: "dial_wss" | "listen_ws", @@ -11861,234 +11760,7 @@ async function createRunnerdBackendWithinSessionClaim( remotePrepared = true; return; } - // Compatibility metadata does not prove artifact identity. Reuse only the - // exact controller-owned bytes when the controller artifact is available. - const matchesControllerRunnerArtifact = async (executable: string): Promise => { - const expected = createHash("sha256") - .update(readFileSync(controllerRunnerBinary)) - .digest("hex"); - try { - const probe = await remoteCommandRunner.execute({ - command: "sh", - args: [ - "-c", - 'test -x "$1" || exit 1; if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1"; else shasum -a 256 "$1"; fi', - "paperclip-runner-artifact", - executable, - ], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 10_000, - }); - return probe.exitCode === 0 && !probe.timedOut && - /^[a-f0-9]{64}\s/.test(probe.stdout) && - probe.stdout.trim().split(/\s+/)[0] === expected; - } catch { - // An unavailable checksum uses the verified staging path. - return false; - } - }; - let runnerArtifactPrepared = false; - if ( - sandboxLeaseAcquisition?.outcome === "resumed" && - existsSync(controllerRunnerBinary) - ) { - runnerArtifactPrepared = await measureNativeRunnerSpan( - input.trace, - "runner.artifact.verify_retained", - () => matchesControllerRunnerArtifact(remoteBinary), - ); - } - const explicitRemoteBinary = input.runnerRemoteBinaryPath?.trim() || null; - if (!runnerArtifactPrepared && mayUsePreinstalledRunnerArtifact(explicitRemoteBinary)) { - const preinstalledRunner = await measureNativeRunnerSpan( - input.trace, - "runner.artifact.discover", - () => discoverPreinstalledExecutable("paperclip-runnerd"), - ); - if (preinstalledRunner) { - try { - await measureNativeRunnerSpan( - input.trace, - "runner.artifact.verify_preinstalled", - () => verifyRemoteRunner(requiredMode, preinstalledRunner), - ); - if (existsSync(controllerRunnerBinary) && - !await matchesControllerRunnerArtifact(preinstalledRunner)) { - throw new Error("runner_remote_preinstalled_artifact_mismatch"); - } - await measureNativeRunnerSpan( - input.trace, - "runner.artifact.link", - () => linkPreinstalledExecutable(preinstalledRunner, remoteBinary), - ); - runnerArtifactPrepared = true; - await input.onLog?.( - "stderr", - "[paperclip-runner] using preinstalled runnerd from the sandbox image\n", - ); - } catch { - runnerArtifactPrepared = false; - } - } - } - if (!runnerArtifactPrepared) { - // Upload the same artifact used for the controller identity. The server - // vendors the runner under vendor/paperclip-runner/bin, so the package - // development fallback cannot locate it in a deployed server. - const sourceBinary = controllerRunnerBinary; - if (!existsSync(sourceBinary)) { - throw new Error("runner_remote_artifact_unavailable"); - } - if (!explicitRemoteBinary && !remotePiCompanion && !useBundledRemoteImageAssets) { - const platform = await remoteCommandRunner.execute({ - command: "sh", - args: ["-c", "uname -s; uname -m"], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 10_000, - }); - const [remoteOs = "", remoteArch = ""] = platform.stdout - .trim() - .split(/\r?\n/); - const localOs = - process.platform === "darwin" - ? "Darwin" - : process.platform === "linux" - ? "Linux" - : process.platform; - const localArch = - process.arch === "x64" - ? "x86_64" - : process.arch === "arm64" - ? "aarch64" - : process.arch; - const archMatches = - remoteArch === localArch || - (localArch === "aarch64" && remoteArch === "arm64"); - if ( - platform.exitCode !== 0 || - platform.timedOut || - remoteOs !== localOs || - !archMatches - ) { - throw new Error( - "runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the remote OS and architecture", - ); - } - } - await stageRemoteRunnerFile({ - target: remoteTarget, - runner: remoteCommandRunner, - sourcePath: sourceBinary, - targetPath: remoteBinary, - mode: 0o700, - }); - } - await measureNativeRunnerSpan(input.trace, "runner.artifact.verify", () => - verifyRemoteRunner(requiredMode), - ); - if (remoteCodexBinary && explicitRemoteCodex) { - if (!existsSync(explicitRemoteCodex)) { - throw new Error("runner_remote_codex_artifact_unavailable"); - } - await stageRemoteRunnerFile({ - target: remoteTarget, - runner: remoteCommandRunner, - sourcePath: explicitRemoteCodex, - targetPath: remoteCodexBinary, - mode: 0o700, - }); - await verifyRemoteCodex(); - } - if (remoteCodexBinary && remoteCodexNpmSpec) { - let usedPreinstalledCodex = false; - const preinstalledCodex = await measureNativeRunnerSpan( - input.trace, - "harness.artifact.discover", - () => discoverPreinstalledExecutable("codex"), - ); - if (preinstalledCodex) { - try { - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.verify_preinstalled", - () => verifyRemoteCodex(preinstalledCodex), - ); - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.link", - () => - linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), - ); - usedPreinstalledCodex = true; - await input.onLog?.( - "stderr", - "[paperclip-runner] using preinstalled Codex from the sandbox image\n", - ); - } catch { - usedPreinstalledCodex = false; - } - } - if (!usedPreinstalledCodex) { - const installRoot = posix.join( - remoteRuntimeRoot!, - "harnesses", - "codex", - ); - const installResult = await remoteCommandRunner.execute({ - command: "npm", - args: [ - "install", - "--prefix", - installRoot, - "--no-audit", - "--no-fund", - remoteCodexNpmSpec, - ], - cwd: remoteTarget.remoteCwd, - bypassSession: true, - timeoutMs: 180_000, - }); - if (installResult.exitCode !== 0 || installResult.timedOut) { - throw new Error("runner_remote_codex_install_failed"); - } - } - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.verify", - () => verifyRemoteCodex(), - ); - } - if (remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) { - const preinstalledCodex = await measureNativeRunnerSpan( - input.trace, - "harness.artifact.discover", - () => discoverPreinstalledExecutable("codex"), - ); - if (!preinstalledCodex) { - throw new Error( - "runner_remote_codex_artifact_unavailable: install codex in the sandbox image or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC", - ); - } - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.verify_preinstalled", - () => verifyRemoteCodex(preinstalledCodex), - ); - await measureNativeRunnerSpan(input.trace, "harness.artifact.link", () => - linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), - ); - await measureNativeRunnerSpan( - input.trace, - "harness.artifact.verify", - () => verifyRemoteCodex(), - ); - await input.onLog?.( - "stderr", - "[paperclip-runner] using preinstalled Codex from the sandbox image\n", - ); - } + await remoteNativeArtifacts?.prepare(requiredMode); if ( requiresRemoteProviderPack && expectedProviderPackManifest && @@ -13712,3 +13384,336 @@ async function createRunnerdBackendWithinSessionClaim( }, } satisfies NativeSessionBackend & { bindManagedSession(session: NativeSession): NativeSession }; } + +export function createRemoteNativeArtifactPreparation(input: { + target: Extract; + runner: CommandManagedRuntimeRunner; + remoteBinary: string; + controllerRunnerBinary: string; + remoteRuntimeRoot: string; + remoteCodexBinary?: string | null; + runnerRemoteBinaryPath?: string | null; + runnerRemoteCodexPath?: string | null; + runnerRemoteCodexNpmSpec?: string | null; + model?: string | null; + reuseRetainedRunner?: boolean; + targetPlatformArtifactVerified?: boolean; + trace?: NativeRunTrace; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; +}) { + const remoteTarget = input.target; + const remoteCommandRunner = input.runner; + const { remoteBinary, controllerRunnerBinary, remoteRuntimeRoot } = input; + const remoteCodexBinary = input.remoteCodexBinary ?? null; + const explicitRemoteCodex = input.runnerRemoteCodexPath?.trim() || null; + const remoteCodexNpmSpec = input.runnerRemoteCodexNpmSpec?.trim() || null; + if (explicitRemoteCodex && remoteCodexNpmSpec) throw new Error("runner_remote_codex_source_conflict"); + const verifyRemoteRunner = async ( + requiredMode: "dial_wss" | "listen_ws", + executable = remoteBinary, + ) => { + if (!remoteTarget || !remoteCommandRunner || !executable) return; + const metadataResult = await remoteCommandRunner.execute({ + command: executable, + args: ["--build-metadata"], + cwd: remoteTarget.remoteCwd, + bypassSession: true, + timeoutMs: 30_000, + }); + if (metadataResult.exitCode !== 0 || metadataResult.timedOut) { + throw new Error("runner_remote_artifact_verification_failed"); + } + let metadata: Record; + try { + metadata = JSON.parse(metadataResult.stdout) as Record; + } catch (error) { + throw new Error("runner_remote_artifact_metadata_invalid", { + cause: error, + }); + } + assertRemoteRunnerBuildMetadata(metadata, requiredMode); + }; + + const reportedCodexVersions = new Set(); + const verifyRemoteCodex = async (executable = remoteCodexBinary) => { + if (!remoteTarget || !remoteCommandRunner || !executable) return; + const versionResult = await remoteCommandRunner.execute({ + command: executable, + args: ["--version"], + cwd: remoteTarget.remoteCwd, + bypassSession: true, + timeoutMs: 30_000, + }); + if (versionResult.exitCode !== 0 || versionResult.timedOut) { + throw new Error("runner_remote_codex_artifact_verification_failed"); + } + const versionOutput = `${versionResult.stdout}\n${versionResult.stderr}`; + const version = parseCodexCliVersion(versionOutput); + if (version !== REMOTE_PROVIDER_PACK_PINS.codex && !reportedCodexVersions.has(version)) { + reportedCodexVersions.add(version); + await input.onLog?.( + "stderr", + `[paperclip-runner] using installed Codex ${version ?? "unrecognized version"} (image install pin ${REMOTE_PROVIDER_PACK_PINS.codex}); compatibility is checked by the native app-server\n`, + ); + } + }; + + const discoverPreinstalledExecutable = async ( + name: "paperclip-runnerd" | "codex", + ) => { + if (!remoteTarget || !remoteCommandRunner) return null; + return discoverRemoteExecutable(remoteCommandRunner, remoteTarget.remoteCwd, name); + }; + + const linkPreinstalledExecutable = async ( + sourcePath: string, + targetPath: string, + ) => { + if (!remoteTarget || !remoteCommandRunner) return; + const escapedSource = sourcePath.replaceAll("'", "'\\''"); + const escapedTarget = targetPath.replaceAll("'", "'\\''"); + const escapedDirectory = posix.dirname(targetPath).replaceAll("'", "'\\''"); + const result = await remoteCommandRunner.execute({ + command: "sh", + args: [ + "-c", + targetPath === remoteCodexBinary + ? buildRemoteCodexLauncherCommand(sourcePath, targetPath) + : `umask 077; mkdir -p '${escapedDirectory}' && ` + + `ln -sfn '${escapedSource}' '${escapedTarget}'`, + ], + cwd: remoteTarget.remoteCwd, + bypassSession: true, + timeoutMs: 10_000, + }); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error("runner_remote_preinstalled_link_failed"); + } + }; + + const prepare = async (requiredMode: "dial_wss" | "listen_ws") => { + const matchesControllerRunnerArtifact = async (executable: string): Promise => { + const expected = createHash("sha256") + .update(readFileSync(controllerRunnerBinary)) + .digest("hex"); + try { + const probe = await remoteCommandRunner.execute({ + command: "sh", + args: [ + "-c", + 'test -x "$1" || exit 1; if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1"; else shasum -a 256 "$1"; fi', + "paperclip-runner-artifact", + executable, + ], + cwd: remoteTarget.remoteCwd, + bypassSession: true, + timeoutMs: 10_000, + }); + return probe.exitCode === 0 && !probe.timedOut && + /^[a-f0-9]{64}\s/.test(probe.stdout) && + probe.stdout.trim().split(/\s+/)[0] === expected; + } catch { + return false; + } + }; + let runnerArtifactPrepared = false; + if ( + input.reuseRetainedRunner === true && + existsSync(controllerRunnerBinary) + ) { + runnerArtifactPrepared = await measureNativeRunnerSpan( + input.trace, + "runner.artifact.verify_retained", + () => matchesControllerRunnerArtifact(remoteBinary), + ); + } + const explicitRemoteBinary = input.runnerRemoteBinaryPath?.trim() || null; + if (!runnerArtifactPrepared && mayUsePreinstalledRunnerArtifact(explicitRemoteBinary)) { + const preinstalledRunner = await measureNativeRunnerSpan( + input.trace, + "runner.artifact.discover", + () => discoverPreinstalledExecutable("paperclip-runnerd"), + ); + if (preinstalledRunner) { + try { + await measureNativeRunnerSpan( + input.trace, + "runner.artifact.verify_preinstalled", + () => verifyRemoteRunner(requiredMode, preinstalledRunner), + ); + if (existsSync(controllerRunnerBinary) && + !await matchesControllerRunnerArtifact(preinstalledRunner)) { + throw new Error("runner_remote_preinstalled_artifact_mismatch"); + } + await measureNativeRunnerSpan( + input.trace, + "runner.artifact.link", + () => linkPreinstalledExecutable(preinstalledRunner, remoteBinary), + ); + runnerArtifactPrepared = true; + await input.onLog?.( + "stderr", + "[paperclip-runner] using preinstalled runnerd from the sandbox image\n", + ); + } catch { + runnerArtifactPrepared = false; + } + } + } + if (!runnerArtifactPrepared) { + const sourceBinary = controllerRunnerBinary; + if (!existsSync(sourceBinary)) { + throw new Error("runner_remote_artifact_unavailable"); + } + if (!explicitRemoteBinary && !input.targetPlatformArtifactVerified) { + const platform = await remoteCommandRunner.execute({ + command: "sh", + args: ["-c", "uname -s; uname -m"], + cwd: remoteTarget.remoteCwd, + bypassSession: true, + timeoutMs: 10_000, + }); + const [remoteOs = "", remoteArch = ""] = platform.stdout + .trim() + .split(/\r?\n/); + const localOs = + process.platform === "darwin" + ? "Darwin" + : process.platform === "linux" + ? "Linux" + : process.platform; + const localArch = + process.arch === "x64" + ? "x86_64" + : process.arch === "arm64" + ? "aarch64" + : process.arch; + const archMatches = + remoteArch === localArch || + (localArch === "aarch64" && remoteArch === "arm64"); + if ( + platform.exitCode !== 0 || + platform.timedOut || + remoteOs !== localOs || + !archMatches + ) { + throw new Error( + "runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the remote OS and architecture", + ); + } + } + await stageRemoteRunnerFile({ + target: remoteTarget, + runner: remoteCommandRunner, + sourcePath: sourceBinary, + targetPath: remoteBinary, + mode: 0o700, + }); + } + await measureNativeRunnerSpan(input.trace, "runner.artifact.verify", () => + verifyRemoteRunner(requiredMode), + ); + if (remoteCodexBinary && explicitRemoteCodex) { + if (!existsSync(explicitRemoteCodex)) { + throw new Error("runner_remote_codex_artifact_unavailable"); + } + await stageRemoteRunnerFile({ + target: remoteTarget, + runner: remoteCommandRunner, + sourcePath: explicitRemoteCodex, + targetPath: remoteCodexBinary, + mode: 0o700, + }); + await verifyRemoteCodex(); + } + if (remoteCodexBinary && remoteCodexNpmSpec) { + let usedPreinstalledCodex = false; + const preinstalledCodex = await measureNativeRunnerSpan( + input.trace, + "harness.artifact.discover", + () => discoverPreinstalledExecutable("codex"), + ); + if (preinstalledCodex) { + try { + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.verify_preinstalled", + () => verifyRemoteCodex(preinstalledCodex), + ); + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.link", + () => + linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), + ); + usedPreinstalledCodex = true; + await input.onLog?.( + "stderr", + "[paperclip-runner] using preinstalled Codex from the sandbox image\n", + ); + } catch { + usedPreinstalledCodex = false; + } + } + if (!usedPreinstalledCodex) { + const installRoot = posix.join( + remoteRuntimeRoot!, + "harnesses", + "codex", + ); + const installResult = await remoteCommandRunner.execute({ + command: "npm", + args: [ + "install", + "--prefix", + installRoot, + "--no-audit", + "--no-fund", + remoteCodexNpmSpec, + ], + cwd: remoteTarget.remoteCwd, + bypassSession: true, + timeoutMs: 180_000, + }); + if (installResult.exitCode !== 0 || installResult.timedOut) { + throw new Error("runner_remote_codex_install_failed"); + } + } + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.verify", + () => verifyRemoteCodex(), + ); + } + if (remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) { + const preinstalledCodex = await measureNativeRunnerSpan( + input.trace, + "harness.artifact.discover", + () => discoverPreinstalledExecutable("codex"), + ); + if (!preinstalledCodex) { + throw new Error( + "runner_remote_codex_artifact_unavailable: install codex in the sandbox image or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC", + ); + } + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.verify_preinstalled", + () => verifyRemoteCodex(preinstalledCodex), + ); + await measureNativeRunnerSpan(input.trace, "harness.artifact.link", () => + linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), + ); + await measureNativeRunnerSpan( + input.trace, + "harness.artifact.verify", + () => verifyRemoteCodex(), + ); + await input.onLog?.( + "stderr", + "[paperclip-runner] using preinstalled Codex from the sandbox image\n", + ); + } + }; + return { prepare, verifyRemoteRunner, verifyRemoteCodex }; +} diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index f4a9a81072..dd40545d7e 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -110,6 +110,7 @@ export const createRunnerdCodexTransport: ( runner.createRunnerdCodexTransport; export const defaultCapabilityRunnerdBinary = runner.defaultCapabilityRunnerdBinary; +export const runnerBinaryTarget = runner.runnerBinaryTarget; export const executeNativeSession = runner.executeNativeSession; export const applyNativeSessionGoalControl = runner.applyNativeSessionGoalControl; @@ -159,3 +160,5 @@ export const CONFIGURED_ENVIRONMENT_KEYS = runner.CONFIGURED_ENVIRONMENT_KEYS; export const GENERATED_RUNTIME_ENVIRONMENT_KEYS = runner.GENERATED_RUNTIME_ENVIRONMENT_KEYS; export const configuredEnvironmentProjection = runner.configuredEnvironmentProjection; export const configuredEnvironment = runner.configuredEnvironment; + +export const probeNativeRunnerEnvironment = runner.probeNativeRunnerEnvironment; diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 05dbdb66b4..37e99d29ce 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -111,6 +111,19 @@ registry therefore discovers that row through the public environments API. This still provides full isolation because every cell starts a new Paperclip instance and database. + +Native-generation Codex on Daytona qualifies its immutable image through one +existing saved-environment probe before agent creation. Automatic Codex requests +also inspect the scoped server discovery default. Legacy and other provider +fixtures keep their existing setup. The probe uses no provider turn, permits one +setup lease per cell, has a 330-second HTTP deadline and a 10-second platform +command budget, and performs no retry. It uses the existing archive/release and +provider-expiry cleanup policy. The returned owned probe lease joins task leases +in the existing runtime-cost estimator; unrelated lease IDs do not gain task +attribution. Failed setup before fixture completion retains unavailable/unknown +cost coverage and its holds. A connection-only probe or name-only snapshot cannot +qualify native defaults. + Daytona creates sandbox environments through the public API. The core fixture keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated warm-continuity fixture uses `reuseLease:true` and diff --git a/tests/runner-e2e/installed-cli.test.ts b/tests/runner-e2e/installed-cli.test.ts index 4020bafe8c..64c4d5d9b3 100644 --- a/tests/runner-e2e/installed-cli.test.ts +++ b/tests/runner-e2e/installed-cli.test.ts @@ -1,11 +1,11 @@ import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdtemp, mkdir, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { chmod, lstat, mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { afterEach, expect, it } from "vitest"; import { runnerMatrix } from "./catalog.js"; -import { assertInstalledCliSelection, installedCliKeys, verifyInstalledCli } from "./installed-cli.js"; +import { assertInstalledCliSelection, installedCliKeys, probeInstalledCodexStartup, verifyInstalledCli } from "./installed-cli.js"; import { buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment } from "./harness-env.js"; const cells = runnerMatrix.filter(e => e.profile.id === "runner-acpx-pi"); const roots: string[] = []; @@ -25,6 +25,20 @@ it("admits all26 current Pi cases through pinned installed public CLI/server wit const { env, server } = await fixture(); expect(cells).toHaveLength(26); for (const cell of cells) expect(await verifyInstalledCli(env, [cell])).toMatchObject({ serverRoot: server, defaultRuntimeResolution: true, qualificationOverride: false }); }); +it("admits existing Codex onboarding, useful-file revision and restart journeys without broadening other cells", async () => { + const { env, server } = await fixture(); + for (const id of ["first-task.runner-codex.local.task-card-accept", "everyday-workflows.runner-codex.local.build-revise", "agent-chat.runner-codex.local.continuity-restart"]) { + const cell = runnerMatrix.find(e => e.id === id)!; + expect(cell).toBeDefined(); + expect(await verifyInstalledCli(env, [cell])).toMatchObject({ serverRoot: server, defaultRuntimeResolution: true, qualificationOverride: false }); + for (const key of ["PAPERCLIP_RUNNER_BINARY", "NODE_PATH"]) expect(() => assertInstalledCliSelection({ ...env, [key]: "foreign" }, [cell])).toThrow("forbids"); + } + for (const id of ["first-task.legacy-codex.local.task-card-accept", "first-task.runner-codex.local.interview-first-response", "everyday-workflows.runner-codex.daytona.build-revise"]) { + const cell = runnerMatrix.find(e => e.id === id)!; + expect(cell).toBeDefined(); + expect(() => assertInstalledCliSelection(env, [cell])).toThrow("explicit supported Pi or Codex"); + } +}); it("qualified providers drop ambient admission while pending Copilot retains its exact gate", () => { for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); for (const agent of ["copilot"]) { @@ -92,3 +106,84 @@ it("startup-only accepts only explicit uncredentialed local hello and zero retri expect(()=>assertInstalledStartupOnly({...f.env,KIMI_MODEL_API_KEY:"dummy-not-a-real-key"},[cell],options)).toThrow("credential inputs"); expect(()=>assertInstalledStartupOnly({},[cell],options)).toThrow("startup-only"); }); + +async function startupFixture(mode: "normal" | "wrong-model" | "ambient-codex" | "startup-and-cleanup-failed" | "cleanup-failed" = "normal") { + const f = await fixture(); + const vendor = join(f.server, "dist/vendor/paperclip-runner"); + const daemon = join(vendor, "bin", `${process.platform}-${process.arch}`, "paperclip-runnerd"); + const codex = join(f.root, "node_modules/@openai/codex/bin/codex"); + const observations = join(f.root, "startup-observations.json"); + await mkdir(join(vendor, "bin", `${process.platform}-${process.arch}`), { recursive: true }); + await mkdir(join(f.server, "dist/services/native-runtime"), { recursive: true }); + await mkdir(join(f.root, "node_modules/@openai/codex/bin"), { recursive: true }); + await writeFile(daemon, "fixture-daemon"); + await writeFile(codex, "#!/bin/sh\nprintf 'codex-cli 0.149.5\\n'\n"); + await chmod(codex, 0o755); + await writeFile(join(f.server, "dist/services/native-runtime/native-codex-runner.js"), `export function resolvePaperclipRunnerBinary() { return ${JSON.stringify(daemon)}; }`); + await writeFile(join(vendor, "index.js"), ` + import {writeFileSync} from "node:fs"; + const state={calls:[],environmentKeys:[],fileAuthPinned:false};let retired=false; + const save=()=>writeFileSync(${JSON.stringify(observations)},JSON.stringify(state)); + export function resolveCodexCommand(){return ${JSON.stringify(mode === "ambient-codex" ? process.execPath : codex)};} + export function createIsolatedCodexAppServerArgs(){return ["app-server"];} + export function createRunnerdCodexTransport(options){ + state.environmentKeys=Object.keys(options.environment).sort(); + state.fileAuthPinned=options.codexArgs.includes('cli_auth_credentials_store="file"'); + state.privateStateDirectory=options.stateDirectory; + save();return {transport:{async close(){retired=true;state.calls.push("transport-close");save();${mode === "startup-and-cleanup-failed" ? 'throw new Error("fixture secondary transport cleanup failure");' : ''}}}, + evidence:()=>({runnerPid:111,codexPid:222,providerDriver:"codex_app_server",runnerExited:retired,diagnostics:["fixture provider bootstrap failed"],providerSessionId:"PRIVATE-PROVIDER-ID"})};} + export class CodexAppServerDriver { + constructor(options){this.options=options;} + async openSession(){state.calls.push("open");save(); + ${mode === "startup-and-cleanup-failed" ? 'throw new Error("fixture original provider startup failure");' : ''} + const model=${JSON.stringify(mode === "wrong-model" ? "wrong-model" : "gpt-5.6-sol")}; + return {ids:()=>({driverSessionId:"PRIVATE-DRIVER-ID",providerSessionId:"PRIVATE-PROVIDER-ID"}), + events:()=>({async *[Symbol.asyncIterator](){yield {eventType:"session.started",payload:{context:{model,sandbox:{permissionProfile:{id:"paperclip-runner-workspace-read-only"}}}}};}}), + startTurn(){state.calls.push("FORBIDDEN-TURN");save();throw new Error("No provider turn is permitted");}, + async close(){retired=true;state.calls.push("session-close");save();${mode === "cleanup-failed" ? 'throw new Error("fixture session cleanup failure");' : ''}}};} + } + `); + return { ...f, observations, expectedRunnerSha256: hash("fixture-daemon") }; +} + +it.skipIf(process.platform === "win32")("installed native startup uses closed homes, file-only auth and no turn or version equality gate", async () => { + const f = await startupFixture(); + const receipt = await probeInstalledCodexStartup({ consumerRoot: f.root, serverRoot: f.server, expectedRunnerSha256: f.expectedRunnerSha256 }); + expect(receipt).toMatchObject({ nativeSessionStarted: true, cleanupConfirmed: true, providerTurns: 0, credentialsUsed: false, + codexVersion: "codex-cli 0.149.5", defaultRuntimeResolution: true, reasoningEffortExercised: false }); + expect(await readFile(f.observations, "utf8").then(JSON.parse)).toMatchObject({ calls: ["open", "session-close"], fileAuthPinned: true, + environmentKeys: ["CODEX_HOME", "HOME", "LC_ALL", "PAPERCLIP_RUNNER_NETWORK_ACCESS", "PATH", "TMPDIR", "TZ"] }); + expect(JSON.stringify(receipt)).not.toMatch(/PRIVATE-(?:DRIVER|PROVIDER)-ID/); +}); + +it.skipIf(process.platform === "win32")("installed startup preserves the primary cause, records bounded diagnostics and attempts independent cleanup", async () => { + const f = await startupFixture("startup-and-cleanup-failed"); + const error = await probeInstalledCodexStartup({ consumerRoot: f.root, serverRoot: f.server, expectedRunnerSha256: f.expectedRunnerSha256 }).catch(error => error); + expect(error).toBeInstanceOf(AggregateError); + expect(error.message).toContain("fixture original provider startup failure"); + expect(error.message).toContain("fixture secondary transport cleanup failure"); + expect(error.message).toContain("fixture provider bootstrap failed"); + expect(error.cause.message).toBe("fixture original provider startup failure"); + expect(error.errors.map((error: Error) => error.message)).toEqual(["fixture original provider startup failure", "fixture secondary transport cleanup failure"]); + expect(error.message).not.toContain("PRIVATE-PROVIDER-ID"); + const observations = JSON.parse(await readFile(f.observations, "utf8")); + expect(observations.calls).toEqual(["open", "transport-close"]); + await expect(lstat(observations.privateStateDirectory)).rejects.toThrow(); +}); + +it.skipIf(process.platform === "win32")("installed startup cannot qualify when session cleanup fails after successful open", async () => { + const f = await startupFixture("cleanup-failed"); + await expect(probeInstalledCodexStartup({ consumerRoot: f.root, serverRoot: f.server, expectedRunnerSha256: f.expectedRunnerSha256 })).rejects.toThrow("fixture session cleanup failure"); + expect(JSON.parse(await readFile(f.observations, "utf8")).calls).toEqual(["open", "session-close", "session-close"]); +}); + +it.skipIf(process.platform === "win32")("installed startup rejects stale daemon/ambient Codex and cleans a mismatched native session", async () => { + const f = await startupFixture(); + await expect(probeInstalledCodexStartup({ consumerRoot: f.root, serverRoot: f.server, expectedRunnerSha256: "0".repeat(64) })).rejects.toThrow("reviewed pin"); + await expect(readFile(f.observations)).rejects.toThrow(); + const ambient = await startupFixture("ambient-codex"); + await expect(probeInstalledCodexStartup({ consumerRoot: ambient.root, serverRoot: ambient.server, expectedRunnerSha256: ambient.expectedRunnerSha256 })).rejects.toThrow("ambient Codex"); + const wrong = await startupFixture("wrong-model"); + await expect(probeInstalledCodexStartup({ consumerRoot: wrong.root, serverRoot: wrong.server, expectedRunnerSha256: wrong.expectedRunnerSha256 })).rejects.toThrow("matching native identity"); + expect(await readFile(wrong.observations, "utf8").then(JSON.parse)).toMatchObject({ calls: ["open", "session-close"] }); +}); diff --git a/tests/runner-e2e/installed-cli.ts b/tests/runner-e2e/installed-cli.ts index 97dc50a49f..a7b84db3b0 100644 --- a/tests/runner-e2e/installed-cli.ts +++ b/tests/runner-e2e/installed-cli.ts @@ -1,21 +1,31 @@ /** Optional published-install lane. No production hook or admission override. */ import { constants } from "node:fs"; -import { lstat, open, realpath } from "node:fs/promises"; +import { lstat, mkdir, mkdtemp, open, realpath, rm } from "node:fs/promises"; import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; import { findPackageJSON } from "node:module"; -import { dirname, isAbsolute, join, resolve } from "node:path"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { tmpdir } from "node:os"; import { pathToFileURL } from "node:url"; import { CREDENTIAL_NAMES, type MatrixExecution } from "./types.js"; export const installedCliKeys = ["PAPERCLIP_RUNNER_E2E_INSTALLED_CLI", "PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256"] as const; const overrideKeys = ["PAPERCLIP_RUNNER_BINARY", "PAPERCLIP_RUNNER_REMOTE_BINARY_PATH", "PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH", "PAPERCLIP_RUNNER_ACPX_QUALIFICATION", "PAPERCLIP_ACPX_BUILTIN_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT", "PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST"] as const; const suites = new Set(["extended-harnesses", "pi-native", "pi-controls", "rich-acp-warm-continuity"]); +const installedCodexCells = new Set([ + "first-task.runner-codex.local.task-card-accept", + "everyday-workflows.runner-codex.local.build-revise", + "agent-chat.runner-codex.local.continuity-restart", +]); export function usesInstalledCli(env: NodeJS.ProcessEnv): boolean { return installedCliKeys.some(key => env[key] !== undefined); } export function assertInstalledCliSelection(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[]): void { if (!usesInstalledCli(env)) return; - if (!installedCliKeys.every(key => Boolean(env[key])) || executions.length === 0 || executions.some(e => e.profile.id !== "runner-acpx-pi" || e.profile.qualificationCandidate !== "pi" || !e.suite.manualOnly || !suites.has(e.suite.id))) throw new Error("Installed CLI proof requires complete pins and explicit supported Pi cases"); + if (!installedCliKeys.every(key => Boolean(env[key])) || executions.length === 0 || executions.some(e => !( + (e.profile.id === "runner-acpx-pi" && e.profile.qualificationCandidate === "pi" && e.suite.manualOnly && suites.has(e.suite.id)) || + (e.profile.id === "runner-codex" && e.profile.provider === "codex" && e.environment.id === "local" && installedCodexCells.has(e.id)) + ))) throw new Error("Installed CLI proof requires complete pins and explicit supported Pi or Codex cases"); for (const key of overrideKeys) if (env[key] !== undefined) throw new Error(`Installed CLI proof forbids ${key}`); if (env.NODE_OPTIONS !== undefined || env.NODE_PATH !== undefined) throw new Error("Installed CLI proof forbids ambient Node injection"); } @@ -51,6 +61,109 @@ export async function verifyInstalledCli(env: NodeJS.ProcessEnv, executions: rea return { schema: "paperclip.e2e.installed-cli-admission/v1", entry, cliRoot, cliSha256, serverRoot, serverEntry, serverSha256, version: cli.version, defaultRuntimeResolution: true, qualificationOverride: false }; } +export async function probeInstalledCodexStartup(input: { + consumerRoot: string; + serverRoot: string; + expectedRunnerSha256: string; + timeoutMs?: number; +}) { + for (const key of [...overrideKeys, "NODE_OPTIONS", "NODE_PATH"]) { + if (process.env[key] !== undefined) throw new Error(`Installed Codex startup forbids ${key}`); + } + const inside = (root: string, filename: string) => { + const name = relative(root, filename); + return name !== "" && name !== ".." && !name.startsWith(`..${sep}`) && !isAbsolute(name); + }; + for (const filename of [input.consumerRoot, input.serverRoot]) { + if (!isAbsolute(filename) || resolve(filename) !== filename || await realpath(filename) !== filename) throw new Error("Installed Codex startup paths must be canonical and unlinked"); + } + if (!inside(join(input.consumerRoot, "node_modules"), input.serverRoot) || !/^[a-f0-9]{64}$/.test(input.expectedRunnerSha256)) throw new Error("Installed Codex startup requires a contained server and reviewed daemon digest"); + const manifest = JSON.parse((await readOwned(join(input.serverRoot, "package.json"), 65536)).toString()); + if (manifest.name !== "@paperclipai/server" || manifest.exports?.["."]?.import !== "./dist/index.js") throw new Error("Installed Codex startup requires the published server"); + const vendorRoot = join(input.serverRoot, "dist/vendor/paperclip-runner"); + const entries = [join(vendorRoot, "index.js"), join(input.serverRoot, "dist/services/native-runtime/native-codex-runner.js")]; + for (const filename of entries) if (await realpath(filename) !== filename) throw new Error("Installed Codex startup runtime escapes its package"); + const [runtime, production] = await Promise.all(entries.map(filename => import(pathToFileURL(filename).href))); + const runnerBinary = await realpath(production.resolvePaperclipRunnerBinary()); + if (runnerBinary !== join(vendorRoot, "bin", `${process.platform}-${process.arch}`, "paperclip-runnerd")) throw new Error("Installed Codex startup requires the normal platform daemon; no source or flat fallback"); + const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); + const runnerSha256 = digest(await readOwned(runnerBinary, 512 * 1024 * 1024)); + if (runnerSha256 !== input.expectedRunnerSha256) throw new Error("Installed Codex startup daemon differs from its reviewed pin"); + const directory = await realpath(await mkdtemp(join(tmpdir(), "pc-codex-startup-"))); + const home = join(directory, "home"), codexHome = join(directory, "codex-home"), workspace = join(directory, "workspace"); + for (const filename of [home, codexHome, workspace]) await mkdir(filename, { mode: 0o700 }); + const environment = { PATH: `${dirname(process.execPath)}:/usr/bin:/bin:/usr/sbin`, HOME: home, CODEX_HOME: codexHome, + TMPDIR: directory, LC_ALL: "C", TZ: "UTC", PAPERCLIP_RUNNER_NETWORK_ACCESS: "disabled" }; + let session: any; + let bundle: any; + let cleanupConfirmed = false; + let startupFailed = false; + let startupFailure: unknown; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new Error("Installed Codex startup timed out")), Math.max(1, Math.min(input.timeoutMs ?? 45000, 120000))); + try { + const codexCommand = await realpath(runtime.resolveCodexCommand(undefined, environment, workspace)); + if (!inside(join(input.consumerRoot, "node_modules"), codexCommand)) throw new Error("Installed Codex startup resolved an ambient Codex executable"); + const codexSha256 = digest(await readOwned(codexCommand, 512 * 1024 * 1024)); + const codexVersion = execFileSync(codexCommand, ["--version"], { cwd: workspace, env: environment, encoding: "utf8", timeout: 10000, maxBuffer: 16384 }).trim(); + if (!codexVersion || codexVersion.length > 200) throw new Error("Installed Codex startup returned no bounded version identity"); + const args = runtime.createIsolatedCodexAppServerArgs(environment); + args.splice(args.indexOf("app-server"), 0, "-c", 'cli_auth_credentials_store="file"'); + bundle = runtime.createRunnerdCodexTransport({ provider: "codex", stateDirectory: directory, environment, + sourceCodexHome: "", runnerBinary, codexCommand, codexArgs: args, closeGraceMs: 10000, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }); + const driver = new runtime.CodexAppServerDriver({ + taskEnvelope: { schema: "paperclip.skillless_task.v1", objective: "Credential-free installed startup proof.", + completionContract: { revision: "installed-startup-v1", criteria: [] }, constraints: [], expectedResultSchema: "paperclip.run_result.v1" }, + model: "gpt-5.6-sol", conversationMode: "direct", requestedCollaborationMode: "plan", approvalPolicy: "never", + includeCollaborationModeInstructions: false, dynamicTools: [], environment, + transportFactory: () => bundle.transport, requireProviderSessionIdentity: true, + }); + session = await driver.openSession({ runId: "installed-startup", normalizedSessionId: "installed-startup", workingDirectory: workspace, signal: controller.signal }); + const event = await session.events()[Symbol.asyncIterator]().next(); + const identity = session.ids(); + const evidence = bundle.evidence(); + if (event.value?.eventType !== "session.started" || event.value.payload?.context?.model !== "gpt-5.6-sol" || + event.value.payload?.context?.sandbox?.permissionProfile?.id !== "paperclip-runner-workspace-read-only" || + !identity.driverSessionId || !identity.providerSessionId || evidence.providerDriver !== "codex_app_server" || + !(evidence.runnerPid > 0) || !(evidence.codexPid > 0)) throw new Error("Installed Codex startup returned no matching native identity or read-only model context"); + await session.close({ reason: "credential-free installed startup complete" }); + session = undefined; + if (!bundle.evidence().runnerExited) throw new Error("Installed Codex startup cleanup is incomplete"); + cleanupConfirmed = true; + return { schema: "paperclip.e2e.installed-codex-startup/v1", platform: `${process.platform}-${process.arch}`, + runnerSha256, codexSha256, codexVersion, providerDriver: "codex_app_server", model: "gpt-5.6-sol", + nativeSessionStarted: true, cleanupConfirmed: true, providerTurns: 0, credentialsUsed: false, + defaultRuntimeResolution: true, reasoningEffortExercised: false }; + } catch (error) { + startupFailed = true; + startupFailure = error; + throw error; + } finally { + clearTimeout(timer); + const cleanupFailures: unknown[] = []; + try { + if (session) await session.close({ reason: "installed startup cleanup" }); + else if (bundle && !cleanupConfirmed) await bundle.transport.close("installed startup cleanup"); + } catch (error) { cleanupFailures.push(error); } + try { + if (!bundle || bundle.evidence().runnerExited) await rm(directory, { recursive: true, force: true }); + } catch (error) { cleanupFailures.push(error); } + if (startupFailed) { + const evidence = bundle?.evidence(); + const bounded = (value: unknown) => String(value).replaceAll(directory, "[private-startup]").slice(0, 2000); + const primaryMessage = bounded(startupFailure instanceof Error ? startupFailure.message : startupFailure); + const diagnostic = { schema: "paperclip.e2e.installed-codex-startup-failure/v1", platform: `${process.platform}-${process.arch}`, + runnerExited: evidence?.runnerExited ?? null, runnerExitCode: evidence?.runnerExitCode ?? null, + runnerSignal: evidence?.runnerSignal ?? null, + diagnostics: (Array.isArray(evidence?.diagnostics) ? evidence.diagnostics : []).slice(-16).map(bounded), + cleanupFailures: cleanupFailures.map(error => bounded(error instanceof Error ? error.message : error)) }; + throw new AggregateError([startupFailure, ...cleanupFailures], `Installed Codex startup failed: ${primaryMessage}\n${JSON.stringify(diagnostic)}`, { cause: startupFailure }); + } + if (cleanupFailures.length) throw new AggregateError(cleanupFailures, "Installed Codex startup cleanup failed", { cause: cleanupFailures[0] }); + } +} + /** This prep lane cannot create agents or read the normal local credential file. */ export function assertInstalledStartupOnly(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[], options: { ids: string[]; maxParallel: number; maxAutomaticRetries: number; headed: boolean; ui: boolean; debug: boolean; list: boolean; matrixJson: boolean }) { assertInstalledCliSelection(env, executions); diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index d3ac65d6d7..a9722e956d 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it } from "vitest"; import type { RunnerApi } from "./api.js"; +import { buildRuntimeUsage } from "./billing.js"; import { runnerMatrix } from "./catalog.js"; -import { setupLiveFixtures } from "./live-fixtures.js"; +import { setupLiveFixtures, selectFixtureRuntimeLeases } from "./live-fixtures.js"; describe("live runner fixtures", () => { it.each(["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile => @@ -163,7 +164,7 @@ describe("live runner fixtures", () => { }, ); - it("installs the Daytona provider through the public API before creating its environment", async () => { + it.each(["legacy-codex", "runner-acpx-claude", "runner-opencode"])("installs Daytona before its %s environment without adding a Codex qualification probe", async profile => { const calls: string[] = []; const api = { async post(path: string, data?: Record) { @@ -186,7 +187,7 @@ describe("live runner fixtures", () => { } if (path.endsWith("/environments")) { expect(calls).toContain("POST /api/plugins/install"); - return { id: "environment-1", driver: "sandbox" }; + return { id: "environment-1", driver: "sandbox", config: data?.config }; } if (path.endsWith("/agents")) { return { id: "agent-1", name: "Agent", companyId: "company-1" }; @@ -204,7 +205,7 @@ describe("live runner fixtures", () => { const execution = runnerMatrix.find( (candidate) => candidate.id === - "core-compatibility.legacy-codex.daytona.message-marker", + `core-compatibility.${profile}.daytona.message-marker`, ); expect(execution).toBeDefined(); @@ -214,7 +215,7 @@ describe("live runner fixtures", () => { executionNonce: "nonce", workspacePath: "/tmp/workspace", credentials: { - OPENAI_API_KEY: "openai-test-value", + [execution!.profile.credential]: "provider-test-value", DAYTONA_API_KEY: "daytona-test-value", }, daytonaImage: @@ -224,12 +225,99 @@ describe("live runner fixtures", () => { expect(calls.indexOf("POST /api/plugins/install")).toBeLessThan( calls.indexOf("POST /api/companies/company-1/environments"), ); + expect(calls.some(call => call.includes("/probe"))).toBe(false); await fixtures.teardown(); expect(calls).toContain( "DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true", ); }); + it.each(["explicit", "auto"])("qualifies the immutable Codex image once before %s agent creation", async mode => { + const original = runnerMatrix.find(row => row.id === "core-compatibility.runner-codex.daytona.message-marker")!; + const execution = { ...original, profile: { ...original.profile, buildAgent(input: Parameters[0]) { + const payload = original.profile.buildAgent(input); + return { ...payload, adapterType: mode === "auto" ? "codex_local" : "paperclip_runner", + runner: mode === "auto" ? "auto" : "paperclip" }; + } } }; + const calls: string[] = []; + const api = { + async get(path: string) { + calls.push(`GET ${path}`); + return [{ type: "codex_local", defaultRunner: "paperclip", supportedRunners: ["paperclip", "legacy"] }]; + }, + async post(path: string, data: any, options: any) { + calls.push(`POST ${path}`); + if (path === "/api/plugins/install") return { id: "plugin", status: "ready" }; + if (path === "/api/companies") return { id: "company", name: "Fixture" }; + if (path.endsWith("/environments")) return { id: "environment", driver: "sandbox", config: data.config }; + if (path === "/api/environments/environment/probe?companyId=company") { + expect(options).toEqual({ timeout: 330_000 }); + return { ok: true, details: { platformQualificationRecorded: true, leaseId: "setup-probe" } }; + } + if (path.endsWith("/agents")) { + expect(calls).toContain("POST /api/environments/environment/probe?companyId=company"); + expect(data.adapterType).toBe(mode === "auto" ? "codex_local" : "paperclip_runner"); + expect(data.runner).toBe(mode === "auto" ? "auto" : "paperclip"); + if (mode === "auto") expect(calls).toContain("GET /api/adapters?companyId=company&environmentId=environment"); + return { id: "agent", companyId: "company", ...data }; + } + throw new Error(`Unexpected POST ${path}`); + }, + async postSensitive() { return { id: "secret" }; }, + async delete() {}, + } as unknown as RunnerApi; + const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture", + credentials: { OPENAI_API_KEY: "fixture-key", DAYTONA_API_KEY: "fixture-provider-key" }, + daytonaImage: `registry.example.test/runtime@sha256:${"a".repeat(64)}` }); + expect(fixtures.setupProbe).toMatchObject({ leaseId: "setup-probe", ...(mode === "auto" ? { defaultRunner: "paperclip" } : {}) }); + expect(calls.filter(call => call.includes("/probe?"))).toHaveLength(1); + expect(calls.filter(call => call.startsWith("GET"))).toHaveLength(mode === "auto" ? 1 : 0); + await fixtures.teardown(); + }); + + it.each(["probe-failed", "not-recorded", "wrong-default"])("stops before automatic Codex creation when image qualification is %s", async failure => { + const original = runnerMatrix.find(row => row.id === "core-compatibility.runner-codex.daytona.message-marker")!; + const execution = { ...original, profile: { ...original.profile, buildAgent(input: Parameters[0]) { + return { ...original.profile.buildAgent(input), adapterType: "codex_local", runner: "auto" }; + } } }; + const calls: string[] = []; + const api = { + async get() { return [{ type: "codex_local", defaultRunner: "legacy", supportedRunners: ["legacy"] }]; }, + async post(path: string, data: any) { + calls.push(path); + if (path === "/api/plugins/install") return { id: "plugin", status: "ready" }; + if (path === "/api/companies") return { id: "company", name: "Fixture" }; + if (path.endsWith("/environments")) return { id: "environment", driver: "sandbox", config: data.config }; + if (path.includes("/probe?")) return { ok: failure !== "probe-failed", + details: { platformQualificationRecorded: failure !== "not-recorded", leaseId: "setup-probe" } }; + throw new Error(`Unexpected POST ${path}`); + }, + async postSensitive() { return { id: "secret" }; }, + async delete(path: string) { calls.push(`DELETE ${path}`); }, + } as unknown as RunnerApi; + await expect(setupLiveFixtures({ api, execution, executionNonce: "nonce", workspacePath: "/tmp/fixture", + credentials: { OPENAI_API_KEY: "fixture-key", DAYTONA_API_KEY: "fixture-provider-key" }, + daytonaImage: `registry.example.test/runtime@sha256:${"a".repeat(64)}` })).rejects.toThrow(failure === "wrong-default" ? "server-owned default" : "not recorded"); + expect(calls.some(call => call.endsWith("/agents"))).toBe(false); + expect(calls.filter(call => call.includes("/probe?"))).toHaveLength(1); + expect(calls).toContain("DELETE /api/environments/environment?destroyReusableSandboxLeases=true"); + }); + + it("includes the owned setup probe in existing runtime estimates while excluding unrelated leases", () => { + const lease = (id: string, duration: number, heartbeatRunId?: string) => ({ id, provider: "daytona", heartbeatRunId, + acquiredAt: "2026-10-12T00:00:00Z", releasedAt: new Date(Date.parse("2026-10-12T00:00:00Z") + duration).toISOString(), + metadata: { cpu: 4, memory: 4, disk: 10 } }); + const leases = [lease("task", 20_000, "selected-run"), lease("setup-probe", 10_000), lease("unrelated", 600_000, "other-run")]; + const selected = selectFixtureRuntimeLeases({ leases, runIds: ["selected-run"], setupProbeLeaseId: "setup-probe" }); + expect(selected.map(value => value.id)).toEqual(["task", "setup-probe"]); + const usage = buildRuntimeUsage({ environmentId: "daytona", runs: [], leases: selected }); + expect(usage.leaseCount).toBe(2); + expect(usage.leaseDurationMs).toBe(30_000); + expect(usage.costStatus).toBe("estimated"); + expect(usage.estimatedListCostUsd).toBeGreaterThan(0); + expect(selectFixtureRuntimeLeases({ leases, runIds: ["selected-run"] }).map(value => value.id)).toEqual(["task"]); + }); + it("creates a primary project workspace for reusable Daytona scope", async () => { const calls: string[] = []; const api = { @@ -246,11 +334,14 @@ describe("live runner fixtures", () => { return { id: "company-1", name: "Runner E2E" }; } if (path.endsWith("/environments")) { - return { id: "environment-1", driver: "sandbox" }; + return { id: "environment-1", driver: "sandbox", config: data?.config }; } if (path.endsWith("/agents")) { return { id: "agent-1", name: "Agent", companyId: "company-1" }; } + if (path.startsWith("/api/environments/environment-1/probe?")) { + return { ok: true, details: { platformQualificationRecorded: true, leaseId: "setup-probe" } }; + } if (path.endsWith("/projects")) { expect(data).toMatchObject({ executionWorkspacePolicy: { @@ -275,6 +366,7 @@ describe("live runner fixtures", () => { } throw new Error(`Unexpected POST ${path}`); }, + async get() { return [{ type: "codex_local", defaultRunner: "paperclip", supportedRunners: ["paperclip", "legacy"] }]; }, async postSensitive(_path: string, data?: Record) { return { id: `secret-${String(data?.key).toLowerCase()}` }; }, @@ -292,7 +384,7 @@ describe("live runner fixtures", () => { executionNonce: "nonce", workspacePath: "/tmp/workspace", credentials: { - OPENAI_API_KEY: "openai-test-value", + [execution!.profile.credential]: "provider-test-value", DAYTONA_API_KEY: "daytona-test-value", }, daytonaImage: diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 753e18762c..ff5e344256 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { installedReleaseDaytonaPlugin } from "./installed-release.js"; import { isManagedHiringCase } from "./chat-cases.js"; import { FixtureRegistry } from "./fixture-registry.js"; +import { isImmutableDaytonaImage } from "./catalog.js"; import { TASK_TITLE_BUDGET_CENTS } from "./task-titles.js"; import { stageGrokSubscriptionFixture } from "./grok-subscription-fixture.js"; import type { RunnerApi } from "./api.js"; @@ -63,6 +64,7 @@ export interface LiveFixtureValues { agent: AgentRecord; project?: ProjectRecord; aiConnection?: ManagedAccountFixture; + setupProbe?: { leaseId: string; durationMs: number; defaultRunner?: string }; onboardingRuntime?: { mode: "production-wizard" | "post-onboarding-runtime-switch"; @@ -79,6 +81,18 @@ function value(resolved: ReadonlyMap, id: string): T { return result as T; } +export function selectFixtureRuntimeLeases(input: { leases: T[]; runIds: string[]; issueId?: string; setupProbeLeaseId?: string }): T[] { + const selectedRunIds = new Set(input.runIds); + const relevant = input.leases.filter(lease => lease.provider === "daytona" && ( + lease.id === input.setupProbeLeaseId || selectedRunIds.size === 0 + || (lease.heartbeatRunId && selectedRunIds.has(lease.heartbeatRunId)) + || (input.issueId && lease.issueId === input.issueId) + )); + return relevant.length > 0 ? relevant : input.leases; +} + async function deleteDaytonaEnvironment(api: RunnerApi, environmentId: string) { const deadlineAt = Date.now() + 120_000; let lastError: unknown; @@ -111,6 +125,7 @@ export async function setupLiveFixtures(input: { }): Promise { const { api, execution } = input; const registry = new FixtureRegistry(); + let setupProbe: LiveFixtureValues["setupProbe"]; if (execution.environment.id === "daytona") { registry.register({ @@ -295,6 +310,33 @@ export async function setupLiveFixtures(input: { secretRefs, executionId: input.executionNonce, }); + const autoCodex = agent.adapterType === "codex_local" && (agent.runner === undefined || agent.runner === "auto"); + const nativeCodex = agent.adapterType === "paperclip_runner" && (agent.adapterConfig as Record).provider === "codex"; + if (environment.driver === "sandbox" && execution.profile.generation === "native" + && execution.profile.provider === "codex" && agent.runner !== "legacy" && (autoCodex || nativeCodex)) { + if (!isImmutableDaytonaImage(String(environment.config?.image ?? "")) || environment.config?.snapshot) { + throw new Error("Native Codex qualification requires this fixture's immutable image-backed environment."); + } + const startedAt = Date.now(); + const probe = await api.post<{ ok: boolean; details?: { platformQualificationRecorded?: boolean; leaseId?: string } }>( + `/api/environments/${environment.id}/probe?companyId=${encodeURIComponent(company.id)}`, + {}, { timeout: 330_000 }, + ); + if (probe.ok !== true || probe.details?.platformQualificationRecorded !== true || !probe.details.leaseId) { + throw new Error("Native Codex image qualification was not recorded; agent creation was not attempted."); + } + setupProbe = { leaseId: probe.details.leaseId, durationMs: Date.now() - startedAt }; + if (autoCodex) { + const adapters = await api.get>( + `/api/adapters?companyId=${encodeURIComponent(company.id)}&environmentId=${encodeURIComponent(environment.id)}`, + ); + const selected = adapters.find(adapter => adapter.type === "codex_local"); + if (selected?.defaultRunner !== "paperclip" || !selected.supportedRunners?.includes("paperclip")) { + throw new Error("The qualified image does not advertise native Codex as its server-owned default."); + } + setupProbe.defaultRunner = selected.defaultRunner; + } + } if (["stock-harness", "native-connection-guidance"].includes(execution.suite.id) || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) { agent.budgetMonthlyCents = 1_000; @@ -362,6 +404,7 @@ export async function setupLiveFixtures(input: { secretRefs: value(setup.values, "secrets"), environment: value(setup.values, "environment"), agent: value(setup.values, "agent"), + ...(setupProbe ? { setupProbe } : {}), ...(setup.values.has("project") ? { project: value(setup.values, "project") } : {}), diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 07ec11372d..d1fc5a8960 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -52,7 +52,7 @@ import { runnerExecutionById } from "./catalog.js"; import { classifyFailure } from "./failure-classifier.js"; import { runnerE2EServerControlPaths } from "./harness-env.js"; import { setupConnectionReview } from "./connection-reviews.js"; -import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js"; +import { setupLiveFixtures, selectFixtureRuntimeLeases, type LiveFixtureValues } from "./live-fixtures.js"; import { evaluateMatcher, persistedFinalRunMessage, type MatcherResult } from "./matchers.js"; import { readRegisteredArtifacts } from "./registered-artifact.js"; import { @@ -696,16 +696,9 @@ for (const execution of executions) { const listed = await api.get( `/api/environments/${fixtures.environment.id}/leases`, ); - const selectedRunIds = new Set(selectedRuns.map((run) => run.id)); - const relevant = listed.filter( - (lease) => - lease.provider === "daytona" && - (selectedRunIds.size === 0 || - (lease.heartbeatRunId && - selectedRunIds.has(lease.heartbeatRunId)) || - (issue && lease.issueId === issue.id)), - ); - runtimeLeases = relevant.length > 0 ? relevant : listed; + runtimeLeases = selectFixtureRuntimeLeases({ leases: listed, + runIds: selectedRuns.map(run => run.id), issueId: issue?.id, + setupProbeLeaseId: fixtures.setupProbe?.leaseId }); }; const cancelActiveRunsForCleanup = async () => { @@ -968,6 +961,8 @@ for (const execution of executions) { companyId: fixtures.company.id, environmentId: fixtures.environment.id, agentId: fixtures.agent.id, + ...(fixtures.setupProbe ? { setupProbe: { durationMs: fixtures.setupProbe.durationMs, + ...(fixtures.setupProbe.defaultRunner ? { defaultRunner: fixtures.setupProbe.defaultRunner } : {}) } } : {}), secretIds: Object.fromEntries( Object.entries(fixtures.secretRefs).map(([name, ref]) => [ name,