diff --git a/.dockerignore b/.dockerignore index f4699e217a..b8cad4152a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,3 +10,22 @@ tmp *.log packages/paperclip-runner/dist packages/paperclip-runner/runner/target +packages/paperclip-runner/devtools +packages/paperclip-runner/docs +packages/paperclip-runner/examples +packages/paperclip-runner/test +packages/paperclip-runner/test-fixtures +packages/paperclip-runner/test-support +packages/paperclip-runner/**/*.md +packages/paperclip-runner/**/*.spec.ts +packages/paperclip-runner/**/*.spec.tsx +packages/paperclip-runner/**/*.test.cjs +packages/paperclip-runner/**/*.test.cts +packages/paperclip-runner/**/*.test.js +packages/paperclip-runner/**/*.test.jsx +packages/paperclip-runner/**/*.test.mjs +packages/paperclip-runner/**/*.test.mts +packages/paperclip-runner/**/*.test.ts +packages/paperclip-runner/**/*.test.tsx +packages/paperclip-runner/runner/crates/*/tests +packages/paperclip-runner/scripts/*-smoke.mjs diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index b7662b4205..9c45df4479 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -42,9 +42,10 @@ permissions: contents: read concurrency: - group: runner-full-stack-e2e-${{ inputs.target_branch || github.event.repository.default_branch }} + group: runner-full-stack-e2e-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch && format('development-{0}', inputs.target_branch) || format('protected-{0}', github.run_id) }} # Development branch campaigns supersede older runs for the same target. - # Preserve every default-branch campaign for its paid audit trail. + # Give protected/default-branch campaigns unique groups because GitHub also + # replaces pending runs when cancel-in-progress is false. cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }} jobs: @@ -59,6 +60,7 @@ jobs: test_runner: ${{ steps.runner.outputs.runner }} max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }} max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }} + playwright_channel: ${{ steps.runner.outputs.playwright_channel }} target_sha: ${{ steps.target.outputs.sha }} target_ref: ${{ steps.target.outputs.ref }} steps: @@ -131,6 +133,7 @@ jobs: echo "runner=$aws_runner" echo "max_parallel_default=100" echo "max_parallel_limit=100" + echo "playwright_channel=chrome" } >> "$GITHUB_OUTPUT" echo '::notice title=Paid runner routing::Using an ephemeral RunsOn Fleet runner' else @@ -138,8 +141,9 @@ jobs: echo "runner=$github_runner" echo "max_parallel_default=32" echo "max_parallel_limit=57" + echo "playwright_channel=" } >> "$GITHUB_OUTPUT" - echo '::notice title=Paid runner routing::RUNNER_E2E_AWS_ENABLED is not true; using the existing paid runner' + echo '::notice title=Paid runner routing::RUNNER_E2E_AWS_ENABLED is not true; using the proven GitHub-hosted runner' fi target_lock: @@ -350,18 +354,21 @@ jobs: source_revision: ${{ steps.image.outputs.source_revision }} content_id: ${{ steps.image.outputs.content_id }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - if: needs.catalog.outputs.needs_daytona == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ needs.authorize.outputs.target_sha }} persist-credentials: false - name: Download resolved target lockfile + if: needs.catalog.outputs.needs_daytona == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.target_lock.outputs.artifact_id }} path: ${{ runner.temp }}/runner-e2e-target-lock - name: Restore resolved target lockfile + if: needs.catalog.outputs.needs_daytona == 'true' env: TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }} @@ -406,9 +413,11 @@ jobs: run: | set -euo pipefail if [ "$NEEDS_DAYTONA" != true ]; then - echo "image=" >> "$GITHUB_OUTPUT" - echo "source_revision=" >> "$GITHUB_OUTPUT" - echo "content_id=" >> "$GITHUB_OUTPUT" + { + echo "image=" + echo "source_revision=" + echo "content_id=" + } >> "$GITHUB_OUTPUT" exit 0 fi [[ "$IMAGE_CONTENT_ID" =~ ^[0-9a-f]{64}$ ]] @@ -452,9 +461,11 @@ jobs: .config.User == "daytona" and (.config.Env | any(startswith("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=")))' \ <<< "$image_config" >/dev/null - echo "image=$immutable" >> "$GITHUB_OUTPUT" - echo "source_revision=$source_revision" >> "$GITHUB_OUTPUT" - echo "content_id=$published_content_id" >> "$GITHUB_OUTPUT" + { + echo "image=$immutable" + echo "source_revision=$source_revision" + echo "content_id=$published_content_id" + } >> "$GITHUB_OUTPUT" build_runner_artifacts: name: Build reusable runner campaign artifacts @@ -833,6 +844,7 @@ jobs: run: node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - name: Download immutable campaign outputs + if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: ${{ needs.build_runner_artifacts.outputs.build_artifact_name }} @@ -846,6 +858,7 @@ jobs: path: runner-e2e-provider-pack - name: Verify and restore campaign outputs + if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' env: NEEDS_RUNNER_TYPESCRIPT: ${{ matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth' }} NEEDS_NATIVE_BINARY: ${{ startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' }} @@ -902,8 +915,28 @@ jobs: if: matrix.profileId == 'legacy-claude' run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.19 - - name: Install Chromium - run: pnpm exec playwright install --with-deps chromium + - name: Qualify preinstalled Chrome + if: needs.authorize.outputs.playwright_channel == 'chrome' + run: | + set -euo pipefail + chrome_path="$(command -v google-chrome)" + test -x "$chrome_path" + google-chrome --version + + - name: Install Chromium headless shell on GitHub-hosted fallback + if: needs.authorize.outputs.playwright_channel != 'chrome' + run: | + set -euo pipefail + for attempt in 1 2 3; do + if pnpm exec playwright install --with-deps --only-shell chromium; then + exit 0 + fi + if [ "$attempt" -eq 3 ]; then + echo "Chromium headless shell installation failed after $attempt attempts." >&2 + exit 1 + fi + sleep "$((attempt * 10))" + done - name: Run paid cell env: @@ -916,6 +949,7 @@ jobs: PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }} PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} + PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }} run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" - name: Upload access-controlled packaged cell evidence @@ -929,13 +963,14 @@ jobs: report: name: Merge and enforce campaign result - if: always() && needs.catalog.result == 'success' + if: always() && !cancelled() && needs.catalog.result == 'success' needs: [authorize, catalog, daytona_image, test] outputs: history_source_ready: ${{ steps.history_source_ready.outputs.ready }} runs-on: ubuntu-latest timeout-minutes: 15 permissions: + actions: read contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -963,22 +998,52 @@ jobs: - run: pnpm install --frozen-lockfile + - name: Resolve workflow job attempts + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + gh api --paginate --slurp \ + "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?filter=all&per_page=100" \ + > runner-e2e-job-pages.json + for attempt in $(seq 1 "${{ github.run_attempt }}"); do + gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/attempts/$attempt" \ + --jq '{run_attempt, run_started_at}' + done > runner-e2e-attempts.jsonl + jq -s '.' runner-e2e-attempts.jsonl > runner-e2e-attempts.json + jq --slurpfile attempts runner-e2e-attempts.json \ + '{jobs: [.[].jobs[]], attempts: $attempts[0]}' \ + runner-e2e-job-pages.json > runner-e2e-jobs.json + - name: Download cell evidence id: download_evidence continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-* + pattern: runner-e2e-${{ github.run_id }}-*-* path: downloaded-runner-e2e - merge-multiple: true + merge-multiple: false - name: Retry cell evidence download after transport failure if: steps.download_evidence.outcome == 'failure' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-* + pattern: runner-e2e-${{ github.run_id }}-*-* path: downloaded-runner-e2e - merge-multiple: true + merge-multiple: false + + - name: Select latest workflow attempt per cell + if: always() + env: + PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e + PAPERCLIP_RUNNER_E2E_SELECTED_ROOT: ${{ github.workspace }}/selected-runner-e2e + PAPERCLIP_RUNNER_E2E_JOBS_JSON: ${{ github.workspace }}/runner-e2e-jobs.json + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }} + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/select-rerun-artifacts.ts - name: Collect blob reports run: | @@ -990,7 +1055,7 @@ jobs: if [ ! -e "$target" ]; then cp "$report" "$target" fi - done < <(find downloaded-runner-e2e -path '*/blob-report/*.zip' -print0) + done < <(find selected-runner-e2e -path '*/blob-report/*.zip' -print0) - name: Merge Playwright HTML and JUnit if: always() @@ -1001,7 +1066,7 @@ jobs: - name: Aggregate normalized campaign results if: always() env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }} PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} @@ -1043,6 +1108,8 @@ jobs: if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true' runs-on: ubuntu-latest timeout-minutes: 15 + outputs: + pages_artifact_name: ${{ steps.pages_artifact_name.outputs.name }} concurrency: group: runner-e2e-history-publish cancel-in-progress: false @@ -1092,10 +1159,16 @@ jobs: RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} run: pnpm test:e2e:runner:history:publish + - name: Resolve Pages artifact name + id: pages_artifact_name + if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' + run: echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}" >> "$GITHUB_OUTPUT" + - name: Package pruned structured dashboard for GitHub Pages if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 with: + name: ${{ steps.pages_artifact_name.outputs.name }} path: runner-e2e-merged-report/normalized pages: @@ -1113,3 +1186,7 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 + with: + # If only this failed job is rerun, GitHub retains the successful + # publisher job's output from the earlier workflow attempt. + artifact_name: ${{ needs.publish_history.outputs.pages_artifact_name }} diff --git a/packages/adapters/opencode-local/src/server/models.test.ts b/packages/adapters/opencode-local/src/server/models.test.ts index b4fde5986a..0de89de8dd 100644 --- a/packages/adapters/opencode-local/src/server/models.test.ts +++ b/packages/adapters/opencode-local/src/server/models.test.ts @@ -18,7 +18,8 @@ describe("openCode models", () => { }); it("returns an empty list when discovery command is unavailable", async () => { - process.env.PAPERCLIP_OPENCODE_COMMAND = "__paperclip_missing_opencode_command__"; + process.env.PAPERCLIP_OPENCODE_COMMAND = + "__paperclip_missing_opencode_command__"; await expect(listOpenCodeModels()).resolves.toEqual([]); }); @@ -29,7 +30,9 @@ describe("openCode models", () => { }); it("accepts a provider/model id without running discovery", () => { - expect(requireOpenCodeModelId("openai/gpt-5.2-codex")).toBe("openai/gpt-5.2-codex"); + expect(requireOpenCodeModelId("openai/gpt-5.2-codex")).toBe( + "openai/gpt-5.2-codex", + ); }); it("rejects malformed provider/model ids before discovery", () => { @@ -42,7 +45,8 @@ describe("openCode models", () => { }); it("proceeds with the configured model when discovery cannot run (probe is best-effort, never fatal)", async () => { - process.env.PAPERCLIP_OPENCODE_COMMAND = "__paperclip_missing_opencode_command__"; + process.env.PAPERCLIP_OPENCODE_COMMAND = + "__paperclip_missing_opencode_command__"; await expect( ensureOpenCodeModelConfiguredAndAvailable({ model: "openai/gpt-5", @@ -51,23 +55,35 @@ describe("openCode models", () => { }); it("skips the availability check when OPENCODE_ALLOW_ALL_MODELS is set in the run env", async () => { - process.env.PAPERCLIP_OPENCODE_COMMAND = "__paperclip_missing_opencode_command__"; + process.env.PAPERCLIP_OPENCODE_COMMAND = + "__paperclip_missing_opencode_command__"; await expect( ensureOpenCodeModelConfiguredAndAvailable({ model: "anthropic/tensorix/deepseek/deepseek-chat-v3.1", env: { OPENCODE_ALLOW_ALL_MODELS: "true" }, }), ).resolves.toEqual([ - { id: "anthropic/tensorix/deepseek/deepseek-chat-v3.1", label: "anthropic/tensorix/deepseek/deepseek-chat-v3.1" }, + { + id: "anthropic/tensorix/deepseek/deepseek-chat-v3.1", + label: "anthropic/tensorix/deepseek/deepseek-chat-v3.1", + }, ]); }); it("honours OPENCODE_ALLOW_ALL_MODELS from the process env", async () => { - process.env.PAPERCLIP_OPENCODE_COMMAND = "__paperclip_missing_opencode_command__"; + process.env.PAPERCLIP_OPENCODE_COMMAND = + "__paperclip_missing_opencode_command__"; process.env.OPENCODE_ALLOW_ALL_MODELS = "1"; await expect( - ensureOpenCodeModelConfiguredAndAvailable({ model: "anthropic/gateway/some-model" }), - ).resolves.toEqual([{ id: "anthropic/gateway/some-model", label: "anthropic/gateway/some-model" }]); + ensureOpenCodeModelConfiguredAndAvailable({ + model: "anthropic/gateway/some-model", + }), + ).resolves.toEqual([ + { + id: "anthropic/gateway/some-model", + label: "anthropic/gateway/some-model", + }, + ]); }); it("still enforces provider/model format when OPENCODE_ALLOW_ALL_MODELS is set", async () => { @@ -120,20 +136,177 @@ describe("openCode models", () => { expect(spy).toHaveBeenCalledTimes(3); }); - it("surfaces the last error once retries are exhausted", async () => { - vi.useFakeTimers(); + it("refreshes a stale non-empty catalog before rejecting the configured model", async () => { const spy = vi .spyOn(serverUtils, "runChildProcess") - .mockResolvedValue({ - exitCode: 1, + .mockResolvedValueOnce({ + exitCode: 0, signal: null, timedOut: false, - stdout: "", - stderr: "queued behind another opencode run", + stdout: "openrouter/example/stale-model\n", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }) + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "Models cache refreshed\n", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }) + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: + "openrouter/example/current-model\nopenrouter/deepseek/deepseek-v4-flash-0731\n", + stderr: "", pid: 1, startedAt: new Date().toISOString(), }); + await expect( + ensureOpenCodeModelConfiguredAndAvailable({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + }), + ).resolves.toContainEqual({ + id: "openrouter/deepseek/deepseek-v4-flash-0731", + label: "openrouter/deepseek/deepseek-v4-flash-0731", + }); + expect(spy).toHaveBeenCalledTimes(3); + expect(spy.mock.calls[0]?.[2]).toEqual(["models"]); + expect(spy.mock.calls[1]?.[2]).toEqual(["models", "--refresh"]); + expect(spy.mock.calls[2]?.[2]).toEqual(["models"]); + }); + + it("still rejects when a refreshed non-empty catalog omits the configured model", async () => { + const spy = vi + .spyOn(serverUtils, "runChildProcess") + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "openrouter/example/stale-model\n", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }) + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "Models cache refreshed\n", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }) + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "openrouter/example/current-model\n", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }); + + await expect( + ensureOpenCodeModelConfiguredAndAvailable({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + }), + ).rejects.toThrow( + "Configured OpenCode model is unavailable: openrouter/deepseek/deepseek-v4-flash-0731", + ); + expect(spy).toHaveBeenCalledTimes(3); + expect(spy.mock.calls[1]?.[2]).toEqual(["models", "--refresh"]); + expect(spy.mock.calls[2]?.[2]).toEqual(["models"]); + }); + + it("still rejects from the original catalog when post-refresh enumeration returns no models", async () => { + const spy = vi + .spyOn(serverUtils, "runChildProcess") + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "openrouter/example/stale-model\n", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }) + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "Models cache refreshed\n", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }) + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }); + + await expect( + ensureOpenCodeModelConfiguredAndAvailable({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + }), + ).rejects.toThrow("Available models: openrouter/example/stale-model"); + expect(spy).toHaveBeenCalledTimes(3); + expect(spy.mock.calls[1]?.[2]).toEqual(["models", "--refresh"]); + expect(spy.mock.calls[2]?.[2]).toEqual(["models"]); + }); + + it("still rejects from the original catalog when refresh fails", async () => { + const warning = vi.spyOn(console, "warn").mockImplementation(() => {}); + const spy = vi + .spyOn(serverUtils, "runChildProcess") + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "openrouter/example/stale-model\n", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }) + .mockRejectedValueOnce(new Error("refresh unavailable")); + + await expect( + ensureOpenCodeModelConfiguredAndAvailable({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + }), + ).rejects.toThrow("Available models: openrouter/example/stale-model"); + expect(spy).toHaveBeenCalledTimes(2); + expect(spy.mock.calls[1]?.[2]).toEqual(["models", "--refresh"]); + expect(warning).toHaveBeenCalledWith( + expect.stringContaining( + 'refresh failed for "openrouter/deepseek/deepseek-v4-flash-0731"', + ), + ); + }); + + it("surfaces the last error once retries are exhausted", async () => { + vi.useFakeTimers(); + const spy = vi.spyOn(serverUtils, "runChildProcess").mockResolvedValue({ + exitCode: 1, + signal: null, + timedOut: false, + stdout: "", + stderr: "queued behind another opencode run", + pid: 1, + startedAt: new Date().toISOString(), + }); + const promise = discoverOpenCodeModels(); const assertion = expect(promise).rejects.toThrow( "`opencode models` failed: queued behind another opencode run", diff --git a/packages/adapters/opencode-local/src/server/models.ts b/packages/adapters/opencode-local/src/server/models.ts index 5af8c203c9..3e576eaa20 100644 --- a/packages/adapters/opencode-local/src/server/models.ts +++ b/packages/adapters/opencode-local/src/server/models.ts @@ -29,14 +29,26 @@ function resolveOpenCodeCommand(input: unknown): string { return asString(input, envOverride); } -const discoveryCache = new Map(); +const discoveryCache = new Map< + string, + { expiresAt: number; models: AdapterModel[] } +>(); const VOLATILE_ENV_KEY_PREFIXES = ["PAPERCLIP_", "npm_", "NPM_"] as const; -const VOLATILE_ENV_KEY_EXACT = new Set(["PWD", "OLDPWD", "SHLVL", "_", "TERM_SESSION_ID", "HOME"]); +const VOLATILE_ENV_KEY_EXACT = new Set([ + "PWD", + "OLDPWD", + "SHLVL", + "_", + "TERM_SESSION_ID", + "HOME", +]); export function requireOpenCodeModelId(input: unknown): string { const model = asString(input, "").trim(); if (!isValidOpenCodeModelId(model)) { - throw new Error("OpenCode requires `adapterConfig.model` in provider/model format."); + throw new Error( + "OpenCode requires `adapterConfig.model` in provider/model format.", + ); } return model; } @@ -84,9 +96,10 @@ export function parseOpenCodeModelsOutput(stdout: string): AdapterModel[] { } function normalizeEnv(input: unknown): Record { - const envInput = typeof input === "object" && input !== null && !Array.isArray(input) - ? (input as Record) - : {}; + const envInput = + typeof input === "object" && input !== null && !Array.isArray(input) + ? (input as Record) + : {}; const env: Record = {}; for (const [key, value] of Object.entries(envInput)) { if (typeof value === "string") env[key] = value; @@ -103,7 +116,11 @@ function hashValue(value: string): string { return createHash("sha256").update(value).digest("hex"); } -function discoveryCacheKey(command: string, cwd: string, env: Record) { +function discoveryCacheKey( + command: string, + cwd: string, + env: Record, +) { const envKey = Object.entries(env) .filter(([key]) => !isVolatileEnvKey(key)) .sort(([a], [b]) => a.localeCompare(b)) @@ -118,11 +135,14 @@ function pruneExpiredDiscoveryCache(now: number) { } } -export async function discoverOpenCodeModels(input: { - command?: unknown; - cwd?: unknown; - env?: unknown; -} = {}): Promise { +export async function discoverOpenCodeModels( + input: { + command?: unknown; + cwd?: unknown; + env?: unknown; + refresh?: boolean; + } = {}, +): Promise { const command = resolveOpenCodeCommand(input.command); const cwd = asString(input.cwd, process.cwd()); const env = normalizeEnv(input.env); @@ -139,7 +159,14 @@ export async function discoverOpenCodeModels(input: { // image). Fall back to process.env.HOME. } // Prevent OpenCode from writing an opencode.json into the working directory. - const runtimeEnv = normalizeEnv(ensurePathInEnv({ ...process.env, ...env, ...(resolvedHome ? { HOME: resolvedHome } : {}), OPENCODE_DISABLE_PROJECT_CONFIG: "true" })); + const runtimeEnv = normalizeEnv( + ensurePathInEnv({ + ...process.env, + ...env, + ...(resolvedHome ? { HOME: resolvedHome } : {}), + OPENCODE_DISABLE_PROJECT_CONFIG: "true", + }), + ); const maxAttempts = MODELS_DISCOVERY_RETRY_DELAYS_MS.length + 1; let lastError: Error | undefined; @@ -148,7 +175,7 @@ export async function discoverOpenCodeModels(input: { const result = await runChildProcess( `opencode-models-${Date.now()}-${Math.random().toString(16).slice(2)}`, command, - ["models"], + ["models", ...(input.refresh ? ["--refresh"] : [])], { cwd, env: runtimeEnv, @@ -159,10 +186,17 @@ export async function discoverOpenCodeModels(input: { ); if (result.timedOut) { - lastError = new Error(`\`opencode models\` timed out after ${MODELS_DISCOVERY_TIMEOUT_MS / 1000}s.`); + lastError = new Error( + `\`opencode models\` timed out after ${MODELS_DISCOVERY_TIMEOUT_MS / 1000}s.`, + ); } else if ((result.exitCode ?? 1) !== 0) { - const detail = firstNonEmptyLine(result.stderr) || firstNonEmptyLine(result.stdout); - lastError = new Error(detail ? `\`opencode models\` failed: ${detail}` : "`opencode models` failed."); + const detail = + firstNonEmptyLine(result.stderr) || firstNonEmptyLine(result.stdout); + lastError = new Error( + detail + ? `\`opencode models\` failed: ${detail}` + : "`opencode models` failed.", + ); } else { return sortModels(parseOpenCodeModelsOutput(result.stdout)); } @@ -175,11 +209,13 @@ export async function discoverOpenCodeModels(input: { throw lastError ?? new Error("`opencode models` failed."); } -export async function discoverOpenCodeModelsCached(input: { - command?: unknown; - cwd?: unknown; - env?: unknown; -} = {}): Promise { +export async function discoverOpenCodeModelsCached( + input: { + command?: unknown; + cwd?: unknown; + env?: unknown; + } = {}, +): Promise { const command = resolveOpenCodeCommand(input.command); const cwd = asString(input.cwd, process.cwd()); const env = normalizeEnv(input.env); @@ -194,6 +230,34 @@ export async function discoverOpenCodeModelsCached(input: { return models; } +async function refreshOpenCodeModelsCached(input: { + command?: unknown; + cwd?: unknown; + env?: unknown; +}): Promise { + const command = resolveOpenCodeCommand(input.command); + const cwd = asString(input.cwd, process.cwd()); + const env = normalizeEnv(input.env); + // OpenCode 1.18.17 uses `models --refresh` only to update its on-disk + // models.dev cache. Its stdout is a confirmation message, not the refreshed + // catalog, so enumerate once more after the refresh under the exact same + // command/cwd/env before deciding whether the configured model exists. + await discoverOpenCodeModels({ + command, + cwd, + env, + refresh: true, + }); + const models = await discoverOpenCodeModels({ command, cwd, env }); + if (models.length > 0) { + discoveryCache.set(discoveryCacheKey(command, cwd, env), { + expiresAt: Date.now() + MODELS_CACHE_TTL_MS, + models, + }); + } + return models; +} + export function isTruthyEnvFlag(value: string | undefined): boolean { if (value === undefined) return false; const v = value.trim().toLowerCase(); @@ -214,7 +278,11 @@ export async function ensureOpenCodeModelConfiguredAndAvailable(input: { // we still enforce the provider/model format above and do not second-guess // the configured model. Prefer the explicit run env, then the process env. const env = normalizeEnv(input.env); - if (isTruthyEnvFlag(env.OPENCODE_ALLOW_ALL_MODELS ?? process.env.OPENCODE_ALLOW_ALL_MODELS)) { + if ( + isTruthyEnvFlag( + env.OPENCODE_ALLOW_ALL_MODELS ?? process.env.OPENCODE_ALLOW_ALL_MODELS, + ) + ) { return [{ id: model, label: model }]; } @@ -250,7 +318,33 @@ export async function ensureOpenCodeModelConfiguredAndAvailable(input: { } if (!models.some((entry) => entry.id === model)) { - const sample = models.slice(0, 12).map((entry) => entry.id).join(", "); + // `opencode models` reads a persistent models.dev cache. Long-lived runner + // hosts can therefore report a stale non-empty catalog even while the + // configured provider serves the model. Refresh once before treating a + // cached miss as authoritative; a successful refresh that still omits the + // model retains the strict availability rejection below. + try { + const refreshedModels = await refreshOpenCodeModelsCached({ + command: input.command, + cwd: input.cwd, + env: input.env, + }); + if (refreshedModels.some((entry) => entry.id === model)) { + return refreshedModels; + } + if (refreshedModels.length > 0) models = refreshedModels; + } catch (err) { + console.warn( + `[opencode-local] Model availability refresh failed for "${model}" (${ + err instanceof Error ? err.message : String(err) + }); preserving the cached availability rejection.`, + ); + } + + const sample = models + .slice(0, 12) + .map((entry) => entry.id) + .join(", "); throw new Error( `Configured OpenCode model is unavailable: ${model}. Available models: ${sample}${models.length > 12 ? ", ..." : ""}`, ); diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 07ce418ba9..d69f6d9739 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -58,7 +58,7 @@ "sideEffects": false, "scripts": { "build": "pnpm run check:protocol-manifest && pnpm run build:typescript && pnpm run check:capability-contract && pnpm run check:capability-inventory && pnpm run check:protocol-coverage && pnpm run check:semantic-contracts && pnpm run check:runner-workflow-traceability && pnpm run build:binary && node scripts/generate-replay-goldens.mjs --check && node scripts/generate-semantic-action-catalog.mjs --check", - "build:typescript": "pnpm run ensure:eval-build-deps && pnpm run check:protocol-types && node ./node_modules/typescript/bin/tsc --version && node ./node_modules/typescript/bin/tsc -p tsconfig.json && node ./node_modules/typescript/bin/tsc -p tsconfig.surfaces.json", + "build:typescript": "pnpm run ensure:eval-build-deps && pnpm run check:protocol-types && node ./node_modules/typescript/bin/tsc --version && node ./node_modules/typescript/bin/tsc -p tsconfig.json && node ./node_modules/typescript/bin/tsc -p tsconfig.surfaces.json && node scripts/build-verified-provider-entrypoints.mjs", "build:rust": "cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins", "build:binary": "cargo build --release --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/stage-runner-binary.mjs", "build:provider-pack": "pnpm run build:typescript && node scripts/build-provider-pack.mjs", @@ -73,7 +73,7 @@ "typecheck:rust": "cargo fmt --manifest-path runner/Cargo.toml --all -- --check && cargo check --manifest-path runner/Cargo.toml --locked --workspace", "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", - "test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/materialize-opencode-binary.test.mjs && vitest run", + "test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs && vitest run", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", "test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::", @@ -144,6 +144,7 @@ "demo:live-console": "pnpm run build:typescript && node scripts/live-console-demo-server.mjs", "smoke:capability:ui": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/capability-issue-thread-smoke.mjs", "smoke:capability:cleanroom": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/capability-clean-room-smoke.mjs", + "smoke:local-provider": "node scripts/run-local-provider-smoke.mjs", "console:live-console": "pnpm run build:typescript && vite --config vite.config.ts --host 127.0.0.1 --port 4180", "console:sdk": "pnpm run build:typescript && vite --config vite.sdk.config.ts --host 127.0.0.1 --port 4181", "browser:dev": "pnpm run build:typescript && pnpm run build:runner-binaries && vite --config vite.config.ts", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index 402ba2e9ee..97ced9b875 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -1,8 +1,9 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::fs::{self, DirBuilder, File}; use std::io::{Read, Write}; +use std::net::TcpListener; use std::path::{Path, PathBuf}; -use std::time::Duration; +use std::time::{Duration, Instant}; #[cfg(unix)] use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; @@ -33,10 +34,12 @@ use crate::provider_events::{ use crate::qualified_launch::verify_launch_artifact; pub const ACPX_PROVIDER_STATE_FILE: &str = "acpx-provider-state.json"; -const ACPX_PROVIDER_STATE_SCHEMA: &str = "paperclip.runner.acpx-provider-state.v2"; +const ACPX_PROVIDER_STATE_SCHEMA: &str = "paperclip.runner.acpx-provider-state.v3"; const MAX_PROVIDER_STATE_BYTES: u64 = 16 * 1024 * 1024; const MAX_PENDING_EVENTS: usize = 8_320; const MAX_EVENTS_PER_POLL: usize = 128; +const PROVIDER_LIFETIME_CONFIRMATION_TIMEOUT: Duration = Duration::from_secs(5); +const PROVIDER_LIFETIME_CONFIRMATION_RETRY: Duration = Duration::from_millis(10); fn initial_event_sequence() -> u64 { 1 @@ -51,6 +54,56 @@ fn event_sequence(value: &str) -> Option { (event_id(sequence) == value).then_some(sequence) } +fn try_acquire_provider_lifetime_fence( + candidates: [u16; 3], +) -> Result>, DurableRunnerError> { + let mut listeners = Vec::with_capacity(2); + for port in candidates { + match TcpListener::bind(("127.0.0.1", port)) { + Ok(listener) => { + listeners.push(listener); + if listeners.len() == 2 { + return Ok(Some(listeners)); + } + } + Err(error) if error.kind() == std::io::ErrorKind::AddrInUse => {} + Err(error) => { + return Err(DurableRunnerError::invalid(format!( + "failed to prove ACPX provider lifetime cleanup: {error}" + ))) + } + } + } + Ok(None) +} + +fn acquire_provider_lifetime_fence( + candidates: [u16; 3], +) -> Result, DurableRunnerError> { + try_acquire_provider_lifetime_fence(candidates)?.ok_or_else(|| { + DurableRunnerError::invalid( + "ACPX original provider lifetime remains active; cleanup is not yet proven", + ) + }) +} + +fn await_provider_lifetime_fence( + candidates: [u16; 3], +) -> Result, DurableRunnerError> { + let deadline = Instant::now() + PROVIDER_LIFETIME_CONFIRMATION_TIMEOUT; + loop { + if let Some(listeners) = try_acquire_provider_lifetime_fence(candidates)? { + return Ok(listeners); + } + if Instant::now() >= deadline { + return Err(DurableRunnerError::invalid( + "ACPX original provider lifetime remains active after suspension; provider exit is not confirmed", + )); + } + std::thread::sleep(PROVIDER_LIFETIME_CONFIRMATION_RETRY); + } +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct AcpxProviderDescriptor { @@ -82,23 +135,26 @@ struct AcpxProviderDescriptor { } impl AcpxProviderDescriptor { - fn validate(&self, context: &AcpxEventProjectionContext) -> Result<(), DurableRunnerError> { + fn validate_session( + &self, + context: &AcpxEventProjectionContext, + ) -> Result<(), DurableRunnerError> { let expected = match self.agent.as_str() { "claude" => ( "claude-sonnet-5", "@agentclientprotocol/claude-agent-acp", "0.70.0", - None, - None, + Some("@anthropic-ai/claude-agent-sdk"), + Some("0.3.232"), "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", ), "codex" => ( "gpt-5.6-sol", "@agentclientprotocol/codex-acp", "1.6.2", - None, - None, - "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79", + Some("@openai/codex"), + Some("0.148.0"), + "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400", ), "pi" => return Err(DurableRunnerError::invalid( "ACPX agent pi is not executable through the verified runnerd provider boundary", @@ -130,13 +186,21 @@ impl AcpxProviderDescriptor { "ACPX permission mode must be pinned by runner policy", )); } - if self.run_id != context.run_id - || self.normalized_session_id != context.normalized_session_id - { + if self.normalized_session_id != context.normalized_session_id { return Err(DurableRunnerError::invalid( "ACPX descriptor identity conflicts with the durable runner identity", )); } + if self.run_id.is_empty() + || self.run_id.len() > 160 + || !self.run_id.bytes().all(|value| { + value.is_ascii_alphanumeric() || matches!(value, b'.' | b'_' | b':' | b'-') + }) + { + return Err(DurableRunnerError::invalid( + "ACPX descriptor run identity is malformed", + )); + } if self.instructions.len() > 1024 * 1024 || self.instructions.contains('\0') { return Err(DurableRunnerError::invalid( "ACPX instructions exceed their bounded contract", @@ -150,6 +214,16 @@ impl AcpxProviderDescriptor { Ok(()) } + fn validate(&self, context: &AcpxEventProjectionContext) -> Result<(), DurableRunnerError> { + self.validate_session(context)?; + if self.run_id != context.run_id { + return Err(DurableRunnerError::invalid( + "ACPX descriptor identity conflicts with the durable runner identity", + )); + } + Ok(()) + } + fn session_config( &self, tool_set: AuthorizedToolSet, @@ -213,7 +287,8 @@ impl AcpxProviderDescriptor { let verified_args = launch_profile .args .iter() - .map(|argument| { + .enumerate() + .map(|(index, argument)| { let path = Path::new(argument); if !path.is_absolute() { return Ok(VerifiedProcessArgument::Literal(argument.clone())); @@ -221,7 +296,13 @@ impl AcpxProviderDescriptor { verified .get(path) .cloned() - .map(VerifiedProcessArgument::Artifact) + .map(|artifact| { + if index == 0 { + VerifiedProcessArgument::CommonJsArtifact(artifact) + } else { + VerifiedProcessArgument::Artifact(artifact) + } + }) .ok_or_else(|| { DurableRunnerError::invalid( "ACPX runner launch profile does not authenticate an absolute argument", @@ -232,7 +313,10 @@ impl AcpxProviderDescriptor { Ok(AcpxSidecarTransportConfig { command: launch_profile.command.clone(), args: launch_profile.args.clone(), - verified_launch: Some(VerifiedProcessLaunch::new(command, verified_args)), + verified_launch: Some( + VerifiedProcessLaunch::new(command, verified_args) + .with_inherited_runtime_executable(), + ), request_timeout: Duration::from_secs(30), shutdown_grace: Duration::from_secs(2), }) @@ -272,6 +356,8 @@ struct AcpxDurableState { #[serde(default)] active_turn_id: Option, #[serde(default)] + provider_exit_unconfirmed: bool, + #[serde(default)] semantic_result: Option, #[serde(default)] pending_events: VecDeque, @@ -293,6 +379,7 @@ impl AcpxDurableState { tool_set, identity: None, active_turn_id: None, + provider_exit_unconfirmed: false, semantic_result: None, pending_events: VecDeque::new(), next_event_sequence: initial_event_sequence(), @@ -304,7 +391,12 @@ impl AcpxDurableState { context: &AcpxEventProjectionContext, expected_launch_profile_digest: &str, ) -> Result<(), DurableRunnerError> { - self.descriptor.validate(context)?; + // The normalized session is the durable provider boundary. A settled + // provider can outlive one heartbeat run and be attached to the next, + // so its persisted descriptor legitimately carries the prior run ID + // until run.attach rotates authority. Fresh command descriptors still + // use validate(), which binds them to the current run. + self.descriptor.validate_session(context)?; if self.launch_profile_digest != expected_launch_profile_digest { return Err(DurableRunnerError::invalid( "ACPX durable launch profile digest does not match runner startup", @@ -334,6 +426,9 @@ impl AcpxDurableState { }) || (matches!(self.lifecycle.as_str(), "turn_starting" | "turn_active") != self.active_turn_id.is_some()) + || (self.provider_exit_unconfirmed + && (!matches!(self.lifecycle.as_str(), "prepared" | "closed") + || self.identity.is_none())) || self .semantic_result .as_ref() @@ -484,6 +579,12 @@ impl AcpxCommandExecutor { let Some(state) = self.state.as_ref() else { return Ok(()); }; + // A replacement runner for a new heartbeat run must first execute + // run.attach. Do not restart the provider under the prior run authority + // or emit prior-run events into the new run while attachment is pending. + if state.descriptor.run_id != self.context.run_id { + return Ok(()); + } if !matches!( state.lifecycle.as_str(), "session_open" | "turn_starting" | "turn_active" | "suspended" @@ -499,6 +600,7 @@ impl AcpxCommandExecutor { .expect("ACPX state remains available during recovery"); state.lifecycle = "closed".to_owned(); state.active_turn_id = None; + state.provider_exit_unconfirmed = true; state.push(NormalizedProviderEvent { event_type: "turn.failed".to_owned(), priority: EventPriority::P0, @@ -508,7 +610,7 @@ impl AcpxCommandExecutor { "status": "failed", "providerTerminalObserved": false, "code": "acpx_active_turn_recovery_closed", - "providerShutdownFailed": false, + "providerShutdownFailed": true, }), })?; state.push(NormalizedProviderEvent { @@ -666,6 +768,7 @@ impl AcpxCommandExecutor { .iter() .all(|event| event.event_type == "session.resumed"); if state.lifecycle == "closed" + || state.provider_exit_unconfirmed || state.identity.is_none() || state.active_turn_id.is_some() || !only_recovery_notice_pending @@ -698,6 +801,15 @@ impl AcpxCommandExecutor { } fn open_session(&mut self) -> Result { + if self + .state + .as_ref() + .is_some_and(|state| state.provider_exit_unconfirmed) + { + return Err(DurableRunnerError::invalid( + "ACPX provider lifetime cleanup is not yet proven", + )); + } if self.session.is_none() { let recovering = self .state @@ -868,6 +980,73 @@ impl AcpxCommandExecutor { }))) } + fn stop_turn_for_suspension( + &mut self, + reason: &str, + ) -> Result { + let turn_id = self + .state + .as_ref() + .and_then(|state| state.active_turn_id.clone()); + let Some(turn_id) = turn_id else { + return Ok(CommandExecution::result(json!({ + "status": "already_settled", + "reason": reason, + }))); + }; + let provider_lifetime_fence_candidates = { + let session = self + .session + .as_mut() + .ok_or_else(|| DurableRunnerError::invalid("ACPX session is unavailable"))?; + let candidates = session.identity().provider_lifetime_fence_candidates; + session + .terminate_active_turn_for_suspension(&turn_id) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to terminate ACPX turn at the suspension boundary: {error}" + )) + })?; + candidates + }; + // Process-group termination reaps the sidecar leader and its ordinary + // descendants, but an escaped provider or guardian can outlive that + // group. Require the inherited listener quorum before making this + // durable session attachable, and retain it through the state write. + self.session = None; + let state = self + .state + .as_mut() + .expect("ACPX state remains available after provider termination"); + state.active_turn_id = None; + // Persist a non-attachable, recoverable boundary before the fallible + // lifetime proof. Terminal cleanup can then retry a timed-out fence + // without reviving the stopped provider. + state.lifecycle = "prepared".to_owned(); + state.provider_exit_unconfirmed = true; + self.save_state()?; + let _provider_lifetime_fence = + await_provider_lifetime_fence(provider_lifetime_fence_candidates)?; + let state = self + .state + .as_mut() + .expect("ACPX state remains available after provider termination"); + state.provider_exit_unconfirmed = false; + if let Err(error) = self.save_state() { + self.state + .as_mut() + .expect("ACPX state remains available after save failure") + .provider_exit_unconfirmed = true; + return Err(error); + } + Ok(CommandExecution::result(json!({ + "status": "stopped", + "providerTurnId": turn_id, + "reason": reason, + "providerExitConfirmed": true, + }))) + } + fn resolve_request(&mut self, payload: &Value) -> Result { let request_id = payload .get("requestId") @@ -980,6 +1159,23 @@ impl AcpxCommandExecutor { state.active_turn_id = None; self.session = None; self.save_state()?; + } else if self.state.as_ref().is_some_and(|state| { + state.lifecycle == "prepared" + && state.identity.is_some() + && !state.provider_exit_unconfirmed + && state.active_turn_id.is_none() + }) { + // turn.stop deliberately leaves an already-reaped provider in a + // non-recoverable `prepared` state while runner.drain crosses the + // durable event barrier. Once the following runner.suspend reaches + // this boundary, publish the exact stopped checkpoint as + // recoverable instead of reporting a no-op success that can never + // emit session.resumed in the replacement runner. + self.state + .as_mut() + .expect("ACPX stopped provider state remains available") + .lifecycle = "suspended".to_owned(); + self.save_state()?; } Ok(CommandExecution::result(json!({"status": "completed"}))) } @@ -1064,6 +1260,16 @@ impl AcpxCommandExecutor { impl CommandExecutor for AcpxCommandExecutor { fn execute(&mut self, command: &Command) -> Result { self.restore()?; + if command.command_type != "run.attach" + && self + .state + .as_ref() + .is_some_and(|state| state.descriptor.run_id != self.context.run_id) + { + return Err(DurableRunnerError::invalid( + "ACPX durable session requires run.attach before commands from a new run", + )); + } match command.command_type.as_str() { "run.prepare" => self.prepare(&command.payload), "run.attach" => { @@ -1087,9 +1293,8 @@ impl CommandExecutor for AcpxCommandExecutor { "code": "provider_command_unavailable", "message": "ACPX does not support steering an active turn", }))), - "turn.interrupt" | "turn.stop" | "run.cancel" => { - self.interrupt_turn(&command.command_type) - } + "turn.interrupt" | "run.cancel" => self.interrupt_turn(&command.command_type), + "turn.stop" => self.stop_turn_for_suspension(&command.command_type), "request.resolve" => self.resolve_request(&command.payload), "semantic_tool.result" => self.deliver_tool_result(&command.payload), "session.snapshot" => self.snapshot(), @@ -1111,6 +1316,14 @@ impl CommandExecutor for AcpxCommandExecutor { } fn poll_events(&mut self) -> Result, DurableRunnerError> { + self.restore()?; + if self + .state + .as_ref() + .is_some_and(|state| state.descriptor.run_id != self.context.run_id) + { + return Ok(Vec::new()); + } self.poll_provider()?; Ok(self .state @@ -1139,6 +1352,30 @@ impl CommandExecutor for AcpxCommandExecutor { } fn shutdown(&mut self) -> Result<(), DurableRunnerError> { + // A replacement durable runner may reach terminal reconciliation + // before any provider command or event poll. Restore the persisted + // session first so cleanup cannot succeed merely because this process + // has no in-memory session yet. + self.restore()?; + let provider_exit_unconfirmed = self + .state + .as_ref() + .is_some_and(|state| state.provider_exit_unconfirmed); + // The prior provider, guardian, and sidecar inherit two listeners from + // this exact three-port set. A replacement can bind any two only after + // the original lifetime has lost quorum. Keep the acquired quorum live + // through the durable state update so no successor can race the proof. + let _provider_lifetime_fence = if self.session.is_none() && provider_exit_unconfirmed { + let candidates = self + .state + .as_ref() + .and_then(|state| state.identity.as_ref()) + .expect("provider cleanup state has a validated identity") + .provider_lifetime_fence_candidates; + Some(acquire_provider_lifetime_fence(candidates)?) + } else { + None + }; if let Some(session) = self.session.as_mut() { session .shutdown("runner process shutdown") @@ -1147,6 +1384,20 @@ impl CommandExecutor for AcpxCommandExecutor { })?; } self.session = None; + if provider_exit_unconfirmed { + let state = self + .state + .as_mut() + .expect("ACPX state exists for replacement cleanup"); + state.provider_exit_unconfirmed = false; + if let Err(error) = self.save_state() { + self.state + .as_mut() + .expect("ACPX state remains available after save failure") + .provider_exit_unconfirmed = true; + return Err(error); + } + } Ok(()) } } @@ -1305,21 +1556,26 @@ mod tests { } fn descriptor(agent: &str) -> Value { - let (model, package, version, digest) = if agent == "claude" { - ( - "claude-sonnet-5", - "@agentclientprotocol/claude-agent-acp", - "0.70.0", - "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", - ) - } else { - ( - "gpt-5.6-sol", - "@agentclientprotocol/codex-acp", - "1.6.2", - "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79", - ) - }; + let (model, package, version, runtime_package, runtime_version, digest) = + if agent == "claude" { + ( + "claude-sonnet-5", + "@agentclientprotocol/claude-agent-acp", + "0.70.0", + json!("@anthropic-ai/claude-agent-sdk"), + json!("0.3.232"), + "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", + ) + } else { + ( + "gpt-5.6-sol", + "@agentclientprotocol/codex-acp", + "1.6.2", + json!("@openai/codex"), + json!("0.148.0"), + "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400", + ) + }; json!({ "kind": "acpx", "provider": "acpx", @@ -1330,8 +1586,8 @@ mod tests { "acpxVersion": "0.13.1", "agentServerPackage": package, "agentServerVersion": version, - "agentRuntimePackage": null, - "agentRuntimeVersion": null, + "agentRuntimePackage": runtime_package, + "agentRuntimeVersion": runtime_version, "commandDigest": digest, "sidecarCommand": "/qualified/node", "sidecarArgs": ["/qualified/acpx-sidecar.js"], @@ -1372,6 +1628,7 @@ mod tests { requested_model: "gpt-5.6-sol".to_owned(), effective_model: "gpt-5.6-sol".to_owned(), permission_mode: Some(AcpxPermissionMode::ApproveReads), + provider_lifetime_fence_candidates: [60_001, 60_002, 60_003], }; let payload = replacement_continuity_payload(&identity, 41, 42, "turn-2"); @@ -1399,7 +1656,7 @@ mod tests { fn binds_sidecar_paths_arguments_and_contents_to_the_runner_profile() { let directory = temporary_directory("launch-binding"); let command = directory.join("node"); - let sidecar = directory.join("sidecar.js"); + let sidecar = directory.join("sidecar.cjs"); write_artifact(&command, b"qualified node", true); write_artifact(&sidecar, b"qualified sidecar", false); let args = vec![sidecar.to_string_lossy().into_owned()]; @@ -1416,7 +1673,11 @@ mod tests { let transport = descriptor.verified_transport(Some(&profile)).unwrap(); assert_eq!(transport.command, profile.command); assert_eq!(transport.args[0], sidecar.to_string_lossy()); - assert!(transport.verified_launch.is_some()); + let verified_launch = transport.verified_launch.as_ref().unwrap(); + assert!(matches!( + verified_launch.arguments().first(), + Some(VerifiedProcessArgument::CommonJsArtifact(_)) + )); let mut drifted_path = descriptor.clone(); drifted_path.sidecar_command = directory.join("other-node"); @@ -1457,6 +1718,102 @@ mod tests { fs::remove_dir_all(directory).unwrap(); } + #[cfg(unix)] + #[test] + fn restores_settled_session_for_explicit_run_attachment_only() { + let directory = temporary_directory("cross-run-attach"); + let runtime = directory.join("runtime"); + let workspace = directory.join("workspace"); + fs::create_dir_all(&runtime).unwrap(); + fs::create_dir_all(&workspace).unwrap(); + fs::set_permissions(&runtime, fs::Permissions::from_mode(0o700)).unwrap(); + fs::set_permissions(&workspace, fs::Permissions::from_mode(0o700)).unwrap(); + let marker = directory.join("provider-started"); + let command = directory.join("sidecar"); + write_artifact( + &command, + format!("#!/bin/sh\ntouch '{}'\n", marker.display()).as_bytes(), + true, + ); + let launch_profile = AcpxLaunchProfile { + authority_digest: format!("sha256:{}", "d".repeat(64)), + command: command.clone(), + args: Vec::new(), + artifacts: vec![artifact(&command)], + }; + let mut descriptor_value = descriptor("codex"); + descriptor_value["sidecarCommand"] = json!(command); + descriptor_value["sidecarArgs"] = json!([]); + descriptor_value["runtimeDirectory"] = json!(runtime); + descriptor_value["cwd"] = json!(workspace); + let original_descriptor: AcpxProviderDescriptor = + serde_json::from_value(descriptor_value.clone()).unwrap(); + let identity = AcpxProviderSessionIdentity { + kind: "acpx".to_owned(), + normalized_session_id: "session-1".to_owned(), + acpx_record_id: "record-1".to_owned(), + backend_session_id: "backend-1".to_owned(), + agent_session_id: "agent-1".to_owned(), + profile_digest: original_descriptor.command_digest.clone(), + workspace_digest: format!("sha256:{}", "a".repeat(64)), + requested_model: original_descriptor.model.clone(), + effective_model: original_descriptor.model.clone(), + permission_mode: Some(original_descriptor.permission_mode), + provider_lifetime_fence_candidates: [60_001, 60_002, 60_003], + }; + let operations = Vec::new(); + let tool_set = AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }; + let launch_profile_digest = launch_profile.canonical_digest().unwrap(); + let mut state = AcpxDurableState::new(original_descriptor, tool_set, launch_profile_digest); + state.lifecycle = "suspended".to_owned(); + state.identity = Some(identity); + let original_config = test_config(&directory, Some(launch_profile.clone())); + let mut original = AcpxCommandExecutor::with_runner_config(&directory, &original_config); + original.state = Some(state); + original.save_state().unwrap(); + + let mut wrong_session_config = original_config.clone(); + wrong_session_config.run_id = "run-2".to_owned(); + wrong_session_config.normalized_session_id = "session-2".to_owned(); + let mut wrong_session = + AcpxCommandExecutor::with_runner_config(&directory, &wrong_session_config); + assert!(wrong_session.restore().is_err()); + + let mut attached_config = original_config.clone(); + attached_config.run_id = "run-2".to_owned(); + let mut attached = AcpxCommandExecutor::with_runner_config(&directory, &attached_config); + attached.restore().unwrap(); + assert!(!marker.exists()); + let non_attach_error = attached + .execute(&Command { + schema: "paperclip.prp.command.v1".to_owned(), + command_id: "command-before-attach".to_owned(), + controller_seq: 1, + command_type: "session.snapshot".to_owned(), + issued_at: "2026-09-01T00:00:00.000Z".to_owned(), + deadline_at: None, + precondition: None, + payload: json!({}), + }) + .unwrap_err(); + assert!(non_attach_error + .to_string() + .contains("requires run.attach before commands from a new run")); + + descriptor_value["runId"] = json!("run-2"); + attached + .attach_run(&json!({"provider": descriptor_value})) + .unwrap(); + assert_eq!(attached.state.as_ref().unwrap().descriptor.run_id, "run-2"); + assert!(!marker.exists()); + fs::remove_dir_all(directory).unwrap(); + } + #[cfg(unix)] #[test] fn active_turn_recovery_closes_without_starting_the_provider() { @@ -1486,6 +1843,8 @@ mod tests { value["runtimeDirectory"] = json!(runtime); value["cwd"] = json!(workspace); let descriptor: AcpxProviderDescriptor = serde_json::from_value(value).unwrap(); + let (provider_lifetime_fence_candidates, original_lifetime_fence) = + reserve_provider_lifetime_fence(); let identity = AcpxProviderSessionIdentity { kind: "acpx".to_owned(), normalized_session_id: "session-1".to_owned(), @@ -1497,6 +1856,7 @@ mod tests { requested_model: descriptor.model.clone(), effective_model: descriptor.model.clone(), permission_mode: Some(descriptor.permission_mode), + provider_lifetime_fence_candidates, }; let operations = Vec::new(); let tool_set = AuthorizedToolSet { @@ -1572,7 +1932,137 @@ mod tests { assert!(!marker.exists()); let events = recovered.poll_events().unwrap(); assert_eq!(events[0].event_type, "turn.failed"); + assert_eq!(events[0].payload["providerShutdownFailed"], true); assert_eq!(events[1].event_type, "run.terminal"); + let cleanup_error = recovered + .shutdown() + .expect_err("cleanup must not succeed while the original lifetime remains active"); + assert!(cleanup_error + .to_string() + .contains("original provider lifetime remains active")); + let persisted: AcpxDurableState = serde_json::from_slice( + &fs::read(recovered.state_path()).expect("read retained ACPX state"), + ) + .expect("parse retained ACPX state"); + assert!(persisted.provider_exit_unconfirmed); + assert!(!marker.exists()); + + drop(original_lifetime_fence); + recovered.shutdown().unwrap(); + let persisted: AcpxDurableState = serde_json::from_slice( + &fs::read(recovered.state_path()).expect("read cleared ACPX state"), + ) + .expect("parse cleared ACPX state"); + assert!(!persisted.provider_exit_unconfirmed); + assert!(!marker.exists()); fs::remove_dir_all(directory).unwrap(); } + + #[test] + fn suspension_waits_for_the_original_provider_lifetime_quorum() { + let (candidates, original_lifetime_fence) = reserve_provider_lifetime_fence(); + let releaser = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(25)); + drop(original_lifetime_fence); + }); + + let confirmed = await_provider_lifetime_fence(candidates) + .expect("suspension must wait until the original lifetime loses quorum"); + assert_eq!(confirmed.len(), 2); + releaser.join().unwrap(); + } + + #[test] + fn unconfirmed_suspension_state_becomes_recoverable_only_after_cleanup_and_suspend() { + let directory = temporary_directory("suspension-fence-pending"); + let (provider_lifetime_fence_candidates, original_lifetime_fence) = + reserve_provider_lifetime_fence(); + let sidecar = directory.join("sidecar"); + write_artifact(&sidecar, b"qualified sidecar", true); + let launch_profile = AcpxLaunchProfile { + authority_digest: format!("sha256:{}", "d".repeat(64)), + command: sidecar.clone(), + args: Vec::new(), + artifacts: vec![artifact(&sidecar)], + }; + let provider_descriptor: AcpxProviderDescriptor = + serde_json::from_value(descriptor("codex")).unwrap(); + let operations = Vec::new(); + let tool_set = AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }; + let launch_profile_digest = launch_profile.canonical_digest().unwrap(); + let mut state = AcpxDurableState::new( + provider_descriptor.clone(), + tool_set, + launch_profile_digest.clone(), + ); + state.lifecycle = "prepared".to_owned(); + state.identity = Some(AcpxProviderSessionIdentity { + kind: "acpx".to_owned(), + normalized_session_id: "session-1".to_owned(), + acpx_record_id: "record-1".to_owned(), + backend_session_id: "backend-1".to_owned(), + agent_session_id: "agent-1".to_owned(), + profile_digest: provider_descriptor.command_digest.clone(), + workspace_digest: format!("sha256:{}", "a".repeat(64)), + requested_model: provider_descriptor.model.clone(), + effective_model: provider_descriptor.model.clone(), + permission_mode: Some(provider_descriptor.permission_mode), + provider_lifetime_fence_candidates, + }); + state.provider_exit_unconfirmed = true; + state.validate(&context(), &launch_profile_digest).unwrap(); + + let config = test_config(&directory, Some(launch_profile)); + let mut executor = AcpxCommandExecutor::with_runner_config(&directory, &config); + executor.state = Some(state); + let open_error = executor.open_session().unwrap_err(); + assert!(open_error + .to_string() + .contains("provider lifetime cleanup is not yet proven")); + let attach_error = executor + .attach_run(&json!({"provider": descriptor("codex")})) + .unwrap_err(); + assert!(attach_error + .to_string() + .contains("requires the same settled ACPX provider profile and session")); + drop(original_lifetime_fence); + executor.shutdown().unwrap(); + let recovered = executor.state.as_ref().unwrap(); + assert_eq!(recovered.lifecycle, "prepared"); + assert!(!recovered.provider_exit_unconfirmed); + + executor.suspend().unwrap(); + let suspended: AcpxDurableState = serde_json::from_slice( + &fs::read(executor.state_path()).expect("read suspended ACPX state"), + ) + .expect("parse suspended ACPX state"); + assert_eq!(suspended.lifecycle, "suspended"); + assert!(!suspended.provider_exit_unconfirmed); + fs::remove_dir_all(directory).unwrap(); + } + + fn reserve_provider_lifetime_fence() -> ([u16; 3], Vec) { + let mut listeners = Vec::new(); + for port in 49_152..=u16::MAX { + if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) { + listeners.push(listener); + if listeners.len() == 3 { + break; + } + } + } + assert_eq!(listeners.len(), 3, "reserve provider lifetime ports"); + let candidates = [ + listeners[0].local_addr().unwrap().port(), + listeners[1].local_addr().unwrap().port(), + listeners[2].local_addr().unwrap().port(), + ]; + drop(listeners.pop()); + (candidates, listeners) + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_checkpoint.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_checkpoint.rs index 68e6526388..0e4c332ce3 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_checkpoint.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_checkpoint.rs @@ -18,7 +18,7 @@ use crate::durable::{ use crate::local_runner::LocalRunnerError; use crate::stable_identity::{is_stable_id, SHORT_STABLE_ID_CHARS}; -const CHECKPOINT_SCHEMA: &str = "paperclip.runner.acpx-suspension-checkpoint.v1"; +const CHECKPOINT_SCHEMA: &str = "paperclip.runner.acpx-suspension-checkpoint.v2"; const CHECKPOINT_DIRECTORY: &str = "acpx-provider"; const CHECKPOINT_FILE: &str = "suspension-checkpoint.json"; const MAX_CHECKPOINT_BYTES: u64 = 1024 * 1024; @@ -48,6 +48,7 @@ struct PersistedAcpxProviderSessionIdentity { requested_model: String, effective_model: String, permission_mode: AcpxPermissionMode, + provider_lifetime_fence_candidates: [u16; 3], } impl PersistedAcpxProviderSessionIdentity { @@ -69,6 +70,7 @@ impl PersistedAcpxProviderSessionIdentity { requested_model: identity.requested_model, effective_model: identity.effective_model, permission_mode, + provider_lifetime_fence_candidates: identity.provider_lifetime_fence_candidates, }) } @@ -84,6 +86,7 @@ impl PersistedAcpxProviderSessionIdentity { requested_model: self.requested_model.clone(), effective_model: self.effective_model.clone(), permission_mode: Some(self.permission_mode), + provider_lifetime_fence_candidates: self.provider_lifetime_fence_candidates, } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs index 793cc2c2bb..a6dc391d47 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs @@ -47,6 +47,7 @@ pub struct AcpxProviderSessionIdentity { pub effective_model: String, #[serde(default)] pub permission_mode: Option, + pub provider_lifetime_fence_candidates: [u16; 3], } #[derive(Clone, Debug)] @@ -186,6 +187,21 @@ impl AcpxProviderSessionIdentity { ))); } } + if self + .provider_lifetime_fence_candidates + .iter() + .any(|port| *port < 49_152) + || self.provider_lifetime_fence_candidates[0] + == self.provider_lifetime_fence_candidates[1] + || self.provider_lifetime_fence_candidates[0] + == self.provider_lifetime_fence_candidates[2] + || self.provider_lifetime_fence_candidates[1] + == self.provider_lifetime_fence_candidates[2] + { + return Err(LocalRunnerError::invalid( + "ACPX provider lifetime fence candidates are invalid", + )); + } Ok(()) } } @@ -679,6 +695,31 @@ impl AcpxProviderSession { } } + /// Reaps an active provider generation at a controller-owned suspension + /// boundary without waiting for the provider's graceful close protocol. + /// + /// A governed Paperclip result can settle the run while the model is still + /// waiting for its semantic-tool callback to unwind. In that state the + /// ordinary sidecar close path may wait for the callback longer than the + /// server process that owns this runner. Process-group termination closes + /// this session's transport authority. The caller must additionally + /// acquire the identity's inherited lifetime-fence quorum before + /// persisting the durable session as attachable. + pub fn terminate_active_turn_for_suspension( + &mut self, + turn_id: &str, + ) -> Result<(), LocalRunnerError> { + self.ensure_open()?; + validate_stable_id(turn_id, DURABLE_STABLE_ID_CHARS, "ACPX turn id")?; + if self.state.active_turn_id() != Some(turn_id) { + return Err(LocalRunnerError::invalid( + "ACPX suspension termination named a stale or inactive turn", + )); + } + self.closed = true; + self.terminate_transport() + } + fn terminate_transport(&mut self) -> Result<(), LocalRunnerError> { if self.transport_terminated { return Ok(()); @@ -825,6 +866,15 @@ fn validate_reserved_terminal_value( } fn reserved_terminal_tool_bridge() -> Result { + let tool_set = reserved_terminal_tool_set()?; + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tool_set).map_err(|error| { + LocalRunnerError::invalid(format!("ACPX reserved terminal tools are invalid: {error}")) + })?; + Ok(bridge) +} + +fn reserved_terminal_tool_set() -> Result { let result_schema: Value = serde_json::from_str(include_str!( "../../../../protocol/schemas/result.schema.json" )) @@ -848,18 +898,33 @@ fn reserved_terminal_tool_bridge() -> Result Vec { + // The authenticated TypeScript bridge installs the trusted terminal tools + // itself and rejects caller attempts to replace either reserved schema. + // Project the durable Rust catalog into the bridge's public tool shape; + // forwarding AuthorizedTool verbatim would expose `operationId` where the + // bridge requires `name` and reject every non-empty catalog at admission. + // Rust keeps its independent reserved receipt ledger and validates terminal + // values after the sidecar reports them. + run_tool_set + .operations + .iter() + .map(|tool| { + json!({ + "name": tool.operation_id, + "description": tool.description, + "inputSchema": tool.input_schema, + }) }) - .map_err(|error| { - LocalRunnerError::invalid(format!("ACPX reserved terminal tools are invalid: {error}")) - })?; - Ok(bridge) + .collect() } fn validate_prp_run_result(value: &Value) -> Result<(), LocalRunnerError> { @@ -891,6 +956,7 @@ fn bootstrap( transport: &mut AcpxSidecarTransport, config: &AcpxProviderSessionConfig, ) -> Result<(AcpxProviderSessionIdentity, AcpxProviderState), LocalRunnerError> { + let sidecar_tools = sidecar_run_tool_operations(&config.tool_set); let initialized = transport.request( GeneratedAcpxSidecarCommand::Initialize, json!({"agent": config.agent, "model": config.model}), @@ -909,7 +975,7 @@ fn bootstrap( "permissionModePinned": config.permission_mode_pinned, "systemInstructions": config.system_instructions, "runtimeContext": Value::Null, - "tools": config.tool_set.operations, + "tools": &sidecar_tools, "expectedIdentity": config.expected_identity, }), )?; @@ -920,7 +986,7 @@ fn bootstrap( json!({ "runId": config.run_id, "catalogRevision": config.catalog_revision, - "tools": config.tool_set.operations, + "tools": &sidecar_tools, }), )?; if attached.get("runId").and_then(Value::as_str) != Some(config.run_id.as_str()) @@ -1119,3 +1185,44 @@ fn with_cleanup_error( )), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sidecar_catalog_leaves_reserved_terminal_tools_to_the_trusted_bridge() { + let operations = vec![AuthorizedTool { + operation_id: "get_task_context".to_owned(), + version: 1, + description: "Read the task context.".to_owned(), + input_schema: json!({"type":"object"}), + response_schema: json!({"type":"object"}), + }]; + let run_tool_set = AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }; + + let sidecar_tools = sidecar_run_tool_operations(&run_tool_set); + assert_eq!( + sidecar_tools + .iter() + .filter_map(|tool| tool.get("name").and_then(Value::as_str)) + .collect::>(), + vec!["get_task_context"] + ); + assert_eq!(run_tool_set.operations.len(), 1); + assert_eq!( + sidecar_tools[0], + json!({ + "name": "get_task_context", + "description": "Read the task context.", + "inputSchema": {"type":"object"}, + }) + ); + assert!(sidecar_tools[0].get("operationId").is_none()); + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index 8d4a0ea456..55f7b25335 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -119,6 +119,8 @@ impl AcpxSidecarTransport { "RUST_BACKTRACE", "PAPERCLIP_NATIVE_MCP_NAME", "PAPERCLIP_NATIVE_MCP_URL", + "PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT", + "PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST", ]; keys.extend_from_slice(credential_keys); Self::start_with_environment_keys(config, &keys) @@ -277,9 +279,10 @@ impl AcpxSidecarTransport { let error = response.error.expect("failed response has validated error"); return Ok(CommandOutcome::Rejected(LocalRunnerError::invalid( format!( - "ACPX sidecar command {} was rejected (retryable={})", + "ACPX sidecar command {} was rejected (retryable={}, classification={})", command.as_str(), error.retryable, + response_error_classification(&error), ), ))); } @@ -595,6 +598,75 @@ fn redact_diagnostic(value: &str) -> String { } } +fn response_error_classification(error: &ResponseError) -> &'static str { + match error.code.as_str() { + "ACP_MODEL_UNSUPPORTED" => return "requested_model_unsupported", + "AGENT_STARTUP_FAILED" => return "agent_startup_failed", + "AGENT_STARTUP_FAILED.UNVERIFIED_MODULE" => return "agent_startup_unverified_module", + "AGENT_STARTUP_FAILED.MODULE_NOT_FOUND" => return "agent_startup_module_not_found", + "AGENT_STARTUP_FAILED.PERMISSION_DENIED" => return "agent_startup_permission_denied", + "AGENT_STARTUP_FAILED.FILE_NOT_FOUND" => return "agent_startup_file_not_found", + "AGENT_STARTUP_FAILED.SYNTAX_ERROR" => return "agent_startup_syntax_error", + "AGENT_STARTUP_FAILED.INVALID_ARGUMENT" => return "agent_startup_invalid_argument", + "AGENT_STARTUP_FAILED.NO_STDERR" => return "agent_startup_no_stderr", + "AGENT_STARTUP_FAILED.SIGNAL" => return "agent_startup_signal", + "AGENT_STARTUP_FAILED.EXIT_NONZERO" => return "agent_startup_exit_nonzero", + "AGENT_STARTUP_FAILED.OTHER" => return "agent_startup_other", + "AGENT_DISCONNECTED" => return "agent_disconnected", + "AUTH_REQUIRED" => return "authentication_required", + "SESSION_RESUME_REQUIRED" => return "session_resume_required", + "SESSION_MODE_REPLAY_FAILED" => return "session_mode_replay_failed", + "SESSION_MODEL_REPLAY_FAILED" => return "session_model_replay_failed", + "SESSION_CONFIG_OPTION_REPLAY_FAILED" => return "session_config_option_replay_failed", + "CLAUDE_ACP_SESSION_CREATE_TIMEOUT" => return "claude_session_create_timeout", + "ACPX_SESSION_HANDSHAKE_TIMEOUT" => return "session_handshake_timeout", + "ACPX_SESSION_ENSURE_FAILED" => return "session_ensure_failed", + "ACPX_SESSION_ENSURE_TYPE_ERROR" => return "session_ensure_type_error", + "ACPX_SESSION_ENSURE_NON_ERROR" => return "session_ensure_non_error", + "ACP_SESSION_INIT_FAILED" => return "acp_session_init_failed", + "NO_SESSION" => return "acpx_no_session", + "TIMEOUT" => return "acpx_timeout", + "PERMISSION_DENIED" => return "acpx_permission_denied", + "PERMISSION_PROMPT_UNAVAILABLE" => return "acpx_permission_prompt_unavailable", + "RUNTIME" => return "acpx_runtime_failure", + "USAGE" => return "acpx_usage_failure", + "ACPX_RUNTIME_ADMISSION_VERIFICATION_TIMEOUT" => { + return "runtime_admission_verification_timeout" + } + "ACPX_SIDECAR_STATUS_READ_TIMEOUT" => return "session_status_read_timeout", + "ACPX_PERSISTED_SESSION_MISSING" => return "persisted_session_missing", + "ACPX_PERSISTED_SESSION_IDENTITY_MISMATCH" => return "persisted_session_identity_mismatch", + "ACPX_MODEL_STATUS_UNAVAILABLE" => return "model_status_unavailable", + "ACPX_MODEL_SELECTION_UNAVAILABLE" => return "model_selection_unavailable", + "ACPX_EFFECTIVE_MODEL_MISMATCH" => return "effective_model_mismatch", + _ => {} + } + match error.message.as_str() { + "ACPX session handshake exceeded its admission deadline" => "session_handshake_timeout", + "ACPX provider lifetime guardian exited before ownership transfer" => { + "provider_guardian_exit" + } + "ACPX provider lifetime guardian ownership timed out" => "provider_guardian_timeout", + "ACPX session handshake and runtime cleanup failed" => "session_handshake_cleanup_failed", + "ACPX runtime initialization and cleanup failed" => "runtime_initialization_cleanup_failed", + _ if error + .message + .starts_with("ACP agent exited before initialize completed") => + { + "agent_startup_failed" + } + _ if error.message.starts_with("Failed to spawn agent command:") => "agent_spawn_failed", + _ if error + .message + .starts_with("ACP agent disconnected during request") => + { + "agent_disconnected" + } + _ if error.message.starts_with("Authentication required") => "authentication_required", + _ => "unclassified", + } +} + #[cfg(test)] mod tests { use super::*; @@ -640,4 +712,88 @@ mod tests { assert!(!message.contains("Q7Z9"), "error leaked input: {message}"); } } + + #[test] + fn classifies_only_allowlisted_internal_sidecar_failures() { + let error = |code: &str, message: &str| ResponseError { + code: code.to_owned(), + message: message.to_owned(), + retryable: false, + }; + assert_eq!( + response_error_classification(&error( + "acpx_sidecar_command_failed", + "ACPX session handshake exceeded its admission deadline", + )), + "session_handshake_timeout" + ); + assert_eq!( + response_error_classification(&error( + "ACPX_SESSION_HANDSHAKE_TIMEOUT", + "bounded provider admission failed", + )), + "session_handshake_timeout" + ); + let admission_failures = [ + ( + "ACPX_RUNTIME_ADMISSION_VERIFICATION_TIMEOUT", + "runtime_admission_verification_timeout", + ), + ("ACPX_SESSION_ENSURE_FAILED", "session_ensure_failed"), + ( + "ACPX_SESSION_ENSURE_TYPE_ERROR", + "session_ensure_type_error", + ), + ("ACPX_SESSION_ENSURE_NON_ERROR", "session_ensure_non_error"), + ("ACP_SESSION_INIT_FAILED", "acp_session_init_failed"), + ("NO_SESSION", "acpx_no_session"), + ("TIMEOUT", "acpx_timeout"), + ("PERMISSION_DENIED", "acpx_permission_denied"), + ( + "PERMISSION_PROMPT_UNAVAILABLE", + "acpx_permission_prompt_unavailable", + ), + ("RUNTIME", "acpx_runtime_failure"), + ("USAGE", "acpx_usage_failure"), + ( + "ACPX_SIDECAR_STATUS_READ_TIMEOUT", + "session_status_read_timeout", + ), + ( + "ACPX_PERSISTED_SESSION_MISSING", + "persisted_session_missing", + ), + ( + "ACPX_PERSISTED_SESSION_IDENTITY_MISMATCH", + "persisted_session_identity_mismatch", + ), + ("ACPX_MODEL_STATUS_UNAVAILABLE", "model_status_unavailable"), + ( + "ACPX_MODEL_SELECTION_UNAVAILABLE", + "model_selection_unavailable", + ), + ("ACPX_EFFECTIVE_MODEL_MISMATCH", "effective_model_mismatch"), + ]; + for (code, classification) in admission_failures { + assert_eq!( + response_error_classification(&error(code, "violet-circuit-4821")), + classification, + ); + } + assert_eq!( + response_error_classification(&error("ACP_MODEL_UNSUPPORTED", "violet-circuit-4821",)), + "requested_model_unsupported" + ); + assert_eq!( + response_error_classification(&error( + "acpx_sidecar_command_failed", + "ACP agent exited before initialize completed (exit=1, signal=null): violet-circuit-4821", + )), + "agent_startup_failed" + ); + assert_eq!( + response_error_classification(&error("VIOLET_CIRCUIT", "violet-circuit-4821")), + "unclassified" + ); + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs index 70053c7310..f20c2333a0 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs @@ -589,6 +589,7 @@ fn bootstrap_success( "requestedModel": model, "effectiveModel": if mode == "bootstrap-wrong-model" { "wrong-model" } else { model }, "permissionMode": params.get("permissionMode"), + "providerLifetimeFenceCandidates": [60001, 60002, 60003], }, "status": {}, }) @@ -614,6 +615,7 @@ fn bootstrap_success( "requestedModel": "gpt-5.6-sol", "effectiveModel": "gpt-5.6-sol", "permissionMode": "approve-reads", + "providerLifetimeFenceCandidates": [60001, 60002, 60003], })}, }), "tool.resolve" => json!({ diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index a2e73441de..6c77d15241 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -514,6 +514,9 @@ fn run() -> Result<(), Box> { .iter() .any(|value| value == "--finish-turn-with-pending-tool"); let require_dynamic_tool = args.iter().any(|value| value == "--require-dynamic-tool"); + let require_completion_contract = args + .iter() + .any(|value| value == "--require-completion-contract"); let expected_canonical_task_context = argument(&args, "--expected-canonical-task-context") .map(|value| serde_json::from_str::(&value)) .transpose()?; @@ -766,6 +769,15 @@ fn run() -> Result<(), Box> { if require_dynamic_tool && !has_task_context_tool(&message) { return Err("thread/start omitted the authorized dynamic tool".into()); } + if require_completion_contract + && message.pointer("/params/completionContract") + != Some(&json!({ + "revision": "revision-1", + "criterionIds": ["criterion-1"], + })) + { + return Err("thread/start omitted the durable completion contract".into()); + } state.thread_id = "codex-thread-1".to_owned(); state.active_turn_id = None; save_state(&state_path, &state)?; @@ -784,6 +796,15 @@ fn run() -> Result<(), Box> { if require_dynamic_tool && !has_task_context_tool(&message) { return Err("thread/resume omitted the authorized dynamic tool".into()); } + if require_completion_contract + && message.pointer("/params/completionContract") + != Some(&json!({ + "revision": "revision-1", + "criterionIds": ["criterion-1"], + })) + { + return Err("thread/resume omitted the durable completion contract".into()); + } let unowned_turn_marker = state_path.with_file_name("resume-unowned-turn"); if resume_unowned_turn_when_marked && unowned_turn_marker.exists() { state.active_turn_id = Some("provider-turn-unowned".to_owned()); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 206aa7694f..41f1685cb0 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -10,10 +10,13 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; +#[cfg(test)] +use crate::durable::QualifiedLaunchArtifact; use crate::durable::{redact_text, OpenCodeLaunchProfile}; use crate::local_runner::LocalRunnerError; use crate::process_supervisor::{ - SupervisedProcess, VerifiedProcessArgument, VerifiedProcessLaunch, + is_node_interpreter, BoundedLogBuffer, ProcessOutput, SupervisedProcess, + VerifiedProcessArgument, VerifiedProcessLaunch, }; use crate::provider_bridge::{AuthorizedTool, DurableReplayFilter, ToolResult}; use crate::provider_events::normalized_codex_terminal_event_type; @@ -38,6 +41,8 @@ const OPENCODE_PROVIDER_ENVIRONMENT_KEYS: &[&str] = &[ "PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH", ]; const TRUSTED_OPENCODE_EXECUTABLE_ARG: &str = "--paperclip-trusted-opencode-executable"; +const MAX_PROVIDER_STDERR_LINES: usize = 32; +const MAX_PROVIDER_STDERR_BYTES: usize = 8 * 1024; const MAX_INSTRUCTIONS_BYTES: usize = 1024 * 1024; const MAX_PENDING_TOOL_REQUESTS: usize = 4_096; const MAX_PENDING_TOOL_REQUEST_BYTES: usize = 16 * 1024 * 1024; @@ -49,6 +54,12 @@ pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096; type QuestionOptionLabels = BTreeMap>; type QuestionSetMapping = (String, Value, QuestionOptionLabels); +#[derive(Clone)] +struct ProviderCompletionContract { + revision: String, + criterion_ids: Vec, +} + fn base64_encode(input: &[u8]) -> String { const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; let mut encoded = String::with_capacity(input.len().div_ceil(3) * 4); @@ -489,6 +500,7 @@ enum AmbiguousTurnMessage { pub struct CodexProvider { process: SupervisedProcess, + stderr_tail: BoundedLogBuffer, config: CodexProviderConfig, authorized_tools: Vec, next_request_id: u64, @@ -518,6 +530,7 @@ pub struct CodexProvider { trace: Option, last_trace_frame_id: Option, opencode_launch_profile: Option, + completion_contract: Option, } impl CodexProvider { @@ -525,7 +538,14 @@ impl CodexProvider { config: &CodexProviderConfig, resume_thread_id: Option<&str>, ) -> Result { - Self::start_with_tools_for_generation(config, std::iter::empty(), resume_thread_id, 1, None) + Self::start_with_tools_for_generation( + config, + std::iter::empty(), + resume_thread_id, + 1, + None, + None, + ) } pub fn start_with_tools( @@ -533,7 +553,14 @@ impl CodexProvider { authorized_tools: impl IntoIterator, resume_thread_id: Option<&str>, ) -> Result { - Self::start_with_tools_for_generation(config, authorized_tools, resume_thread_id, 1, None) + Self::start_with_tools_for_generation( + config, + authorized_tools, + resume_thread_id, + 1, + None, + None, + ) } pub(crate) fn start_with_tools_for_generation( @@ -542,6 +569,7 @@ impl CodexProvider { resume_thread_id: Option<&str>, process_generation: u64, opencode_launch_profile: Option<&OpenCodeLaunchProfile>, + completion_contract: Option<(&str, &[String])>, ) -> Result { config.validate()?; if process_generation == 0 { @@ -591,21 +619,7 @@ impl CodexProvider { "OpenCode launch does not match the runner-owned qualified profile", )); } - let command = verify_launch_artifact(&profile.command, "OpenCode proxy command") - .map_err(|error| LocalRunnerError::invalid(error.to_string()))?; - let proxy = verify_launch_artifact(&profile.proxy_script, "OpenCode proxy script") - .map_err(|error| LocalRunnerError::invalid(error.to_string()))?; - let executable = - verify_launch_artifact(&profile.executable, "OpenCode provider executable") - .map_err(|error| LocalRunnerError::invalid(error.to_string()))?; - let launch = VerifiedProcessLaunch::new( - command, - vec![ - VerifiedProcessArgument::Artifact(proxy), - VerifiedProcessArgument::Literal(TRUSTED_OPENCODE_EXECUTABLE_ARG.to_owned()), - VerifiedProcessArgument::ExecutableArtifact(executable), - ], - ); + let launch = verified_opencode_launch(profile)?; SupervisedProcess::spawn_verified_with_environment_keys( &launch, Duration::from_secs(2), @@ -623,6 +637,10 @@ impl CodexProvider { }; let mut provider = Self { process, + stderr_tail: BoundedLogBuffer::new( + MAX_PROVIDER_STDERR_LINES, + MAX_PROVIDER_STDERR_BYTES, + ), config: config.clone(), authorized_tools, next_request_id: 1, @@ -652,6 +670,12 @@ impl CodexProvider { trace: ProviderTraceSink::from_environment(), last_trace_frame_id: None, opencode_launch_profile: opencode_launch_profile.cloned(), + completion_contract: completion_contract.map(|(revision, criterion_ids)| { + ProviderCompletionContract { + revision: revision.to_owned(), + criterion_ids: criterion_ids.to_vec(), + } + }), }; let initialized = provider.request( "initialize", @@ -681,6 +705,17 @@ impl CodexProvider { let params_object = params .as_object_mut() .expect("Codex thread parameters are an object"); + if config.provider == "opencode" { + if let Some(contract) = provider.completion_contract.as_ref() { + params_object.insert( + "completionContract".to_owned(), + json!({ + "revision": contract.revision, + "criterionIds": contract.criterion_ids, + }), + ); + } + } let method = if let Some(thread_id) = resume_thread_id { params_object.insert("threadId".to_owned(), json!(thread_id)); "thread/resume" @@ -840,6 +875,7 @@ impl CodexProvider { let completed_turn_authority = self.completed_turn_authority.clone(); let completion_reconciliation_pending = self.completion_reconciliation_pending; let durable_tool_call_replays = self.durable_tool_call_replays; + let completion_contract = self.completion_contract.clone(); // Exact turn identities may be forgotten only after the provider // process that could emit them is gone. Resume the same thread in a @@ -852,6 +888,12 @@ impl CodexProvider { Some(&thread_id), next_generation, self.opencode_launch_profile.as_ref(), + completion_contract.as_ref().map(|contract| { + ( + contract.revision.as_str(), + contract.criterion_ids.as_slice(), + ) + }), )?; replacement.durable_tool_call_replays = durable_tool_call_replays; if replacement.active_provider_turn_id.is_some() { @@ -1795,14 +1837,29 @@ impl CodexProvider { .map_err(ProviderRequestError::Ambiguous)?; loop { let line = self - .process - .receive_stdout_line(Duration::from_secs(30)) - .map_err(ProviderRequestError::Ambiguous)? - .ok_or_else(|| { - ProviderRequestError::Ambiguous(LocalRunnerError::invalid(format!( - "Codex {method} response timed out" - ))) - })?; + .receive_provider_stdout_line(Duration::from_secs(30)) + .map_err(ProviderRequestError::Ambiguous)?; + let Some(line) = line else { + let exit = self + .process + .try_wait() + .map_err(ProviderRequestError::Ambiguous)?; + if exit.is_some() { + self.drain_provider_diagnostics(Duration::from_millis(50)); + } + let diagnostic_suffix = self.provider_diagnostic_suffix(); + let message = if let Some(exit) = exit { + format!( + "Codex {method} process exited before responding (exitCode={:?}, signal={:?}){diagnostic_suffix}", + exit.exit_code, exit.signal + ) + } else { + format!("Codex {method} response timed out{diagnostic_suffix}") + }; + return Err(ProviderRequestError::Ambiguous(LocalRunnerError::invalid( + message, + ))); + }; let trace_frame_id = self.trace_inbound(&line); let message = parse_provider_message(&line).map_err(|error| { if let (Some(trace), Some(frame_id)) = (self.trace.as_mut(), trace_frame_id) { @@ -1870,6 +1927,87 @@ impl CodexProvider { self.pending_message_bytes = next_retained_bytes; } } + + fn receive_provider_stdout_line( + &mut self, + timeout: Duration, + ) -> Result, LocalRunnerError> { + let deadline = std::time::Instant::now() + timeout; + loop { + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + return Ok(None); + } + match self.process.recv_timeout(remaining) { + Ok(ProcessOutput::Stdout(line)) => return Ok(Some(line)), + Ok(ProcessOutput::Stderr(line)) => { + self.stderr_tail.push(redact_text(&line)); + } + Ok(ProcessOutput::StdoutError(message)) => { + return Err(LocalRunnerError::invalid(message)); + } + Ok(ProcessOutput::StdoutClosed) => return Ok(None), + Ok(ProcessOutput::StderrClosed) => {} + Err(mpsc::RecvTimeoutError::Timeout) => return Ok(None), + Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(None), + } + } + } + + fn drain_provider_diagnostics(&mut self, max_wait: Duration) { + let deadline = std::time::Instant::now() + max_wait; + loop { + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + break; + } + match self.process.recv_timeout(remaining) { + Ok(ProcessOutput::Stderr(line)) => { + self.stderr_tail.push(redact_text(&line)); + } + Ok(ProcessOutput::StderrClosed) + | Err(mpsc::RecvTimeoutError::Timeout) + | Err(mpsc::RecvTimeoutError::Disconnected) => break, + Ok(ProcessOutput::Stdout(_)) + | Ok(ProcessOutput::StdoutError(_)) + | Ok(ProcessOutput::StdoutClosed) => {} + } + } + } + + fn provider_diagnostic_suffix(&self) -> String { + let diagnostics = self.stderr_tail.snapshot().lines.join("\n"); + if diagnostics.is_empty() { + String::new() + } else { + format!(" stderrTail={diagnostics:?}") + } + } +} + +fn verified_opencode_launch( + profile: &OpenCodeLaunchProfile, +) -> Result { + let command = verify_launch_artifact(&profile.command, "OpenCode proxy command") + .map_err(|error| LocalRunnerError::invalid(error.to_string()))?; + let proxy = verify_launch_artifact(&profile.proxy_script, "OpenCode proxy script") + .map_err(|error| LocalRunnerError::invalid(error.to_string()))?; + let executable = verify_launch_artifact(&profile.executable, "OpenCode provider executable") + .map_err(|error| LocalRunnerError::invalid(error.to_string()))?; + let proxy = if is_node_interpreter(&profile.command.path) { + VerifiedProcessArgument::CommonJsArtifact(proxy) + } else { + // Qualified test and alternate proxy commands own their ordinary + // argv contract. Only Node understands the runner-owned CommonJS + // descriptor loader flags. + VerifiedProcessArgument::Artifact(proxy) + }; + let args = vec![ + proxy, + VerifiedProcessArgument::Literal(TRUSTED_OPENCODE_EXECUTABLE_ARG.to_owned()), + VerifiedProcessArgument::ExecutableArtifact(executable), + ]; + Ok(VerifiedProcessLaunch::new(command, args)) } fn json_size(value: &Value, label: &str) -> Result { @@ -2583,6 +2721,53 @@ fn codex_question_response( mod tests { use super::*; + fn qualified_artifact(path: &Path) -> QualifiedLaunchArtifact { + QualifiedLaunchArtifact { + path: path.to_owned(), + sha256: format!("sha256:{:x}", Sha256::digest(fs::read(path).unwrap())), + } + } + + #[test] + fn verified_opencode_proxy_executes_the_descriptor_safe_commonjs_bundle() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "paperclip-opencode-launch-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&directory).unwrap(); + let command = directory.join("node"); + let proxy = directory.join("proxy.cjs"); + let executable = directory.join("opencode"); + fs::write(&command, b"qualified node").unwrap(); + fs::write(&proxy, b"module.exports = {};\n").unwrap(); + fs::write(&executable, b"qualified opencode").unwrap(); + let profile = OpenCodeLaunchProfile { + command: qualified_artifact(&command), + proxy_script: qualified_artifact(&proxy), + executable: qualified_artifact(&executable), + }; + + let launch = verified_opencode_launch(&profile).unwrap(); + assert!(matches!( + launch.arguments().first(), + Some(VerifiedProcessArgument::CommonJsArtifact(_)) + )); + assert!(matches!( + launch.arguments().get(1), + Some(VerifiedProcessArgument::Literal(argument)) + if argument == TRUSTED_OPENCODE_EXECUTABLE_ARG + )); + assert!(matches!( + launch.arguments().get(2), + Some(VerifiedProcessArgument::ExecutableArtifact(_)) + )); + fs::remove_dir_all(directory).unwrap(); + } + #[test] fn admits_only_exact_local_facade_provider_driver_pairs() { let mut config = CodexProviderConfig { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index bd59e77132..6d0ae23dfc 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -6,7 +6,7 @@ use serde_json::{json, Value}; use super::state::{ Command, CommandDisposition, DurableState, DurableStateStore, EventPriority, - StoredCommandResult, + PendingTerminalDelivery, StoredCommandResult, }; use super::transport::{ current_unix_ms, validate_control_identity, AuthenticatedTransport, ConnectionMetadata, @@ -45,6 +45,8 @@ enum CommandLifecycle { Shutdown, } +const TERMINAL_RESULT_ACK_TIMEOUT: Duration = Duration::from_secs(2); + fn sleep_for_reconnect(base: Duration, max_delay: Duration, attempt: &mut u32) { let multiplier = 1_u128 << (*attempt).min(5); let uncapped = base.as_millis().saturating_mul(multiplier); @@ -89,7 +91,7 @@ fn connection_attempt_deadline( } impl CommandLifecycle { - fn for_completed(command: &Command) -> Self { + fn for_terminal(command: &Command) -> Self { match command.command_type.as_str() { "runner.suspend" => Self::Suspend, "runner.shutdown" => Self::Shutdown, @@ -141,7 +143,9 @@ pub fn run_durable_runner( config.validate()?; let store = DurableStateStore::new(&config.state_dir)?; let (mut state, recovered) = store.load_or_create(&config)?; - if state.lifecycle == "revoked" || state.lifecycle == "stopped" { + if state.lifecycle == "revoked" + || (state.lifecycle == "stopped" && state.pending_terminal_delivery.is_none()) + { return Ok(()); } if recovered { @@ -265,6 +269,18 @@ pub fn run_durable_runner( if let Some(acked_source_seq) = welcome.acked_source_seq { state.apply_ack(acked_source_seq)?; } + let connection = welcome.connection; + if state.pending_terminal_delivery.is_some() { + return reconcile_pending_terminal_delivery( + &mut state, + &store, + &config, + &mut executor, + &mut transport, + &connection, + &welcome.pending_commands, + ); + } state.lifecycle = "ready".to_owned(); state.recoverable_failure = None; store.save(&state)?; @@ -275,25 +291,71 @@ pub fn run_durable_runner( for command in welcome.pending_commands { let (result, lifecycle) = process_command(&mut state, &store, &config, &mut executor, &command)?; + if let Some(durable_lifecycle) = lifecycle.durable_state() { + persist_lifecycle_before_command_delivery( + &mut state, + &store, + durable_lifecycle, + &result, + )?; + } lifecycle_after_reply = lifecycle_after_reply.merge(lifecycle); if let Err(error) = transport.send_json(&command_result_envelope(&state, &result)) { + if lifecycle.durable_state().is_some() { + return stop_after_terminal_result_delivery_failure( + &mut state, + &store, + &mut executor, + error, + ); + } state.record_diagnostic(error.to_string()); disconnected = true; break; } + if lifecycle.durable_state().is_some() { + if let Err(error) = wait_for_terminal_result_ack( + &mut transport, + &mut state, + &store, + &connection, + &result, + ) { + return stop_after_terminal_result_delivery_failure( + &mut state, + &store, + &mut executor, + error, + ); + } + // A terminal lifecycle command is the final command this + // process may accept. Flush its already-durable outbox below, + // then release the executor without observing later commands. + break; + } } - if !disconnected && send_outbox(&mut transport, &state, &mut sent_source_seq).is_err() { - state.record_diagnostic("outbox delivery failed; unacknowledged suffix will replay"); - disconnected = true; + if !disconnected { + if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) { + state.record_diagnostic( + "outbox delivery failed; unacknowledged suffix remains durable", + ); + if lifecycle_after_reply.durable_state().is_some() { + // The terminal result was delivered above. Never reconnect + // this process and overwrite its durable terminal state as + // ready merely to retry a later outbox frame. + store.save(&state)?; + let _ = executor.shutdown(); + return Err(error); + } + disconnected = true; + } } if let Some(durable_lifecycle) = lifecycle_after_reply .durable_state() .filter(|_| !disconnected) { - executor.shutdown()?; - state.lifecycle = durable_lifecycle.to_owned(); - store.save(&state)?; - return Ok(()); + debug_assert_eq!(state.lifecycle, durable_lifecycle); + return finish_terminal_transition_after_ack(&mut state, &store, &mut executor); } if disconnected { disconnected_since.get_or_insert_with(Instant::now); @@ -304,8 +366,6 @@ pub fn run_durable_runner( sleep_before_deadline(config.reconnect_delay, reconnect_deadline); continue; } - - let connection = welcome.connection; loop { if started.elapsed() >= config.max_runtime { break; @@ -367,21 +427,70 @@ pub fn run_durable_runner( })?; let (result, lifecycle) = process_command(&mut state, &store, &config, &mut executor, &command)?; - let delivery = transport - .send_json(&command_result_envelope(&state, &result)) - .and_then(|()| send_outbox(&mut transport, &state, &mut sent_source_seq)); - if let Err(error) = delivery { + if let Some(durable_lifecycle) = lifecycle.durable_state() { + persist_lifecycle_before_command_delivery( + &mut state, + &store, + durable_lifecycle, + &result, + )?; + } + if let Err(error) = + transport.send_json(&command_result_envelope(&state, &result)) + { + if lifecycle.durable_state().is_some() { + return stop_after_terminal_result_delivery_failure( + &mut state, + &store, + &mut executor, + error, + ); + } disconnected_since.get_or_insert_with(Instant::now); state.record_diagnostic(error.to_string()); state.reconnect_count = state.reconnect_count.saturating_add(1); store.save(&state)?; break; } - if let Some(durable_lifecycle) = lifecycle.durable_state() { - executor.shutdown()?; - state.lifecycle = durable_lifecycle.to_owned(); + if lifecycle.durable_state().is_some() { + if let Err(error) = wait_for_terminal_result_ack( + &mut transport, + &mut state, + &store, + &connection, + &result, + ) { + return stop_after_terminal_result_delivery_failure( + &mut state, + &store, + &mut executor, + error, + ); + } + } + if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) { + state.record_diagnostic( + "outbox delivery failed; unacknowledged suffix remains durable", + ); store.save(&state)?; - return Ok(()); + if lifecycle.durable_state().is_some() { + // The controller has accepted this terminal result. + // Stop even though a later outbox frame failed so a + // reconnect cannot restore the runner to ready. + let _ = executor.shutdown(); + return Err(error); + } + disconnected_since.get_or_insert_with(Instant::now); + state.reconnect_count = state.reconnect_count.saturating_add(1); + break; + } + if let Some(durable_lifecycle) = lifecycle.durable_state() { + debug_assert_eq!(state.lifecycle, durable_lifecycle); + return finish_terminal_transition_after_ack( + &mut state, + &store, + &mut executor, + ); } } Some("revoke") => { @@ -396,10 +505,13 @@ pub fn run_durable_runner( "revoke must advance the authenticated revocation epoch", )); } - state.lifecycle = "revoked".to_owned(); state.record_diagnostic("connection capability was revoked"); - executor.shutdown()?; - store.save(&state)?; + persist_lifecycle_before_shutdown( + &mut state, + &store, + &mut executor, + "revoked", + )?; return Ok(()); } Some("ping") => { @@ -431,6 +543,207 @@ pub fn run_durable_runner( } } +fn persist_lifecycle_before_shutdown( + state: &mut DurableState, + store: &DurableStateStore, + executor: &mut E, + lifecycle: &str, +) -> Result<(), DurableRunnerError> { + state.lifecycle = lifecycle.to_owned(); + store.save(state)?; + executor.shutdown() +} + +fn persist_lifecycle_before_command_delivery( + state: &mut DurableState, + store: &DurableStateStore, + lifecycle: &str, + result: &StoredCommandResult, +) -> Result<(), DurableRunnerError> { + // A terminal command result is already durable before this boundary. Save + // its matching lifecycle before exposing that result to the controller, + // then let the caller deliver the result before fallible provider cleanup. + // Recovery can therefore never observe a ready runner after the controller + // has already observed its terminal command result. + state.lifecycle = lifecycle.to_owned(); + state.pending_terminal_delivery = Some(PendingTerminalDelivery { + command_id: result.command_id.clone(), + controller_seq: result.controller_seq, + command_type: result.command_type.clone(), + lifecycle: lifecycle.to_owned(), + }); + store.save(state) +} + +fn complete_terminal_delivery_after_cleanup( + state: &mut DurableState, + store: &DurableStateStore, +) -> Result<(), DurableRunnerError> { + let pending = state.pending_terminal_delivery.as_ref().ok_or_else(|| { + DurableRunnerError::invalid("terminal cleanup has no durable recovery fence") + })?; + if state.lifecycle != pending.lifecycle { + return Err(DurableRunnerError::invalid( + "terminal cleanup does not match its durable recovery fence", + )); + } + state.pending_terminal_delivery = None; + store.save(state) +} + +fn finish_terminal_transition_after_ack( + state: &mut DurableState, + store: &DurableStateStore, + executor: &mut E, +) -> Result<(), DurableRunnerError> { + // Keep the durable fence through provider cleanup. If cleanup fails, a + // replacement may authenticate only to retry terminal reconciliation and + // cannot restore the suspended runner to ready. + executor.shutdown()?; + complete_terminal_delivery_after_cleanup(state, store) +} + +fn wait_for_terminal_result_ack( + transport: &mut AuthenticatedTransport, + state: &mut DurableState, + store: &DurableStateStore, + connection: &ConnectionMetadata, + result: &StoredCommandResult, +) -> Result<(), DurableRunnerError> { + let deadline = Instant::now() + TERMINAL_RESULT_ACK_TIMEOUT; + while Instant::now() < deadline { + let Some(message) = transport.receive_json()? else { + continue; + }; + validate_control_identity(&message, state, Some(connection))?; + match message.get("kind").and_then(Value::as_str) { + Some("command_result_ack") => { + let payload = message + .get("payload") + .and_then(Value::as_object) + .ok_or_else(|| { + DurableRunnerError::invalid( + "terminal command result acknowledgement payload is required", + ) + })?; + if payload.get("commandId").and_then(Value::as_str) + != Some(result.command_id.as_str()) + || payload.get("commandType").and_then(Value::as_str) + != Some(result.command_type.as_str()) + || payload.get("controllerSeq").and_then(Value::as_u64) + != Some(result.controller_seq) + || payload.get("status").and_then(Value::as_str) != Some(result.status.as_str()) + { + return Err(DurableRunnerError::invalid( + "terminal command result acknowledgement changed its durable identity", + )); + } + return Ok(()); + } + Some("ack") => { + let acked = message + .pointer("/payload/ackedSourceSeq") + .and_then(Value::as_u64) + .ok_or_else(|| DurableRunnerError::invalid("ACK cursor is required"))?; + state.apply_ack(acked)?; + store.save(state)?; + } + Some("ping") => transport.send_json(&control_envelope( + state, + connection, + "pong", + json!({ + "lifecycle": state.lifecycle, + "ackedSourceSeq": state.acked_source_seq, + "outboxBytes": state.outbox_bytes(), + }), + ))?, + _ => { + return Err(DurableRunnerError::invalid( + "controller sent a non-acknowledgement after a terminal command result", + )); + } + } + } + Err(DurableRunnerError::invalid( + "terminal command result acknowledgement timed out", + )) +} + +fn reconcile_pending_terminal_delivery( + state: &mut DurableState, + store: &DurableStateStore, + config: &DurableRunnerConfig, + executor: &mut E, + transport: &mut AuthenticatedTransport, + connection: &ConnectionMetadata, + pending_commands: &[Command], +) -> Result<(), DurableRunnerError> { + let pending = state.pending_terminal_delivery.clone().ok_or_else(|| { + DurableRunnerError::invalid("terminal result reconciliation has no durable fence") + })?; + if let Some(command) = pending_commands + .iter() + .find(|command| command.command_id == pending.command_id) + { + if command.controller_seq != pending.controller_seq + || command.command_type != pending.command_type + { + return Err(DurableRunnerError::invalid( + "controller changed the pending terminal command identity", + )); + } + let (result, lifecycle) = process_command(state, store, config, executor, command)?; + if lifecycle.durable_state() != Some(pending.lifecycle.as_str()) { + return Err(DurableRunnerError::invalid( + "pending terminal command did not replay its durable lifecycle", + )); + } + if let Err(error) = transport.send_json(&command_result_envelope(state, &result)) { + return stop_after_terminal_result_delivery_failure(state, store, executor, error); + } + if let Err(error) = + wait_for_terminal_result_ack(transport, state, store, connection, &result) + { + return stop_after_terminal_result_delivery_failure(state, store, executor, error); + } + } else { + // An authenticated welcome is the controller's authoritative pending + // set. Absence means the prior write reached the controller even if + // the runner did not observe transport success before it exited. + state.record_diagnostic( + "controller confirmed the pending terminal result was already delivered", + ); + store.save(state)?; + } + + let mut sent_source_seq = state.acked_source_seq; + if let Err(error) = send_outbox(transport, state, &mut sent_source_seq) { + state.record_diagnostic("outbox delivery failed after terminal result reconciliation"); + store.save(state)?; + let _ = executor.shutdown(); + return Err(error); + } + finish_terminal_transition_after_ack(state, store, executor) +} + +fn stop_after_terminal_result_delivery_failure( + state: &mut DurableState, + store: &DurableStateStore, + executor: &mut E, + error: DurableRunnerError, +) -> Result<(), DurableRunnerError> { + // The terminal transition was committed before the attempted delivery. + // Its result may or may not have reached the controller, but reconnecting + // this process would overwrite that durable state as ready and admit work + // after shutdown/suspend. Leave the result journaled for reconciliation by + // a future authorized process instead. + state.record_diagnostic(error.to_string()); + store.save(state)?; + let _ = executor.shutdown(); + Err(error) +} + fn poll_executor_events( state: &mut DurableState, store: &DurableStateStore, @@ -476,10 +789,10 @@ fn process_command( ) -> Result<(StoredCommandResult, CommandLifecycle), DurableRunnerError> { match state.begin_command(command)? { CommandDisposition::Replay(result) => { - let lifecycle = if result.status == "completed" { - CommandLifecycle::for_completed(command) - } else { + let lifecycle = if result.status == "pending" { CommandLifecycle::Continue + } else { + CommandLifecycle::for_terminal(command) }; return Ok((result, lifecycle)); } @@ -492,13 +805,36 @@ fn process_command( // in the effect window, recovery returns an indeterminate result and never // executes the same logical command twice. store.save(state)?; - let execution = executor.execute(command)?; + let execution = match executor.execute(command) { + Ok(execution) => execution, + Err(error) => { + // An executor-returned error is a terminal observation, not crash + // ambiguity. Commit it before replying so recovery can replay the + // original provider/bootstrap failure without executing the + // command twice. A process death inside execute still leaves the + // pre-effect marker pending and remains indeterminate on recovery. + let message = error.to_string(); + state.record_diagnostic(format!( + "{} command failed: {message}", + command.command_type + )); + let result = state.fail_command( + command, + json!({ + "code": "command_execution_failed", + "message": message, + }), + )?; + store.save(state)?; + return Ok((result, CommandLifecycle::for_terminal(command))); + } + }; for (event_type, priority, payload) in execution.events { state.enqueue_event(config, event_type, priority, payload)?; } let result = state.complete_command(command, execution.result)?; store.save(state)?; - Ok((result, CommandLifecycle::for_completed(command))) + Ok((result, CommandLifecycle::for_terminal(command))) } fn send_outbox( @@ -566,6 +902,16 @@ mod tests { calls: usize, } + struct FailingExecutor { + calls: usize, + } + + struct ShutdownFailingExecutor; + + struct ShutdownCountingExecutor { + shutdown_calls: usize, + } + struct RetainingEventExecutor { events: VecDeque, fail_acknowledgement: bool, @@ -578,6 +924,38 @@ mod tests { } } + impl CommandExecutor for FailingExecutor { + fn execute(&mut self, _command: &Command) -> Result { + self.calls += 1; + Err(DurableRunnerError::invalid( + "provider bootstrap rejected authorization=Bearer test-secret", + )) + } + } + + impl CommandExecutor for ShutdownFailingExecutor { + fn execute(&mut self, _command: &Command) -> Result { + Ok(CommandExecution::result(json!({"status": "completed"}))) + } + + fn shutdown(&mut self) -> Result<(), DurableRunnerError> { + Err(DurableRunnerError::invalid( + "simulated terminal cleanup failure", + )) + } + } + + impl CommandExecutor for ShutdownCountingExecutor { + fn execute(&mut self, _command: &Command) -> Result { + Ok(CommandExecution::result(json!({"status": "completed"}))) + } + + fn shutdown(&mut self) -> Result<(), DurableRunnerError> { + self.shutdown_calls += 1; + Ok(()) + } + } + impl CommandExecutor for RetainingEventExecutor { fn execute(&mut self, _command: &Command) -> Result { Ok(CommandExecution::result(json!({"status": "completed"}))) @@ -640,6 +1018,142 @@ mod tests { } } + #[test] + fn terminal_lifecycle_is_durable_before_fallible_cleanup() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-terminal-before-cleanup-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let mut executor = ShutdownFailingExecutor; + + let error = persist_lifecycle_before_shutdown(&mut state, &store, &mut executor, "stopped") + .expect_err("cleanup failure remains observable"); + let (recovered, existed) = store.load_or_create(&config).unwrap(); + + assert!(error.to_string().contains("terminal cleanup failure")); + assert!(existed); + assert_eq!(recovered.lifecycle, "stopped"); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn terminal_result_delivery_failure_stops_without_reopening_lifecycle() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-terminal-result-delivery-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let mut executor = ShutdownCountingExecutor { shutdown_calls: 0 }; + let command = command("runner.shutdown"); + let (result, lifecycle) = + process_command(&mut state, &store, &config, &mut executor, &command).unwrap(); + persist_lifecycle_before_command_delivery( + &mut state, + &store, + lifecycle.durable_state().unwrap(), + &result, + ) + .unwrap(); + + let error = stop_after_terminal_result_delivery_failure( + &mut state, + &store, + &mut executor, + DurableRunnerError::invalid("simulated result delivery failure"), + ) + .expect_err("terminal result delivery failure remains observable"); + let (recovered, existed) = store.load_or_create(&config).unwrap(); + + assert!(error.to_string().contains("result delivery failure")); + assert!(existed); + assert_eq!(recovered.lifecycle, "stopped"); + assert_eq!( + recovered + .pending_terminal_delivery + .as_ref() + .map(|pending| pending.command_id.as_str()), + Some("command_1") + ); + assert_eq!(executor.shutdown_calls, 1); + assert!(recovered + .diagnostics + .iter() + .any(|diagnostic| diagnostic.contains("result delivery failure"))); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn successful_terminal_cleanup_clears_the_recovery_fence() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-terminal-result-delivered-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let mut executor = CountingExecutor { calls: 0 }; + let command = command("runner.suspend"); + let (result, lifecycle) = + process_command(&mut state, &store, &config, &mut executor, &command).unwrap(); + + persist_lifecycle_before_command_delivery( + &mut state, + &store, + lifecycle.durable_state().unwrap(), + &result, + ) + .unwrap(); + assert!(state.pending_terminal_delivery.is_some()); + complete_terminal_delivery_after_cleanup(&mut state, &store).unwrap(); + let (recovered, existed) = store.load_or_create(&config).unwrap(); + + assert!(existed); + assert_eq!(recovered.lifecycle, "suspended"); + assert!(recovered.pending_terminal_delivery.is_none()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn failed_terminal_cleanup_keeps_the_recovery_fence() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-terminal-cleanup-failed-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let mut executor = ShutdownFailingExecutor; + let command = command("runner.suspend"); + let (result, lifecycle) = + process_command(&mut state, &store, &config, &mut executor, &command).unwrap(); + persist_lifecycle_before_command_delivery( + &mut state, + &store, + lifecycle.durable_state().unwrap(), + &result, + ) + .unwrap(); + + let error = finish_terminal_transition_after_ack(&mut state, &store, &mut executor) + .expect_err("cleanup failure remains fenced"); + let (recovered, existed) = store.load_or_create(&config).unwrap(); + + assert!(error.to_string().contains("terminal cleanup failure")); + assert!(existed); + assert_eq!(recovered.lifecycle, "suspended"); + assert!(recovered.pending_terminal_delivery.is_some()); + fs::remove_dir_all(directory).unwrap(); + } + #[test] fn event_batch_keeps_accepted_prefix_and_unacknowledged_suffix() { let directory = std::env::temp_dir().join(format!( @@ -758,6 +1272,140 @@ mod tests { fs::remove_dir_all(directory).unwrap(); } + #[test] + fn executor_failure_is_durable_and_does_not_become_indeterminate() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-command-failure-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let mut executor = FailingExecutor { calls: 0 }; + let command = command("session.open"); + + let (failed, failed_lifecycle) = + process_command(&mut state, &store, &config, &mut executor, &command).unwrap(); + let (mut recovered, existed) = store.load_or_create(&config).unwrap(); + let replay = process_command(&mut recovered, &store, &config, &mut executor, &command) + .unwrap() + .0; + + assert!(existed); + assert_eq!(executor.calls, 1); + assert_eq!(failed_lifecycle, CommandLifecycle::Continue); + assert_eq!(failed, replay); + assert_eq!(failed.status, "failed"); + assert_eq!(failed.result["code"], "command_execution_failed"); + assert_eq!( + failed.result["message"], + "provider bootstrap rejected authorization=Bearer [REDACTED]" + ); + assert!(recovered.diagnostics.iter().any(|diagnostic| { + diagnostic + == "session.open command failed: provider bootstrap rejected authorization=Bearer [REDACTED]" + })); + assert!(recovered + .diagnostics + .iter() + .all(|diagnostic| !diagnostic.contains("test-secret"))); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn failed_lifecycle_commands_replay_their_terminal_transition() { + for (command_type, expected_lifecycle) in [ + ("runner.suspend", CommandLifecycle::Suspend), + ("runner.shutdown", CommandLifecycle::Shutdown), + ] { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-failed-lifecycle-{}-{}", + command_type.replace('.', "-"), + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let mut executor = FailingExecutor { calls: 0 }; + let command = command(command_type); + + let (failed, first_lifecycle) = + process_command(&mut state, &store, &config, &mut executor, &command).unwrap(); + let (mut recovered, _) = store.load_or_create(&config).unwrap(); + let (replay, replay_lifecycle) = + process_command(&mut recovered, &store, &config, &mut executor, &command).unwrap(); + + assert_eq!(failed.status, "failed"); + assert_eq!(failed, replay); + assert_eq!(first_lifecycle, expected_lifecycle); + assert_eq!(replay_lifecycle, expected_lifecycle); + assert_eq!(executor.calls, 1); + fs::remove_dir_all(directory).unwrap(); + } + } + + #[test] + fn process_death_after_journaling_remains_indeterminate_without_reexecution() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-command-indeterminate-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let command = command("session.open"); + + assert_eq!( + state.begin_command(&command).unwrap(), + CommandDisposition::Execute + ); + store.save(&state).unwrap(); + + let (mut recovered, existed) = store.load_or_create(&config).unwrap(); + let mut executor = CountingExecutor { calls: 0 }; + let replay = process_command(&mut recovered, &store, &config, &mut executor, &command) + .unwrap() + .0; + + assert!(existed); + assert_eq!(executor.calls, 0); + assert_eq!(replay.status, "indeterminate"); + assert_eq!(replay.result["code"], "execution_indeterminate"); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn indeterminate_lifecycle_command_still_stops_after_recovery_delivery() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-lifecycle-indeterminate-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let command = command("runner.shutdown"); + + assert_eq!( + state.begin_command(&command).unwrap(), + CommandDisposition::Execute + ); + store.save(&state).unwrap(); + + let (mut recovered, _) = store.load_or_create(&config).unwrap(); + let mut executor = CountingExecutor { calls: 0 }; + let (result, lifecycle) = + process_command(&mut recovered, &store, &config, &mut executor, &command).unwrap(); + + assert_eq!(result.status, "indeterminate"); + assert_eq!(lifecycle, CommandLifecycle::Shutdown); + assert_eq!(executor.calls, 0); + fs::remove_dir_all(directory).unwrap(); + } + #[test] fn completed_shutdown_replay_still_stops_after_delivery() { let directory = std::env::temp_dir().join(format!( diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs index 187a8b8348..c1e29bef7d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs @@ -151,6 +151,15 @@ pub struct StoredCommandResult { pub result: Value, } +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PendingTerminalDelivery { + pub(crate) command_id: String, + pub(crate) controller_seq: u64, + pub(crate) command_type: String, + pub(crate) lifecycle: String, +} + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] struct ExecutorEventReceipt { @@ -189,6 +198,8 @@ pub struct DurableState { #[serde(default)] pub processed_command_fingerprints: BTreeMap, #[serde(default)] + pub(crate) pending_terminal_delivery: Option, + #[serde(default)] executor_event_receipts: BTreeMap, pub diagnostics: Vec, pub backpressure: bool, @@ -217,6 +228,7 @@ impl DurableState { outbox: Vec::new(), processed_commands: BTreeMap::new(), processed_command_fingerprints: BTreeMap::new(), + pending_terminal_delivery: None, executor_event_receipts: BTreeMap::new(), diagnostics: Vec::new(), backpressure: false, @@ -537,6 +549,28 @@ impl DurableState { command: &Command, result: Value, ) -> Result { + self.finish_command(command, "completed", result) + } + + pub fn fail_command( + &mut self, + command: &Command, + result: Value, + ) -> Result { + self.finish_command(command, "failed", result) + } + + fn finish_command( + &mut self, + command: &Command, + status: &str, + result: Value, + ) -> Result { + if status != "completed" && status != "failed" { + return Err(DurableRunnerError::invalid( + "durable command terminal status is unsupported", + )); + } { let stored = self .processed_commands @@ -568,7 +602,7 @@ impl DurableState { .processed_commands .get_mut(&command.command_id) .expect("pending command was checked above"); - stored.status = "completed".to_owned(); + stored.status = status.to_owned(); stored.result = sanitized_result; Ok(stored.clone()) } @@ -903,7 +937,7 @@ fn validate_binding( || command.controller_seq > state.last_controller_command_seq || !matches!( command.status.as_str(), - "pending" | "completed" | "indeterminate" + "pending" | "completed" | "failed" | "indeterminate" ) { return Err(DurableRunnerError::invalid( @@ -943,6 +977,29 @@ fn validate_binding( && receipt.source_seq <= state.highest_source_seq() && executor_receipt_sequences.insert(receipt.source_seq) }); + let pending_terminal_delivery_is_valid = + state + .pending_terminal_delivery + .as_ref() + .map_or(true, |pending| { + let expected_lifecycle = match pending.command_type.as_str() { + "runner.suspend" => "suspended", + "runner.shutdown" => "stopped", + _ => return false, + }; + pending.lifecycle == expected_lifecycle + && state.lifecycle == expected_lifecycle + && pending.controller_seq == state.last_controller_command_seq + && state + .processed_commands + .get(&pending.command_id) + .is_some_and(|result| { + result.command_id == pending.command_id + && result.controller_seq == pending.controller_seq + && result.command_type == pending.command_type + && result.status != "pending" + }) + }); command_sequences.sort_unstable(); let command_cursors_are_valid = match (command_sequences.first(), command_sequences.last()) { (None, None) => state.compacted_through_controller_seq == state.last_controller_command_seq, @@ -969,6 +1026,7 @@ fn validate_binding( || !command_cursors_are_valid || !command_fingerprints_are_valid || !executor_event_receipts_are_valid + || !pending_terminal_delivery_is_valid { return Err(DurableRunnerError::invalid( "durable state cursors, bounds, or journals are inconsistent", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs index 4827c7cc8a..ddbeec4a01 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs @@ -2458,6 +2458,22 @@ mod tests { ); let shutdown_result = receive_secure(&mut second, &mut second_secure, &server_config); assert_eq!(shutdown_result["kind"], "command_result"); + send_secure( + &mut second, + &mut second_secure, + &server_config, + &control( + &server_state, + "connection_2", + "command_result_ack", + json!({ + "commandId": "command_shutdown", + "commandType": "runner.shutdown", + "controllerSeq": 2, + "status": "completed", + }), + ), + ); }); let session_open_calls = Arc::new(AtomicUsize::new(0)); @@ -2479,6 +2495,7 @@ mod tests { let final_state: DurableState = serde_json::from_slice(&state_bytes).unwrap(); assert_eq!(final_state.acked_source_seq, 1); assert!(final_state.outbox.is_empty()); + assert!(final_state.pending_terminal_delivery.is_none()); assert_eq!(final_state.reconnect_count, 1); std::fs::remove_dir_all(directory).unwrap(); } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs index 69ec37b5c7..5957b4c97d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs @@ -1696,6 +1696,10 @@ impl CommandExecutor for ManagedProviderCommandExecutor { } fn shutdown(&mut self) -> Result<(), DurableRunnerError> { + // A replacement runner has no live provider object until durable state + // is restored. Require that restoration before accepting terminal + // cleanup so a persisted remote session cannot be abandoned silently. + self.restore()?; if let Some(provider) = self.provider.as_mut() { provider.shutdown().map_err(|error| { DurableRunnerError::invalid(format!( diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs index 040bdea015..959b06c8d1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs @@ -177,6 +177,11 @@ impl CommandExecutor for NativeProviderCommandExecutor { } fn shutdown(&mut self) -> Result<(), DurableRunnerError> { + // Terminal delivery can be reconciled by a replacement runner whose + // executor has not processed a provider command. Select the durable + // provider authority before cleanup so an absent in-memory selection + // can never turn the cleanup fence into a successful no-op. + self.select_recovery()?; if let Some(executor) = self.selected.as_mut() { executor.shutdown() } else { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs index 6d4c96ec84..6ac380da5d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs @@ -15,7 +15,7 @@ use std::os::unix::process::CommandExt; use std::os::fd::AsRawFd; #[cfg(target_os = "macos")] -use std::os::unix::fs::{DirBuilderExt, FileExt, MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::os::unix::fs::{FileExt, MetadataExt, OpenOptionsExt, PermissionsExt}; #[cfg(target_os = "macos")] use uuid::Uuid; @@ -26,6 +26,14 @@ use sha2::{Digest, Sha256}; use crate::local_runner::LocalRunnerError; const PROCESS_OUTPUT_QUEUE_CAPACITY: usize = 256; +const VERIFIED_RUNTIME_EXECUTABLE_ENV: &str = "PAPERCLIP_VERIFIED_RUNTIME_EXECUTABLE"; +const VERIFIED_COMMONJS_ARTIFACT_LOADER: &str = r#"const fs=require("node:fs");const Module=require("node:module");const filename=process.argv[1];const source=fs.readFileSync(filename,"utf8").replace(/^#![^\r\n]*(?:\r?\n|$)/,"");const artifact=new Module(filename);artifact.filename=filename;artifact.paths=[];artifact._compile(source,filename);"#; + +pub(crate) fn is_node_interpreter(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| matches!(name, "node" | "nodejs" | "node.exe")) +} #[derive(Clone, Debug)] pub struct VerifiedProcessArtifact { @@ -185,6 +193,7 @@ fn snapshot_error(display_path: &Path, error: impl std::fmt::Display) -> LocalRu pub enum VerifiedProcessArgument { Literal(String), Artifact(VerifiedProcessArtifact), + CommonJsArtifact(VerifiedProcessArtifact), ExecutableArtifact(VerifiedProcessArtifact), } @@ -192,11 +201,26 @@ pub enum VerifiedProcessArgument { pub struct VerifiedProcessLaunch { program: VerifiedProcessArtifact, args: Vec, + inherit_runtime_executable: bool, } impl VerifiedProcessLaunch { pub fn new(program: VerifiedProcessArtifact, args: Vec) -> Self { - Self { program, args } + Self { + program, + args, + inherit_runtime_executable: false, + } + } + + pub fn with_inherited_runtime_executable(mut self) -> Self { + self.inherit_runtime_executable = true; + self + } + + #[cfg(test)] + pub(crate) fn arguments(&self) -> &[VerifiedProcessArgument] { + &self.args } #[cfg(any(target_os = "linux", target_os = "macos"))] @@ -221,6 +245,13 @@ impl VerifiedProcessLaunch { inherited.push(fd); args.push(path.to_string_lossy().into_owned()); } + VerifiedProcessArgument::CommonJsArtifact(artifact) => { + let (fd, path) = inherited_artifact(artifact)?; + inherited.push(fd); + args.push("--eval".to_owned()); + args.push(VERIFIED_COMMONJS_ARTIFACT_LOADER.to_owned()); + args.push(path.to_string_lossy().into_owned()); + } VerifiedProcessArgument::ExecutableArtifact(artifact) => { #[cfg(target_os = "linux")] { @@ -259,7 +290,6 @@ struct InheritedCommand { #[cfg(target_os = "macos")] struct TemporaryExecutable { path: PathBuf, - directory: PathBuf, _file: File, } @@ -267,7 +297,6 @@ struct TemporaryExecutable { impl Drop for TemporaryExecutable { fn drop(&mut self) { let _ = fs::remove_file(&self.path); - let _ = fs::remove_dir(&self.directory); } } @@ -275,29 +304,31 @@ impl Drop for TemporaryExecutable { fn materialize_executable( artifact: &VerifiedProcessArtifact, ) -> Result { - let directory = std::env::temp_dir().join(format!( + let directory = artifact.display_path.parent().ok_or_else(|| { + LocalRunnerError::invalid(format!( + "verified process artifact {} has no parent directory", + artifact.display_path.display() + )) + })?; + let directory_metadata = fs::symlink_metadata(directory) + .map_err(|error| snapshot_error(&artifact.display_path, error))?; + if !directory_metadata.is_dir() || directory_metadata.permissions().mode() & 0o022 != 0 { + return Err(LocalRunnerError::invalid(format!( + "verified process artifact directory {} must be a directory that is not group- or world-writable", + directory.display() + ))); + } + let path = directory.join(format!( ".paperclip-verified-executable-{}", Uuid::new_v4().simple() )); - let mut directory_builder = fs::DirBuilder::new(); - directory_builder.mode(0o700); - directory_builder - .create(&directory) - .map_err(|error| snapshot_error(&artifact.display_path, error))?; - let path = directory.join("launch"); - let writable = OpenOptions::new() + let mut writable = OpenOptions::new() .read(true) .write(true) .create_new(true) .mode(0o700) - .open(&path); - let mut writable = match writable { - Ok(file) => file, - Err(error) => { - let _ = fs::remove_dir(&directory); - return Err(snapshot_error(&artifact.display_path, error)); - } - }; + .open(&path) + .map_err(|error| snapshot_error(&artifact.display_path, error))?; let result = (|| { let length = artifact .file @@ -344,13 +375,11 @@ fn materialize_executable( drop(writable); Ok(TemporaryExecutable { path: path.clone(), - directory: directory.clone(), _file: file, }) })(); if result.is_err() { let _ = fs::remove_file(path); - let _ = fs::remove_dir(directory); } result } @@ -590,6 +619,7 @@ impl SupervisedProcess { shutdown_grace, max_line_bytes, additional_environment_keys, + None, ) } @@ -608,6 +638,9 @@ impl SupervisedProcess { shutdown_grace, max_line_bytes, additional_environment_keys, + launch + .inherit_runtime_executable + .then_some(inherited.program.as_path()), ); #[cfg(target_os = "macos")] if let Ok(process) = result.as_mut() { @@ -637,6 +670,7 @@ impl SupervisedProcess { shutdown_grace: Duration, max_line_bytes: usize, additional_environment_keys: &[&str], + verified_runtime_executable: Option<&Path>, ) -> Result { let mut command = Command::new(program); command @@ -666,6 +700,13 @@ impl SupervisedProcess { command.env(key, value); } } + if let Some(executable) = verified_runtime_executable { + // Node reports a sealed memfd launch as `/memfd:... (deleted)` via + // process.execPath. Give descriptor-loaded runtimes the inherited, + // authenticated executable path so their governed child launches + // can reopen the same immutable image instead of that dead alias. + command.env(VERIFIED_RUNTIME_EXECUTABLE_ENV, executable); + } #[cfg(unix)] command.process_group(0); @@ -777,6 +818,8 @@ impl SupervisedProcess { })?; // The group leader can exit while descendants remain alive. Reap the // leader first, then clear any remaining members of its private group. + // A caller that must exclude escaped or re-parented descendants still + // needs a separately inherited lifetime fence. #[cfg(unix)] signal_process_group(self.process_group_id, "KILL"); self.finished = true; @@ -853,3 +896,49 @@ fn exit_fact(status: ExitStatus) -> ProcessExitFact { } } } + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn verified_artifact(path: &Path, bytes: &[u8]) -> VerifiedProcessArtifact { + fs::write(path, bytes).unwrap(); + let digest = format!("sha256:{:x}", Sha256::digest(bytes)); + VerifiedProcessArtifact::snapshot_verified( + path.to_owned(), + File::open(path).unwrap(), + &digest, + ) + .unwrap() + } + + #[test] + fn commonjs_artifacts_use_the_bounded_descriptor_loader() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "paperclip-commonjs-launch-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&directory).unwrap(); + let program = verified_artifact(&directory.join("node"), b"node"); + let script = verified_artifact(&directory.join("sidecar.cjs"), b"module.exports = {};\n"); + let launch = VerifiedProcessLaunch::new( + program, + vec![VerifiedProcessArgument::CommonJsArtifact(script)], + ); + + let inherited = launch.inherited_command().unwrap(); + + assert_eq!(inherited.args[0], "--eval"); + assert_eq!(inherited.args[1], VERIFIED_COMMONJS_ARTIFACT_LOADER); + #[cfg(target_os = "linux")] + assert!(inherited.args[2].starts_with("/proc/self/fd/")); + #[cfg(target_os = "macos")] + assert!(inherited.args[2].starts_with("/dev/fd/")); + fs::remove_dir_all(directory).unwrap(); + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 6151c8cced..d2b634c015 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -236,13 +236,142 @@ fn semantic_result_event( } } +fn validate_opencode_run_result( + state: &CodexProviderState, + params: &Value, +) -> Result<(Value, String, String), DurableRunnerError> { + if state.config.provider != "opencode" { + return Err(DurableRunnerError::invalid( + "paperclip/runResult is reserved for the verified OpenCode provider", + )); + } + if state.lifecycle != "turn_active" || state.active_provider_turn_id.is_none() { + return Err(DurableRunnerError::invalid( + "OpenCode emitted paperclip/runResult without an active provider turn", + )); + } + if params.get("threadId").and_then(Value::as_str) != state.thread_id.as_deref() + || params.get("turnId").and_then(Value::as_str) != state.active_provider_turn_id.as_deref() + { + return Err(DurableRunnerError::invalid( + "OpenCode paperclip/runResult is not bound to the active provider turn", + )); + } + let result = params.get("result").cloned().ok_or_else(|| { + DurableRunnerError::invalid("OpenCode paperclip/runResult omitted its result") + })?; + let schema: Value = serde_json::from_str(include_str!( + "../../../../protocol/schemas/result.schema.json" + )) + .map_err(|_| DurableRunnerError::invalid("embedded Paperclip result schema is invalid"))?; + let validator = jsonschema::validator_for(&schema).map_err(|_| { + DurableRunnerError::invalid("embedded Paperclip result schema cannot compile") + })?; + if !validator.is_valid(&result) { + return Err(DurableRunnerError::invalid( + "OpenCode paperclip/runResult failed the Paperclip result schema", + )); + } + let contract = state.completion_contract.as_ref().ok_or_else(|| { + DurableRunnerError::invalid("OpenCode paperclip/runResult has no bound completion contract") + })?; + let claim = result.get("completionClaim").ok_or_else(|| { + DurableRunnerError::invalid("OpenCode paperclip/runResult omitted its completion claim") + })?; + if claim.get("contractRevision").and_then(Value::as_str) != Some(contract.revision.as_str()) { + return Err(DurableRunnerError::invalid( + "OpenCode paperclip/runResult changed its completion contract revision", + )); + } + let criteria = claim + .get("criteria") + .and_then(Value::as_array) + .ok_or_else(|| { + DurableRunnerError::invalid( + "OpenCode paperclip/runResult omitted its completion criteria", + ) + })?; + let mut reported_criterion_ids = HashSet::new(); + for criterion in criteria { + let criterion_id = criterion + .get("criterionId") + .and_then(Value::as_str) + .ok_or_else(|| { + DurableRunnerError::invalid( + "OpenCode paperclip/runResult has an invalid completion criterion", + ) + })?; + if !reported_criterion_ids.insert(criterion_id) { + return Err(DurableRunnerError::invalid( + "OpenCode paperclip/runResult repeated a completion criterion", + )); + } + } + if reported_criterion_ids.len() != contract.criterion_ids.len() + || !contract + .criterion_ids + .iter() + .all(|criterion_id| reported_criterion_ids.contains(criterion_id.as_str())) + { + return Err(DurableRunnerError::invalid( + "OpenCode paperclip/runResult changed its bound completion criteria", + )); + } + let disposition = result + .get("reportedWorkDisposition") + .and_then(Value::as_str) + .expect("the validated result schema requires a disposition") + .to_owned(); + let fingerprint = semantic_value_digest(&result); + Ok((result, fingerprint, disposition)) +} + +fn normalize_provider_notification( + state: &mut CodexProviderState, + method: &str, + params: &Value, +) -> Result, DurableRunnerError> { + if method != "paperclip/runResult" { + return Ok(normalize_codex_notification(method, params) + .into_iter() + .map(|event| relabel_provider_event(event, &state.config.provider)) + .collect()); + } + let (result, fingerprint, disposition) = validate_opencode_run_result(state, params)?; + match ( + state.active_provider_result_fingerprint.as_deref(), + state.active_provider_result_disposition.as_deref(), + ) { + (None, None) => { + state.active_provider_result_fingerprint = Some(fingerprint); + state.active_provider_result_disposition = Some(disposition); + Ok(vec![NormalizedProviderEvent { + event_type: "run.result.proposed".to_owned(), + priority: EventPriority::P0, + payload: result, + }]) + } + (Some(existing_fingerprint), Some(existing_disposition)) + if existing_fingerprint == fingerprint && existing_disposition == disposition => + { + Ok(Vec::new()) + } + _ => Err(DurableRunnerError::invalid( + "OpenCode emitted conflicting paperclip/runResult notifications for one turn", + )), + } +} + fn terminal_events(state: &CodexProviderState, event_type: &str) -> Vec { let Some(contract) = state.completion_contract.as_ref() else { return Vec::new(); }; let succeeded = event_type == "turn.completed"; let cancelled = matches!(event_type, "turn.cancelled" | "turn.interrupted"); - let disposition = if succeeded { "done" } else { "needs_review" }; + let disposition = state + .active_provider_result_disposition + .as_deref() + .unwrap_or(if succeeded { "done" } else { "needs_review" }); let provider = state.config.provider.as_str(); let provider_name = if provider == "opencode" { "OpenCode" @@ -308,18 +437,20 @@ fn terminal_events(state: &CodexProviderState, event_type: &str) -> Vec, + #[serde(default)] + active_provider_result_fingerprint: Option, + #[serde(default)] + active_provider_result_disposition: Option, last_agent_message: Option, #[serde(default)] pending_events: VecDeque, @@ -466,6 +601,8 @@ impl CodexProviderState { receipt_limit_interrupt_accepted: false, receipt_limit_interrupt_attempts: 0, receipt_limit_interrupt_deadline_unix_ms: None, + active_provider_result_fingerprint: None, + active_provider_result_disposition: None, last_agent_message: None, pending_events: VecDeque::new(), queued_events: VecDeque::new(), @@ -566,6 +703,25 @@ impl CodexProviderState { || self .receipt_limit_interrupt_deadline_unix_ms .is_some_and(|deadline| deadline == 0 || !self.receipt_limit_interrupt_pending) + || self.active_provider_result_fingerprint.is_some() + != self.active_provider_result_disposition.is_some() + || self + .active_provider_result_fingerprint + .as_ref() + .is_some_and(|fingerprint| { + fingerprint.len() != 71 + || !fingerprint.starts_with("sha256:") + || !fingerprint[7..] + .chars() + .all(|character| character.is_ascii_hexdigit()) + }) + || self + .active_provider_result_disposition + .as_deref() + .is_some_and(|disposition| { + !matches!(disposition, "done" | "blocked" | "needs_review" | "yielded") + || self.config.provider != "opencode" + }) || (matches!( self.lifecycle.as_str(), "prepared" | "session_open" | "closed" @@ -776,6 +932,8 @@ impl CodexProviderState { self.completed_turn_process_generation = None; self.completed_provider_turn_id = None; self.ambiguous_turn_start_pending = false; + self.active_provider_result_fingerprint = None; + self.active_provider_result_disposition = None; self.last_agent_message = None; } self.lifecycle = if self.active_provider_turn_id.is_some() { @@ -1001,6 +1159,12 @@ impl CodexCommandExecutor { Some(&thread_id), process_generation, self.opencode_launch_profile.as_ref(), + state.completion_contract.as_ref().map(|contract| { + ( + contract.revision.as_str(), + contract.criterion_ids.as_slice(), + ) + }), ) .map_err(|error| { DurableRunnerError::invalid(format!( @@ -1050,6 +1214,8 @@ impl CodexCommandExecutor { state.receipt_limit_interrupt_accepted = false; state.receipt_limit_interrupt_attempts = 0; state.receipt_limit_interrupt_deadline_unix_ms = None; + state.active_provider_result_fingerprint = None; + state.active_provider_result_disposition = None; state.last_agent_message = None; state.lifecycle = "closed".to_owned(); let _ = state.push_terminal_event(NormalizedProviderEvent { @@ -1102,6 +1268,8 @@ impl CodexCommandExecutor { state.receipt_limit_interrupt_accepted = false; state.receipt_limit_interrupt_attempts = 0; state.receipt_limit_interrupt_deadline_unix_ms = None; + state.active_provider_result_fingerprint = None; + state.active_provider_result_disposition = None; state.last_agent_message = None; state.lifecycle = "closed".to_owned(); // Closing the provider is the safety boundary. Preserve that @@ -1395,6 +1563,12 @@ impl CodexCommandExecutor { state.thread_id.as_deref(), process_generation, self.opencode_launch_profile.as_ref(), + state.completion_contract.as_ref().map(|contract| { + ( + contract.revision.as_str(), + contract.criterion_ids.as_slice(), + ) + }), ) .map_err(|error| { DurableRunnerError::invalid(format!("failed to start Codex provider: {error}")) @@ -1498,6 +1672,8 @@ impl CodexCommandExecutor { next_state.receipt_limit_interrupt_accepted = false; next_state.receipt_limit_interrupt_attempts = 0; next_state.receipt_limit_interrupt_deadline_unix_ms = None; + next_state.active_provider_result_fingerprint = None; + next_state.active_provider_result_disposition = None; next_state.last_agent_message = None; if let Some(provider) = self.provider.as_mut() { provider.shutdown().map_err(|error| { @@ -1618,6 +1794,8 @@ impl CodexCommandExecutor { state.receipt_limit_interrupt_accepted = false; state.receipt_limit_interrupt_attempts = 0; state.receipt_limit_interrupt_deadline_unix_ms = None; + state.active_provider_result_fingerprint = None; + state.active_provider_result_disposition = None; state.last_agent_message = None; state.lifecycle = "closed".to_owned(); let provider_label = state.config.provider.clone(); @@ -1842,6 +2020,8 @@ impl CodexCommandExecutor { state.completed_turn_authoritative = false; state.completed_turn_process_generation = None; state.completed_provider_turn_id = None; + state.active_provider_result_fingerprint = None; + state.active_provider_result_disposition = None; state.last_agent_message = None; } self.save_state()?; @@ -1878,6 +2058,8 @@ impl CodexCommandExecutor { state.receipt_limit_interrupt_accepted = false; state.receipt_limit_interrupt_attempts = 0; state.receipt_limit_interrupt_deadline_unix_ms = None; + state.active_provider_result_fingerprint = None; + state.active_provider_result_disposition = None; state.last_agent_message = None; state.lifecycle = "turn_active".to_owned(); let provider_label = state.config.provider.clone(); @@ -1938,6 +2120,83 @@ impl CodexCommandExecutor { }))) } + fn stop_turn_for_suspension( + &mut self, + reason: &str, + ) -> Result { + self.restore_provider_if_needed()?; + let provider_turn_id = self + .state + .as_ref() + .and_then(|state| state.active_provider_turn_id.clone()); + let Some(provider_turn_id) = provider_turn_id else { + return Ok(CommandExecution::result(json!({ + "status": "already_settled", + "reason": reason, + }))); + }; + + // The cooperative interrupt is useful to the provider, but its RPC + // acknowledgement is not proof that an active turn stopped. A + // controller issues turn.stop only while closing a run whose result is + // already durable, so terminate the exact process generation before + // publishing the provider state as attachable by a successor run. + let interrupt_accepted = self.interrupt_turn(reason).is_ok(); + let provider_shutdown_failed = self + .provider + .as_mut() + .is_some_and(|provider| provider.shutdown().is_err()); + if provider_shutdown_failed { + return Err(DurableRunnerError::invalid( + "failed to prove provider termination at the suspension boundary", + )); + } + self.provider = None; + + let identity = self.event_identity()?; + let state = self + .state + .as_mut() + .expect("Codex state remains available after provider termination"); + state.settle_active_provider_turn_identity()?; + let settled = state + .tool_bridge + .settle_turn("provider_turn_stopped_for_suspension") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to settle semantic tools at the suspension boundary: {error}" + )) + })?; + for result in settled { + state.push_terminal_event(semantic_result_event(&identity, &result))?; + } + state.active_provider_turn_id = None; + state.ambiguous_turn_start_pending = false; + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + state.active_provider_result_fingerprint = None; + state.active_provider_result_disposition = None; + state.last_agent_message = None; + // Do not let the runner.drain command that follows turn.stop restore a + // fresh provider process. `prepared` retains the durable thread while + // deferring the only authorized restart to the successor run.attach. + state.lifecycle = "prepared".to_owned(); + self.save_state()?; + Ok(CommandExecution::result(json!({ + "status": "stopped", + "providerTurnId": provider_turn_id, + "reason": reason, + "interruptAccepted": interrupt_accepted, + "providerExitConfirmed": true, + }))) + } + fn steer_turn(&mut self, payload: &Value) -> Result { let text = payload .get("text") @@ -2388,6 +2647,8 @@ impl CodexCommandExecutor { state.receipt_limit_interrupt_accepted = false; state.receipt_limit_interrupt_attempts = 0; state.receipt_limit_interrupt_deadline_unix_ms = None; + state.active_provider_result_fingerprint = None; + state.active_provider_result_disposition = None; state.lifecycle = "closed".to_owned(); let thread_id = state.thread_id.clone(); let provider_name = state.config.provider.clone(); @@ -2486,29 +2747,7 @@ impl CodexCommandExecutor { } else { None }; - let provider_name = self - .state - .as_ref() - .map(|state| state.config.provider.clone()) - .unwrap_or_else(|| "codex".to_owned()); - let normalized = normalize_codex_notification(&method, ¶ms) - .into_iter() - .map(|event| relabel_provider_event(event, &provider_name)) - .collect::>(); - let normalized_event_count = normalized.len(); - let terminal_event_type = normalized - .iter() - .find(|event| event.event_type.starts_with("turn.")) - .map(|event| event.event_type.clone()) - .filter(|event_type| { - matches!( - event_type.as_str(), - "turn.completed" - | "turn.failed" - | "turn.cancelled" - | "turn.interrupted" - ) - }); + let terminal_event_type = normalized_terminal_type.map(str::to_owned); let identity = self.event_identity.clone(); let state = self .state @@ -2532,6 +2771,8 @@ impl CodexCommandExecutor { })?; state.reconcile_active_provider_turn(Some(provider_turn_id)); } + let normalized = normalize_provider_notification(state, &method, ¶ms)?; + let normalized_event_count = normalized.len(); if terminal_event_type.is_some() { state.settle_active_provider_turn_identity()?; let settled = state @@ -2748,9 +2989,8 @@ impl CommandExecutor for CodexCommandExecutor { "session.open" => self.open_session(), "turn.start" => self.start_turn(&command.payload), "turn.steer" => self.steer_turn(&command.payload), - "turn.interrupt" | "turn.stop" | "run.cancel" => { - self.interrupt_turn(&command.command_type) - } + "turn.interrupt" | "run.cancel" => self.interrupt_turn(&command.command_type), + "turn.stop" => self.stop_turn_for_suspension(&command.command_type), "request.resolve" => self.resolve_request(&command.payload), "semantic_tool.result" => self.deliver_semantic_result(&command.payload), "session.snapshot" => self.snapshot(), @@ -2798,6 +3038,11 @@ impl CommandExecutor for CodexCommandExecutor { } fn shutdown(&mut self) -> Result<(), DurableRunnerError> { + // Terminal-result recovery can invoke shutdown on a fresh executor. + // Loading the durable provider identity here ensures that cleanup is + // attempted against the persisted session instead of reporting a + // successful no-op from an empty in-memory provider slot. + self.restore()?; if let Some(provider) = self.provider.as_mut() { provider.shutdown().map_err(|error| { DurableRunnerError::invalid(format!("failed to stop Codex provider: {error}")) @@ -2812,6 +3057,158 @@ impl CommandExecutor for CodexCommandExecutor { mod tests { use super::*; + fn opencode_result_state() -> CodexProviderState { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "opencode".to_owned(), + driver: "opencode_server".to_owned(), + provider_version: "1.18.17".to_owned(), + command: PathBuf::from("node"), + args: Vec::new(), + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: Some("openrouter/model".to_owned()), + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + Some(CompletionContractBinding { + revision: "revision-1".to_owned(), + criterion_ids: vec!["criterion-1".to_owned()], + }), + ProviderToolBridge::default(), + ); + state.thread_id = Some("thread-1".to_owned()); + state.active_provider_turn_id = Some("turn-1".to_owned()); + state.lifecycle = "turn_active".to_owned(); + state + } + + fn valid_opencode_result() -> Value { + json!({ + "schema": "paperclip.run_result.v1", + "reportedWorkDisposition": "done", + "summary": "Finished the requested work.", + "completionClaim": { + "contractRevision": "revision-1", + "objectiveSatisfied": true, + "criteria": [{ + "criterionId": "criterion-1", + "status": "satisfied", + "evidenceRefs": ["provider:opencode:agent-message"], + }], + "remainingWork": [], + }, + "evidence": [{"ref": "provider:opencode:agent-message"}], + "verification": [], + "attentionRequests": [], + "artifacts": [], + }) + } + + #[test] + fn preserves_one_verified_opencode_result_before_its_terminal() { + let mut state = opencode_result_state(); + let params = json!({ + "threadId": "thread-1", + "turnId": "turn-1", + "itemId": "semantic-result", + "result": valid_opencode_result(), + }); + + let result_events = + normalize_provider_notification(&mut state, "paperclip/runResult", ¶ms).unwrap(); + let replay_events = + normalize_provider_notification(&mut state, "paperclip/runResult", ¶ms).unwrap(); + let terminal = terminal_events(&state, "turn.completed"); + + assert_eq!(result_events.len(), 1); + assert_eq!(result_events[0].event_type, "run.result.proposed"); + assert_eq!(result_events[0].priority, EventPriority::P0); + assert!(replay_events.is_empty()); + assert_eq!(terminal.len(), 1); + assert_eq!(terminal[0].event_type, "run.terminal"); + assert_eq!(terminal[0].payload["reportedWorkDisposition"], "done"); + assert!(state.validate().is_ok()); + } + + #[test] + fn rejects_unbound_conflicting_or_spoofed_opencode_results() { + let params = |result: Value| { + json!({ + "threadId": "thread-1", + "turnId": "turn-1", + "itemId": "semantic-result", + "result": result, + }) + }; + + let mut wrong_revision = opencode_result_state(); + let mut result = valid_opencode_result(); + result["completionClaim"]["contractRevision"] = json!("revision-2"); + assert!(normalize_provider_notification( + &mut wrong_revision, + "paperclip/runResult", + ¶ms(result), + ) + .unwrap_err() + .to_string() + .contains("contract revision")); + + let mut malformed = opencode_result_state(); + assert!(normalize_provider_notification( + &mut malformed, + "paperclip/runResult", + ¶ms(json!({"schema": "paperclip.run_result.v1"})), + ) + .unwrap_err() + .to_string() + .contains("failed the Paperclip result schema")); + + let mut wrong_criteria = opencode_result_state(); + let mut result = valid_opencode_result(); + result["completionClaim"]["criteria"][0]["criterionId"] = json!("criterion-2"); + assert!(normalize_provider_notification( + &mut wrong_criteria, + "paperclip/runResult", + ¶ms(result), + ) + .unwrap_err() + .to_string() + .contains("bound completion criteria")); + + let mut conflicting = opencode_result_state(); + normalize_provider_notification( + &mut conflicting, + "paperclip/runResult", + ¶ms(valid_opencode_result()), + ) + .unwrap(); + let mut result = valid_opencode_result(); + result["summary"] = json!("A conflicting second result."); + assert!(normalize_provider_notification( + &mut conflicting, + "paperclip/runResult", + ¶ms(result), + ) + .unwrap_err() + .to_string() + .contains("conflicting")); + + let mut spoofed = opencode_result_state(); + spoofed.config.provider = "codex".to_owned(); + assert!(normalize_provider_notification( + &mut spoofed, + "paperclip/runResult", + ¶ms(valid_opencode_result()), + ) + .unwrap_err() + .to_string() + .contains("reserved for the verified OpenCode provider")); + } + #[test] fn opencode_terminal_fallback_uses_its_actual_provider_identity() { let mut state = CodexProviderState::new( @@ -2890,6 +3287,8 @@ mod tests { receipt_limit_interrupt_accepted: false, receipt_limit_interrupt_attempts: 0, receipt_limit_interrupt_deadline_unix_ms: None, + active_provider_result_fingerprint: None, + active_provider_result_disposition: None, last_agent_message: None, pending_events: VecDeque::new(), queued_events: VecDeque::new(), diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_checkpoint.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_checkpoint.rs index c5897506e0..0cea4bb818 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_checkpoint.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_checkpoint.rs @@ -74,6 +74,7 @@ fn identity() -> AcpxProviderSessionIdentity { requested_model: "gpt-5.6-sol".to_owned(), effective_model: "gpt-5.6-sol".to_owned(), permission_mode: Some(AcpxPermissionMode::ApproveReads), + provider_lifetime_fence_candidates: [60_001, 60_002, 60_003], } } @@ -193,6 +194,11 @@ fn fails_closed_on_unknown_or_oversized_checkpoint_files() { .as_object_mut() .unwrap() .remove("permissionMode"); + let mut missing_lifetime_fence = valid.clone(); + missing_lifetime_fence["identity"] + .as_object_mut() + .unwrap() + .remove("providerLifetimeFenceCandidates"); let mut invalid_run = valid.clone(); invalid_run["runId"] = json!("run 1"); let mut invalid_session = valid; @@ -202,6 +208,7 @@ fn fails_closed_on_unknown_or_oversized_checkpoint_files() { top_level_unknown, nested_unknown, missing_permission, + missing_lifetime_fence, invalid_run, invalid_session, ] { diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs index e0ae0f7f17..3fe5e42780 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs @@ -62,6 +62,7 @@ fn expected_identity() -> AcpxProviderSessionIdentity { requested_model: "gpt-5.6-sol".to_owned(), effective_model: "gpt-5.6-sol".to_owned(), permission_mode: Some(AcpxPermissionMode::ApproveReads), + provider_lifetime_fence_candidates: [60_001, 60_002, 60_003], } } @@ -100,6 +101,12 @@ fn validates_qualified_policy_and_tool_catalog_before_spawning() { let mut invalid_tools = config("bootstrap"); invalid_tools.tool_set.catalog_digest = "invalid".to_owned(); assert!(start_error(&invalid_tools).contains("authorized tools")); + + let mut invalid_lifetime_fence = config("bootstrap"); + let mut invalid_identity = expected_identity(); + invalid_identity.provider_lifetime_fence_candidates = [60_001, 60_001, 60_003]; + invalid_lifetime_fence.expected_identity = Some(invalid_identity); + assert!(start_error(&invalid_lifetime_fence).contains("lifetime fence candidates")); } #[test] diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_suspend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_suspend.rs index 1054261f3e..75d6fed02f 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_suspend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_suspend.rs @@ -59,6 +59,19 @@ fn rejects_suspension_during_an_active_turn_without_closing_the_session() { session.shutdown("test complete").unwrap(); } +#[test] +fn reaps_an_active_provider_generation_at_the_suspension_boundary() { + let mut session = AcpxProviderSession::start(&config("suspend")).unwrap(); + session + .start_turn("turn-1", "Please help", &std::env::temp_dir()) + .unwrap(); + + session + .terminate_active_turn_for_suspension("turn-1") + .unwrap(); + assert!(session.shutdown("already terminated").is_ok()); +} + #[test] fn fails_closed_when_the_suspension_acknowledgement_does_not_match() { for mode in ["suspend-wrong-ack", "suspend-wrong-identity"] { diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs index bbead687b5..2299a71e62 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs @@ -162,6 +162,7 @@ fn keeps_valid_command_rejections_separate_from_protocol_failures() { .expect_err("fake command should be rejected"); let message = error.to_string(); assert!(message.contains("was rejected")); + assert!(message.contains("classification=unclassified")); assert!(!message.contains("Q7Z9")); assert!(!message.contains("violet-circuit-4821")); assert!(!message.contains("unavailable")); diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 0723b7338e..27c6d57fc9 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -354,7 +354,8 @@ fn codex_dynamic_tool_round_trips_through_the_provider_boundary() { let mut delivered = false; let mut completed = false; - for _ in 0..32 { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { match provider.poll().expect("poll semantic tool event") { Some(CodexProviderEvent::ToolCall { call_id, diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index 815eedea69..2a2db7a20b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -15,7 +15,7 @@ use serde_json::{json, Value}; use sha2::{Digest, Sha256}; const CODEX_ACPX_DIGEST: &str = - "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79"; + "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400"; fn temporary_directory(label: &str) -> PathBuf { let nonce = SystemTime::now() @@ -95,7 +95,7 @@ fn opencode_config(state_dir: &Path) -> DurableRunnerConfig { fs::write( &proxy_script, format!( - "#!/bin/sh\nexec '{}' --state-file '{}' --call-log '{}'\n", + "#!/bin/sh\nexec '{}' --state-file '{}' --call-log '{}' --require-completion-contract\n", env!("CARGO_BIN_EXE_fake-codex-app-server"), state_dir.join("fake-opencode-state.json").display(), state_dir.join("fake-opencode-calls.log").display(), @@ -116,6 +116,14 @@ fn opencode_config(state_dir: &Path) -> DurableRunnerConfig { config } +fn opencode_call_count(state_dir: &Path, method: &str) -> usize { + fs::read_to_string(state_dir.join("fake-opencode-calls.log")) + .unwrap_or_default() + .lines() + .filter(|line| *line == method) + .count() +} + fn command(sequence: u64, command_type: &str, payload: Value) -> Command { Command { schema: "paperclip.prp.command.v1".to_owned(), @@ -135,6 +143,11 @@ fn prepare_payload(directory: &Path, agent: &str) -> Value { fn prepare_payload_with_mode(directory: &Path, agent: &str, mode: &str) -> Value { let operations = Vec::new(); + let (runtime_package, runtime_version) = if agent == "codex" { + (json!("@openai/codex"), json!("0.148.0")) + } else { + (Value::Null, Value::Null) + }; json!({ "authorizedTools": { "schema": "paperclip.runner.authorized-tools.v1", @@ -152,8 +165,8 @@ fn prepare_payload_with_mode(directory: &Path, agent: &str, mode: &str) -> Value "acpxVersion": "0.13.1", "agentServerPackage": "@agentclientprotocol/codex-acp", "agentServerVersion": "1.6.2", - "agentRuntimePackage": null, - "agentRuntimeVersion": null, + "agentRuntimePackage": runtime_package, + "agentRuntimeVersion": runtime_version, "commandDigest": CODEX_ACPX_DIGEST, "sidecarCommand": env!("CARGO_BIN_EXE_fake-acpx-sidecar"), "sidecarArgs": [ @@ -419,6 +432,37 @@ fn executes_opencode_through_the_local_facade_without_codex_event_labels() { fs::remove_dir_all(directory).unwrap(); } +#[test] +fn replacement_shutdown_restores_the_persisted_provider_before_cleanup() { + let directory = temporary_directory("opencode-replacement-shutdown"); + let config = opencode_config(&directory); + let mut first = NativeProviderCommandExecutor::with_runner_config(&directory, &config); + + first + .execute(&command( + 1, + "run.prepare", + opencode_prepare_payload(&directory), + )) + .unwrap(); + first + .execute(&command(2, "session.open", json!({}))) + .unwrap(); + first.shutdown().unwrap(); + drop(first); + + let resumes_before_cleanup = opencode_call_count(&directory, "thread/resume"); + let mut replacement = NativeProviderCommandExecutor::with_runner_config(&directory, &config); + replacement.shutdown().unwrap(); + + assert_eq!( + opencode_call_count(&directory, "thread/resume"), + resumes_before_cleanup + 1, + "a replacement executor must restore the persisted provider before terminal cleanup", + ); + fs::remove_dir_all(directory).unwrap(); +} + #[test] fn rejects_a_mutable_opencode_command_outside_the_runner_launch_profile() { let directory = temporary_directory("opencode-command-override"); @@ -485,7 +529,12 @@ fn rejects_opencode_launch_profile_drift_across_fresh_recovery() { .contains("launch profile changed across durable recovery")); assert_eq!(fs::read(&state_path).unwrap(), state_before_recovery); - recovered.shutdown().unwrap(); + let shutdown_error = recovered + .shutdown() + .expect_err("invalid recovered launch authority also blocks cleanup"); + assert!(shutdown_error + .to_string() + .contains("launch profile changed across durable recovery")); fs::remove_dir_all(directory).unwrap(); } diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/process_supervisor.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/process_supervisor.rs index d3671ecccf..d1fac57753 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/process_supervisor.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/process_supervisor.rs @@ -1,6 +1,8 @@ #![cfg(unix)] use std::fs::{self, File}; +#[cfg(target_os = "macos")] +use std::io::Read; use std::io::Write; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; @@ -35,6 +37,21 @@ fn sha256(contents: &str) -> String { format!("sha256:{:x}", Sha256::digest(contents.as_bytes())) } +#[cfg(target_os = "macos")] +fn sha256_file(path: &Path) -> String { + let mut file = File::open(path).unwrap(); + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let count = file.read(&mut buffer).unwrap(); + if count == 0 { + break; + } + digest.update(&buffer[..count]); + } + format!("sha256:{:x}", digest.finalize()) +} + #[test] fn verified_launch_uses_open_command_and_script_after_atomic_path_replacement() { let directory = std::env::temp_dir().join(format!( @@ -48,7 +65,7 @@ fn verified_launch_uses_open_command_and_script_after_atomic_path_replacement() fs::create_dir(&directory).unwrap(); let command = directory.join("command"); let script = directory.join("script"); - let original_command = "#!/bin/sh\nprintf '%s\\n' old-command\nexec /bin/sh \"$1\"\n"; + let original_command = "#!/bin/sh\nprintf '%s\\n' old-command\nprintf '%s\\n' \"$PAPERCLIP_VERIFIED_RUNTIME_EXECUTABLE\"\nexec /bin/sh \"$1\"\n"; let original_script = "#!/bin/sh\nprintf '%s\\n' old-script\n"; write_executable(&command, original_command); write_executable(&script, original_script); @@ -68,7 +85,8 @@ fn verified_launch_uses_open_command_and_script_after_atomic_path_replacement() ) .unwrap(), )], - ); + ) + .with_inherited_runtime_executable(); let replacement_command = directory.join("replacement-command"); let replacement_script = directory.join("replacement-script"); @@ -97,6 +115,21 @@ fn verified_launch_uses_open_command_and_script_after_atomic_path_replacement() .as_deref(), Some("old-command") ); + let inherited_runtime = process + .receive_stdout_line(Duration::from_secs(1)) + .unwrap() + .expect("verified launch should identify its inherited runtime"); + #[cfg(target_os = "linux")] + assert!(inherited_runtime.starts_with("/proc/self/fd/")); + #[cfg(target_os = "macos")] + { + assert!(inherited_runtime.contains(".paperclip-verified-executable-")); + assert_eq!( + Path::new(&inherited_runtime).parent(), + command.parent(), + "macOS verified launches must preserve loader-relative runtime layout" + ); + } assert_eq!( process .receive_stdout_line(Duration::from_secs(1)) @@ -108,6 +141,55 @@ fn verified_launch_uses_open_command_and_script_after_atomic_path_replacement() fs::remove_dir_all(directory).unwrap(); } +#[cfg(target_os = "macos")] +#[test] +fn verified_launch_preserves_homebrew_node_loader_layout() { + let resolved = Command::new("/usr/bin/which") + .arg("node") + .output() + .expect("node lookup should run"); + assert!( + resolved.status.success(), + "node should be available on PATH" + ); + let node = fs::canonicalize( + String::from_utf8(resolved.stdout) + .expect("node path should be UTF-8") + .trim(), + ) + .expect("node path should resolve"); + let launch = VerifiedProcessLaunch::new( + VerifiedProcessArtifact::snapshot_verified( + node.clone(), + File::open(&node).unwrap(), + &sha256_file(&node), + ) + .unwrap(), + vec![ + VerifiedProcessArgument::Literal("--eval".to_owned()), + VerifiedProcessArgument::Literal("console.log(process.execPath)".to_owned()), + ], + ); + + let mut process = SupervisedProcess::spawn_verified_with_environment_keys( + &launch, + Duration::from_millis(50), + 1024, + &[], + ) + .expect("verified Node should start with its loader-relative libraries"); + let executed_node = process + .receive_stdout_line(Duration::from_secs(2)) + .unwrap() + .expect("Node should report its executable path"); + assert_eq!( + Path::new(&executed_node).parent(), + node.parent(), + "verified Node must execute beside the authenticated runtime" + ); + process.wait().unwrap(); +} + fn spawn_linger_process() -> (SupervisedProcess, u32, u64) { let harness = PathBuf::from(env!("CARGO_BIN_EXE_fake-harness")); let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")) diff --git a/packages/paperclip-runner/scripts/build-provider-pack.mjs b/packages/paperclip-runner/scripts/build-provider-pack.mjs index bb5f6e4869..b616674819 100644 --- a/packages/paperclip-runner/scripts/build-provider-pack.mjs +++ b/packages/paperclip-runner/scripts/build-provider-pack.mjs @@ -4,6 +4,7 @@ import { chmodSync, copyFileSync, existsSync, + mkdirSync, mkdtempSync, readdirSync, readFileSync, @@ -24,16 +25,14 @@ const outputRoot = resolve( outputArgument ?? join(packageRoot, "provider-pack"), ); if ( - outputRoot === workspaceRoot - || outputRoot === packageRoot - || outputRoot === "/" + outputRoot === workspaceRoot || + outputRoot === packageRoot || + outputRoot === "/" ) { throw new Error(`Refusing unsafe provider-pack output path: ${outputRoot}`); } -const temporaryParent = mkdtempSync( - join(tmpdir(), "paperclip-provider-pack-"), -); +const temporaryParent = mkdtempSync(join(tmpdir(), "paperclip-provider-pack-")); const temporaryRoot = join(temporaryParent, "pack"); function canonicalJson(value) { @@ -56,8 +55,9 @@ function sha256File(path) { function sha256Tree(root) { const hash = createHash("sha256"); const visit = (directory, prefix = "") => { - const entries = readdirSync(directory, { withFileTypes: true }) - .sort((left, right) => left.name.localeCompare(right.name)); + const entries = readdirSync(directory, { withFileTypes: true }).sort( + (left, right) => left.name.localeCompare(right.name), + ); for (const entry of entries) { const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name; const absolutePath = join(directory, entry.name); @@ -67,9 +67,13 @@ function sha256Tree(root) { } else if (entry.isFile()) { hash.update(`file\0${relativePath}\0${sha256File(absolutePath)}\n`); } else if (entry.isSymbolicLink()) { - hash.update(`symlink\0${relativePath}\0${readlinkSync(absolutePath)}\n`); + hash.update( + `symlink\0${relativePath}\0${readlinkSync(absolutePath)}\n`, + ); } else { - throw new Error(`Provider pack tree contains unsupported entry ${relativePath}`); + throw new Error( + `Provider pack tree contains unsupported entry ${relativePath}`, + ); } } }; @@ -123,6 +127,32 @@ try { throw new Error(`pnpm deploy failed with exit code ${deployed.status}`); } + // Reuse the already-qualified build interpreter instead of introducing a + // package-manager lifecycle hook or a second binary supply chain. The pack + // manifest binds the copied bytes, platform, architecture, and minimum + // version before any provider is launched. + const minimumNodeVersion = [24, 11, 0]; + const actualNodeVersion = process.versions.node.split(".").map(Number); + if ( + actualNodeVersion[0] < minimumNodeVersion[0] || + (actualNodeVersion[0] === minimumNodeVersion[0] && + (actualNodeVersion[1] < minimumNodeVersion[1] || + (actualNodeVersion[1] === minimumNodeVersion[1] && + actualNodeVersion[2] < minimumNodeVersion[2]))) + ) { + throw new Error("Provider pack build Node is older than 24.11.0"); + } + const stableNodeRoot = join(temporaryRoot, "node_modules", "node"); + if (existsSync(stableNodeRoot)) { + throw new Error( + "Provider pack deployment unexpectedly claimed the stable Node path", + ); + } + const stableNodeCommand = join(stableNodeRoot, "bin", "node"); + mkdirSync(dirname(stableNodeCommand), { recursive: true, mode: 0o755 }); + copyFileSync(process.execPath, stableNodeCommand); + chmodSync(stableNodeCommand, 0o755); + // pnpm's generated .bin shims embed the temporary deployment directory in // NODE_PATH. That makes an otherwise identical provider pack hash differ on // every build and leaks a nonexistent host path after relocation. Replace @@ -187,13 +217,16 @@ try { ); } - const opencodeProxyPath = "dist/cli/opencode-app-server-proxy.js"; - const acpxSidecarPath = "dist/cli/acpx-runtime-sidecar.js"; + const opencodeProxyPath = "dist/cli/opencode-app-server-proxy.cjs"; + const acpxSidecarPath = "dist/cli/acpx-runtime-sidecar.cjs"; const opencodeCommand = "node_modules/.bin/opencode"; const opencodeExecutable = "node_modules/opencode-ai/bin/opencode.exe"; const nodeCommand = "node_modules/node/bin/node"; const productionLock = "pnpm-lock.yaml"; - copyFileSync(join(workspaceRoot, "pnpm-lock.yaml"), join(temporaryRoot, productionLock)); + copyFileSync( + join(workspaceRoot, "pnpm-lock.yaml"), + join(temporaryRoot, productionLock), + ); for (const relativePath of [ nodeCommand, productionLock, @@ -207,16 +240,14 @@ try { } } - const opencodeProxySha = sha256File( - join(temporaryRoot, opencodeProxyPath), - ); + const opencodeProxySha = sha256File(join(temporaryRoot, opencodeProxyPath)); const acpxSidecarSha = sha256File(join(temporaryRoot, acpxSidecarPath)); const distDigest = sha256Tree(join(temporaryRoot, "dist")); const configuredRevision = process.env.PAPERCLIP_RUNNER_SOURCE_REVISION?.trim(); const revision = - configuredRevision - ?? execFileSync("git", ["rev-parse", "HEAD"], { + configuredRevision ?? + execFileSync("git", ["rev-parse", "HEAD"], { cwd: workspaceRoot, encoding: "utf8", }).trim(); @@ -225,14 +256,12 @@ try { } const dirty = configuredRevision ? false - : spawnSync( - "git", - ["diff", "--quiet", "--", "packages/paperclip-runner"], - { cwd: workspaceRoot }, - ).status !== 0; + : spawnSync("git", ["diff", "--quiet", "--", "packages/paperclip-runner"], { + cwd: workspaceRoot, + }).status !== 0; const payload = { pins: { - nodeMinimum: "24.11.0", + nodeMinimum: minimumNodeVersion.join("."), codex: "0.148.0", opencode: "1.18.17", acpx: "0.13.1", @@ -253,7 +282,7 @@ try { claude: "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", codex: - "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79", + "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400", }, artifacts: { nodeCommand: { diff --git a/packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs b/packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs new file mode 100644 index 0000000000..acf7b56083 --- /dev/null +++ b/packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs @@ -0,0 +1,106 @@ +import { chmod } from "node:fs/promises"; +import { builtinModules } from "node:module"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { build } from "esbuild"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +export const verifiedProviderEntrypoints = Object.freeze([ + Object.freeze({ + name: "acpx-runtime-sidecar", + source: resolve(packageRoot, "src/cli/acpx-runtime-sidecar.ts"), + output: resolve(packageRoot, "dist/cli/acpx-runtime-sidecar.js"), + verifiedOutput: resolve(packageRoot, "dist/cli/acpx-runtime-sidecar.cjs"), + }), + Object.freeze({ + name: "opencode-app-server-proxy", + source: resolve(packageRoot, "src/cli/opencode-app-server-proxy.ts"), + output: resolve(packageRoot, "dist/cli/opencode-app-server-proxy.js"), + verifiedOutput: resolve( + packageRoot, + "dist/cli/opencode-app-server-proxy.cjs", + ), + }), +]); + +const nodeBuiltins = new Set([ + ...builtinModules, + ...builtinModules.map((name) => `node:${name}`), +]); + +function assertSelfContainedBundle(entrypoint, result) { + const outputs = Object.entries(result.metafile.outputs).filter( + ([, output]) => output.entryPoint !== undefined, + ); + if (outputs.length !== 1) { + throw new Error( + `${entrypoint.name} bundle emitted ${outputs.length} entrypoint outputs instead of one`, + ); + } + const imports = outputs[0][1].imports; + for (const dependency of imports) { + if (!dependency.external || !nodeBuiltins.has(dependency.path)) { + throw new Error( + `${entrypoint.name} bundle retained a non-builtin import: ${dependency.path}`, + ); + } + } +} + +export async function bundleVerifiedProviderEntrypoints({ write = true } = {}) { + const results = []; + for (const entrypoint of verifiedProviderEntrypoints) { + const buildBundle = async (outfile, format) => { + const result = await build({ + entryPoints: [entrypoint.source], + outfile, + bundle: true, + platform: "node", + format, + target: "node24", + packages: "bundle", + splitting: false, + sourcemap: false, + legalComments: "none", + metafile: true, + treeShaking: true, + write, + logLevel: "silent", + banner: + format === "cjs" + ? { + js: 'const __paperclipVerifiedEntrypointUrl = require("node:url").pathToFileURL(__filename).href;', + } + : undefined, + define: + format === "cjs" + ? { + "import.meta.dirname": "__dirname", + "import.meta.url": "__paperclipVerifiedEntrypointUrl", + } + : undefined, + }); + assertSelfContainedBundle(entrypoint, result); + return result; + }; + const result = await buildBundle(entrypoint.output, "esm"); + const verifiedResult = await buildBundle(entrypoint.verifiedOutput, "cjs"); + if (write && process.platform !== "win32") { + await Promise.all([ + chmod(entrypoint.output, 0o755), + chmod(entrypoint.verifiedOutput, 0o755), + ]); + } + results.push({ entrypoint, result, verifiedResult }); + } + return results; +} + +const invokedPath = process.argv[1] + ? pathToFileURL(resolve(process.argv[1])).href + : null; +if (invokedPath === import.meta.url) { + await bundleVerifiedProviderEntrypoints(); +} diff --git a/packages/paperclip-runner/scripts/build-verified-provider-entrypoints.test.mjs b/packages/paperclip-runner/scripts/build-verified-provider-entrypoints.test.mjs new file mode 100644 index 0000000000..223af55817 --- /dev/null +++ b/packages/paperclip-runner/scripts/build-verified-provider-entrypoints.test.mjs @@ -0,0 +1,40 @@ +import assert from "node:assert/strict"; +import { stat } from "node:fs/promises"; +import test from "node:test"; + +import { + bundleVerifiedProviderEntrypoints, + verifiedProviderEntrypoints, +} from "./build-verified-provider-entrypoints.mjs"; + +test("provider entrypoints include self-contained ESM and descriptor-safe CommonJS bundles", async () => { + const bundles = await bundleVerifiedProviderEntrypoints({ write: false }); + assert.equal(bundles.length, verifiedProviderEntrypoints.length); + for (const { entrypoint, result, verifiedResult } of bundles) { + for (const bundle of [result, verifiedResult]) { + assert.equal(bundle.outputFiles?.length, 1, entrypoint.name); + const source = bundle.outputFiles[0].text; + assert.match(source, /^#!\/usr\/bin\/env node\n/); + } + assert.doesNotMatch( + verifiedResult.outputFiles[0].text, + /\bimport\.meta\b/, + entrypoint.name, + ); + } +}); + +test("written provider entrypoints satisfy qualified launch permissions", async (t) => { + if (process.platform === "win32") { + t.skip("POSIX launch permissions do not apply on Windows"); + return; + } + await bundleVerifiedProviderEntrypoints(); + for (const entrypoint of verifiedProviderEntrypoints) { + for (const output of [entrypoint.output, entrypoint.verifiedOutput]) { + const mode = (await stat(output)).mode; + assert.equal(mode & 0o022, 0, entrypoint.name); + assert.notEqual(mode & 0o100, 0, entrypoint.name); + } + } +}); diff --git a/packages/paperclip-runner/scripts/local-provider-smoke-environment.mjs b/packages/paperclip-runner/scripts/local-provider-smoke-environment.mjs new file mode 100644 index 0000000000..a1200497d4 --- /dev/null +++ b/packages/paperclip-runner/scripts/local-provider-smoke-environment.mjs @@ -0,0 +1,15 @@ +export async function withIsolatedProfileCredentials(input) { + for (const name of input.providerCredentialNames) { + delete input.environment[name]; + } + for (const [name, value] of Object.entries(input.profileCredentials)) { + input.environment[name] = value; + } + try { + return await input.run(); + } finally { + for (const name of input.providerCredentialNames) { + delete input.environment[name]; + } + } +} diff --git a/packages/paperclip-runner/scripts/local-provider-smoke-environment.test.mjs b/packages/paperclip-runner/scripts/local-provider-smoke-environment.test.mjs new file mode 100644 index 0000000000..1a60070aa5 --- /dev/null +++ b/packages/paperclip-runner/scripts/local-provider-smoke-environment.test.mjs @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { withIsolatedProfileCredentials } from "./local-provider-smoke-environment.mjs"; + +test("a later smoke profile cannot observe another provider credential", async () => { + const environment = { + PATH: "/bin", + OPENAI_API_KEY: "credential-from-an-earlier-profile", + }; + const providerCredentialNames = ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"]; + + await assert.rejects( + withIsolatedProfileCredentials({ + environment, + providerCredentialNames, + profileCredentials: { + ANTHROPIC_API_KEY: "credential-for-current-profile", + }, + run: async () => { + assert.equal(environment.OPENAI_API_KEY, undefined); + assert.equal( + environment.ANTHROPIC_API_KEY, + "credential-for-current-profile", + ); + assert.equal(environment.PATH, "/bin"); + throw new Error("provider failed"); + }, + }), + /provider failed/, + ); + + assert.deepEqual(environment, { PATH: "/bin" }); +}); diff --git a/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs b/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs new file mode 100644 index 0000000000..a45dbcf80b --- /dev/null +++ b/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs @@ -0,0 +1,344 @@ +#!/usr/bin/env node + +import { access, mkdir, mkdtemp, readdir, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; + +import { withIsolatedProfileCredentials } from "./local-provider-smoke-environment.mjs"; + +const PROFILE_IDS = [ + "runner-acpx-claude", + "runner-acpx-codex", + "runner-codex", + "runner-opencode", +]; + +function parseProfiles(args) { + const selected = []; + for (let index = 0; index < args.length; index += 1) { + if (args[index] !== "--profile" || !args[index + 1]) { + throw new Error( + `Usage: pnpm smoke:local-provider -- --profile <${PROFILE_IDS.join("|")}|all>`, + ); + } + selected.push(args[++index]); + } + if (selected.length === 0) return ["runner-acpx-claude"]; + const expanded = selected.flatMap((profile) => + profile === "all" ? PROFILE_IDS : [profile], + ); + for (const profile of expanded) { + if (!PROFILE_IDS.includes(profile)) { + throw new Error(`Unsupported local provider smoke profile: ${profile}`); + } + } + return [...new Set(expanded)]; +} + +function liveCandidate(id, candidateSlots) { + const candidates = candidateSlots.flatMap((slot) => slot.candidates); + if (id === "runner-acpx-claude") { + return candidates.find( + (candidate) => candidate.id === "acpx-claude-sonnet", + ); + } + if (id === "runner-acpx-codex") { + return candidates.find((candidate) => candidate.id === "acpx-codex-sol"); + } + if (id === "runner-codex") { + const source = candidates.find( + (candidate) => candidate.id === "codex-luna", + ); + return ( + source && { + ...source, + id: "runner-codex-sol", + model: "gpt-5.6-sol", + } + ); + } + const source = candidates.find( + (candidate) => candidate.id === "opencode-kimi", + ); + return ( + source && { + ...source, + id: "runner-opencode-deepseek", + model: "openrouter/deepseek/deepseek-v4-flash-0731", + } + ); +} + +function failedChecks(observation) { + const smokeChecks = new Set([ + "terminal-authority", + "first-visible-progress", + "substantive-response", + "expected-assistant-text", + "no-empty-comment", + "terminal-presentation", + ]); + return [...observation.lifecycle.checks, ...observation.presentation.checks] + .filter((check) => smokeChecks.has(check.id) && !check.passed) + .map((check) => check.id); +} + +function safeErrorMessage(error, credentialValues) { + let message = error instanceof Error ? error.message : String(error); + for (const credential of credentialValues) { + if (credential.length > 0) + message = message.split(credential).join("[REDACTED]"); + } + return message + .replace(/\bsk-[A-Za-z0-9_-]{8,}\b/g, "[REDACTED]") + .replace(/\bBearer\s+\S+/gi, "Bearer [REDACTED]") + .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]") + .slice(0, 1_000); +} + +async function filesUnder(directory, include, skipDirectory = () => false) { + const files = []; + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + if (!skipDirectory(path)) { + files.push(...(await filesUnder(path, include, skipDirectory))); + } + } else if (entry.isFile() && include(path)) { + files.push(path); + } + } + return files; +} + +async function assertArtifactsFresh(label, sources, artifacts) { + const sourceTimes = await Promise.all( + sources.map(async (source) => (await stat(source)).mtimeMs), + ); + const artifactTimes = await Promise.all( + artifacts.map(async (artifact) => (await stat(artifact)).mtimeMs), + ); + if (Math.max(...sourceTimes) > Math.min(...artifactTimes)) { + throw new Error( + `${label} smoke artifacts are older than their source. Rebuild the targeted artifacts before running the smoke.`, + ); + } +} + +const profiles = parseProfiles(process.argv.slice(2)); +const packageRoot = resolve(import.meta.dirname, ".."); +const runnerd = resolve( + packageRoot, + "runner", + "target", + "debug", + process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd", +); +const requiredArtifacts = [ + runnerd, + resolve(packageRoot, "dist", "eval", "index.js"), + ...(profiles.some((profile) => profile.startsWith("runner-acpx-")) + ? [resolve(packageRoot, "dist", "cli", "acpx-runtime-sidecar.cjs")] + : []), + ...(profiles.includes("runner-opencode") + ? [resolve(packageRoot, "dist", "cli", "opencode-app-server-proxy.cjs")] + : []), +]; +await Promise.all(requiredArtifacts.map((artifact) => access(artifact))).catch( + () => { + throw new Error( + "Local provider smoke artifacts are missing. Run build:typescript and build:runner-binaries once.", + ); + }, +); + +const typescriptSources = await filesUnder( + resolve(packageRoot, "src"), + (path) => path.endsWith(".ts") && !path.endsWith(".test.ts"), + (path) => + /\/(?:browser|devtools|issue-thread|react|scenarios|standalone)$/u.test( + path, + ), +); +await assertArtifactsFresh( + "TypeScript", + [ + resolve(packageRoot, "package.json"), + resolve(packageRoot, "tsconfig.json"), + ...typescriptSources, + ], + requiredArtifacts.filter((artifact) => artifact !== runnerd), +); +const rustSources = await filesUnder( + resolve(packageRoot, "runner"), + (path) => + path.endsWith(".rs") || + path.endsWith("Cargo.toml") || + path.endsWith("Cargo.lock"), + (path) => /\/(?:target|tests|benches|examples)$/u.test(path), +); +await assertArtifactsFresh("runnerd", rustSources, [runnerd]); + +const smokeRoot = await mkdtemp( + join(tmpdir(), "paperclip-local-provider-smoke-"), +); +if (!basename(smokeRoot).startsWith("paperclip-local-provider-smoke-")) { + throw new Error("Refusing an unrecognized local provider smoke root"); +} +await Promise.all( + ["tmp", "home", "paperclip-home", "codex-home", "claude-home"].map( + (directory) => mkdir(join(smokeRoot, directory), { recursive: true }), + ), +); + +const ambientEnvironment = { ...process.env }; +for (const name of Object.keys(process.env)) delete process.env[name]; +for (const name of [ + "PATH", + "SystemRoot", + "ComSpec", + "PATHEXT", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TZ", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "NODE_EXTRA_CA_CERTS", +]) { + if (ambientEnvironment[name] !== undefined) { + process.env[name] = ambientEnvironment[name]; + } +} +Object.assign(process.env, { + TMPDIR: join(smokeRoot, "tmp"), + HOME: join(smokeRoot, "home"), + PAPERCLIP_HOME: join(smokeRoot, "paperclip-home"), + CODEX_HOME: join(smokeRoot, "codex-home"), + CLAUDE_CONFIG_DIR: join(smokeRoot, "claude-home"), + NO_BROWSER: "1", + PAPERCLIP_OPEN_ON_LISTEN: "false", +}); + +let cleanupPromise; +const cleanup = () => { + cleanupPromise ??= rm(smokeRoot, { recursive: true, force: true }); + return cleanupPromise; +}; +const exitAfterCleanup = (status) => { + void cleanup().finally(() => process.exit(status)); +}; +process.once("SIGINT", () => exitAfterCleanup(130)); +process.once("SIGTERM", () => exitAfterCleanup(143)); + +let failed = false; +try { + // Import only after the disposable homes are authoritative so no provider + // module can snapshot the developer's normal Paperclip or provider state. + const { + RUNNER_LIVE_CANDIDATE_SLOTS, + executeLiveRunnerWorkflow, + runnerWorkflowCase, + } = await import("../dist/eval/index.js"); + const candidates = new Map( + profiles.map((profile) => [ + profile, + liveCandidate(profile, RUNNER_LIVE_CANDIDATE_SLOTS), + ]), + ); + const credentialsByProfile = new Map(); + for (const [profile, candidate] of candidates) { + if (!candidate) throw new Error(`Missing live candidate for ${profile}`); + const missingCredentials = []; + const profileCredentials = {}; + for (const name of candidate.qualification.requiredEnvironment) { + const value = ambientEnvironment[name]?.trim(); + if (value) profileCredentials[name] = value; + else missingCredentials.push(name); + } + if (missingCredentials.length > 0) { + throw new Error( + `${profile} requires ${missingCredentials.join(", ")} in the smoke process environment`, + ); + } + credentialsByProfile.set(profile, profileCredentials); + } + const providerCredentialNames = new Set( + [...credentialsByProfile.values()].flatMap((credentials) => + Object.keys(credentials), + ), + ); + const credentialValues = new Set( + [...credentialsByProfile.values()].flatMap((credentials) => + Object.values(credentials), + ), + ); + const evalCase = runnerWorkflowCase("completion-robustness"); + for (const profile of profiles) { + const candidate = candidates.get(profile); + const workspace = join(smokeRoot, `workspace-${profile}`); + await mkdir(workspace, { recursive: true }); + const entry = { + // Avoid a three-segment dotted identity: durable redaction correctly + // treats that shape as a possible JWT and refuses to rewrite IDs. + executionId: `local-smoke-${profile}-${String(Date.now())}`, + caseId: evalCase.id, + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + try { + const observation = await withIsolatedProfileCredentials({ + environment: process.env, + providerCredentialNames, + profileCredentials: credentialsByProfile.get(profile), + run: () => + executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase, + workingDirectory: workspace, + // This smoke proves the provider launch/message/semantic-terminal path. + // Usage conformance remains covered by the dedicated eval campaign. + allowMissingUsage: true, + expectedAssistantText: "PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK", + promptOverride: + "Reply with exactly PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK and no other text. Do not call tools.", + runnerBinary: runnerd, + }), + }); + const failures = failedChecks(observation); + const passed = failures.length === 0; + failed ||= !passed; + process.stdout.write( + `${JSON.stringify({ + profile, + status: passed ? "passed" : "failed", + classification: passed + ? "message_completed" + : observation.classification, + settlementMs: observation.metrics.settlementMs ?? null, + totalTokens: observation.metrics.totalTokens ?? null, + costUsd: observation.metrics.costUsd ?? null, + failedChecks: failures, + failureCode: observation.failure?.code ?? null, + })}\n`, + ); + } catch (error) { + failed = true; + process.stderr.write( + `${JSON.stringify({ + profile, + status: "failed", + infrastructureError: safeErrorMessage(error, credentialValues), + })}\n`, + ); + } + } +} finally { + await cleanup(); +} + +if (failed) process.exitCode = 1; diff --git a/packages/paperclip-runner/src/backends/codex-native-backend.ts b/packages/paperclip-runner/src/backends/codex-native-backend.ts index 1653c75a6f..5a586d62c2 100644 --- a/packages/paperclip-runner/src/backends/codex-native-backend.ts +++ b/packages/paperclip-runner/src/backends/codex-native-backend.ts @@ -6,12 +6,15 @@ import type { } from "../contracts/native-session-backend.js"; import type { CodexAppServerTransport } from "../drivers/codex/app-server-transport.js"; import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver.js"; +import type { CodexWorkingDirectoryAuthority } from "../drivers/codex/codex-boundaries.js"; import { HarnessDriverBackend } from "./harness-driver-backend.js"; import { nativeSystemInstructions, nativeTaskConstraints } from "./runtime-context.js"; export interface CodexNativeSessionBackendOptions { /** Effective provider environment, including the assigned workspace boundary. */ environment?: NodeJS.ProcessEnv; + /** Filesystem that authoritatively admits the workspace path. */ + workingDirectoryAuthority?: CodexWorkingDirectoryAuthority; runnerInstanceId?: string; onSpawn?: (meta: { pid: number; @@ -90,8 +93,20 @@ function createTransportBackedNativeSessionBackend( input: NativeExecutionInput, options: CodexNativeSessionBackendOptions, ): NativeSessionBackend { + if ( + options.workingDirectoryAuthority === "remote_runner" && + !options.transportFactory + ) { + throw new Error( + "Remote runner workspace authority requires a runnerd transport", + ); + } const driverIdentity = transportDriverIdentity(input); const isCodex = input.provider.kind === "codex"; + const supportsCollaborativePlanning = + isCodex || + input.provider.kind === "opencode" || + input.provider.kind === "acpx"; if ( input.provider.kind === "codex" && input.provider.approvalPolicy !== undefined @@ -113,14 +128,18 @@ function createTransportBackedNativeSessionBackend( baseInstructions: nativeSystemInstructions(input), includeSkillInstructions: isCodex && "runtimeContext" in input, requestedCollaborationMode: - isCodex && "executionMode" in input ? input.executionMode : "default", + supportsCollaborativePlanning && "executionMode" in input + ? input.executionMode + : "default", taskEnvelope: createCodexTaskEnvelope({ objective: input.completionContract.contract.objective, contractRevision: input.completionContract.contract.revision, criteria: input.completionContract.contract.criteria, constraints: [ "Work only inside the supplied working directory.", - ...(isCodex && "executionMode" in input && input.executionMode === "plan" + ...(supportsCollaborativePlanning && + "executionMode" in input && + input.executionMode === "plan" ? [ "Use native plan collaboration mode and do not modify workspace files.", "Treat the supplied Paperclip planning context as the canonical pinned base revision.", @@ -139,11 +158,14 @@ function createTransportBackedNativeSessionBackend( dynamicTools: options.dynamicTools, dynamicToolHandler: options.dynamicToolHandler, environment: options.environment, + workingDirectoryAuthority: options.workingDirectoryAuthority, driverIdentity, capabilities: isCodex ? {} : { steering: false, goals: false, threadLineage: false }, - collaborationModes: isCodex ? ["default", "plan"] : ["default"], + collaborationModes: supportsCollaborativePlanning + ? ["default", "plan"] + : ["default"], requireProviderSessionIdentity: options.transportFactory !== undefined, })); } diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index 6a066dd4d1..c94f4c3023 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -30,6 +30,7 @@ const providerIdentity = { workspaceDigest: "sha256:workspace", requestedModel: "claude-sonnet-4-20250514", effectiveModel: "claude-sonnet-4-20250514", + providerLifetimeFenceCandidates: [60_001, 60_002, 60_003] as const, }; function prpEvent(sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record): PrpEvent { diff --git a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts index 57f4074912..185805b736 100644 --- a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts +++ b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; import type { NativeExecutionInput } from "../contracts/native-execution.js"; +import { + FakeCodexTransport, + WORKSPACE, +} from "../drivers/codex/codex-app-server-driver.test-support.js"; import { createNativeSessionBackend } from "../index.js"; import { createCodexNativeSessionBackend } from "./codex-native-backend.js"; @@ -91,11 +95,16 @@ function acpxExecution( agentServerVersion: agent === "codex" ? "1.6.2" : agent === "pi" ? "0.0.33" : "0.70.0", agentRuntimePackage: - agent === "pi" ? "@earendil-works/pi-coding-agent" : null, - agentRuntimeVersion: agent === "pi" ? "0.84.2" : null, + agent === "pi" + ? "@earendil-works/pi-coding-agent" + : agent === "codex" + ? "@openai/codex" + : "@anthropic-ai/claude-agent-sdk", + agentRuntimeVersion: + agent === "pi" ? "0.84.2" : agent === "codex" ? "0.148.0" : "0.3.232", commandDigest: agent === "codex" - ? "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79" + ? "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400" : agent === "pi" ? "sha256:8c696f38296d53d0061fa11534570c5ddd951b63532aed30e0f1fcc676dc169f" : "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", @@ -121,6 +130,23 @@ function opencodeExecution(): NativeExecutionInput { }; } +function planningExecution(input: NativeExecutionInput): NativeExecutionInput { + return { + ...input, + schema: "paperclip.native-execution-input.v2", + task: { ...input.task, workMode: "planning" }, + executionMode: "plan", + planningContext: { + documentId: null, + baseRevisionId: null, + baseRevisionNumber: 0, + markdown: "", + sha256: "empty-plan", + reviewContext: {}, + }, + }; +} + function managedExecution( kind: "claude_managed" | "aws_agentcore", ): NativeExecutionInput { @@ -168,12 +194,16 @@ function managedExecution( profileId: "profile", region: "us-east-1", accountId: "123456789012", - harnessArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:harness/test", + harnessArn: + "arn:aws:bedrock-agentcore:us-east-1:123456789012:harness/test", harnessVersion: "1", - endpointArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:endpoint/test", + endpointArn: + "arn:aws:bedrock-agentcore:us-east-1:123456789012:endpoint/test", endpointQualifier: "1", - agentRuntimeArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/test", - memoryArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:memory/test", + agentRuntimeArn: + "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/test", + memoryArn: + "arn:aws:bedrock-agentcore:us-east-1:123456789012:memory/test", memoryId: "memory", invocationRoleArn: "arn:aws:iam::123456789012:role/runner", contextBucket: "context-bucket", @@ -221,9 +251,9 @@ describe("native backend factory", () => { ); it("requires an explicit runtime root for OpenCode", () => { - expect(() => - createNativeSessionBackend(opencodeExecution()), - ).toThrow("OpenCode native backend requires an instance runtime directory"); + expect(() => createNativeSessionBackend(opencodeExecution())).toThrow( + "OpenCode native backend requires an instance runtime directory", + ); }); it("routes OpenCode through runnerd when a durable transport is supplied", async () => { @@ -242,13 +272,124 @@ describe("native backend factory", () => { resume: true, interruption: true, dynamicTools: true, + collaborationModes: ["default", "plan"], }, }); }); + it("defers remote ACPX workspace admission to the runner filesystem", async () => { + const remoteWorkspace = "/home/daytona/paperclip-workspace"; + const transport = new FakeCodexTransport(); + const request = transport.request.bind(transport); + transport.request = async (method, params) => { + const response = await request(method, params); + if (method !== "thread/start" && method !== "thread/resume") { + return response; + } + return { + ...response, + cwd: remoteWorkspace, + thread: { + ...(response.thread as Record), + cwd: remoteWorkspace, + }, + }; + }; + const backend = createNativeSessionBackend(acpxExecution("claude"), { + codexTransportFactory: () => transport, + workingDirectoryAuthority: "remote_runner", + environment: { + HOME: remoteWorkspace, + CODEX_HOME: `${remoteWorkspace}/.codex`, + PAPERCLIP_WORKSPACE_CWD: remoteWorkspace, + }, + }); + + const session = await backend.openSession({ + identity: { + runId: "run", + sessionId: "session", + companyId: "company", + issueId: "issue", + agentId: "agent", + }, + workingDirectory: remoteWorkspace, + }); + + expect( + transport.calls.find((call) => call.method === "thread/start")?.params, + ).toMatchObject({ cwd: remoteWorkspace }); + await session.close({ reason: "test complete" }); + }); + + it("does not allow remote workspace authority without runnerd", () => { + expect(() => + createNativeSessionBackend(execution(), { + workingDirectoryAuthority: "remote_runner", + environment: { + PAPERCLIP_WORKSPACE_CWD: "/home/daytona/paperclip-workspace", + }, + }), + ).toThrow("requires a runnerd transport"); + }); + it.each([ - ["claude_managed" as const, "claude_managed_agents_api", "managed-agents-2026-04-01"], - ["aws_agentcore" as const, "aws_agentcore_harness_api", "aws-agentcore-harness-v1"], + ["OpenCode", opencodeExecution()], + ["ACPX Codex", acpxExecution("codex")], + ["ACPX Claude", acpxExecution("claude")], + ])( + "opens %s planning runs through the runner-managed plan contract", + async (_label, input) => { + const transport = new FakeCodexTransport(); + const backend = createNativeSessionBackend(planningExecution(input), { + codexTransportFactory: () => transport, + environment: { + ...process.env, + PAPERCLIP_WORKSPACE_CWD: WORKSPACE, + }, + }); + + await expect(backend.descriptor()).resolves.toMatchObject({ + capabilities: { collaborationModes: ["default", "plan"] }, + }); + const session = await backend.openSession({ + identity: { + runId: "run", + sessionId: "session", + companyId: "company", + issueId: "issue", + agentId: "agent", + }, + workingDirectory: WORKSPACE, + }); + + await expect( + session.startTurn({ + message: { role: "user", text: "Author a plan." }, + requestedCollaborationMode: "plan", + }), + ).resolves.toMatchObject({ effectiveCollaborationMode: "plan" }); + expect( + transport.calls.find((call) => call.method === "thread/start")?.params, + ).toMatchObject({ permissions: "paperclip-runner-workspace-read-only" }); + expect( + transport.calls.find((call) => call.method === "turn/start")?.params, + ).toMatchObject({ collaborationMode: { mode: "plan" } }); + await session.close({ reason: "test complete" }); + }, + ); + + it.each([ + [ + "claude_managed" as const, + "claude_managed_agents_api", + "managed-agents-2026-04-01", + ], + [ + "aws_agentcore" as const, + "aws_agentcore_harness_api", + "aws-agentcore-harness-v1", + ], ])("routes %s through runnerd", async (kind, name, version) => { const backend = createNativeSessionBackend(managedExecution(kind), { codexTransportFactory: () => { @@ -320,6 +461,7 @@ describe("native backend factory", () => { resume: true, interruption: true, dynamicTools: true, + collaborationModes: ["default", "plan"], }, }); }, diff --git a/packages/paperclip-runner/src/backends/native-backend-factory.ts b/packages/paperclip-runner/src/backends/native-backend-factory.ts index 7be58e4156..8f18951044 100644 --- a/packages/paperclip-runner/src/backends/native-backend-factory.ts +++ b/packages/paperclip-runner/src/backends/native-backend-factory.ts @@ -47,6 +47,7 @@ export function createNativeSessionBackend( dynamicTools: options.dynamicTools, dynamicToolHandler: options.dynamicToolHandler, environment: options.environment, + workingDirectoryAuthority: options.workingDirectoryAuthority, transportFactory: options.codexTransportFactory, }); } @@ -100,6 +101,7 @@ export function createNativeSessionBackend( dynamicTools: options.dynamicTools, dynamicToolHandler: options.dynamicToolHandler, environment: options.environment, + workingDirectoryAuthority: options.workingDirectoryAuthority, transportFactory: options.codexTransportFactory, }); } diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts index 24bc74d381..b783aaac03 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts @@ -29,6 +29,21 @@ afterEach(async () => { }); describe("qualified ACPX runtime sidecar", () => { + it("shuts down without using readline after stdin closes", async () => { + const sidecar = startSidecar(); + sidecar.write(initializeRequest(1, "codex")); + await expect( + sidecar.next((frame) => frame.id === 1), + ).resolves.toMatchObject({ + id: 1, + ok: true, + }); + + await sidecar.close(); + + expect(sidecar.stderr()).not.toContain("ERR_USE_AFTER_CLOSE"); + }); + it("keeps session admission closed while any cleanup owner remains", () => { const cleanup = Promise.resolve(); diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts index 78ee146639..2922d635a5 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts @@ -23,6 +23,7 @@ import { type NormalizedAcpForm, } from "../drivers/acpx/acp-question-adapter.js"; import { openCodexAcpxRuntime } from "../drivers/acpx/codex-runtime-adapter.js"; +import { acpxProviderSessionIdentity } from "../drivers/acpx/recovery-identity.js"; import { resolveQualifiedAcpxProfile, type QualifiedAcpxAgent, @@ -54,6 +55,7 @@ import { validatePrpStructuredRunResult } from "../protocol/replay-contract.js"; import type { RunnerToolCall } from "../drivers/runner-tool-bridge.js"; import { acpxBootstrapBlockedError, + acpxSidecarErrorCode, enqueueAcpxSidecarInput, recordAcpxBootstrapFailure, } from "./acpx-sidecar-input.js"; @@ -120,6 +122,7 @@ let pendingInput = Promise.resolve(); let bootstrapFailure: Error | null = null; let initializedAgent: QualifiedAcpxAgent | null = null; let initializedModel: string | null = null; +let inputClosed = false; const tools = new Map(); const inputs = new Map(); @@ -136,6 +139,7 @@ lines.on("line", (line) => { ); }); lines.on("close", () => { + inputClosed = true; requestShutdown("sidecar stdin closed"); }); process.once("SIGTERM", () => { @@ -148,7 +152,7 @@ process.once("SIGINT", () => { function requestShutdown(reason: string): void { if (shutdownRequested) return; shutdownRequested = true; - lines.pause(); + if (!inputClosed) lines.pause(); pendingInput = enqueueAcpxSidecarInput( pendingInput, () => shutdown(reason), @@ -180,15 +184,19 @@ async function receiveLine(line: string): Promise { } catch (error) { const normalized = error instanceof Error ? error : new Error(String(error)); + const normalizedRecord = record(normalized); bootstrapFailure = recordAcpxBootstrapFailure( bootstrapFailure, request.command, normalized, ); response(request.id, false, undefined, { - code: safeCode(record(normalized).code, "acpx_sidecar_command_failed"), + code: safeCode( + acpxSidecarErrorCode(normalized), + "acpx_sidecar_command_failed", + ), message: safeMessage(normalized), - retryable: record(normalized).retryable === true, + retryable: normalizedRecord.retryable === true, }); } } @@ -275,7 +283,10 @@ async function dispatch( startedAt: new Date().toISOString(), }); return { - identity: opened.identity, + identity: acpxProviderSessionIdentity( + openedHost.identity(), + openedHost.binding(), + ), sidecarPid: process.pid, status: opened.status, }; @@ -390,7 +401,10 @@ async function dispatch( if (request.command === "session.read") { const activeHost = requireHost(); return { - identity: activeHost.identity(), + identity: acpxProviderSessionIdentity( + activeHost.identity(), + activeHost.binding(), + ), status: sanitizeRuntimeStatus( await readSidecarHostStatusWithin(activeHost), ), @@ -399,7 +413,10 @@ async function dispatch( if (request.command === "session.snapshot") { const activeHost = requireHost(); return { - identity: activeHost.identity(), + identity: acpxProviderSessionIdentity( + activeHost.identity(), + activeHost.binding(), + ), status: sanitizeRuntimeStatus( await readSidecarHostStatusWithin(activeHost), ), @@ -418,7 +435,10 @@ async function dispatch( // still serialized, and retainActiveHostCleanup keeps admission closed // until one sequential close proves ownership was released. const activeHost = requireHost({ allowCleanupRetry: true }); - const identity = activeHost.identity(); + const identity = acpxProviderSessionIdentity( + activeHost.identity(), + activeHost.binding(), + ); await closeSidecarHostForCommand( activeHost, boundedOptionalText(request.params.reason, "Paperclip suspension", 4_000), @@ -1006,9 +1026,30 @@ function parseExpectedIdentity(value: unknown): AcpxExpectedSessionIdentity { ...(input.permissionMode === undefined ? {} : { permissionMode: requiredPermissionMode(input.permissionMode) }), + providerLifetimeFenceCandidates: requiredFenceCandidates( + input.providerLifetimeFenceCandidates, + ), }; } +function requiredFenceCandidates( + value: unknown, +): readonly [number, number, number] { + if ( + !Array.isArray(value) || + value.length !== 3 || + value.some( + (port) => !Number.isSafeInteger(port) || port < 49_152 || port > 65_535, + ) || + new Set(value).size !== 3 + ) { + throw new Error( + "providerLifetimeFenceCandidates must be three distinct private ports", + ); + } + return Object.freeze([...value]) as readonly [number, number, number]; +} + function requiredPermissionMode( value: unknown, ): AcpxSidecarOpenParams["permissionMode"] { diff --git a/packages/paperclip-runner/src/cli/acpx-sidecar-input.test.ts b/packages/paperclip-runner/src/cli/acpx-sidecar-input.test.ts index c9f2f613f6..4ce9ef20a2 100644 --- a/packages/paperclip-runner/src/cli/acpx-sidecar-input.test.ts +++ b/packages/paperclip-runner/src/cli/acpx-sidecar-input.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { acpxBootstrapBlockedError, + acpxSidecarErrorCode, enqueueAcpxSidecarInput, recordAcpxBootstrapFailure, } from "./acpx-sidecar-input.js"; @@ -112,4 +113,64 @@ describe("ACPX sidecar input sequencing", () => { ).toBeNull(); expect(acpxBootstrapBlockedError(null, "turn.start")).toBeNull(); }); + + it("preserves stable ACPX error identities without copying startup stderr", () => { + const missingModule = Object.assign(new Error("provider exited"), { + detailCode: "AGENT_STARTUP_FAILED", + stderrSummary: + "Error [ERR_MODULE_NOT_FOUND]: violet-circuit-4821 was not found", + exitCode: 1, + }); + const opaqueExit = Object.assign(new Error("provider exited"), { + detailCode: "AGENT_STARTUP_FAILED", + stderrSummary: "violet-circuit-4821", + exitCode: 1, + }); + const model = Object.assign(new Error("model rejected"), { + code: "ACP_MODEL_UNSUPPORTED", + detailCode: "AGENT_STARTUP_FAILED", + }); + const genericStartup = Object.assign(new Error("provider exited"), { + outputCode: "RUNTIME", + detailCode: "AGENT_STARTUP_FAILED", + stderrSummary: "Error [ERR_MODULE_NOT_FOUND]: package was not found", + exitCode: 1, + }); + const genericRuntime = Object.assign(new Error("provider rejected"), { + outputCode: "RUNTIME", + }); + const nestedHandshake = new AggregateError( + [ + Object.assign(new Error("admission deadline"), { + name: "AcpxSessionHandshakeTimeoutError", + }), + ], + "runtime initialization cleanup failed", + ); + const codedWrapper = Object.assign(new Error("opaque wrapper"), { + code: "ERR_UNCLASSIFIED_WRAPPER", + cause: missingModule, + }); + + expect(acpxSidecarErrorCode(missingModule)).toBe( + "AGENT_STARTUP_FAILED.MODULE_NOT_FOUND", + ); + expect(acpxSidecarErrorCode(opaqueExit)).toBe( + "AGENT_STARTUP_FAILED.EXIT_NONZERO", + ); + expect(acpxSidecarErrorCode(opaqueExit)).not.toContain( + "violet-circuit-4821", + ); + expect(acpxSidecarErrorCode(model)).toBe("ACP_MODEL_UNSUPPORTED"); + expect(acpxSidecarErrorCode(genericStartup)).toBe( + "AGENT_STARTUP_FAILED.MODULE_NOT_FOUND", + ); + expect(acpxSidecarErrorCode(genericRuntime)).toBe("RUNTIME"); + expect(acpxSidecarErrorCode(codedWrapper)).toBe( + "AGENT_STARTUP_FAILED.MODULE_NOT_FOUND", + ); + expect(acpxSidecarErrorCode(nestedHandshake)).toBe( + "ACPX_SESSION_HANDSHAKE_TIMEOUT", + ); + }); }); diff --git a/packages/paperclip-runner/src/cli/acpx-sidecar-input.ts b/packages/paperclip-runner/src/cli/acpx-sidecar-input.ts index f6f4a116c6..9f508bb289 100644 --- a/packages/paperclip-runner/src/cli/acpx-sidecar-input.ts +++ b/packages/paperclip-runner/src/cli/acpx-sidecar-input.ts @@ -28,3 +28,138 @@ export function acpxBootstrapBlockedError( ) : null; } + +/** + * Preserve only stable ACPX/provider error identities across the sidecar + * boundary. Startup stderr can contain credentials or provider output, so it + * contributes a closed category and is never copied into the code itself. + */ +export function acpxSidecarErrorCode(error: Error): string { + const pending: Error[] = [error]; + const observed = new Set(); + while (pending.length > 0 && observed.size < 16) { + const current = pending.shift()!; + if (observed.has(current)) continue; + observed.add(current); + const code = directAcpxSidecarErrorCode(current); + if (code !== null) return code; + + const details = current as Error & Record; + if (current instanceof AggregateError) { + for (const nested of current.errors) { + if (nested instanceof Error) pending.push(nested); + } + } + if (details.cause instanceof Error) pending.push(details.cause); + } + return "acpx_sidecar_command_failed"; +} + +function directAcpxSidecarErrorCode(error: Error): string | null { + const details = error as Error & Record; + // AcpxOperationalError publishes its presentation category as outputCode; + // Node/system errors conventionally use code. Accept the ACPX field first + // while retaining the latter for closed launch failures. + const outputCode = + typeof details.outputCode === "string" + ? details.outputCode + : typeof details.code === "string" + ? details.code + : null; + const detailCode = + typeof details.detailCode === "string" ? details.detailCode : null; + // ACPX output errors may carry both a broad presentation code (for example, + // RUNTIME) and the stable operational identity that produced it. Preserve + // the latter across the sidecar boundary; otherwise a provider bootstrap + // failure is reduced to an unclassified generic runtime rejection. + const code = + detailCode !== null && + (outputCode === null || GENERIC_ACPX_OUTPUT_CODES.has(outputCode)) + ? detailCode + : (outputCode ?? detailCode); + if (code === null) { + return error.name === "AcpxSessionHandshakeTimeoutError" || + error.message === "ACPX session handshake exceeded its admission deadline" + ? "ACPX_SESSION_HANDSHAKE_TIMEOUT" + : null; + } + if (!STABLE_ACPX_SIDECAR_CODES.has(code)) return null; + if (code !== "AGENT_STARTUP_FAILED") return code; + + const stderr = + typeof details.stderrSummary === "string" ? details.stderrSummary : ""; + if (/ERR_ACPX_UNVERIFIED_MODULE/.test(stderr)) { + return "AGENT_STARTUP_FAILED.UNVERIFIED_MODULE"; + } + if (/ERR_MODULE_NOT_FOUND|Cannot find (?:module|package)/i.test(stderr)) { + return "AGENT_STARTUP_FAILED.MODULE_NOT_FOUND"; + } + if (/\bEACCES\b|permission denied/i.test(stderr)) { + return "AGENT_STARTUP_FAILED.PERMISSION_DENIED"; + } + if (/\bENOENT\b|no such file or directory/i.test(stderr)) { + return "AGENT_STARTUP_FAILED.FILE_NOT_FOUND"; + } + if (/SyntaxError|unexpected token/i.test(stderr)) { + return "AGENT_STARTUP_FAILED.SYNTAX_ERROR"; + } + if (/ERR_INVALID_ARG|invalid argument/i.test(stderr)) { + return "AGENT_STARTUP_FAILED.INVALID_ARGUMENT"; + } + if (!stderr.trim()) return "AGENT_STARTUP_FAILED.NO_STDERR"; + if (typeof details.signal === "string" && details.signal) { + return "AGENT_STARTUP_FAILED.SIGNAL"; + } + if ( + typeof details.exitCode === "number" && + Number.isInteger(details.exitCode) && + details.exitCode !== 0 + ) { + return "AGENT_STARTUP_FAILED.EXIT_NONZERO"; + } + return "AGENT_STARTUP_FAILED.OTHER"; +} + +const GENERIC_ACPX_OUTPUT_CODES = new Set([ + "NO_SESSION", + "TIMEOUT", + "PERMISSION_DENIED", + "PERMISSION_PROMPT_UNAVAILABLE", + "RUNTIME", + "USAGE", +]); + +const STABLE_ACPX_SIDECAR_CODES = new Set([ + "ACP_MODEL_UNSUPPORTED", + "ACP_SESSION_INIT_FAILED", + "AGENT_DISCONNECTED", + "AGENT_STARTUP_FAILED", + "AGENT_STARTUP_FAILED.EXIT_NONZERO", + "AGENT_STARTUP_FAILED.FILE_NOT_FOUND", + "AGENT_STARTUP_FAILED.INVALID_ARGUMENT", + "AGENT_STARTUP_FAILED.MODULE_NOT_FOUND", + "AGENT_STARTUP_FAILED.NO_STDERR", + "AGENT_STARTUP_FAILED.OTHER", + "AGENT_STARTUP_FAILED.PERMISSION_DENIED", + "AGENT_STARTUP_FAILED.SIGNAL", + "AGENT_STARTUP_FAILED.SYNTAX_ERROR", + "AGENT_STARTUP_FAILED.UNVERIFIED_MODULE", + "AUTH_REQUIRED", + "CLAUDE_ACP_SESSION_CREATE_TIMEOUT", + "SESSION_CONFIG_OPTION_REPLAY_FAILED", + "SESSION_MODEL_REPLAY_FAILED", + "SESSION_MODE_REPLAY_FAILED", + "SESSION_RESUME_REQUIRED", + "ACPX_EFFECTIVE_MODEL_MISMATCH", + "ACPX_MODEL_SELECTION_UNAVAILABLE", + "ACPX_MODEL_STATUS_UNAVAILABLE", + "ACPX_PERSISTED_SESSION_IDENTITY_MISMATCH", + "ACPX_PERSISTED_SESSION_MISSING", + "ACPX_RUNTIME_ADMISSION_VERIFICATION_TIMEOUT", + "ACPX_SESSION_ENSURE_FAILED", + "ACPX_SESSION_ENSURE_NON_ERROR", + "ACPX_SESSION_ENSURE_TYPE_ERROR", + "ACPX_SESSION_HANDSHAKE_TIMEOUT", + "ACPX_SIDECAR_STATUS_READ_TIMEOUT", + ...GENERIC_ACPX_OUTPUT_CODES, +]); diff --git a/packages/paperclip-runner/src/cli/acpx-sidecar-lifecycle.ts b/packages/paperclip-runner/src/cli/acpx-sidecar-lifecycle.ts index 902798507b..d5c12111db 100644 --- a/packages/paperclip-runner/src/cli/acpx-sidecar-lifecycle.ts +++ b/packages/paperclip-runner/src/cli/acpx-sidecar-lifecycle.ts @@ -7,6 +7,15 @@ export interface OpenedAcpxSidecarHost { const FAILED_ADMISSION_CLOSE_TIMEOUT_MS = 8_000; const ACTIVE_HOST_CLEANUP_ATTEMPTS = 4; +class AcpxSidecarStatusReadTimeoutError extends Error { + readonly code = "ACPX_SIDECAR_STATUS_READ_TIMEOUT"; + + constructor() { + super("ACPX session status read exceeded its timeout"); + this.name = "AcpxSidecarStatusReadTimeoutError"; + } +} + export function hasSidecarSessionOwnership( host: unknown, activeHostCleanup: Promise | null, @@ -42,8 +51,7 @@ export async function readSidecarHostStatusWithin( host.status(), new Promise((_resolve, reject) => { timer = setTimeout( - () => - reject(new Error("ACPX session status read exceeded its timeout")), + () => reject(new AcpxSidecarStatusReadTimeoutError()), timeoutMs, ); timer.unref(); @@ -129,9 +137,7 @@ export function recoverAndCombineSidecarHostCleanup( prior: Promise | null, ): Promise { const recovered = recoverSidecarHostCleanup(host, cleanup); - return prior - ? combineSidecarHostCleanups([prior, recovered]) - : recovered; + return prior ? combineSidecarHostCleanups([prior, recovered]) : recovered; } export function reportAuthoritativeSidecarHostCleanupFailure( diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index da3017e853..a3c7b6792c 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -432,6 +432,7 @@ export interface AcpxSessionIdentity { effectiveModel: string; /** Missing on legacy snapshots; those used the historical approve-reads behavior. */ permissionMode?: "approve-all" | "approve-reads" | "deny-all"; + providerLifetimeFenceCandidates: readonly [number, number, number]; } export type PersistedHarnessProviderIdentity = AcpxSessionIdentity; diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 155e93b4ef..bcf5b70085 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -123,20 +123,22 @@ it("pins the OpenCode launch profile in runner startup arguments and restarts", handle.restart("replacement-ticket"); expect(launches).toHaveLength(2); for (const launch of launches) { - expect(launch.args).toEqual(expect.arrayContaining([ - "--opencode-proxy-command", - profile.command, - "--opencode-proxy-command-sha256", - profile.commandSha256, - "--opencode-proxy-script", - profile.proxyScript, - "--opencode-proxy-script-sha256", - profile.proxyScriptSha256, - "--opencode-executable", - profile.executable, - "--opencode-executable-sha256", - profile.executableSha256, - ])); + expect(launch.args).toEqual( + expect.arrayContaining([ + "--opencode-proxy-command", + profile.command, + "--opencode-proxy-command-sha256", + profile.commandSha256, + "--opencode-proxy-script", + profile.proxyScript, + "--opencode-proxy-script-sha256", + profile.proxyScriptSha256, + "--opencode-executable", + profile.executable, + "--opencode-executable-sha256", + profile.executableSha256, + ]), + ); } }); @@ -192,6 +194,53 @@ it("preserves an explicit OpenCode permission mode at the runner spawn boundary" expect(launches[0]!.environment.PAPERCLIP_OPENCODE_COMMAND).toBeUndefined(); }); +it("preserves the controller-selected ACPX provider package root", () => { + const launches: RunnerProcessLaunchSpec[] = []; + spawnRunner({ + connection: { mode: "connect", connectUrl: "ws://127.0.0.1:43127" }, + stateDirectory: "/tmp/paperclip-runner-test", + identity, + ticket: "bootstrap-ticket", + maxOutboxBytes: 256 * 1024, + p0ReserveBytes: 64 * 1024, + runnerVersion: expectedRunnerVersion, + runnerDigest: expectedRunnerDigest, + environment: { + PATH: "/bin", + PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT: "/verified/provider-pack", + PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST: + "/verified/provider-pack/package.json", + NODE_PATH: "/untrusted/modules", + }, + processLauncher: (spec) => { + launches.push(spec); + return { + child: { + pid: 42, + exitCode: null, + signalCode: null, + kill: () => true, + }, + completion: Promise.resolve({ + code: 0, + signal: null, + stdout: "", + stderr: "", + }), + }; + }, + }); + + expect(launches).toHaveLength(1); + expect(launches[0]!.environment.PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT).toBe( + "/verified/provider-pack", + ); + expect( + launches[0]!.environment.PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST, + ).toBe("/verified/provider-pack/package.json"); + expect(launches[0]!.environment.NODE_PATH).toBeUndefined(); +}); + it("preserves file-backed AWS workload identity at the runner spawn boundary", () => { const launches: RunnerProcessLaunchSpec[] = []; spawnRunner({ @@ -925,4 +974,63 @@ describe.sequential("DurablePrpControlPlane", () => { rmSync(root, { recursive: true, force: true }); } }); + + it("acknowledges terminal command results after persisting them", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-terminal-ack-")); + const controlPlane = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + }); + try { + await controlPlane.start(); + const command = controlPlane.queueCommand( + "runner.suspend", + {}, + "command-suspend-1", + ); + const client = await authenticate( + controlPlane, + controlPlane.issueBootstrapTicket(), + ); + const terminalResult = { + protocol: "paperclip.runner", + version: 1, + kind: "command_result", + payload: { + commandId: command.commandId, + commandType: command.type, + controllerSeq: command.controllerSeq, + status: "completed", + result: { suspended: true }, + }, + }; + + sendSecure(client!, terminalResult); + await expect(receiveSecure(client!)).resolves.toMatchObject({ + kind: "command_result_ack", + payload: { + commandId: "command-suspend-1", + commandType: "runner.suspend", + controllerSeq: command.controllerSeq, + status: "completed", + }, + }); + expect(controlPlane.store.state.commands).toMatchObject([ + { commandId: "command-suspend-1", status: "completed" }, + ]); + + sendSecure(client!, terminalResult); + await expect(receiveSecure(client!)).resolves.toMatchObject({ + kind: "command_result_ack", + payload: { commandId: "command-suspend-1" }, + }); + expect(controlPlane.store.state.duplicateCommandResults).toBe(1); + client?.socket.destroy(); + } finally { + await controlPlane.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); }); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 62e66596fe..0ae8d34dc5 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -295,21 +295,33 @@ function canonicalJson( depth = 0, ): string { state.nodes += 1; - if (depth > MAX_CANONICAL_JSON_DEPTH || state.nodes > MAX_CANONICAL_JSON_NODES) { + if ( + depth > MAX_CANONICAL_JSON_DEPTH || + state.nodes > MAX_CANONICAL_JSON_NODES + ) { throw new Error("durable_prp_canonical_json_too_large"); } - if (value === null || typeof value === "boolean" || typeof value === "string") { + if ( + value === null || + typeof value === "boolean" || + typeof value === "string" + ) { return JSON.stringify(value) ?? "null"; } if (typeof value === "number") { - if (!Number.isFinite(value)) throw new Error("durable_prp_canonical_json_invalid"); + if (!Number.isFinite(value)) + throw new Error("durable_prp_canonical_json_invalid"); return JSON.stringify(value) ?? "null"; } if (typeof value !== "object" || ancestors.has(value)) { throw new Error("durable_prp_canonical_json_invalid"); } const prototype = Object.getPrototypeOf(value); - if (!Array.isArray(value) && prototype !== Object.prototype && prototype !== null) { + if ( + !Array.isArray(value) && + prototype !== Object.prototype && + prototype !== null + ) { throw new Error("durable_prp_canonical_json_invalid"); } ancestors.add(value); @@ -1511,10 +1523,7 @@ export class DurablePrpControlPlane { this.#welcome(connection, leaseToken); } - #welcome( - connection: AuthorityConnection, - leaseToken: string | null, - ): void { + #welcome(connection: AuthorityConnection, leaseToken: string | null): void { const lease = connection.lease; if (lease === null || connection.connectionId === null) { connection.close(); @@ -1666,15 +1675,42 @@ export class DurablePrpControlPlane { } this.#store.state.duplicateCommandResults += 1; this.#store.save(); + this.#ackTerminalCommandResult(connection, command); this.#sendNextCommand(connection); return; } command.status = status; command.result = structuredClone(result); this.#store.save(); + this.#ackTerminalCommandResult(connection, command); this.#sendNextCommand(connection); } + #ackTerminalCommandResult( + connection: AuthorityConnection, + command: DurableRecoveryCoreCommand, + ): void { + if ( + command.type !== "runner.suspend" && + command.type !== "runner.shutdown" + ) { + return; + } + connection.sendJson( + this.#controlEnvelope( + connection, + `command_result_ack_${command.controllerSeq}`, + "command_result_ack", + { + commandId: command.commandId, + commandType: command.type, + controllerSeq: command.controllerSeq, + status: command.status, + }, + ), + ); + } + async #event( connection: AuthorityConnection, envelope: Record, @@ -1892,6 +1928,8 @@ const runnerExplicitProviderEnvironmentKeys = [ "PAPERCLIP_NATIVE_MCP_URL", "PAPERCLIP_NATIVE_MCP_TOKEN", "PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH", + "PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT", + "PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST", "PAPERCLIP_ACPX_PROVIDER_RECOVERY_POLICY", "PAPERCLIP_PROVIDER_TRACE_PATH", "PAPERCLIP_PROVIDER_TRACE_MAX_BYTES", @@ -1956,26 +1994,30 @@ export function spawnRunner(options: { environment?: NodeJS.ProcessEnv; processLauncher?: (spec: RunnerProcessLaunchSpec) => RunnerProcessHandle; }): RunnerProcessHandle { - const connection = options.connection ?? (options.connectUrl - ? { mode: "connect" as const, connectUrl: options.connectUrl } - : null); - if (connection === null) throw new Error("runner process connection is required"); - const connectionArgs = connection.mode === "connect" - ? [ - "--connect-url", - connection.connectUrl, - ...(connection.caBundlePath === undefined - ? [] - : ["--ca-bundle-path", connection.caBundlePath]), - ] - : [ - "--listen-address", - connection.listenAddress, - "--listen-port", - String(connection.listenPort), - "--listen-path", - connection.listenPath, - ]; + const connection = + options.connection ?? + (options.connectUrl + ? { mode: "connect" as const, connectUrl: options.connectUrl } + : null); + if (connection === null) + throw new Error("runner process connection is required"); + const connectionArgs = + connection.mode === "connect" + ? [ + "--connect-url", + connection.connectUrl, + ...(connection.caBundlePath === undefined + ? [] + : ["--ca-bundle-path", connection.caBundlePath]), + ] + : [ + "--listen-address", + connection.listenAddress, + "--listen-port", + String(connection.listenPort), + "--listen-path", + connection.listenPath, + ]; const args = [ ...connectionArgs, "--state-dir", @@ -2048,7 +2090,10 @@ export function spawnRunner(options: { if (options.lifecyclePolicy !== undefined) { args.push("--lifecycle-mode", options.lifecyclePolicy.mode); if (options.lifecyclePolicy.mode === "warm") { - args.push("--idle-timeout-ms", String(options.lifecyclePolicy.idleTimeoutMs)); + args.push( + "--idle-timeout-ms", + String(options.lifecyclePolicy.idleTimeoutMs), + ); } } @@ -2059,7 +2104,9 @@ export function spawnRunner(options: { restart: (ticket) => spawnRunner({ ...options, ticket }), }); if (options.processLauncher !== undefined) { - return withRestart(options.processLauncher({ command, args, cwd: packageRoot, environment })); + return withRestart( + options.processLauncher({ command, args, cwd: packageRoot, environment }), + ); } const child = spawn(command, args, { @@ -2075,10 +2122,14 @@ export function spawnRunner(options: { child.stderr.setEncoding("utf8").on("data", (chunk: string) => { stderr = `${stderr}${chunk}`.slice(-16_384); }); - const completion = new Promise((resolveCompletion, rejectCompletion) => { - child.once("error", rejectCompletion); - child.once("exit", (code, signal) => resolveCompletion({ code, signal, stdout, stderr })); - }); + const completion = new Promise( + (resolveCompletion, rejectCompletion) => { + child.once("error", rejectCompletion); + child.once("exit", (code, signal) => + resolveCompletion({ code, signal, stdout, stderr }), + ); + }, + ); return withRestart({ child, completion }); } diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts index 028f2b0d94..d2ed9b172f 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts @@ -2642,7 +2642,7 @@ function recoveryWorkspaceLease( function fakeHost(createTurn: () => AcpxRuntimeTurn, onClose: () => void) { return { identity: () => ({ - schema: "paperclip.runner.acpx-identity.v1" as const, + schema: "paperclip.runner.acpx-identity.v2" as const, normalizedSessionId: "session-1", acpxRecordId: "record-1", backendSessionId: "backend-1", @@ -2652,6 +2652,7 @@ function fakeHost(createTurn: () => AcpxRuntimeTurn, onClose: () => void) { requestedModel: "gpt-5.6-sol", effectiveModel: "gpt-5.6-sol", permissionMode: "approve-reads" as const, + providerLifetimeFenceCandidates: [60_001, 60_002, 60_003] as const, }), binding: () => ({ normalizedSessionId: "session-1", diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts index 34e3ab9638..62343cfeb6 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts @@ -1154,6 +1154,8 @@ class CodexAcpxSession implements HarnessSession { requestedModel: identity.requestedModel, effectiveModel: identity.effectiveModel, permissionMode: identity.permissionMode, + providerLifetimeFenceCandidates: + identity.providerLifetimeFenceCandidates, }, semanticResult: this.#semanticResult && @@ -1871,7 +1873,10 @@ function validateRecoverySnapshot(snapshot: PersistedHarnessSession): void { (identity.permissionMode !== undefined && !["approve-all", "approve-reads", "deny-all"].includes( identity.permissionMode, - )) + )) || + !validProviderLifetimeFenceCandidates( + identity.providerLifetimeFenceCandidates, + ) ) { throw new Error("persisted Codex ACPX session identity is inconsistent"); } @@ -2001,6 +2006,19 @@ function validateRecoverySnapshot(snapshot: PersistedHarnessSession): void { } } +function validProviderLifetimeFenceCandidates( + value: unknown, +): value is readonly [number, number, number] { + return ( + Array.isArray(value) && + value.length === 3 && + value.every( + (port) => Number.isSafeInteger(port) && port >= 49_152 && port <= 65_535, + ) && + new Set(value).size === 3 + ); +} + function isCompletedTerminal(terminalFingerprint: string): boolean { try { const value: unknown = JSON.parse(terminalFingerprint); diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts index 6e31392e06..4f8d8616f8 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts @@ -70,6 +70,9 @@ describe("managed Codex credentials", () => { }); expect(lease.mode).toBe("inline_json"); + expect(lease.lifetimeFenceCandidates).toEqual( + credentialLeasePorts(await realpath(fixture.home)), + ); expect(lease.lifetimeFenceFds).toHaveLength(2); expect(lease.lifetimeFenceFds.every(Number.isSafeInteger)).toBe(true); expect(lease.lifetimeFenceFds[0]).not.toBe(lease.lifetimeFenceFds[1]); diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts index 452ecd7352..a7daf19b5a 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts @@ -12,6 +12,8 @@ import { import { createServer, type Server } from "node:net"; import { isAbsolute, join, resolve } from "node:path"; +import { verifiedRuntimeExecutableHandoff } from "./verified-runtime-executable.js"; + const MAX_CODEX_CREDENTIAL_BYTES = 256 * 1024; const PRIVATE_FILE_MODE = 0o600; const MAX_DIRECTORY_SYNC_ATTEMPTS = 8; @@ -46,6 +48,7 @@ try { interface CredentialHomeLock { assertHeld(): void; + candidatePorts(): readonly [number, number, number]; inheritanceFds(): readonly [number, number]; activateLifetimeOwner(pid: number): Promise; release(): Promise; @@ -112,6 +115,8 @@ export type ManagedCodexCredentialMode = "api_key" | "inline_json" | "managed_file"; export interface AcpxProviderLifetimeLease { + /** Exact kernel quorum candidates used to prove this provider has exited. */ + readonly lifetimeFenceCandidates: readonly [number, number, number]; /** Duplicate both quorum listeners into the provider lifetime sentinel. */ readonly lifetimeFenceFds: readonly [number, number]; /** Validate the guardian while the provider-lifetime quorum is still held. */ @@ -119,8 +124,7 @@ export interface AcpxProviderLifetimeLease { close(): Promise; } -export interface ManagedCodexCredentialLease - extends AcpxProviderLifetimeLease { +export interface ManagedCodexCredentialLease extends AcpxProviderLifetimeLease { readonly path: string; readonly mode: ManagedCodexCredentialMode; } @@ -138,6 +142,7 @@ export async function acquireAcpxProviderLifetimeLease(input: { let closeAttempt: Promise | null = null; let lifetimeOwnerAttempt: Promise | null = null; return Object.freeze({ + lifetimeFenceCandidates: lock.candidatePorts(), lifetimeFenceFds: lock.inheritanceFds(), async activateLifetimeOwner(pid: number): Promise { if (closed || closeAttempt !== null) { @@ -362,10 +367,11 @@ async function acquireCredentialHomeLock( // contenders cannot both reach quorum; one unrelated occupied listener is // tolerated without probing or trusting the process behind it. const servers: Server[] = []; + const candidatePorts = credentialLeasePorts(home); let invalid: Error | null = null; let released = false; try { - for (const port of credentialLeasePorts(home)) { + for (const port of candidatePorts) { const server = createServer((socket) => socket.destroy()); try { await listenForCredentialLease(server, port); @@ -430,6 +436,9 @@ async function acquireCredentialHomeLock( throw new Error("Managed Codex credential ownership was lost"); } }, + candidatePorts(): readonly [number, number, number] { + return candidatePorts; + }, inheritanceFds(): readonly [number, number] { this.assertHeld(); return inheritanceFds; @@ -465,7 +474,7 @@ async function acquireCredentialHomeLock( }); } -function credentialLeasePorts(home: string): readonly number[] { +function credentialLeasePorts(home: string): readonly [number, number, number] { const userScope = typeof process.getuid === "function" ? String(process.getuid()) : "win32"; const digest = createHash("sha256") @@ -476,11 +485,13 @@ function credentialLeasePorts(home: string): readonly number[] { .digest(); const start = digest.readUInt16BE(0) % CREDENTIAL_LEASE_PORT_COUNT; const step = (digest.readUInt16BE(2) | 1) % CREDENTIAL_LEASE_PORT_COUNT; - return Array.from( - { length: CREDENTIAL_LEASE_CANDIDATES }, - (_, index) => - CREDENTIAL_LEASE_PORT_MIN + - ((start + index * step) % CREDENTIAL_LEASE_PORT_COUNT), + return Object.freeze( + Array.from( + { length: CREDENTIAL_LEASE_CANDIDATES }, + (_, index) => + CREDENTIAL_LEASE_PORT_MIN + + ((start + index * step) % CREDENTIAL_LEASE_PORT_COUNT), + ) as [number, number, number], ); } @@ -652,6 +663,7 @@ function credentialLease( return Object.freeze({ path, mode, + lifetimeFenceCandidates: lock.candidatePorts(), lifetimeFenceFds: lock.inheritanceFds(), async activateLifetimeOwner(pid: number): Promise { if (closed || closeAttempt !== null) { @@ -1128,8 +1140,9 @@ async function runDirectorySyncHelper(directory: string): Promise { } let child: ChildProcess; try { + const runtimeHandoff = verifiedRuntimeExecutableHandoff(3); child = spawn( - process.execPath, + runtimeHandoff.executable, [ "--input-type=module", "--eval", @@ -1140,7 +1153,10 @@ async function runDirectorySyncHelper(directory: string): Promise { // The helper imports only Node built-ins. Do not inherit loader hooks or // any credential-bearing process environment into the durability worker. env: {}, - stdio: "ignore", + stdio: + runtimeHandoff.sourceFd === null + ? "ignore" + : ["ignore", "ignore", "ignore", runtimeHandoff.sourceFd], windowsHide: true, }, ); diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts index 2d73db9f73..0be26c2410 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts @@ -101,9 +101,9 @@ describe("Codex ACPX runtime adapter", () => { }); }); - it.each([["claude" as const, "claude-sonnet-5"]])( + it.each([["claude" as const, "claude-sonnet-5", "sonnet"]])( "opens the qualified %s session through the verified lease", - async (agent, model) => { + async (agent, model, providerModel) => { const runtime = fakeRuntime(); const command = fakeCommand(); const options = openOptions(command); @@ -132,7 +132,7 @@ describe("Codex ACPX runtime adapter", () => { expect(runtime.ensureSession).toHaveBeenCalledWith( expect.objectContaining({ agent, - sessionOptions: expect.objectContaining({ model }), + sessionOptions: expect.objectContaining({ model: providerModel }), }), ); }, @@ -199,26 +199,39 @@ describe("Codex ACPX runtime adapter", () => { expect(assertWorkspaceHeld).toHaveBeenCalledOnce(); expect(command.spawn).not.toHaveBeenCalled(); }); - it("maps status, model selection, and state-preserving close", async () => { + it("reads verified status from durable state without draining live updates", async () => { const runtime = fakeRuntime(); - vi.mocked(runtime.getStatus!).mockResolvedValue({ - models: { - currentModelId: "gpt-5.6-sol", - availableModelIds: ["gpt-5.6-sol"], + vi.mocked(runtime.getStatus!).mockReturnValue(new Promise(() => {})); + const durableRecord = { + acpxRecordId: "record-1", + acpSessionId: "backend-1", + agentSessionId: "agent-1", + acpx: { + current_model_id: "gpt-5.6-sol", + available_models: ["gpt-5.6-sol"], }, - }); + } as never; + const durableStore: AcpSessionStore = { + load: vi.fn(async () => structuredClone(durableRecord)), + save: vi.fn(), + }; const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { createRegistry: () => registry(), - createStore: () => store(), + createStore: () => durableStore, createRuntime: () => runtime, }); - expect(await port.getStatus()).toEqual({ + expect(await port.getStatus()).toMatchObject({ + acpxRecordId: "record-1", + backendSessionId: "backend-1", + agentSessionId: "agent-1", models: { currentModelId: "gpt-5.6-sol", availableModelIds: ["gpt-5.6-sol"], }, }); + expect(durableStore.load).toHaveBeenCalledWith("record-1"); + expect(runtime.getStatus).not.toHaveBeenCalled(); await port.setModel?.("gpt-5.6-sol"); expect(runtime.setConfigOption).toHaveBeenCalledWith({ handle: HANDLE, @@ -1273,14 +1286,15 @@ describe("Codex ACPX runtime adapter", () => { const signal = new AbortController().signal; const onElicitation = vi.fn(); - expect( - port.startTurn({ - text: "Complete the task.", - requestId: "turn-1", - signal, - onElicitation, - }), - ).toBe(turn); + const admittedTurn = port.startTurn({ + text: "Complete the task.", + requestId: "turn-1", + signal, + onElicitation, + }); + expect(admittedTurn.requestId).toBe(turn.requestId); + await expect(admittedTurn.promptStarted).resolves.toBeUndefined(); + await expect(admittedTurn.result).resolves.toEqual({ status: "completed" }); expect(runtime.startTurn).toHaveBeenCalledWith({ handle: HANDLE, text: "Complete the task.", @@ -1291,6 +1305,61 @@ describe("Codex ACPX runtime adapter", () => { }); }); + it("admits a verified provider that starts with the first recovered turn", async () => { + const runtime = fakeRuntime(); + const child = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + let runtimeOptions: AcpRuntimeOptions | undefined; + let resolvePromptStarted: (() => void) | undefined; + const promptStarted = new Promise((resolve) => { + resolvePromptStarted = resolve; + }); + const rawTurn = { + requestId: "turn-recovered", + promptStarted, + events: { async *[Symbol.asyncIterator]() {} }, + result: new Promise(() => undefined), + cancel: vi.fn(), + closeStream: vi.fn(), + }; + vi.mocked(runtime.startTurn).mockImplementation(() => { + queueMicrotask(() => { + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + resolvePromptStarted?.(); + }); + return rawTurn; + }); + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, + createRuntime: (options) => { + runtimeOptions = options; + return runtime; + }, + }); + + const turn = port.startTurn({ + text: "Resume the task.", + requestId: "turn-recovered", + }); + await expect(turn.promptStarted).resolves.toBeUndefined(); + expect(command.spawn).toHaveBeenCalledTimes(1); + expect(() => + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }), + ).toThrow("provider spawned after ownership admission was sealed"); + }); + it("projects only ephemeral MCP bindings and applies fail-closed permissions", async () => { const runtime = fakeRuntime(); let runtimeOptions: AcpRuntimeOptions | undefined; @@ -1415,26 +1484,45 @@ describe("Codex ACPX runtime adapter", () => { }, ); - it("fails closed and closes the session when ACPX omits recovery identity", async () => { + it("uses the real ACP session when no second agent identity is advertised", async () => { const runtime = fakeRuntime({ ...HANDLE, agentSessionId: undefined }); - await expect( - openCodexAcpxRuntime(openOptions(fakeCommand()), { - createRegistry: () => registry(), - createStore: () => store(), - createRuntime: () => runtime, - }), - ).rejects.toThrow("ACPX runtime omitted agentSessionId"); - expect(runtime.close).toHaveBeenCalledWith({ - handle: { ...HANDLE, agentSessionId: undefined }, - reason: "ACPX runtime identity validation failed", - discardPersistentState: false, + const durableStore: AcpSessionStore = { + load: vi.fn(async () => + structuredClone({ + acpxRecordId: "record-1", + acpSessionId: "backend-1", + acpx: { current_model_id: "gpt-5.6-sol" }, + } as never), + ), + save: vi.fn(), + }; + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => durableStore, + createRuntime: () => runtime, }); + + await expect(port.identity()).resolves.toEqual({ + acpxRecordId: "record-1", + backendSessionId: "backend-1", + agentSessionId: "backend-1", + }); + await expect(port.getStatus()).resolves.toMatchObject({ + backendSessionId: "backend-1", + agentSessionId: "backend-1", + models: { currentModelId: "gpt-5.6-sol" }, + }); + expect(runtime.close).not.toHaveBeenCalled(); }); it("bounds invalid-identity cleanup before terminating the provider", async () => { vi.useFakeTimers(); try { - const runtime = fakeRuntime({ ...HANDLE, agentSessionId: undefined }); + const runtime = fakeRuntime({ + ...HANDLE, + backendSessionId: undefined, + agentSessionId: undefined, + }); vi.mocked(runtime.close).mockImplementation( () => new Promise(() => undefined), ); @@ -1453,7 +1541,11 @@ describe("Codex ACPX runtime adapter", () => { args: ["--stdio"], options: {}, }); - return { ...HANDLE, agentSessionId: undefined }; + return { + ...HANDLE, + backendSessionId: undefined, + agentSessionId: undefined, + }; }), }), }); @@ -2064,6 +2156,7 @@ describe("Codex ACPX runtime adapter", () => { }, }), ).rejects.toBe(failure); + expect(failure).toMatchObject({ code: "ACPX_SESSION_ENSURE_FAILED" }); expect(runtime.close).toHaveBeenCalledOnce(); const recoveredClose = vi.mocked(runtime.close).mock.calls[0]![0]; expect(recoveredClose).toMatchObject({ diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts index 087bb6e628..13c122bcc2 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts @@ -17,12 +17,14 @@ import type { AcpxRuntimePort, AcpxRuntimePortIdentity, AcpxRuntimePortOpenOptions, + AcpxRuntimeTurn, } from "./runtime-host.js"; import { assertVerifiedAcpxProviderPlatform, awaitVerifiedAcpxProviderExit, awaitVerifiedAcpxProviderOwnership, } from "./installation-integrity.js"; +import type { AcpxModelStatus } from "./model-verification.js"; import { decideAcpxPermission } from "./permission-policy.js"; const VERIFIED_COMMAND_SENTINEL = "paperclip-verified-acpx-command"; @@ -50,7 +52,10 @@ export const DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS = // only after the exact attempt reaches a terminal outcome. const activeRuntimeCleanupOwners = new Set>(); const activeCodexRuntimeCleanupOwners = new Set>(); -const SESSION_HANDSHAKE_TIMEOUT_MS = 8_000; +// Provider initialization may include a cold native app-server start on a +// minimally provisioned runner. Keep admission finite while allowing the +// qualified runtime enough time to complete that local handshake. +const SESSION_HANDSHAKE_TIMEOUT_MS = 30_000; class AcpxRuntimeCloseTimeoutError extends Error { constructor() { @@ -67,6 +72,8 @@ class AcpxRuntimeCloseFinalTimeoutError extends Error { } class AcpxSessionHandshakeTimeoutError extends Error { + readonly code = "ACPX_SESSION_HANDSHAKE_TIMEOUT"; + constructor() { super("ACPX session handshake exceeded its admission deadline"); this.name = "AcpxSessionHandshakeTimeoutError"; @@ -282,20 +289,28 @@ export async function openQualifiedAcpxRuntime( runtimeCloseTimeoutMs, ); - const handshake = Promise.resolve().then(() => - runtime.ensureSession({ - sessionKey: options.providerSessionKey, - agent: options.profile.agent, - mode: "persistent", - cwd: options.cwd, - sessionOptions: { - model: options.profile.qualificationModel, - ...(options.systemInstructions - ? { systemPrompt: { append: options.systemInstructions } } - : {}), - }, - }), - ); + const handshake = Promise.resolve() + .then(() => + runtime.ensureSession({ + sessionKey: options.providerSessionKey, + agent: options.profile.agent, + mode: "persistent", + cwd: options.cwd, + sessionOptions: { + // ACP session construction receives the provider-native selector. + // The caller-facing canonical model was already pinned when the + // qualified profile was resolved and is restored at the status + // boundary after the provider reports this selector. + model: options.profile.reportedModelId, + ...(options.systemInstructions + ? { systemPrompt: { append: options.systemInstructions } } + : {}), + }, + }), + ) + .catch((error: unknown) => { + throw classifySessionEnsureFailure(error); + }); let handle: AcpRuntimeHandle | null = null; let lateCleanup: Promise | null = null; try { @@ -361,6 +376,7 @@ export async function openQualifiedAcpxRuntime( runtime, handle, requireIdentity(handle), + baseStore, children, runtimeCloseTimeoutMs, ); @@ -386,6 +402,22 @@ export async function openQualifiedAcpxRuntime( /** Backward-compatible name retained for existing Codex-only consumers. */ export const openCodexAcpxRuntime = openQualifiedAcpxRuntime; +function classifySessionEnsureFailure(error: unknown): Error { + if (error instanceof Error) { + const details = error as Error & Record; + if (typeof details.code !== "string" || details.code.length === 0) { + details.code = + error instanceof TypeError + ? "ACPX_SESSION_ENSURE_TYPE_ERROR" + : "ACPX_SESSION_ENSURE_FAILED"; + } + return error; + } + return Object.assign(new Error("ACPX session ensure rejected a non-error"), { + code: "ACPX_SESSION_ENSURE_NON_ERROR", + }); +} + function raceRuntimeHandshakeWithAbort( handshake: Promise, signal: AbortSignal, @@ -769,6 +801,7 @@ function runtimePort( runtime: AcpRuntime, handle: AcpRuntimeHandle, identity: AcpxRuntimePortIdentity, + sessionStore: AcpSessionStore, children: SpawnedChildSet, runtimeCloseTimeoutMs: number, ): AcpxRuntimePort { @@ -828,7 +861,7 @@ function runtimePort( } if ( lateReconciliationAttempts >= - MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS + MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS ) { return; } @@ -999,10 +1032,7 @@ function runtimePort( return structuredClone(identity); }, async getStatus() { - if (!runtime.getStatus) { - throw new Error("The pinned ACPX runtime cannot report session status"); - } - return structuredClone(await runtime.getStatus({ handle })); + return await persistedRuntimeStatus(sessionStore, handle, identity); }, ...(runtime.setConfigOption ? { @@ -1016,20 +1046,119 @@ function runtimePort( } : {}), startTurn(input) { - return runtime.startTurn({ - handle, - text: input.text, - mode: "prompt", - requestId: input.requestId, - ...(input.signal ? { signal: input.signal } : {}), - ...(input.onElicitation ? { onElicitation: input.onElicitation } : {}), - }); + const finishOwnershipAdmission = + children.beginLifetimeOwnershipAdmission(); + let turn: AcpxRuntimeTurn; + try { + turn = runtime.startTurn({ + handle, + text: input.text, + mode: "prompt", + requestId: input.requestId, + ...(input.signal ? { signal: input.signal } : {}), + ...(input.onElicitation + ? { onElicitation: input.onElicitation } + : {}), + }); + } catch (error) { + void finishOwnershipAdmission().catch(() => undefined); + throw error; + } + return turnWithVerifiedLifetimeOwnership(turn, finishOwnershipAdmission); }, close: closeRuntime, }; return port; } +function turnWithVerifiedLifetimeOwnership( + turn: AcpxRuntimeTurn, + finishOwnershipAdmission: () => Promise, +): AcpxRuntimeTurn { + // A persisted ACPX session can be loaded without starting an agent process. + // Keep the narrowly scoped turn admission open until either the provider has + // accepted the prompt or the turn has already terminalized. The synchronous + // stable-empty seal in SpawnedChildSet then rejects every later spawn. + const reachedAdmissionBoundary = Promise.race([ + turn.promptStarted.then( + () => undefined, + () => undefined, + ), + turn.result.then( + () => undefined, + () => undefined, + ), + ]); + const ownershipVerified = reachedAdmissionBoundary.then(() => + finishOwnershipAdmission(), + ); + void ownershipVerified.catch(() => undefined); + return { + requestId: turn.requestId, + promptStarted: ownershipVerified.then(() => turn.promptStarted), + events: eventsAfterLifetimeOwnership(turn.events, ownershipVerified), + result: ownershipVerified.then(() => turn.result), + cancel: (input) => turn.cancel(input), + closeStream: (input) => turn.closeStream(input), + }; +} + +async function* eventsAfterLifetimeOwnership( + events: AsyncIterable, + ownershipVerified: Promise, +): AsyncIterable { + await ownershipVerified; + yield* events; +} + +async function persistedRuntimeStatus( + sessionStore: AcpSessionStore, + handle: AcpRuntimeHandle, + identity: AcpxRuntimePortIdentity, +): Promise { + const recordId = handle.acpxRecordId ?? handle.sessionKey; + const record = await sessionStore.load(recordId); + if (!record) { + throw Object.assign( + new Error("The pinned ACPX runtime omitted its persisted session record"), + { code: "ACPX_PERSISTED_SESSION_MISSING" }, + ); + } + const persistedAgentSessionId = + nonEmptyRuntimeIdentity(record.agentSessionId) ?? record.acpSessionId; + if ( + record.acpxRecordId !== identity.acpxRecordId || + record.acpSessionId !== identity.backendSessionId || + persistedAgentSessionId !== identity.agentSessionId + ) { + throw Object.assign( + new Error("The persisted ACPX session identity changed after admission"), + { code: "ACPX_PERSISTED_SESSION_IDENTITY_MISMATCH" }, + ); + } + const currentModelId = record.acpx?.current_model_id; + const availableModelIds = record.acpx?.available_models; + return { + summary: [ + `session=${record.acpxRecordId}`, + `backendSessionId=${record.acpSessionId}`, + `agentSessionId=${persistedAgentSessionId}`, + record.closed === true ? "closed" : "open", + ].join(" "), + acpxRecordId: record.acpxRecordId, + backendSessionId: record.acpSessionId, + agentSessionId: persistedAgentSessionId, + ...(currentModelId === undefined && !availableModelIds?.length + ? {} + : { + models: { + ...(currentModelId === undefined ? {} : { currentModelId }), + availableModelIds: availableModelIds ? [...availableModelIds] : [], + }, + }), + }; +} + function runtimeCloseOutcome( runtime: AcpRuntime, input: Parameters[0], @@ -1127,9 +1256,7 @@ function delay(timeoutMs: number): Promise { return new Promise((resolve) => setTimeout(resolve, timeoutMs)); } -type ProviderExitOutcome = - | { exited: true } - | { exited: false; error: unknown }; +type ProviderExitOutcome = { exited: true } | { exited: false; error: unknown }; class ProviderExitObservation { #outcome: ProviderExitOutcome | null = null; @@ -1229,19 +1356,40 @@ class SpawnedChildSet { } async verifyLifetimeOwnership(): Promise { - for (;;) { - const ownership = this.#lifetimeOwnership.splice(0); - if (ownership.length === 0) { - // This check and seal are synchronous. Any spawn added while an - // earlier batch was pending is observed by the next loop iteration; - // no later provider can race admission after the stable-empty point. - this.#lifetimeOwnershipSealed = true; - return; + try { + for (;;) { + const ownership = this.#lifetimeOwnership.splice(0); + if (ownership.length === 0) { + // This check and seal are synchronous. Any spawn added while an + // earlier batch was pending is observed by the next loop iteration; + // no later provider can race admission after the stable-empty point. + this.#lifetimeOwnershipSealed = true; + return; + } + await Promise.all(ownership); } - await Promise.all(ownership); + } catch (error) { + this.#lifetimeOwnershipSealed = true; + throw error; } } + beginLifetimeOwnershipAdmission(): () => Promise { + if (this.#sealed) { + throw new Error("ACPX provider ownership admission is closed"); + } + if (!this.#lifetimeOwnershipSealed) { + throw new Error("ACPX provider ownership admission is already active"); + } + this.#lifetimeOwnershipSealed = false; + let finished = false; + return async () => { + if (finished) return; + finished = true; + await this.verifyLifetimeOwnership(); + }; + } + #track(child: ChildProcess, providerExit: ProviderExitObservation): void { this.#children.add(child); const onError = (error: unknown) => this.#errors.add(error); @@ -1442,17 +1590,21 @@ function pushUnique(errors: unknown[], error: unknown): void { } function requireIdentity(handle: AcpRuntimeHandle): AcpxRuntimePortIdentity { - const identity = { - acpxRecordId: handle.acpxRecordId, - backendSessionId: handle.backendSessionId, - agentSessionId: handle.agentSessionId, - }; - for (const [name, value] of Object.entries(identity)) { - if (typeof value !== "string" || value.length === 0) { - throw new Error(`ACPX runtime omitted ${name}`); - } + const acpxRecordId = nonEmptyRuntimeIdentity(handle.acpxRecordId); + if (!acpxRecordId) throw new Error("ACPX runtime omitted acpxRecordId"); + const backendSessionId = nonEmptyRuntimeIdentity(handle.backendSessionId); + if (!backendSessionId) { + throw new Error("ACPX runtime omitted backendSessionId"); } - return identity as AcpxRuntimePortIdentity; + return { + acpxRecordId, + backendSessionId, + // ACPX agents do not all advertise a distinct native thread identity. + // In that case the backend ID is the real ACP protocol session, so retain + // it explicitly rather than inventing a Paperclip-owned identifier. + agentSessionId: + nonEmptyRuntimeIdentity(handle.agentSessionId) ?? backendSessionId, + }; } function definedEnvironment( diff --git a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts index 1dab74e06a..cecf96be27 100644 --- a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts @@ -24,6 +24,7 @@ import { resolveQualifiedAcpxProfile } from "./qualified-profiles.js"; import { awaitVerifiedAcpxProviderExit, awaitVerifiedAcpxProviderOwnership, + createAcpxPackageJsonResolver, guardSnapshotModuleLookup, guardSnapshotModuleResolution, reapCurrentProviderProcessGroup, @@ -48,6 +49,145 @@ afterEach(async () => { }); describe("ACPX installation integrity", () => { + it("anchors dynamic provider package resolution at an explicit root", async () => { + const parent = await mkdtemp( + join(tmpdir(), "paperclip-acpx-package-parent-"), + ); + temporaryDirectories.push(parent); + const root = join(parent, "provider-pack"); + const providerDirectory = join(root, "node_modules", "qualified-provider"); + const providerPackageJson = join(providerDirectory, "package.json"); + await mkdir(providerDirectory, { recursive: true }); + await Promise.all([ + writeFile(join(root, "package.json"), JSON.stringify({ private: true })), + writeFile( + providerPackageJson, + JSON.stringify({ name: "qualified-provider", version: "1.0.0" }), + ), + ]); + + expect(createAcpxPackageJsonResolver(root)("qualified-provider")).toBe( + providerPackageJson, + ); + + const nestedDependencyDirectory = join( + providerDirectory, + "node_modules", + "qualified-dependency", + ); + const nestedDependencyPackageJson = join( + nestedDependencyDirectory, + "package.json", + ); + await mkdir(nestedDependencyDirectory, { recursive: true }); + await writeFile( + nestedDependencyPackageJson, + JSON.stringify({ + name: "qualified-dependency", + version: "1.0.0", + exports: "./index.js", + }), + ); + await writeFile(join(nestedDependencyDirectory, "index.js"), "export {};"); + expect( + createAcpxPackageJsonResolver(root)( + "qualified-dependency", + providerPackageJson, + ), + ).toBe(nestedDependencyPackageJson); + expect(() => + createAcpxPackageJsonResolver("relative/provider-pack"), + ).toThrow("explicit normalized absolute path"); + expect(() => createAcpxPackageJsonResolver(undefined)).toThrow( + "explicit normalized absolute path", + ); + + const runnerPackage = join(root, "packages", "paperclip-runner"); + const runnerManifest = join(runnerPackage, "package.json"); + const pnpmProviderDirectory = join( + root, + "node_modules", + ".pnpm", + "qualified-provider@1.0.0", + "node_modules", + "pnpm-provider", + ); + await Promise.all([ + mkdir(join(runnerPackage, "node_modules"), { recursive: true }), + mkdir(pnpmProviderDirectory, { recursive: true }), + ]); + await Promise.all([ + writeFile(runnerManifest, JSON.stringify({ private: true })), + writeFile( + join(pnpmProviderDirectory, "package.json"), + JSON.stringify({ name: "pnpm-provider", version: "1.0.0" }), + ), + ]); + await symlink( + pnpmProviderDirectory, + join(runnerPackage, "node_modules", "pnpm-provider"), + ); + expect( + createAcpxPackageJsonResolver(root, runnerManifest)("pnpm-provider"), + ).toBe(join(pnpmProviderDirectory, "package.json")); + + const outsideManifest = join(parent, "outside-package.json"); + await writeFile(outsideManifest, JSON.stringify({ private: true })); + expect(() => createAcpxPackageJsonResolver(root, outsideManifest)).toThrow( + "manifest resolves outside the selected provider root", + ); + + const ancestorProviderDirectory = join( + parent, + "node_modules", + "ancestor-provider", + ); + await mkdir(ancestorProviderDirectory, { recursive: true }); + await writeFile( + join(ancestorProviderDirectory, "package.json"), + JSON.stringify({ name: "ancestor-provider", version: "1.0.0" }), + ); + expect(() => + createAcpxPackageJsonResolver(root)("ancestor-provider"), + ).toThrow("outside the selected provider root"); + + const outsideProviderDirectory = join(parent, "outside-provider"); + await mkdir(outsideProviderDirectory); + await writeFile( + join(outsideProviderDirectory, "package.json"), + JSON.stringify({ name: "linked-provider", version: "1.0.0" }), + ); + await symlink( + outsideProviderDirectory, + join(root, "node_modules", "linked-provider"), + ); + expect(() => + createAcpxPackageJsonResolver(root)("linked-provider"), + ).toThrow("outside the selected provider root"); + }); + + it("does not fall back through the server package for a missing rooted dependency", async () => { + const fixture = await installationFixture(); + const nestedRuntimeDirectory = join( + fixture.serverDirectory, + "node_modules", + "@earendil-works", + "pi-coding-agent", + ); + await mkdir(nestedRuntimeDirectory, { recursive: true }); + await writeFile( + join(nestedRuntimeDirectory, "package.json"), + JSON.stringify({ version: "0.84.2" }), + ); + + await expect( + verifyQualifiedAcpxInstallation(fixture.profile, (packageName) => { + if (packageName === "pi-acp") return fixture.serverPackageJsonPath; + throw new Error("rooted package is absent"); + }), + ).rejects.toThrow("rooted package is absent"); + }); + it("rejects an unregistered provider exit proof", async () => { await expect( awaitVerifiedAcpxProviderExit({} as ChildProcess), @@ -274,6 +414,178 @@ describe("ACPX installation integrity", () => { }); }); + it("pins Claude ACP direct dependencies outside its package root", async () => { + const fixture = await installationFixture(); + const command = [ + 'import { qualifiedValue } from "@anthropic-ai/claude-agent-sdk";', + "process.stdout.write(qualifiedValue);", + ].join("\n"); + const dependencyRoot = join(fixture.root, "qualified-dependencies"); + const dependencyFixtures = [ + { + name: "@agentclientprotocol/sdk", + version: "1.3.0", + directory: join(dependencyRoot, "agentclient-sdk"), + }, + { + name: "@anthropic-ai/claude-agent-sdk", + version: "0.3.232", + directory: join(dependencyRoot, "claude-agent-sdk"), + }, + { + name: "zod", + version: "4.4.3", + directory: join(dependencyRoot, "zod"), + }, + ] as const; + await Promise.all([ + writeFile(fixture.commandPath, command), + mkdir(join(fixture.serverDirectory, "node_modules", "@anthropic-ai"), { + recursive: true, + }), + ...dependencyFixtures.map((dependency) => + mkdir(dependency.directory, { recursive: true }), + ), + ]); + await Promise.all([ + writeFile( + fixture.serverPackageJsonPath, + JSON.stringify({ + name: "@agentclientprotocol/claude-agent-acp", + version: "0.70.0", + type: "module", + bin: "bin/server.js", + dependencies: { + "@agentclientprotocol/sdk": "1.3.0", + "@anthropic-ai/claude-agent-sdk": "0.3.232", + zod: "^3.25.0 || ^4.0.0", + }, + }), + ), + ...dependencyFixtures.map((dependency) => + writeFile( + join(dependency.directory, "package.json"), + JSON.stringify({ + name: dependency.name, + version: dependency.version, + type: "module", + exports: "./index.js", + }), + ), + ), + writeFile( + join(dependencyFixtures[1].directory, "index.js"), + 'export const qualifiedValue = "qualified-claude-dependency";', + ), + ]); + await symlink( + dependencyFixtures[1].directory, + join( + fixture.serverDirectory, + "node_modules", + "@anthropic-ai", + "claude-agent-sdk", + ), + ); + const paths = new Map([ + ["@agentclientprotocol/claude-agent-acp", fixture.serverPackageJsonPath], + ...dependencyFixtures.map( + (dependency) => + [ + dependency.name, + join(dependency.directory, "package.json"), + ] as const, + ), + ]); + const profile = { + ...resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"), + agentRuntimePackage: null, + agentRuntimeVersion: null, + commandDigest: `sha256:${createHash("sha256").update(command).digest("hex")}`, + }; + const installation = await verifyQualifiedAcpxInstallation( + profile, + (packageName) => { + const resolved = paths.get(packageName); + if (!resolved) throw new Error(`unexpected package ${packageName}`); + return resolved; + }, + ); + + await expectPinnedOutput( + (await installation.openCommand()).spawn(), + "qualified-claude-dependency", + ); + }); + + it("rejects drift in Claude ACP's qualified dependency versions", async () => { + const fixture = await installationFixture(); + await writeFile( + fixture.serverPackageJsonPath, + JSON.stringify({ + version: "0.70.0", + type: "module", + bin: "bin/server.js", + dependencies: { + "@agentclientprotocol/sdk": "1.3.0", + "@anthropic-ai/claude-agent-sdk": "0.3.232", + zod: "^3.25.0 || ^4.0.0", + }, + }), + ); + const dependencyPackage = join(fixture.root, "dependency", "package.json"); + await mkdir(dirname(dependencyPackage), { recursive: true }); + await writeFile( + dependencyPackage, + JSON.stringify({ version: "unexpected" }), + ); + + await expect( + verifyQualifiedAcpxInstallation( + { + ...resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"), + agentRuntimePackage: null, + agentRuntimeVersion: null, + commandDigest: fixture.profile.commandDigest, + }, + (packageName) => + packageName === "@agentclientprotocol/claude-agent-acp" + ? fixture.serverPackageJsonPath + : dependencyPackage, + ), + ).rejects.toThrow( + "ACPX claude dependency package version mismatch for @agentclientprotocol/sdk", + ); + }); + + it.runIf(process.platform === "linux" && process.arch === "x64")( + "resolves and pins the installed Claude ACP dependency graph", + async () => { + const profile = resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"); + const installation = await verifyQualifiedAcpxInstallation(profile); + expect(installation.agentServerPackageJsonPath).toContain( + "/@agentclientprotocol/claude-agent-acp/package.json", + ); + expect(installation.agentRuntimePackageJsonPath).toContain( + "/@anthropic-ai/claude-agent-sdk/package.json", + ); + await (await installation.openCommand()).close(); + }, + ); + + it.runIf(process.platform === "linux" && process.arch === "x64")( + "resolves and pins the qualified Codex native runtime through its transitive packages", + async () => { + const profile = resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol"); + const installation = await verifyQualifiedAcpxInstallation(profile); + expect(installation.agentRuntimePackageJsonPath).toContain( + "/@openai/codex/package.json", + ); + const command = await installation.openCommand(); + await command.close(); + }, + ); + it("rejects package version and executable digest drift", async () => { const fixture = await installationFixture(); await writeFile( diff --git a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts index e628af617d..becb0b496c 100644 --- a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts +++ b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts @@ -4,7 +4,7 @@ import { type ChildProcess, type SpawnOptionsWithoutStdio, } from "node:child_process"; -import { constants } from "node:fs"; +import { constants, realpathSync } from "node:fs"; import { lstat, open, @@ -21,19 +21,73 @@ import { isAbsolute, relative, resolve, + sep, } from "node:path"; import type { Readable, Writable } from "node:stream"; import type { QualifiedAcpxProfile } from "./qualified-profiles.js"; +import { + VERIFIED_RUNTIME_EXECUTABLE_ENV, + verifiedRuntimeExecutableHandoff, +} from "./verified-runtime-executable.js"; const MAX_PACKAGE_JSON_BYTES = 256 * 1024; const MAX_AGENT_COMMAND_BYTES = 16 * 1024 * 1024; +const MAX_RUNTIME_EXECUTABLE_BYTES = 384 * 1024 * 1024; const COMMAND_SOURCE_FD = 3; const COMMAND_DIRECTORY_FD = 4; const DEPENDENCY_ANCESTOR_FD_START = 5; const MAX_DEPENDENCY_ANCESTORS = 64; const PROVIDER_WATCHDOG_HANDSHAKE_TIMEOUT_MS = 2_000; const PROVIDER_GUARDIAN_HANDSHAKE_TIMEOUT_MS = 5_000; +const VERIFIED_PROVIDER_RUNTIME_TARGET_ENV = + "PAPERCLIP_ACPX_VERIFIED_PROVIDER_RUNTIME_TARGET"; + +const QUALIFIED_CLAUDE_LINUX_X64_RUNTIME = Object.freeze({ + runtimePackageName: "@anthropic-ai/claude-agent-sdk", + runtimePackageVersion: "0.3.232", + packageName: "@anthropic-ai/claude-agent-sdk-linux-x64", + packageVersion: "0.3.232", + dependencyDeclaration: "0.3.232", + relativeExecutable: "claude", + executableDigest: + "sha256:61d23f8749136907d586d5b11831ea8a5234d4c1dea40a5e55c33b52e204c6d1", + environmentVariable: "CLAUDE_CODE_EXECUTABLE", +}); + +const QUALIFIED_CODEX_LINUX_X64_RUNTIME = Object.freeze({ + runtimePackageName: "@openai/codex", + runtimePackageVersion: "0.148.0", + packageName: "@openai/codex-linux-x64", + packageVersion: "0.148.0-linux-x64", + dependencyDeclaration: "npm:@openai/codex@0.148.0-linux-x64", + relativeExecutable: "vendor/x86_64-unknown-linux-musl/bin/codex", + executableDigest: + "sha256:ac2cfed85fb647d61e0150b8548102b330e4799d9d81ad5d354de701edf6b074", + environmentVariable: "CODEX_PATH", +}); + +// Claude's ACP server is not a self-contained bundle: its entrypoint imports +// these three packages directly from pnpm's real store paths. Keep that exact +// package graph version-bound and descriptor-pinned instead of granting the +// provider ambient access to the workspace's complete node_modules ancestry. +const QUALIFIED_CLAUDE_PROVIDER_DEPENDENCIES = Object.freeze([ + Object.freeze({ + packageName: "@agentclientprotocol/sdk", + packageVersion: "1.3.0", + dependencyDeclaration: "1.3.0", + }), + Object.freeze({ + packageName: "@anthropic-ai/claude-agent-sdk", + packageVersion: "0.3.232", + dependencyDeclaration: "0.3.232", + }), + Object.freeze({ + packageName: "zod", + packageVersion: "4.4.3", + dependencyDeclaration: "^3.25.0 || ^4.0.0", + }), +]); const PROVIDER_LIFETIME_WATCHDOG_SOURCE = ` const fs = require("node:fs"); @@ -68,14 +122,22 @@ export const PROVIDER_LIFETIME_GUARDIAN_SOURCE = ` const fs = require("node:fs"); const { spawn } = require("node:child_process"); const WATCHDOG_SOURCE = ${JSON.stringify(PROVIDER_LIFETIME_WATCHDOG_SOURCE)}; +const runtimeExecutable = process.env.${VERIFIED_RUNTIME_EXECUTABLE_ENV} || process.execPath; const dependencyAncestorCount = Number.parseInt(process.argv[4], 10); +const providerRuntimeExecutableCount = Number.parseInt(process.argv[8], 10); if (!Number.isSafeInteger(dependencyAncestorCount) || dependencyAncestorCount < 0 || dependencyAncestorCount > ${MAX_DEPENDENCY_ANCESTORS}) throw new Error("ACPX provider dependency ancestry is invalid"); -const OWNER_FD = ${DEPENDENCY_ANCESTOR_FD_START} + dependencyAncestorCount; +if (providerRuntimeExecutableCount !== 0 && providerRuntimeExecutableCount !== 1) throw new Error("ACPX provider runtime executable count is invalid"); +const PROVIDER_RUNTIME_EXECUTABLE_FD = ${DEPENDENCY_ANCESTOR_FD_START} + dependencyAncestorCount; +const OWNER_FD = PROVIDER_RUNTIME_EXECUTABLE_FD + providerRuntimeExecutableCount; const OWNERSHIP_FD = OWNER_FD + 1; const PROVIDER_EXIT_FD = OWNERSHIP_FD + 1; const CREDENTIAL_FENCE_FD_START = PROVIDER_EXIT_FD + 1; +const VERIFIED_RUNTIME_FD = CREDENTIAL_FENCE_FD_START + 2; const dependencyAncestorFds = Array.from({ length: dependencyAncestorCount }, (_, index) => ${DEPENDENCY_ANCESTOR_FD_START} + index); -const PROVIDER_GUARDIAN_FD = ${DEPENDENCY_ANCESTOR_FD_START} + dependencyAncestorCount; +const runtimeDescriptorMatch = /^\\/proc\\/self\\/fd\\/([0-9]+)$/.exec(runtimeExecutable); +const runtimeDescriptorFd = runtimeDescriptorMatch === null ? null : Number.parseInt(runtimeDescriptorMatch[1], 10); +if (runtimeDescriptorFd !== null && runtimeDescriptorFd !== VERIFIED_RUNTIME_FD) throw new Error("ACPX verified runtime descriptor is misplaced"); +if (runtimeDescriptorFd !== null) fs.fstatSync(runtimeDescriptorFd); let provider; let watchdog; let reaped = false; @@ -112,7 +174,7 @@ const startProvider = () => { if (provider || reaped || shutdownStarted) return; try { provider = spawn( - process.execPath, + runtimeExecutable, ["--eval", process.argv[1], ...process.argv.slice(2)], { cwd: process.cwd(), @@ -122,7 +184,7 @@ const startProvider = () => { // The provider observes this guardian-owned pipe directly. Kernel EOF // therefore revokes it even when SIGKILL/OOM prevents our JS reap path. // It also inherits both quorum fences until that self-reap completes. - stdio: [0, 1, 2, ${COMMAND_SOURCE_FD}, ${COMMAND_DIRECTORY_FD}, ...dependencyAncestorFds, "pipe", PROVIDER_EXIT_FD, CREDENTIAL_FENCE_FD_START, CREDENTIAL_FENCE_FD_START + 1], + stdio: [0, 1, 2, ${COMMAND_SOURCE_FD}, ${COMMAND_DIRECTORY_FD}, ...dependencyAncestorFds, ...(providerRuntimeExecutableCount === 1 ? [PROVIDER_RUNTIME_EXECUTABLE_FD] : []), "pipe", PROVIDER_EXIT_FD, CREDENTIAL_FENCE_FD_START, CREDENTIAL_FENCE_FD_START + 1, ...(runtimeDescriptorFd === null ? [] : ["ignore", runtimeDescriptorFd])], windowsHide: true, }, ); @@ -152,12 +214,17 @@ try { // its live identity if this guardian is killed before it can run its reap. // Its private owner pipe reaches kernel EOF on guardian death even while the // provider is stopped and unable to process its own guardian-loss callback. - watchdog = spawn(process.execPath, ["--eval", WATCHDOG_SOURCE], { + const watchdogStdio = ["ignore", "ignore", "ignore", "pipe", "pipe"]; + if (runtimeDescriptorFd !== null) { + while (watchdogStdio.length < runtimeDescriptorFd) watchdogStdio.push("ignore"); + watchdogStdio.push(runtimeDescriptorFd); + } + watchdog = spawn(runtimeExecutable, ["--eval", WATCHDOG_SOURCE], { cwd: process.cwd(), detached: false, env: {}, shell: false, - stdio: ["ignore", "ignore", "ignore", "pipe", "pipe"], + stdio: watchdogStdio, windowsHide: true, }); const watchdogOwnerPipe = watchdog.stdio[3]; @@ -196,7 +263,123 @@ try { const providerGuardianOwnership = new WeakMap>(); const providerExitProof = new WeakMap>(); -export type AcpxPackageJsonResolver = (packageName: string) => string; +export type AcpxPackageJsonResolver = ( + packageName: string, + issuerPackageJsonPath?: string, +) => string; + +export function createAcpxPackageJsonResolver( + providerPackageRoot: string | undefined, + providerPackageManifest?: string, +): AcpxPackageJsonResolver { + const root = providerPackageRoot?.trim(); + if ( + !root || + !isAbsolute(root) || + root.includes("\0") || + resolve(root) !== root + ) { + throw new Error( + "ACPX provider package root must be an explicit normalized absolute path", + ); + } + const manifest = ( + providerPackageManifest ?? resolve(root, "package.json") + ).trim(); + if ( + !manifest || + !isAbsolute(manifest) || + manifest.includes("\0") || + resolve(manifest) !== manifest + ) { + throw new Error( + "ACPX provider package manifest must be an explicit normalized absolute path", + ); + } + const canonicalRoot = realpathSync(root); + const canonicalManifest = realpathSync(manifest); + if (!pathIsInside(canonicalRoot, canonicalManifest)) { + throw new Error( + "ACPX provider package manifest resolves outside the selected provider root", + ); + } + const canonicalNodeModules = realpathSync( + resolve(canonicalRoot, "node_modules"), + ); + if (!pathIsInside(canonicalRoot, canonicalNodeModules)) { + throw new Error( + "ACPX provider node_modules resolves outside the selected provider root", + ); + } + return (packageName, issuerPackageJsonPath) => { + const canonicalIssuer = + issuerPackageJsonPath === undefined + ? canonicalManifest + : realpathSync(issuerPackageJsonPath); + if (!pathIsInside(canonicalRoot, canonicalIssuer)) { + throw new Error( + `ACPX provider package issuer for ${packageName} resolves outside the selected provider root`, + ); + } + const packageJsonPath = realpathSync( + resolvePackageJsonFromIssuer(packageName, canonicalIssuer), + ); + if (!pathIsInside(canonicalNodeModules, packageJsonPath)) { + throw new Error( + `ACPX provider package ${packageName} resolves outside the selected provider root`, + ); + } + return packageJsonPath; + }; +} + +function resolvePackageJsonFromIssuer( + packageName: string, + issuerPackageJsonPath: string, +): string { + const issuerRequire = createRequire(issuerPackageJsonPath); + try { + return issuerRequire.resolve(`${packageName}/package.json`); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code !== "ERR_PACKAGE_PATH_NOT_EXPORTED" + ) + throw error; + } + + const packageSegments = packageName.split("/"); + if ( + packageSegments.length < 1 || + packageSegments.length > 2 || + packageSegments.some((segment) => segment.length === 0) + ) { + throw new Error(`ACPX provider package name is invalid: ${packageName}`); + } + let directory = dirname(realpathSync(issuerRequire.resolve(packageName))); + for (let count = 0; count < MAX_DEPENDENCY_ANCESTORS; count += 1) { + const matchesPackage = + basename(directory) === packageSegments.at(-1) && + (packageSegments.length === 1 || + basename(dirname(directory)) === packageSegments[0]); + if (matchesPackage) return resolve(directory, "package.json"); + const parent = dirname(directory); + if (parent === directory) break; + directory = parent; + } + throw new Error( + `ACPX provider package manifest could not be located for ${packageName}`, + ); +} + +function pathIsInside(root: string, candidate: string): boolean { + const candidateRelativePath = relative(root, candidate); + return ( + candidateRelativePath !== "" && + candidateRelativePath !== ".." && + !candidateRelativePath.startsWith(`..${sep}`) && + !isAbsolute(candidateRelativePath) + ); +} export interface VerifiedAcpxInstallation { readonly commandDigest: string; @@ -285,6 +468,21 @@ interface VerifiedAcpxCommandIdentity { changedNanoseconds: string; } +interface VerifiedAcpxRuntimeExecutable { + path: string; + digest: string; + identity: VerifiedAcpxCommandIdentity; + environmentVariable: "CLAUDE_CODE_EXECUTABLE" | "CODEX_PATH"; +} + +interface AcpxPackageMetadata { + version?: string; + bin?: unknown; + type?: unknown; + dependencies?: unknown; + optionalDependencies?: unknown; +} + interface VerifiedAcpxDirectoryIdentity { device: string; inode: string; @@ -353,14 +551,16 @@ export async function verifyQualifiedAcpxInstallation( let runtimePackageJsonPath: string | null = null; let runtimePackageFormat: AcpxCommandFormat | null = null; + let runtimePackage: AcpxPackageMetadata | null = null; + let runtimeExecutable: VerifiedAcpxRuntimeExecutable | null = null; if (profile.agentRuntimePackage !== null) { if (profile.agentRuntimeVersion === null) { throw new Error("Qualified ACPX runtime package omitted its version"); } runtimePackageJsonPath = await realpath( - resolvePackageJson(profile.agentRuntimePackage), + resolvePackageJson(profile.agentRuntimePackage, serverPackageJsonPath), ); - const runtimePackage = await readPackageJson( + runtimePackage = await readPackageJson( runtimePackageJsonPath, profile.agentRuntimePackage, ); @@ -370,10 +570,58 @@ export async function verifyQualifiedAcpxInstallation( ); } runtimePackageFormat = packageModuleFormat(runtimePackage.type); + runtimeExecutable = await verifyQualifiedRuntimeExecutable({ + profile, + runtimePackage, + runtimePackageJsonPath, + resolvePackageJson, + }); } else if (profile.agentRuntimeVersion !== null) { throw new Error("Qualified ACPX runtime version omitted its package"); } + const supplementalPackages: Array<{ + directory: string; + format: AcpxCommandFormat; + }> = []; + if (profile.agent === "claude") { + const declaredDependencies = serverPackage.dependencies; + if ( + typeof declaredDependencies !== "object" || + declaredDependencies === null || + Array.isArray(declaredDependencies) + ) { + throw new Error("ACPX claude package omitted its qualified dependencies"); + } + for (const expected of QUALIFIED_CLAUDE_PROVIDER_DEPENDENCIES) { + if ( + (declaredDependencies as Record)[ + expected.packageName + ] !== expected.dependencyDeclaration + ) { + throw new Error( + `ACPX claude package dependency mismatch for ${expected.packageName}`, + ); + } + const dependencyPackageJsonPath = await realpath( + resolvePackageJson(expected.packageName, serverPackageJsonPath), + ); + const dependencyPackage = await readPackageJson( + dependencyPackageJsonPath, + expected.packageName, + ); + if (dependencyPackage.version !== expected.packageVersion) { + throw new Error( + `ACPX claude dependency package version mismatch for ${expected.packageName}: expected ${expected.packageVersion}, received ${dependencyPackage.version ?? "unknown"}`, + ); + } + supplementalPackages.push({ + directory: dirname(dependencyPackageJsonPath), + format: packageModuleFormat(dependencyPackage.type), + }); + } + } + const serverDependencyAncestors = await inspectDependencyAncestors( commandDirectory, packageDirectory, @@ -404,6 +652,22 @@ export async function verifyQualifiedAcpxInstallation( dependencyAncestorFormats.push(runtimePackageFormat ?? "commonjs"); } } + for (const supplemental of supplementalPackages) { + if ( + supplemental.directory !== commandDirectory && + !dependencyAncestors.some( + (ancestor) => ancestor.path === supplemental.directory, + ) + ) { + dependencyAncestors.push( + await inspectExplicitDependencyRoot( + supplemental.directory, + `${profile.agent} dependency`, + ), + ); + dependencyAncestorFormats.push(supplemental.format); + } + } if (dependencyAncestors.length > MAX_DEPENDENCY_ANCESTORS) { throw new Error("ACPX provider dependency ancestry exceeds its bound"); } @@ -432,9 +696,24 @@ export async function verifyQualifiedAcpxInstallation( ); } let currentDependencyAncestors: FileHandle[] = []; + let currentRuntimeExecutable: FileHandle | null = null; try { currentDependencyAncestors = await openDependencyAncestors(dependencyAncestors); + if (runtimeExecutable !== null) { + const current = await openVerifiedRuntimeExecutable( + runtimeExecutable.path, + runtimeExecutable.digest, + profile.agent, + ); + if (!sameIdentity(current.identity, runtimeExecutable.identity)) { + await current.handle.close(); + throw new Error( + "ACPX provider runtime executable identity changed after verification", + ); + } + currentRuntimeExecutable = current.handle; + } const current = await inspectCommand( commandPath, commandDigest, @@ -456,11 +735,16 @@ export async function verifyQualifiedAcpxInstallation( serverDependencyAncestorCount, serverPackageFormat, dependencyAncestorFormats, + currentRuntimeExecutable, + runtimeExecutable?.environmentVariable ?? null, ); } catch (error) { await Promise.all([ currentDirectory.handle.close(), ...currentDependencyAncestors.map((handle) => handle.close()), + ...(currentRuntimeExecutable === null + ? [] + : [currentRuntimeExecutable.close()]), ]); throw error; } @@ -468,14 +752,29 @@ export async function verifyQualifiedAcpxInstallation( }); } -function defaultPackageJsonResolver(packageName: string): string { - return createRequire(import.meta.url).resolve(`${packageName}/package.json`); +function defaultPackageJsonResolver( + packageName: string, + issuerPackageJsonPath?: string, +): string { + const providerPackageRoot = process.env.PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT; + if (providerPackageRoot !== undefined) { + return createAcpxPackageJsonResolver( + providerPackageRoot, + process.env.PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST, + )(packageName, issuerPackageJsonPath); + } + // Source-mode and direct runtimes still have a stable module URL. The + // descriptor-backed runner sidecar always receives the explicit root above. + return resolvePackageJsonFromIssuer( + packageName, + issuerPackageJsonPath ?? import.meta.url, + ); } async function readPackageJson( packageJsonPath: string, packageName: string, -): Promise<{ version?: string; bin?: unknown; type?: unknown }> { +): Promise { const bytes = await readBoundedRegularFile( packageJsonPath, MAX_PACKAGE_JSON_BYTES, @@ -490,7 +789,98 @@ async function readPackageJson( if (typeof value !== "object" || value === null || Array.isArray(value)) { throw new Error(`ACPX package ${packageName} has invalid package metadata`); } - return value as { version?: string; bin?: unknown; type?: unknown }; + return value as AcpxPackageMetadata; +} + +async function verifyQualifiedRuntimeExecutable(input: { + profile: QualifiedAcpxProfile; + runtimePackage: AcpxPackageMetadata; + runtimePackageJsonPath: string; + resolvePackageJson: AcpxPackageJsonResolver; +}): Promise { + const qualification = + input.profile.agent === "claude" + ? QUALIFIED_CLAUDE_LINUX_X64_RUNTIME + : input.profile.agent === "codex" + ? QUALIFIED_CODEX_LINUX_X64_RUNTIME + : null; + if (qualification === null) return null; + if ( + input.profile.agentRuntimePackage !== qualification.runtimePackageName || + input.profile.agentRuntimeVersion !== qualification.runtimePackageVersion + ) { + throw new Error( + `ACPX ${input.profile.agent} runtime does not match its qualified profile`, + ); + } + if (process.platform !== "linux" || process.arch !== "x64") { + throw new Error( + `ACPX ${input.profile.agent} verified runtime executable requires qualified Linux x64`, + ); + } + + const optionalDependencies = input.runtimePackage.optionalDependencies; + if ( + typeof optionalDependencies !== "object" || + optionalDependencies === null || + Array.isArray(optionalDependencies) || + (optionalDependencies as Record)[ + qualification.packageName + ] !== qualification.dependencyDeclaration + ) { + throw new Error( + `ACPX ${input.profile.agent} runtime omitted its qualified Linux executable package`, + ); + } + + const executablePackageJsonPath = await realpath( + input.resolvePackageJson( + qualification.packageName, + input.runtimePackageJsonPath, + ), + ); + const executablePackage = await readPackageJson( + executablePackageJsonPath, + qualification.packageName, + ); + if (executablePackage.version !== qualification.packageVersion) { + throw new Error( + `ACPX ${input.profile.agent} runtime executable package version mismatch: expected ${qualification.packageVersion}, received ${executablePackage.version ?? "unknown"}`, + ); + } + + const packageDirectory = dirname(executablePackageJsonPath); + const unresolvedExecutablePath = resolve( + packageDirectory, + qualification.relativeExecutable, + ); + if (!isInside(packageDirectory, unresolvedExecutablePath)) { + throw new Error( + `ACPX ${input.profile.agent} runtime executable escapes its package`, + ); + } + const executableDirectory = await realpath(dirname(unresolvedExecutablePath)); + if (!isInsideOrEqual(packageDirectory, executableDirectory)) { + throw new Error( + `ACPX ${input.profile.agent} runtime executable escapes its package`, + ); + } + const executablePath = resolve( + executableDirectory, + basename(unresolvedExecutablePath), + ); + const verified = await openVerifiedRuntimeExecutable( + executablePath, + qualification.executableDigest, + input.profile.agent, + ); + await verified.handle.close(); + return { + path: executablePath, + digest: qualification.executableDigest, + identity: verified.identity, + environmentVariable: qualification.environmentVariable, + }; } async function readBoundedRegularFile( @@ -582,6 +972,96 @@ async function inspectCommand( } } +async function openVerifiedRuntimeExecutable( + executablePath: string, + expectedDigest: string, + agent: string, +): Promise<{ handle: FileHandle; identity: VerifiedAcpxCommandIdentity }> { + const lexicalBefore = await lstat(executablePath, { bigint: true }).catch( + () => null, + ); + if ( + lexicalBefore === null || + lexicalBefore.isSymbolicLink() || + !lexicalBefore.isFile() + ) { + throw new Error( + `ACPX ${agent} runtime executable must be a real regular file`, + ); + } + + let handle: FileHandle; + try { + handle = await open( + executablePath, + verifiedExecutableOpenFlags(process.platform, constants.O_NOFOLLOW), + ); + } catch { + throw new Error( + `ACPX ${agent} runtime executable could not be opened as a no-follow regular file`, + ); + } + + try { + const before = await handle.stat({ bigint: true }); + if ( + !before.isFile() || + before.size < 1n || + before.size > BigInt(MAX_RUNTIME_EXECUTABLE_BYTES) || + (before.mode & 0o111n) === 0n + ) { + throw new Error( + `ACPX ${agent} runtime executable must be a bounded executable file`, + ); + } + const hash = createHash("sha256"); + const buffer = Buffer.alloc(1024 * 1024); + let position = 0; + try { + while (position < Number(before.size)) { + const { bytesRead } = await handle.read( + buffer, + 0, + Math.min(buffer.length, Number(before.size) - position), + position, + ); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } + } finally { + buffer.fill(0); + } + const after = await handle.stat({ bigint: true }); + const lexicalAfter = await lstat(executablePath, { bigint: true }).catch( + () => null, + ); + const beforeIdentity = fileIdentity(before); + const afterIdentity = fileIdentity(after); + if ( + position !== Number(before.size) || + lexicalAfter === null || + lexicalAfter.isSymbolicLink() || + !lexicalAfter.isFile() || + !sameIdentity(fileIdentity(lexicalBefore), fileIdentity(lexicalAfter)) || + !sameIdentity(fileIdentity(lexicalAfter), afterIdentity) || + !sameIdentity(beforeIdentity, afterIdentity) + ) { + throw new Error( + `ACPX ${agent} runtime executable changed while it was verified`, + ); + } + const digest = `sha256:${hash.digest("hex")}`; + if (digest !== expectedDigest) { + throw new Error(`ACPX ${agent} runtime executable digest mismatch`); + } + return { handle, identity: afterIdentity }; + } catch (error) { + await handle.close(); + throw error; + } +} + /** Fail closed where Node cannot atomically refuse a final symlink component. */ export function verifiedExecutableOpenFlags( platform: NodeJS.Platform, @@ -764,6 +1244,9 @@ function commandLease( serverDependencyAncestorCount: number, serverPackageFormat: AcpxCommandFormat, dependencyAncestorFormats: readonly AcpxCommandFormat[], + providerRuntimeExecutable: FileHandle | null, + providerRuntimeEnvironmentVariable: + VerifiedAcpxRuntimeExecutable["environmentVariable"] | null, ): VerifiedAcpxCommandLease { let consumed = false; let directoriesReleased = false; @@ -773,6 +1256,9 @@ function commandLease( await Promise.all([ commandDirectory.close(), ...dependencyAncestors.map((handle) => handle.close()), + ...(providerRuntimeExecutable === null + ? [] + : [providerRuntimeExecutable.close()]), ]); }; const releaseDirectoriesBestEffort = (): void => { @@ -803,8 +1289,13 @@ function commandLease( : format === "module" ? MODULE_SNAPSHOT_BOOTSTRAP : COMMONJS_SNAPSHOT_BOOTSTRAP; - const providerOwnershipFd = - DEPENDENCY_ANCESTOR_FD_START + dependencyAncestors.length + 1; + const providerRuntimeExecutableCount = + providerRuntimeExecutable === null ? 0 : 1; + const providerGuardianFd = + DEPENDENCY_ANCESTOR_FD_START + + dependencyAncestors.length + + providerRuntimeExecutableCount; + const providerOwnershipFd = providerGuardianFd + 1; const providerExitFd = providerOwnershipFd + 1; if ( guarded && @@ -818,8 +1309,34 @@ function commandLease( ) { throw new Error("ACPX provider credential fence is invalid"); } + const runtimeTargetFd = guarded + ? providerExitFd + 3 + : DEPENDENCY_ANCESTOR_FD_START + + dependencyAncestors.length + + providerRuntimeExecutableCount; + const runtimeHandoff = + verifiedRuntimeExecutableHandoff(runtimeTargetFd); + const environment = sanitizedNodeEnvironment(options.env); + if (runtimeHandoff.environmentValue === undefined) { + delete environment[VERIFIED_RUNTIME_EXECUTABLE_ENV]; + } else { + environment[VERIFIED_RUNTIME_EXECUTABLE_ENV] = + runtimeHandoff.environmentValue; + } + if ( + (providerRuntimeExecutable === null) !== + (providerRuntimeEnvironmentVariable === null) + ) { + throw new Error("ACPX provider runtime executable lease is invalid"); + } + if (providerRuntimeEnvironmentVariable === null) { + delete environment[VERIFIED_PROVIDER_RUNTIME_TARGET_ENV]; + } else { + environment[VERIFIED_PROVIDER_RUNTIME_TARGET_ENV] = + providerRuntimeEnvironmentVariable; + } child = spawnChildProcess( - process.execPath, + runtimeHandoff.executable, guarded ? [ // Keep resolved module URLs on the retained descriptor paths @@ -834,6 +1351,7 @@ function commandLease( String(serverDependencyAncestorCount), serverPackageFormat, JSON.stringify(dependencyAncestorFormats), + String(providerRuntimeExecutableCount), ...args, ] : [ @@ -846,6 +1364,7 @@ function commandLease( String(serverDependencyAncestorCount), serverPackageFormat, JSON.stringify(dependencyAncestorFormats), + String(providerRuntimeExecutableCount), ...args, ], { @@ -855,7 +1374,7 @@ function commandLease( // both credential quorum listeners inherited, and pins the PGID // until its single whole-group reap. detached: process.platform !== "win32", - env: sanitizedNodeEnvironment(options.env), + env: environment, shell: false, stdio: guarded ? [ @@ -865,10 +1384,16 @@ function commandLease( "pipe", commandDirectory.fd, ...dependencyAncestors.map((handle) => handle.fd), + ...(providerRuntimeExecutable === null + ? [] + : [providerRuntimeExecutable.fd]), "pipe", "pipe", "pipe", ...lifetime.credentialFenceFds, + ...(runtimeHandoff.sourceFd === null + ? [] + : [runtimeHandoff.sourceFd]), ] : [ "pipe", @@ -877,6 +1402,12 @@ function commandLease( "pipe", commandDirectory.fd, ...dependencyAncestors.map((handle) => handle.fd), + ...(providerRuntimeExecutable === null + ? [] + : [providerRuntimeExecutable.fd]), + ...(runtimeHandoff.sourceFd === null + ? [] + : [runtimeHandoff.sourceFd]), ], }, ); @@ -1094,13 +1625,19 @@ function snapshotBootstrap(format: AcpxCommandFormat, guarded = false): string { "const serverDependencyAncestorCount = Number.parseInt(process.argv[4], 10);", "const serverPackageFormat = process.argv[5];", "const dependencyAncestorFormats = JSON.parse(process.argv[6]);", + "const providerRuntimeExecutableCount = Number.parseInt(process.argv[7], 10);", + `const providerRuntimeEnvironmentVariable = process.env.${VERIFIED_PROVIDER_RUNTIME_TARGET_ENV};`, + `delete process.env.${VERIFIED_PROVIDER_RUNTIME_TARGET_ENV};`, 'if (process.platform !== "linux") throw new Error("ACPX provider relative module loading requires Linux descriptor-pinned paths");', `if (!Number.isSafeInteger(dependencyAncestorCount) || dependencyAncestorCount < 0 || dependencyAncestorCount > ${MAX_DEPENDENCY_ANCESTORS}) throw new Error("ACPX provider dependency ancestry is invalid");`, 'if (!Number.isSafeInteger(serverDependencyAncestorCount) || serverDependencyAncestorCount < 0 || serverDependencyAncestorCount > dependencyAncestorCount) throw new Error("ACPX provider package ancestry is invalid");', 'if ((serverPackageFormat !== "module" && serverPackageFormat !== "commonjs") || !Array.isArray(dependencyAncestorFormats) || dependencyAncestorFormats.length !== dependencyAncestorCount || dependencyAncestorFormats.some((value) => value !== "module" && value !== "commonjs")) throw new Error("ACPX provider package formats are invalid");', + 'if (providerRuntimeExecutableCount !== 0 && providerRuntimeExecutableCount !== 1) throw new Error("ACPX provider runtime executable count is invalid");', + `const providerRuntimeExecutableFd = ${DEPENDENCY_ANCESTOR_FD_START} + dependencyAncestorCount;`, + 'if (providerRuntimeExecutableCount === 1) { if (providerRuntimeEnvironmentVariable !== "CODEX_PATH" && providerRuntimeEnvironmentVariable !== "CLAUDE_CODE_EXECUTABLE") throw new Error("ACPX provider runtime environment target is invalid"); fs.fstatSync(providerRuntimeExecutableFd); process.env[providerRuntimeEnvironmentVariable] = "/proc/" + process.pid + "/fd/" + providerRuntimeExecutableFd; } else if (providerRuntimeEnvironmentVariable !== undefined) throw new Error("ACPX provider runtime environment target is unexpected");', ...(guarded ? [ - `const guardianFd = ${DEPENDENCY_ANCESTOR_FD_START} + dependencyAncestorCount;`, + `const guardianFd = ${DEPENDENCY_ANCESTOR_FD_START} + dependencyAncestorCount + providerRuntimeExecutableCount;`, 'const guardian = fs.createReadStream("", { fd: guardianFd, autoClose: false });', `const reapCurrentProviderProcessGroup = ${reapCurrentProviderProcessGroup.toString()};`, "const killProviderProcess = process.kill.bind(process);", @@ -1122,7 +1659,7 @@ function snapshotBootstrap(format: AcpxCommandFormat, guarded = false): string { "const directoryUrl = pathToFileURL(`${directory}/`).href;", "const pinnedTarget = new URL(commandName, directoryUrl).href;", 'const target = process.platform === "linux" ? pinnedTarget : pathToFileURL(commandPath).href;', - "process.argv.splice(1, 6, fileURLToPath(target));", + "process.argv.splice(1, 7, fileURLToPath(target));", `const dependencyDirectoryUrls = Array.from({ length: dependencyAncestorCount }, (_, index) => pathToFileURL("/proc/self/fd/" + (${DEPENDENCY_ANCESTOR_FD_START} + index) + "/").href);`, 'const canonicalRootUrl = (url) => pathToFileURL(fs.realpathSync(fileURLToPath(url))).href.replace(/\\/?$/, "/");', 'const canonicalDirectoryUrl = process.platform === "linux" ? canonicalRootUrl(directoryUrl) : directoryUrl;', diff --git a/packages/paperclip-runner/src/drivers/acpx/model-verification.test.ts b/packages/paperclip-runner/src/drivers/acpx/model-verification.test.ts index cd4cf1317a..5fee5bc04e 100644 --- a/packages/paperclip-runner/src/drivers/acpx/model-verification.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/model-verification.test.ts @@ -24,7 +24,7 @@ describe("ACPX qualified model verification", () => { expect(setModel).not.toHaveBeenCalled(); }); - it("selects Claude's canonical model and normalizes its ACP selector", async () => { + it("accepts and normalizes Claude's qualified ACP selector", async () => { const setModel = vi.fn(async () => undefined); const getStatus = vi.fn(async () => ({ models: { @@ -33,6 +33,34 @@ describe("ACPX qualified model verification", () => { }, })); + await expect( + requireVerifiedAcpxModel( + { getStatus, setModel }, + resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"), + ), + ).resolves.toMatchObject({ + models: { + currentModelId: "claude-sonnet-5", + availableModelIds: ["default", "claude-sonnet-5", "opus"], + }, + }); + expect(setModel).not.toHaveBeenCalled(); + expect(getStatus).toHaveBeenCalledTimes(1); + }); + + it("selects Claude's profile-pinned ACP selector from a stale default", async () => { + let selected = false; + const setModel = vi.fn(async (model: string) => { + expect(model).toBe("sonnet"); + selected = true; + }); + const getStatus = vi.fn(async () => ({ + models: { + currentModelId: selected ? "sonnet" : "default", + availableModelIds: ["default", "sonnet", "opus"], + }, + })); + await expect( requireVerifiedAcpxModel( { getStatus, setModel }, @@ -45,7 +73,7 @@ describe("ACPX qualified model verification", () => { }, }); expect(setModel).toHaveBeenCalledTimes(1); - expect(setModel).toHaveBeenCalledWith("claude-sonnet-5"); + expect(setModel).toHaveBeenCalledWith("sonnet"); expect(getStatus).toHaveBeenCalledTimes(2); }); diff --git a/packages/paperclip-runner/src/drivers/acpx/model-verification.ts b/packages/paperclip-runner/src/drivers/acpx/model-verification.ts index 2eb49ac6e4..e178dcabe1 100644 --- a/packages/paperclip-runner/src/drivers/acpx/model-verification.ts +++ b/packages/paperclip-runner/src/drivers/acpx/model-verification.ts @@ -22,30 +22,41 @@ export async function requireVerifiedAcpxModel( profile: QualifiedAcpxProfile, ): Promise { if (!control.getStatus) { - throw new Error("ACPX agent cannot verify its effective model"); + throw acpxModelVerificationError( + "ACPX_MODEL_STATUS_UNAVAILABLE", + "ACPX agent cannot verify its effective model", + ); } const requestedModel = profile.qualificationModel; + const providerModel = profile.reportedModelId; let status = await control.getStatus(); - const mustSelectCanonical = - profile.reportedModelId !== requestedModel || - status.models?.currentModelId !== requestedModel; - if (mustSelectCanonical) { + if (status.models?.currentModelId !== providerModel) { if (!control.setModel) { - throw new Error( - "ACPX agent cannot verify its canonical model through ACP config options", + throw acpxModelVerificationError( + "ACPX_MODEL_SELECTION_UNAVAILABLE", + "ACPX agent cannot verify its qualified model through ACP config options", ); } - await control.setModel(requestedModel); + // The caller-facing model is already pinned by resolveQualifiedAcpxProfile. + // Select the immutable ACP-facing identifier from that same profile: some + // providers expose a stable selector (for example Claude's `sonnet`) while + // Paperclip publishes the canonical model name after verification. + await control.setModel(providerModel); status = await control.getStatus(); } - if (status.models?.currentModelId !== profile.reportedModelId) { - throw new Error( - `ACPX effective model mismatch: requested ${requestedModel}, expected ACP selector ${profile.reportedModelId}, received ${status.models?.currentModelId ?? "unverified"}`, + if (status.models?.currentModelId !== providerModel) { + throw acpxModelVerificationError( + "ACPX_EFFECTIVE_MODEL_MISMATCH", + `ACPX effective model mismatch: requested ${requestedModel}, expected ACP selector ${providerModel}, received ${status.models?.currentModelId ?? "unverified"}`, ); } return normalizeQualifiedModelStatus(status, profile); } +function acpxModelVerificationError(code: string, message: string): Error { + return Object.assign(new Error(message), { code }); +} + function normalizeQualifiedModelStatus( status: AcpxModelStatus, profile: QualifiedAcpxProfile, diff --git a/packages/paperclip-runner/src/drivers/acpx/qualified-profiles.test.ts b/packages/paperclip-runner/src/drivers/acpx/qualified-profiles.test.ts index 142115db99..3b8fa9135c 100644 --- a/packages/paperclip-runner/src/drivers/acpx/qualified-profiles.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/qualified-profiles.test.ts @@ -23,4 +23,18 @@ describe("qualified ACPX profiles", () => { resolveQualifiedAcpxProfile("codex", "some-other-model"), ).toThrow("requires exact model"); }); + + it("binds Codex ACP to the CLI runtime it launches", () => { + expect(QUALIFIED_ACPX_PROFILES.codex).toMatchObject({ + agentRuntimePackage: "@openai/codex", + agentRuntimeVersion: "0.148.0", + }); + }); + + it("binds Claude ACP to the SDK and native CLI runtime it launches", () => { + expect(QUALIFIED_ACPX_PROFILES.claude).toMatchObject({ + agentRuntimePackage: "@anthropic-ai/claude-agent-sdk", + agentRuntimeVersion: "0.3.232", + }); + }); }); diff --git a/packages/paperclip-runner/src/drivers/acpx/qualified-profiles.ts b/packages/paperclip-runner/src/drivers/acpx/qualified-profiles.ts index 403a334110..b953e19f5f 100644 --- a/packages/paperclip-runner/src/drivers/acpx/qualified-profiles.ts +++ b/packages/paperclip-runner/src/drivers/acpx/qualified-profiles.ts @@ -19,12 +19,13 @@ export interface QualifiedAcpxProfile { readonly commandDigest: string; readonly qualificationModel: string; /** - * Model identifier the pinned ACP server reports after accepting the exact - * qualification model. Most agents echo the requested model. Claude's ACP - * server deliberately exposes its stable SDK selector (`sonnet`) while the - * SDK resolves that selector to the canonical wire model - * (`claude-sonnet-5`). Paperclip verifies the exact request was accepted - * before treating this identifier as the qualified effective model. + * Model identifier the pinned ACP server accepts and reports. Profile + * resolution first binds the caller's exact canonical model request. Most + * agents use that same identifier at the ACP boundary; Claude exposes its + * stable SDK selector (`sonnet`) while the SDK resolves it to the canonical + * wire model (`claude-sonnet-5`). Paperclip selects only this profile-pinned + * identifier and verifies the provider reports it before publishing the + * canonical model as the qualified effective model. */ readonly reportedModelId: string; readonly permissionPolicy: "interactive"; @@ -62,8 +63,8 @@ export const QUALIFIED_ACPX_PROFILES: Readonly< agentProfileVersion: 1, agentServerPackage: "@agentclientprotocol/claude-agent-acp", agentServerVersion: "0.70.0", - agentRuntimePackage: null, - agentRuntimeVersion: null, + agentRuntimePackage: "@anthropic-ai/claude-agent-sdk", + agentRuntimeVersion: "0.3.232", commandDigest: "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", qualificationModel: "claude-sonnet-5", @@ -78,10 +79,10 @@ export const QUALIFIED_ACPX_PROFILES: Readonly< agentProfileVersion: 1, agentServerPackage: "@agentclientprotocol/codex-acp", agentServerVersion: "1.6.2", - agentRuntimePackage: null, - agentRuntimeVersion: null, + agentRuntimePackage: "@openai/codex", + agentRuntimeVersion: "0.148.0", commandDigest: - "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79", + "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400", qualificationModel: "gpt-5.6-sol", reportedModelId: "gpt-5.6-sol", permissionPolicy: "interactive", diff --git a/packages/paperclip-runner/src/drivers/acpx/recovery-identity.test.ts b/packages/paperclip-runner/src/drivers/acpx/recovery-identity.test.ts index e14b02a7cd..4e053e5f55 100644 --- a/packages/paperclip-runner/src/drivers/acpx/recovery-identity.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/recovery-identity.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { resolveQualifiedAcpxProfile } from "./qualified-profiles.js"; import { ACPX_IDENTITY_RECORD_SCHEMA, + acpxProviderSessionIdentity, createAcpxIdentityRecord, createAcpxRecoveryBinding, verifyExpectedAcpxIdentity, @@ -41,6 +42,19 @@ describe("ACPX recovery identity", () => { normalizedSessionId: "session-1", permissionMode: "approve-reads", }); + expect(acpxProviderSessionIdentity(record, fixture.binding)).toEqual({ + kind: "acpx", + normalizedSessionId: "session-1", + acpxRecordId: fixture.expected.acpxRecordId, + backendSessionId: fixture.expected.backendSessionId, + agentSessionId: fixture.expected.agentSessionId, + profileDigest: fixture.binding.commandDigest, + workspaceDigest: fixture.binding.workspaceDigest, + requestedModel: fixture.binding.requestedModel, + effectiveModel: fixture.binding.effectiveModel, + permissionMode: "approve-reads", + providerLifetimeFenceCandidates: [60_001, 60_002, 60_003], + }); expect(() => verifyExpectedAcpxIdentity(fixture.expected, fixture.binding, record), ).not.toThrow(); @@ -127,7 +141,7 @@ describe("ACPX recovery identity", () => { { ...fixture.expected, normalizedSessionId: otherBinding.normalizedSessionId, - profileDigest: otherBinding.profileDigest, + profileDigest: otherBinding.commandDigest, workspaceDigest: otherBinding.workspaceDigest, }, otherBinding, @@ -163,17 +177,6 @@ describe("ACPX recovery identity", () => { expect(() => verifyExpectedAcpxIdentity(fixture.expected, fixture.binding, earlyV1), ).toThrow(/persisted runtime record/); - expect(() => - verifyExpectedAcpxIdentity( - { - ...fixture.expected, - profileDigest: fixture.input.profile.commandDigest, - }, - fixture.binding, - earlyV1, - ), - ).toThrow(/immutable session configuration/); - const changedBinding = await createAcpxRecoveryBinding({ ...fixture.input, profile: { @@ -184,11 +187,12 @@ describe("ACPX recovery identity", () => { expect(changedBinding.profileDigest).not.toBe( fixture.binding.profileDigest, ); + expect(changedBinding.commandDigest).toBe(fixture.binding.commandDigest); expect(() => verifyExpectedAcpxIdentity( { ...fixture.expected, - profileDigest: changedBinding.profileDigest, + profileDigest: changedBinding.commandDigest, }, changedBinding, earlyV1, @@ -207,7 +211,7 @@ describe("ACPX recovery identity", () => { expect(() => verifyExpectedAcpxIdentity(fixture.expected, fixture.binding, { ...createAcpxIdentityRecord(fixture.expected, fixture.binding), - schema: "paperclip.runner.acpx-identity.v2", + schema: "paperclip.runner.acpx-identity.v1", }), ).toThrow(/Unsupported ACPX identity record schema/); const missingPermissionMode = createAcpxIdentityRecord( @@ -222,6 +226,24 @@ describe("ACPX recovery identity", () => { missingPermissionMode, ), ).toThrow(/permission mode is invalid/); + const missingFenceCandidates = createAcpxIdentityRecord( + fixture.expected, + fixture.binding, + ) as Partial>; + delete missingFenceCandidates.providerLifetimeFenceCandidates; + expect(() => + verifyExpectedAcpxIdentity( + fixture.expected, + fixture.binding, + missingFenceCandidates, + ), + ).toThrow(/lifetime fence candidates are invalid/); + expect(() => + verifyExpectedAcpxIdentity(fixture.expected, fixture.binding, { + ...createAcpxIdentityRecord(fixture.expected, fixture.binding), + providerLifetimeFenceCandidates: [60_001, 60_002, 60_004], + }), + ).toThrow(/does not match the persisted runtime record/); await expect( createAcpxRecoveryBinding({ @@ -262,11 +284,12 @@ async function recoveryFixture() { acpxRecordId: "record-1", backendSessionId: "backend-1", agentSessionId: "agent-1", - profileDigest: binding.profileDigest, + profileDigest: binding.commandDigest, workspaceDigest: binding.workspaceDigest, requestedModel: binding.requestedModel, effectiveModel: binding.effectiveModel, permissionMode: binding.permissionMode, + providerLifetimeFenceCandidates: [60_001, 60_002, 60_003] as const, }; return { root, workspace, input, binding, expected }; } diff --git a/packages/paperclip-runner/src/drivers/acpx/recovery-identity.ts b/packages/paperclip-runner/src/drivers/acpx/recovery-identity.ts index f10836f2a7..e256632d95 100644 --- a/packages/paperclip-runner/src/drivers/acpx/recovery-identity.ts +++ b/packages/paperclip-runner/src/drivers/acpx/recovery-identity.ts @@ -7,13 +7,14 @@ import type { AcpxExpectedSessionIdentity } from "./sidecar-protocol.js"; import type { QualifiedAcpxProfile } from "./qualified-profiles.js"; export const ACPX_IDENTITY_RECORD_SCHEMA = - "paperclip.runner.acpx-identity.v1" as const; + "paperclip.runner.acpx-identity.v2" as const; export interface AcpxRecoveryBinding { normalizedSessionId: string; workspacePath: string; workspaceDigest: string; runtimeRoot: string; + commandDigest: string; profileDigest: string; requestedModel: string; effectiveModel: string; @@ -32,6 +33,7 @@ export interface AcpxIdentityRecord { requestedModel: string; effectiveModel: string; permissionMode: NativeAcpxPermissionMode; + providerLifetimeFenceCandidates: readonly [number, number, number]; } export async function createAcpxRecoveryBinding(input: { @@ -87,6 +89,7 @@ export async function createAcpxRecoveryBinding(input: { workspacePath, workspaceDigest, runtimeRoot, + commandDigest: input.profile.commandDigest, profileDigest, requestedModel: input.requestedModel, effectiveModel: input.requestedModel, @@ -111,12 +114,40 @@ export function createAcpxIdentityRecord( requestedModel: binding.requestedModel, effectiveModel: binding.effectiveModel, permissionMode: binding.permissionMode, + providerLifetimeFenceCandidates: Object.freeze([ + ...expected.providerLifetimeFenceCandidates, + ]) as readonly [number, number, number], }; } +/** Project the private persisted record into the PRP sidecar wire identity. */ +export function acpxProviderSessionIdentity( + record: AcpxIdentityRecord, + binding: AcpxRecoveryBinding, +): AcpxExpectedSessionIdentity { + const identity: AcpxExpectedSessionIdentity = { + kind: "acpx", + normalizedSessionId: record.normalizedSessionId, + acpxRecordId: record.acpxRecordId, + backendSessionId: record.backendSessionId, + agentSessionId: record.agentSessionId, + // The PRP provider contract historically names this field + // `profileDigest`, but it attests the qualified executable digest. Keep + // the broader immutable-profile digest private in the persisted record. + profileDigest: binding.commandDigest, + workspaceDigest: record.workspaceDigest, + requestedModel: record.requestedModel, + effectiveModel: record.effectiveModel, + permissionMode: record.permissionMode, + providerLifetimeFenceCandidates: record.providerLifetimeFenceCandidates, + }; + verifyExpectedAcpxIdentity(identity, binding, record); + return identity; +} + /** * Verify both the controller-provided identity and a persisted runtime record. - * Only the complete v1 record is recoverable. Draft schema-less and + * Only the complete v2 record is recoverable. Draft schema-less and * command-digest records cannot prove every immutable session binding, so * callers must fail closed and start a fresh provider session for them. */ @@ -128,7 +159,7 @@ export function verifyExpectedAcpxIdentity( validateExpected(expected); if ( expected.normalizedSessionId !== binding.normalizedSessionId || - expected.profileDigest !== binding.profileDigest || + expected.profileDigest !== binding.commandDigest || expected.workspaceDigest !== binding.workspaceDigest || expected.requestedModel !== binding.requestedModel || expected.effectiveModel !== binding.effectiveModel || @@ -150,7 +181,11 @@ export function verifyExpectedAcpxIdentity( record.workspaceDigest !== binding.workspaceDigest || record.requestedModel !== binding.requestedModel || record.effectiveModel !== binding.effectiveModel || - record.permissionMode !== binding.permissionMode + record.permissionMode !== binding.permissionMode || + !sameFenceCandidates( + record.providerLifetimeFenceCandidates, + expected.providerLifetimeFenceCandidates, + ) ) { throw new Error( "ACPX recovery identity does not match the persisted runtime record", @@ -171,6 +206,7 @@ function parsePersistedRecord(value: unknown): AcpxIdentityRecord { "requestedModel", "effectiveModel", "permissionMode", + "providerLifetimeFenceCandidates", ]); return validatedRecord(record); } @@ -196,6 +232,7 @@ function validatedRecord(value: Record): AcpxIdentityRecord { if (!isPermissionMode(value.permissionMode)) { throw new Error("ACPX identity permission mode is invalid"); } + validateFenceCandidates(value.providerLifetimeFenceCandidates); return value as unknown as AcpxIdentityRecord; } @@ -223,6 +260,29 @@ function validateExpected(expected: AcpxExpectedSessionIdentity): void { ) { throw new Error("Expected ACPX permission mode is invalid"); } + validateFenceCandidates(expected.providerLifetimeFenceCandidates); +} + +function validateFenceCandidates( + value: unknown, +): asserts value is readonly [number, number, number] { + if ( + !Array.isArray(value) || + value.length !== 3 || + value.some( + (port) => !Number.isSafeInteger(port) || port < 49_152 || port > 65_535, + ) || + new Set(value).size !== 3 + ) { + throw new Error("ACPX provider lifetime fence candidates are invalid"); + } +} + +function sameFenceCandidates( + left: readonly [number, number, number], + right: readonly [number, number, number], +): boolean { + return left.every((port, index) => port === right[index]); } async function resolveWorkspace(value: string): Promise { diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts index b5e714b066..5fb1a709e1 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts @@ -395,11 +395,18 @@ describe("ACPX runtime host", () => { dependencies, ); expect(host.identity()).toMatchObject({ - schema: "paperclip.runner.acpx-identity.v1", + schema: "paperclip.runner.acpx-identity.v2", acpxRecordId: "record-1", requestedModel: "gpt-5.6-sol", permissionMode: "approve-reads", }); + const lifetimeFenceCandidates = + host.identity().providerLifetimeFenceCandidates; + expect(lifetimeFenceCandidates).toHaveLength(3); + expect(new Set(lifetimeFenceCandidates).size).toBe(3); + expect( + lifetimeFenceCandidates.every((port) => port >= 49_152 && port <= 65_535), + ).toBe(true); expect(capturedEnvironment.OPENAI_API_KEY).toBe("launch-secret"); expect(host.persistedEnvironment().OPENAI_API_KEY).toBeUndefined(); expect(host.persistedEnvironment().HTTPS_PROXY).toBeUndefined(); @@ -544,7 +551,7 @@ describe("ACPX runtime host", () => { const fixture = await hostFixture(); let selected = false; const setModel = vi.fn(async (model: string) => { - expect(model).toBe("claude-sonnet-5"); + expect(model).toBe("sonnet"); selected = true; }); const runtime = runtimePort({ @@ -596,6 +603,7 @@ describe("ACPX runtime host", () => { requestedModel: "claude-sonnet-5", effectiveModel: "claude-sonnet-5", permissionMode: "approve-reads", + providerLifetimeFenceCandidates: [60_001, 60_002, 60_003], }, }, fixture.dependencies({ openRuntime }), @@ -719,6 +727,8 @@ describe("ACPX runtime host", () => { code: "ENOENT", }); }); + // File removal precedes kernel lease release. Wait for the lease itself so + // this assertion cannot race between those two ordered cleanup steps. const contender = await waitForAcpxOperation(() => stageManagedCodexCredential({ agentHomeDirectory: credentialHome, @@ -1222,6 +1232,7 @@ describe("ACPX runtime host", () => { path: string; mode: "inline_json"; lifetimeFenceFds: readonly [number, number]; + lifetimeFenceCandidates: readonly [number, number, number]; activateLifetimeOwner(pid: number): Promise; close(): Promise; }>(); @@ -1267,6 +1278,7 @@ describe("ACPX runtime host", () => { path: lateCredentialPath, mode: "inline_json", lifetimeFenceFds: [42, 43], + lifetimeFenceCandidates: [60_001, 60_002, 60_003], activateLifetimeOwner: async () => undefined, close: lateCredentialClose, }); @@ -1411,6 +1423,7 @@ describe("ACPX runtime host", () => { path: join(fixture.root, "auth.json"), mode: "inline_json", lifetimeFenceFds: [42, 43], + lifetimeFenceCandidates: [60_001, 60_002, 60_003], activateLifetimeOwner: async () => undefined, close: credentialClose, }), diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts index 3c9b54c137..0836cbcf40 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts @@ -48,6 +48,8 @@ const RUNTIME_ADMISSION_VERIFICATION_TIMEOUT_MS = 8_000; const activeRuntimeHostCleanupOwners = new Set>(); class AcpxRuntimeAdmissionTimeoutError extends Error { + readonly code = "ACPX_RUNTIME_ADMISSION_VERIFICATION_TIMEOUT"; + constructor() { super("ACPX runtime admission verification exceeded its deadline"); this.name = "AcpxRuntimeAdmissionTimeoutError"; @@ -126,11 +128,7 @@ export type AcpxSemanticToolSession = Omit; export interface AcpxRetainedCleanupFailure { resource: - | "credential" - | "provider_lifetime" - | "command" - | "runtime" - | "tool_bridge"; + "credential" | "provider_lifetime" | "command" | "runtime" | "tool_bridge"; attempt: number; error: unknown; } @@ -462,11 +460,13 @@ export class AcpxRuntimeHost { kind: "acpx", normalizedSessionId: binding.normalizedSessionId, ...runtimeIdentity, - profileDigest: binding.profileDigest, + profileDigest: binding.commandDigest, workspaceDigest: binding.workspaceDigest, requestedModel: binding.requestedModel, effectiveModel: binding.effectiveModel, permissionMode: binding.permissionMode, + providerLifetimeFenceCandidates: + admittedLifetime.lifetimeFenceCandidates, }; const identity = createAcpxIdentityRecord(observedIdentity, binding); if (options.expectedIdentity) { diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts index 9b97e54ba2..cd3eb07e8d 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts @@ -83,6 +83,14 @@ describe("ACPX runtime sandbox", () => { sandbox.persistedEnvironment.PAPERCLIP_NATIVE_MCP_TOKEN, ).toBeUndefined(); expect(sandbox.persistedEnvironment.HOME).toBe(sandbox.homeDirectory); + if (agent === "codex") { + const config = await readFile( + join(sandbox.agentHomeDirectory, "config.toml"), + "utf8", + ); + expect(config).toBe("[features]\nshell_snapshot = false\n"); + expect(config).not.toContain("provider-secret"); + } expect(await readFile(sandbox.workspaceRecordPath, "utf8")).toBe( `${fixture.binding.workspacePath}\n`, ); diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts index 384f58544c..3b4069507e 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.ts @@ -376,6 +376,22 @@ export async function prepareAcpxRuntimeSandbox(input: { })}\n`, ); } + if (input.agent === "codex") { + await writePrivateFile( + join(agentHomeDirectory, "config.toml"), + [ + // Codex shell snapshots serialize the provider process environment. + // The ACPX sidecar receives a short-lived managed credential only so + // it can authenticate the provider; that value must never become + // durable runtime state. Keep this identical to the proven native + // Codex isolation policy: broader shell-environment filtering can + // also affect provider startup and belongs at the launch boundary. + "[features]", + "shell_snapshot = false", + "", + ].join("\n"), + ); + } const sanitizedSpawnInput = createSanitizedAcpxSpawnInput( input.environment, diff --git a/packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.ts b/packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.ts index 95c1f831de..72cc629690 100644 --- a/packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.ts +++ b/packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.ts @@ -64,6 +64,7 @@ export interface AcpxExpectedSessionIdentity { requestedModel: string; effectiveModel: string; permissionMode?: NativeAcpxPermissionMode; + providerLifetimeFenceCandidates: readonly [number, number, number]; } export function parseAcpxSidecarRequest(value: unknown): AcpxSidecarRequest { diff --git a/packages/paperclip-runner/src/drivers/acpx/verified-runtime-executable.test.ts b/packages/paperclip-runner/src/drivers/acpx/verified-runtime-executable.test.ts new file mode 100644 index 0000000000..2c8eea22ea --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/verified-runtime-executable.test.ts @@ -0,0 +1,178 @@ +import { spawnSync } from "node:child_process"; +import { closeSync, openSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +import { + VERIFIED_RUNTIME_EXECUTABLE_ENV, + verifiedRuntimeExecutable, + verifiedRuntimeExecutableHandoff, +} from "./verified-runtime-executable.js"; + +describe("verified runtime executable", () => { + it("preserves an inherited Linux descriptor for an explicit child handoff", () => { + expect( + verifiedRuntimeExecutable( + { [VERIFIED_RUNTIME_EXECUTABLE_ENV]: "/proc/self/fd/17" }, + "linux", + 4321, + "/usr/bin/node", + ), + ).toBe("/proc/self/fd/17"); + }); + + it("rejects a deleted live-process alias as a verified descendant runtime", () => { + expect(() => + verifiedRuntimeExecutable( + { [VERIFIED_RUNTIME_EXECUTABLE_ENV]: "/proc/self/exe" }, + "linux", + 8765, + "/usr/bin/node", + ), + ).toThrow("descriptor is invalid"); + }); + + it("rejects ancestor descriptor paths at the verified boundary", () => { + expect(() => + verifiedRuntimeExecutable( + { [VERIFIED_RUNTIME_EXECUTABLE_ENV]: "/proc/4321/fd/17" }, + "linux", + 8765, + "/usr/bin/node", + ), + ).toThrow("descriptor is invalid"); + }); + + it("rejects mutable Linux paths at the verified boundary", () => { + expect(() => + verifiedRuntimeExecutable( + { [VERIFIED_RUNTIME_EXECUTABLE_ENV]: "/usr/bin/node" }, + "linux", + 4321, + "/usr/bin/node", + ), + ).toThrow("descriptor is invalid"); + }); + + it("uses process identity only when no verified runtime was supplied", () => { + expect(verifiedRuntimeExecutable({}, "linux", 4321, "/usr/bin/node")).toBe( + "/usr/bin/node", + ); + }); + + it("remaps the authenticated Linux descriptor into the child stdio table", () => { + expect( + verifiedRuntimeExecutableHandoff( + 29, + { [VERIFIED_RUNTIME_EXECUTABLE_ENV]: "/proc/self/fd/17" }, + "linux", + 4321, + "/usr/bin/node", + ), + ).toEqual({ + executable: "/proc/self/fd/29", + environmentValue: "/proc/self/fd/29", + sourceFd: 17, + }); + }); + + it("does not invent a descriptor handoff for an ambient runtime", () => { + expect( + verifiedRuntimeExecutableHandoff(29, {}, "linux", 4321, "/usr/bin/node"), + ).toEqual({ + executable: "/usr/bin/node", + environmentValue: undefined, + sourceFd: null, + }); + }); + + it("rejects standard and invalid child descriptor targets", () => { + for (const targetFd of [-1, 0, 2, 3.5, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => + verifiedRuntimeExecutableHandoff( + targetFd, + { [VERIFIED_RUNTIME_EXECUTABLE_ENV]: "/proc/self/fd/17" }, + "linux", + 4321, + "/usr/bin/node", + ), + ).toThrow("target descriptor is invalid"); + } + }); + + it.runIf(process.platform === "linux")( + "keeps the authenticated runtime executable across two child generations", + () => { + const sourceFd = openSync(process.execPath, "r"); + try { + const targetFd = 10; + const handoff = verifiedRuntimeExecutableHandoff( + targetFd, + { + [VERIFIED_RUNTIME_EXECUTABLE_ENV]: `/proc/self/fd/${sourceFd}`, + }, + "linux", + ); + const stdio: Array<"ignore" | "pipe" | number> = [ + "ignore", + "pipe", + "pipe", + ]; + while (stdio.length < targetFd) stdio.push("ignore"); + stdio.push(handoff.sourceFd!); + const child = spawnSync( + handoff.executable, + [ + "--eval", + `const { spawnSync } = require("node:child_process"); +const fd = ${targetFd}; +const stdio = ["ignore", "pipe", "pipe"]; +while (stdio.length < fd) stdio.push("ignore"); +stdio.push(fd); +const nested = spawnSync("/proc/self/fd/" + fd, ["--eval", "process.stdout.write('nested-ok')"], { stdio }); +if (nested.status !== 0) throw nested.error || new Error(nested.stderr.toString()); +process.stdout.write(nested.stdout);`, + ], + { + env: { + [VERIFIED_RUNTIME_EXECUTABLE_ENV]: handoff.environmentValue!, + }, + stdio, + encoding: "utf8", + }, + ); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(0); + expect(child.stderr).toBe(""); + expect(child.stdout).toBe("nested-ok"); + } finally { + closeSync(sourceFd); + } + }, + ); + + it("accepts only the authenticated live process image on macOS", () => { + expect( + verifiedRuntimeExecutable( + { + [VERIFIED_RUNTIME_EXECUTABLE_ENV]: + "/private/tmp/.paperclip-verified-executable/launch", + }, + "darwin", + 4321, + "/private/tmp/.paperclip-verified-executable/launch", + ), + ).toBe("/private/tmp/.paperclip-verified-executable/launch"); + }); + + it("rejects a different environment-supplied executable on macOS", () => { + expect(() => + verifiedRuntimeExecutable( + { [VERIFIED_RUNTIME_EXECUTABLE_ENV]: "/tmp/attacker/node" }, + "darwin", + 4321, + "/private/tmp/.paperclip-verified-executable/launch", + ), + ).toThrow("path is invalid"); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/acpx/verified-runtime-executable.ts b/packages/paperclip-runner/src/drivers/acpx/verified-runtime-executable.ts new file mode 100644 index 0000000000..aff593281e --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/verified-runtime-executable.ts @@ -0,0 +1,94 @@ +import { isAbsolute, resolve } from "node:path"; + +export const VERIFIED_RUNTIME_EXECUTABLE_ENV = + "PAPERCLIP_VERIFIED_RUNTIME_EXECUTABLE"; + +export interface VerifiedRuntimeExecutableHandoff { + executable: string; + environmentValue: string | undefined; + sourceFd: number | null; +} + +/** + * Recover the runner-authenticated executable inherited by a descriptor-loaded + * sidecar. Linux descendants must explicitly inherit this descriptor: Node + * resolves process.execPath and /proc/self/exe to a deleted memfd alias that a + * later exec cannot reopen. + */ +export function verifiedRuntimeExecutable( + environment: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, + _currentPid: number = process.pid, + fallback: string = process.execPath, +): string { + const configured = environment[VERIFIED_RUNTIME_EXECUTABLE_ENV]; + if (configured === undefined) return fallback; + + if (platform === "linux") { + if (/^\/proc\/self\/fd\/[0-9]+$/.test(configured)) return configured; + throw new Error("Verified runtime executable descriptor is invalid"); + } + + if (platform === "darwin") { + if ( + !isAbsolute(fallback) || + resolve(fallback) !== fallback || + configured !== fallback + ) { + throw new Error("Verified runtime executable path is invalid"); + } + // The Rust supervisor materializes the authenticated runtime as a private, + // read-only executable and starts this process from that exact pathname. + // Descendants may inherit the handoff variable, but they cannot nominate a + // different absolute path and have it treated as verified. + return fallback; + } + + throw new Error( + "Verified runtime executable is unsupported on this platform", + ); +} + +/** + * Project the current verified runtime into a chosen child descriptor. The + * caller must place sourceFd at child targetFd in its stdio table. + */ +export function verifiedRuntimeExecutableHandoff( + targetFd: number, + environment: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, + currentPid: number = process.pid, + fallback: string = process.execPath, +): VerifiedRuntimeExecutableHandoff { + if (!Number.isSafeInteger(targetFd) || targetFd < 3) { + throw new Error("Verified runtime executable target descriptor is invalid"); + } + const executable = verifiedRuntimeExecutable( + environment, + platform, + currentPid, + fallback, + ); + const configured = environment[VERIFIED_RUNTIME_EXECUTABLE_ENV]; + if (platform !== "linux" || configured === undefined) { + return { + executable, + environmentValue: configured === undefined ? undefined : executable, + sourceFd: null, + }; + } + const match = /^\/proc\/self\/fd\/([0-9]+)$/.exec(configured); + if (match === null) { + throw new Error("Verified runtime executable descriptor is invalid"); + } + const sourceFd = Number.parseInt(match[1]!, 10); + if (!Number.isSafeInteger(sourceFd) || sourceFd < 3) { + throw new Error("Verified runtime executable descriptor is invalid"); + } + const childExecutable = `/proc/self/fd/${targetFd}`; + return { + executable: childExecutable, + environmentValue: childExecutable, + sourceFd, + }; +} diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 7087754906..eed1977f56 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -54,8 +54,10 @@ import type { } from "./codex-driver-types.js"; import { boundedText, + canonicalJson, codexSemanticToolSpecs, differingJsonPaths, + parseProviderIdentity, record, text, } from "./codex-driver-values.js"; @@ -199,6 +201,7 @@ export class CodexAppServerDriver implements HarnessDriver { const workingDirectory = validateWorkingDirectory( input.workingDirectory, this.#options.environment, + this.#options.workingDirectoryAuthority, ); const transport = this.#transport(); const cancellation = bootstrapCancellation(transport, input.signal); @@ -326,6 +329,7 @@ export class CodexAppServerDriver implements HarnessDriver { const workingDirectory = validateWorkingDirectory( text(existingThread.cwd), this.#options.environment, + this.#options.workingDirectoryAuthority, ); const response = await cancellation.wait(transport.request("thread/resume", { threadId: snapshot.driverSessionId, @@ -370,6 +374,17 @@ export class CodexAppServerDriver implements HarnessDriver { reason: "provider resumed a different provider session", }; } + if ( + snapshot.providerIdentity !== undefined && + canonicalJson(opened.providerIdentity) !== + canonicalJson(snapshot.providerIdentity) + ) { + await cancellation.wait(cancellation.close()); + return { + recovered: false, + reason: "provider resumed with a different tagged session identity", + }; + } const checkpointedActiveTurnId = snapshot.activeTurnId ?? null; // A terminal fingerprint is the durable provider fact. A crash can // persist it before the following active-turn clear reaches the same @@ -668,9 +683,11 @@ export class CodexAppServerDriver implements HarnessDriver { "Codex thread response changed the assigned working directory", ); } + const providerIdentity = parseProviderIdentity(thread.providerIdentity); return { threadId, providerSessionId, + ...(providerIdentity === undefined ? {} : { providerIdentity }), collaborationMode, context: { protocolVersion: CODEX_CODEX_PROTOCOL_VERSION, diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts index ee99046110..0f8dd8d2a7 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts @@ -44,6 +44,45 @@ import { } from "./codex-app-server-driver.test-support.js"; describe("Codex app-server Codex driver", () => { + it("persists and verifies the tagged runnerd provider identity on recovery", async () => { + const providerIdentity = { + kind: "acpx", + normalizedSessionId: "normalized-tagged-recovery", + acpxRecordId: "acpx-record-1", + backendSessionId: "backend-session-1", + agentSessionId: "agent-session-1", + profileDigest: `sha256:${"a".repeat(64)}`, + workspaceDigest: `sha256:${"b".repeat(64)}`, + requestedModel: "gpt-5.6-sol", + effectiveModel: "gpt-5.6-sol", + permissionMode: "approve-all", + providerLifetimeFenceCandidates: [60_001, 60_002, 60_003], + }; + const first = new FakeCodexTransport( + "thread-1", + "provider-session-1", + providerIdentity, + ); + const second = new FakeCodexTransport("thread-1", "provider-session-1", { + ...providerIdentity, + backendSessionId: "backend-session-2", + }); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-tagged-recovery", + normalizedSessionId: "normalized-tagged-recovery", + workingDirectory: WORKSPACE, + }); + const snapshot = await original.snapshot(); + expect(snapshot.providerIdentity).toEqual(providerIdentity); + await original.close({ reason: "transport lost" }); + + await expect(driver.recoverSession?.(snapshot)).resolves.toEqual({ + recovered: false, + reason: "provider resumed with a different tagged session identity", + }); + }); + it("resumes and reconciles the exact provider thread after transport loss", async () => { const first = new FakeCodexTransport(); const second = new FakeCodexTransport(); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts index a916308719..533c1bec4f 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts @@ -110,6 +110,7 @@ export class FakeCodexTransport implements CodexAppServerTransport { constructor( readonly threadId = "thread-1", readonly providerSessionId = "provider-session-1", + readonly providerIdentity?: Record, ) {} async request( @@ -148,6 +149,9 @@ export class FakeCodexTransport implements CodexAppServerTransport { thread: { id: this.threadId, sessionId: this.providerSessionId, + ...(this.providerIdentity === undefined + ? {} + : { providerIdentity: structuredClone(this.providerIdentity) }), modelProvider: "openai", cwd: WORKSPACE, turns: [], diff --git a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.test.ts index 052a70673d..8c7cbc881d 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.test.ts @@ -120,6 +120,47 @@ describe("Codex value and workspace boundaries", () => { } }); + it("defers provider-owned workspace existence without weakening its assignment", () => { + const remoteWorkspace = "/home/daytona/paperclip-workspace"; + const remoteEnvironment = { + HOME: remoteWorkspace, + CODEX_HOME: `${remoteWorkspace}/.codex`, + PAPERCLIP_WORKSPACE_CWD: remoteWorkspace, + }; + + expect( + validateCodexWorkingDirectory( + remoteWorkspace, + remoteEnvironment, + "remote_runner", + ), + ).toBe(remoteWorkspace); + expect(() => + validateCodexWorkingDirectory(remoteWorkspace, remoteEnvironment), + ).toThrow("must exist before provider admission"); + expect(() => + validateCodexWorkingDirectory( + `${remoteWorkspace}/nested`, + remoteEnvironment, + "remote_runner", + ), + ).toThrow("does not match the assigned workspace"); + expect(() => + validateCodexWorkingDirectory( + `${remoteWorkspace}/../escape`, + remoteEnvironment, + "remote_runner", + ), + ).toThrow("must be a normalized absolute path"); + expect(() => + validateCodexWorkingDirectory( + "/", + { PAPERCLIP_WORKSPACE_CWD: "/" }, + "remote_runner", + ), + ).toThrow("filesystem root"); + }); + it("bounds retained values and redacts protected diagnostics", () => { const bounded = boundedCodexPayload({ short: "ok", diff --git a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts index bb813f12b2..16be810799 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts @@ -4,6 +4,7 @@ import { dirname, isAbsolute, parse, + posix, relative, resolve, sep, @@ -28,6 +29,9 @@ const SENSITIVE_HOST_HOME_DIRECTORIES = [ ".ssh", ] as const; +export type CodexWorkingDirectoryAuthority = + "local_filesystem" | "remote_runner"; + function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -37,10 +41,14 @@ function record(value: unknown): Record { export function validateCodexWorkingDirectory( workingDirectory: string, environment: NodeJS.ProcessEnv = process.env, + authority: CodexWorkingDirectoryAuthority = "local_filesystem", ): string { if (workingDirectory.trim().length === 0) { throw new Error("Codex working directory is required"); } + if (authority === "remote_runner") { + return validateRemoteRunnerWorkingDirectory(workingDirectory, environment); + } const requested = resolve(workingDirectory); let resolved: string; try { @@ -109,6 +117,47 @@ export function validateCodexWorkingDirectory( return resolved; } +function validateRemoteRunnerWorkingDirectory( + workingDirectory: string, + environment: NodeJS.ProcessEnv, +): string { + if ( + !posix.isAbsolute(workingDirectory) || + posix.normalize(workingDirectory) !== workingDirectory || + /[\u0000-\u001f\u007f]/u.test(workingDirectory) + ) { + throw new Error( + "Remote Codex working directory must be a normalized absolute path", + ); + } + if (workingDirectory === posix.parse(workingDirectory).root) { + throw new Error("Codex working directory cannot be a filesystem root"); + } + const configuredRoot = environment.PAPERCLIP_WORKSPACE_CWD?.trim(); + if (!configuredRoot) { + throw new Error( + "Remote Codex working directory requires an assigned workspace", + ); + } + if ( + !posix.isAbsolute(configuredRoot) || + posix.normalize(configuredRoot) !== configuredRoot + ) { + throw new Error( + "Assigned remote workspace must be a normalized absolute path", + ); + } + // The controller cannot inspect a provider-owned filesystem. Pin the facade + // to the exact remote workspace while runnerd validates existence, type, and + // canonical identity inside the authoritative filesystem before launch. + if (workingDirectory !== configuredRoot) { + throw new Error( + "Remote Codex working directory does not match the assigned workspace", + ); + } + return workingDirectory; +} + function canonicalConfiguredPath(value: string | undefined): string | null { const configured = value?.trim(); if (!configured) return null; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts b/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts index f19b1ef4f0..e79103009c 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts @@ -2,6 +2,7 @@ import type { HarnessRuntimeRequest, HarnessRuntimeRequestResolution, HarnessThreadLineageEntry, + PersistedHarnessProviderIdentity, PersistedHarnessSession, } from "../../contracts/harness-driver.js"; import type { @@ -9,6 +10,7 @@ import type { CodexTaskEnvelope, } from "../../contracts/codex.js"; import type { CodexAppServerTransport } from "./app-server-transport.js"; +import type { CodexWorkingDirectoryAuthority } from "./codex-boundaries.js"; import type { CodexQuestionResponseContext } from "./codex-question-adapter.js"; export interface CodexAppServerDriverOptions { @@ -40,6 +42,8 @@ export interface CodexAppServerDriverOptions { arguments: unknown; }) => Promise; environment?: NodeJS.ProcessEnv; + /** Filesystem that authoritatively admits the workspace path. */ + workingDirectoryAuthority?: CodexWorkingDirectoryAuthority; now?: () => Date; runnerInstanceId?: string; onDiagnostic?: (message: string) => void; @@ -84,6 +88,7 @@ export interface TerminalReplayConflict { export interface OpenedCodexThread { threadId: string; providerSessionId: string | null; + providerIdentity?: PersistedHarnessProviderIdentity; collaborationMode: Record | null; context: CodexModelContextSnapshot; lineage: HarnessThreadLineageEntry; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts b/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts index 1aa59070ff..4064019c6c 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts @@ -1,3 +1,4 @@ +import type { PersistedHarnessProviderIdentity } from "../../contracts/harness-driver.js"; import type { NativeUserMessage } from "../../contracts/types.js"; import { CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, @@ -21,6 +22,75 @@ export function text(value: unknown, fallback = ""): string { return typeof value === "string" ? value : fallback; } +export function parseProviderIdentity( + value: unknown, +): PersistedHarnessProviderIdentity | undefined { + const identity = record(value); + if (identity.kind !== "acpx") return undefined; + const requiredStrings = [ + "normalizedSessionId", + "acpxRecordId", + "backendSessionId", + "agentSessionId", + "profileDigest", + "workspaceDigest", + "requestedModel", + "effectiveModel", + ] as const; + if ( + requiredStrings.some( + (key) => + typeof identity[key] !== "string" || + identity[key].length === 0 || + identity[key].length > 240, + ) + ) { + throw new Error("ACPX provider identity is incomplete"); + } + const permissionMode = identity.permissionMode; + if ( + permissionMode !== undefined && + permissionMode !== "approve-all" && + permissionMode !== "approve-reads" && + permissionMode !== "deny-all" + ) { + throw new Error( + "ACPX provider identity contains an invalid permission mode", + ); + } + const fenceCandidates = identity.providerLifetimeFenceCandidates; + if ( + !Array.isArray(fenceCandidates) || + fenceCandidates.length !== 3 || + fenceCandidates.some( + (candidate) => + !Number.isInteger(candidate) || + candidate < 49_152 || + candidate > 65_535, + ) || + new Set(fenceCandidates).size !== 3 + ) { + throw new Error("ACPX provider identity contains invalid lifetime fences"); + } + return { + kind: "acpx", + normalizedSessionId: identity.normalizedSessionId as string, + acpxRecordId: identity.acpxRecordId as string, + backendSessionId: identity.backendSessionId as string, + agentSessionId: identity.agentSessionId as string, + profileDigest: identity.profileDigest as string, + workspaceDigest: identity.workspaceDigest as string, + requestedModel: identity.requestedModel as string, + effectiveModel: identity.effectiveModel as string, + ...(permissionMode === undefined ? {} : { permissionMode }), + providerLifetimeFenceCandidates: fenceCandidates as [ + number, + number, + number, + ], + }; +} + export function boundedText( value: unknown, fallback = "unknown", diff --git a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts index 3092fd8d8a..eaf526ed19 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts @@ -634,6 +634,9 @@ export class CodexHarnessSession extends CodexSessionState implements HarnessSes driverKind: this.driverKind, driverSessionId: this.opened.threadId, providerSessionId: this.opened.providerSessionId, + ...(this.opened.providerIdentity === undefined + ? {} + : { providerIdentity: structuredClone(this.opened.providerIdentity) }), runId: this.runId, normalizedSessionId: this.normalizedSessionId, activeTurnId: this.activeTurnId, diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts index 7705595f44..1f62eb9232 100644 --- a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts +++ b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts @@ -219,6 +219,57 @@ describe("live workflow executor infrastructure failures", () => { } }); + it("keeps launch-only smoke exceptions explicit and rejects a wrong marker", async () => { + liveSessionMocks.snapshot.mockReturnValue({ + sessionId: "session-smoke-marker", + authority: {}, + mockState: JSON.stringify({ tasks: [] }), + transcript: [ + { + id: "assistant-smoke-marker", + role: "assistant", + text: "a different response", + }, + ], + evidence: [], + authorizationRecords: [], + attempts: [], + usageLedger: [], + stateHistory: [], + workspaceDiffs: [], + }); + const candidate = RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!; + const entry: RunnerLiveScheduleEntry = { + executionId: "local-smoke-marker", + caseId: "final-response", + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + + const observation = await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase: runnerWorkflowCase(entry.caseId), + allowMissingUsage: true, + expectedAssistantText: "PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK", + promptOverride: "Return the smoke marker.", + }); + + expect(liveSessionMocks.sendMessage).toHaveBeenCalledWith( + "Return the smoke marker.", + { allowMissingUsage: true }, + ); + expect(observation.presentation.checks).toContainEqual( + expect.objectContaining({ + id: "expected-assistant-text", + passed: false, + }), + ); + }); + it("fails the candidate budget and stops before a paid continuation", async () => { liveSessionMocks.snapshot.mockReturnValue({ sessionId: "session-budget-test", diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.ts index 9448d613d5..6f3b54fa95 100644 --- a/packages/paperclip-runner/src/eval/live-workflow-executor.ts +++ b/packages/paperclip-runner/src/eval/live-workflow-executor.ts @@ -479,6 +479,10 @@ export async function executeLiveRunnerWorkflow(input: { candidate: RunnerLiveEvalCandidate; evalCase: RunnerWorkflowEvalCase; workingDirectory?: string; + allowMissingUsage?: boolean; + expectedAssistantText?: string; + promptOverride?: string; + runnerBinary?: string; }): Promise { const runtimeRoot = await mkdtemp( join(tmpdir(), "paperclip-runner-live-eval-"), @@ -487,6 +491,9 @@ export async function executeLiveRunnerWorkflow(input: { const store = new InMemoryCapabilityLiveSessionStore(); const transportOptions = { environment: candidateTransportEnvironment(input.candidate, tracePath), + ...(input.runnerBinary === undefined + ? {} + : { runnerBinary: input.runnerBinary }), }; let service = new CapabilityLiveSessionService({ store, transportOptions }); let session: CapabilityLiveSession | null = null; @@ -558,7 +565,9 @@ export async function executeLiveRunnerWorkflow(input: { capabilities: capabilityFixtureRunCapabilities(LIVE_GRANTS), explicitClaims: [...LIVE_GRANTS], runId: input.entry.executionId, - sessionId: `session-${input.entry.executionId}`, + // Durable session identity is validation-bearing and must not look like + // credential material (for example a `session-...` token). + sessionId: `eval-${input.entry.executionId}`, attemptId: `attempt-${input.entry.executionId}`, turnTimeoutMs: input.candidate.budget.maxLatencyMs, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 }, @@ -573,11 +582,20 @@ export async function executeLiveRunnerWorkflow(input: { const settled = await Promise.allSettled([pending]); if (settled[0]?.status === "fulfilled") turns.push(settled[0].value); } else { - turns.push(await session.sendMessage(promptFor(input.evalCase))); + turns.push( + await session.sendMessage( + input.promptOverride ?? promptFor(input.evalCase), + { + allowMissingUsage: input.allowMissingUsage, + }, + ), + ); await settleInteractions(input.evalCase, session, turns, withinBudget); if (input.evalCase.id === "steering-causality" && withinBudget()) { turns.push( - await session.sendMessage(continuationPrompt(input.evalCase)), + await session.sendMessage(continuationPrompt(input.evalCase), { + allowMissingUsage: input.allowMissingUsage, + }), ); } if (input.evalCase.id === "restart-recovery" && withinBudget()) { @@ -590,7 +608,9 @@ export async function executeLiveRunnerWorkflow(input: { session = await service.restore(sessionId); subscribeToSession(session); turns.push( - await session.sendMessage(continuationPrompt(input.evalCase)), + await session.sendMessage(continuationPrompt(input.evalCase), { + allowMissingUsage: input.allowMissingUsage, + }), ); } } @@ -799,6 +819,16 @@ export async function executeLiveRunnerWorkflow(input: { assistantTexts.some((text) => text.trim().length >= 2), "provider emitted no user-facing response", ), + ...(input.expectedAssistantText === undefined + ? [] + : [ + check( + "expected-assistant-text", + assistantTexts.length === 1 && + assistantTexts[0]?.trim() === input.expectedAssistantText, + "provider response did not exactly match the smoke marker", + ), + ]), check( "no-empty-comment", assistantTexts.every((text) => text.trim().length > 0), diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index 9bb3c04829..dfd5cacefe 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -1448,7 +1448,11 @@ export class CapabilityLiveSession { return this.snapshot(); } - async sendMessage(message: string): Promise { + async sendMessage( + message: string, + /** Launch-only diagnostics may opt out; qualification campaigns must not. */ + options: { allowMissingUsage?: boolean } = {}, + ): Promise { const value = message.trim(); if (value.length === 0) throw new Error("Capability live messages cannot be empty"); if (this.#status === "suspended" || this.#transport === null) { @@ -1650,7 +1654,10 @@ export class CapabilityLiveSession { }, }); } - await this.#captureTurnUsage(result.turnId, result.status !== "completed"); + await this.#captureTurnUsage( + result.turnId, + result.status !== "completed" || options.allowMissingUsage === true, + ); await this.#persist(); await this.#afterTurnSettled(); return { ...result, snapshot: this.snapshot() }; diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index ca5bc841d9..dc7cf8a319 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -11,6 +11,7 @@ import { import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; import { expect, it } from "vitest"; @@ -33,6 +34,7 @@ import { import { releaseMaterializedNativeRuntimeSkills } from "../drivers/runtime-context-materializer.js"; import { + authorizedToolSetForProvider, createCapabilityRunnerdCodexTransport, createCapabilityRunnerdProviderEnvironment, defaultCapabilityRunnerdBinary, @@ -45,6 +47,7 @@ import { rehydrateRunnerdUsageNotification, rehydrateRunnerdWorkspaceChangeNotification, runnerdLaunchProfileInternals, + runnerdRecoveryInternals, resolveRunnerdAcpxPermissionMode, resolveRunnerdSessionIdentity, resolveSourceCodexHome, @@ -54,6 +57,100 @@ import { withCodexCollaborationRuntimeInstructions, } from "./runnerd-codex-transport.js"; +it("replays the durable run attachment outcome and latest provider identity", () => { + expect( + runnerdRecoveryInternals.recoveredRunAttachment({ + commands: [ + { commandId: "prepare", type: "run.prepare", status: "completed" }, + { commandId: "attach", type: "run.attach", status: "failed" }, + ], + committedEvents: [{ eventType: "session.started" }], + }), + ).toEqual({ + commandId: "attach", + status: "failed", + providerIdentityEventIndex: -1, + }); + + expect( + runnerdRecoveryInternals.recoveredRunAttachment({ + commands: [ + { commandId: "attach", type: "run.attach", status: "completed" }, + ], + committedEvents: [ + { eventType: "runner.reconciled" }, + { eventType: "session.started" }, + { eventType: "runner.diagnostic" }, + { eventType: "session.resumed" }, + ], + }), + ).toEqual({ + commandId: "attach", + status: "completed", + providerIdentityEventIndex: 3, + }); +}); + +it("identifies an active provider turn that must stop before suspension", () => { + expect( + runnerdRecoveryInternals.providerDrainStateFromSnapshot({ + activeProviderTurnId: "provider-turn-1", + pendingEvents: [{ eventType: "item.started" }], + queuedEvents: [{ eventType: "item.completed" }], + }), + ).toEqual({ + pendingEventCount: 2, + activeProviderTurnId: "provider-turn-1", + providerSettled: false, + }); + + expect( + runnerdRecoveryInternals.providerDrainStateFromSnapshot({ + activeTurnId: "acpx-turn-1", + pendingEvents: [], + }), + ).toEqual({ + pendingEventCount: 0, + activeProviderTurnId: "acpx-turn-1", + providerSettled: false, + }); + + expect( + runnerdRecoveryInternals.providerDrainStateFromSnapshot({ + activeProviderTurnId: null, + ambiguousTurnStartPending: false, + pendingEvents: [], + queuedEvents: [], + }), + ).toEqual({ + pendingEventCount: 0, + activeProviderTurnId: null, + providerSettled: true, + }); +}); + +it("keeps ACPX terminal tools under the reserved runner-owned catalog", () => { + const tools = [ + { + name: "get_task_context", + description: "Read the task context.", + inputSchema: { type: "object" }, + }, + ...codexSemanticToolSpecs(), + ]; + + expect(authorizedToolSetForProvider("acpx", tools)).toMatchObject({ + operations: [{ operationId: "get_task_context" }], + }); + expect(authorizedToolSetForProvider("codex", tools)).toMatchObject({ + operations: [ + { operationId: "get_task_context" }, + { operationId: "paperclip_block" }, + { operationId: "paperclip_finish" }, + ], + }); +}); + it("defaults runnerd ACPX permissions to approve reads", () => { expect(resolveRunnerdAcpxPermissionMode(undefined)).toBe("approve-reads"); expect(resolveRunnerdAcpxPermissionMode("deny-all")).toBe("deny-all"); @@ -85,7 +182,7 @@ it("rejects caller-selected local ACPX artifacts even when they are self-hashed" } }); -it.each(["acpx-runtime-sidecar.js", "opencode-app-server-proxy.js"] as const)( +it.each(["acpx-runtime-sidecar.cjs", "opencode-app-server-proxy.cjs"] as const)( "resolves the %s local provider artifact from verified build-owned output", async (artifact) => { const directory = await mkdtemp( @@ -117,6 +214,31 @@ it.each(["acpx-runtime-sidecar.js", "opencode-app-server-proxy.js"] as const)( }, ); +it("derives the ACPX package authority only from the verified dist/cli layout", () => { + const runnerPackageRoot = fileURLToPath(new URL("../..", import.meta.url)); + expect( + runnerdLaunchProfileInternals.acpxProviderPackageAuthority( + resolve(runnerPackageRoot, "dist/cli/acpx-runtime-sidecar.cjs"), + ), + ).toEqual({ + root: resolve(runnerPackageRoot, "../.."), + manifest: resolve(runnerPackageRoot, "package.json"), + }); + expect( + runnerdLaunchProfileInternals.acpxProviderPackageAuthority( + "/provider-pack/dist/cli/acpx-runtime-sidecar.cjs", + ), + ).toEqual({ + root: "/provider-pack", + manifest: "/provider-pack/package.json", + }); + expect(() => + runnerdLaunchProfileInternals.acpxProviderPackageAuthority( + "/unverified/acpx-runtime-sidecar.cjs", + ), + ).toThrow("ACPX sidecar must use the provider package dist/cli layout"); +}); + it("requires a provider-pack authority for remote ACPX artifact hashes", () => { expect(() => runnerdLaunchProfileInternals.acpxRunnerLaunchProfile( @@ -409,6 +531,9 @@ it.each([ environment: { PATH: "/bin", ...credentialEnvironment, + PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT: "/attacker/package-root", + PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST: + "/attacker/package-root/package.json", PAPERCLIP_API_KEY: "must-not-reach-provider", DATABASE_URL: "must-not-reach-provider", }, @@ -424,6 +549,8 @@ it.each([ codexHome: "/isolated/codex-home", runtimeContextPath: "/isolated/runtime-context.json", hasRuntimeContext: true, + acpxSidecarPath: + "/verified/provider-pack/dist/cli/acpx-runtime-sidecar.cjs", }); expect(environment).toMatchObject({ @@ -432,6 +559,9 @@ it.each([ PAPERCLIP_RUN_ID: "run-1", PAPERCLIP_NORMALIZED_SESSION_ID: "session-1", PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH: "/isolated/runtime-context.json", + PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT: "/verified/provider-pack", + PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST: + "/verified/provider-pack/package.json", }); for (const key of allowed) expect(environment[key]).toBe(credentialEnvironment[key]); @@ -1960,3 +2090,50 @@ it("rejects the notification stream promptly when runnerd exits after accepting await rm(stateDirectory, { recursive: true, force: true }); } }, 30_000); + +it("persists an active provider as settled before bounded suspension", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-active-suspension-"), + ); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--linger-after-turn-start"), + stateDirectory, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("initialize", {}); + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + await bundle.transport.request("turn/start", { + input: [{ type: "text", text: "Wait for another instruction." }], + }); + const notifications = bundle.transport + .notifications() + [Symbol.asyncIterator](); + await expect(notifications.next()).resolves.toMatchObject({ + value: { method: "turn/started" }, + }); + await bundle.transport.close(); + + const providerState = JSON.parse( + await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + "utf8", + ), + ) as Record; + expect(providerState).toMatchObject({ + lifecycle: "prepared", + activeProviderTurnId: null, + }); + } finally { + await bundle.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 8cc5ac63a0..29bd8f830d 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -60,7 +60,10 @@ import { releaseMaterializedNativeRuntimeSkills, } from "../drivers/runtime-context-materializer.js"; -const packageRoot = fileURLToPath(new URL("../..", import.meta.url)); +// URL directory conversion preserves a trailing separator while path-derived +// build artifacts do not. Normalize once so a source build cannot be +// misclassified as an external provider pack by a string-only comparison. +const packageRoot = resolve(fileURLToPath(new URL("../..", import.meta.url))); const executableSuffix = process.platform === "win32" ? ".exe" : ""; const MAX_NOTIFICATION_COUNT = 2_048; const MAX_NOTIFICATION_BYTES = 4 * 1024 * 1024; @@ -288,6 +291,66 @@ function rotatedRunAttachPayload( return payload; } +function recoveredRunAttachment(state: { + commands: readonly { + commandId: string; + type: string; + status: string; + }[]; + committedEvents: readonly { eventType: string }[]; +}): { + commandId: string; + status: string; + providerIdentityEventIndex: number; +} | null { + const command = [...state.commands] + .reverse() + .find((candidate) => candidate.type === "run.attach"); + if (!command) return null; + let providerIdentityEventIndex = -1; + if (command.status === "completed") { + for (let index = state.committedEvents.length - 1; index >= 0; index -= 1) { + const eventType = state.committedEvents[index]?.eventType; + if ( + eventType === "harness.ready" || + eventType === "session.started" || + eventType === "session.resumed" + ) { + providerIdentityEventIndex = index; + break; + } + } + } + return { + commandId: command.commandId, + status: command.status, + providerIdentityEventIndex, + }; +} + +function providerDrainStateFromSnapshot(state: Record): { + pendingEventCount: number; + activeProviderTurnId: string | null; + providerSettled: boolean; +} { + const pending = Array.isArray(state.pendingEvents) + ? state.pendingEvents.length + : 0; + const queued = Array.isArray(state.queuedEvents) + ? state.queuedEvents.length + : 0; + const activeProviderTurnId = + [state.activeProviderTurnId, state.activeTurnId].find( + (value): value is string => typeof value === "string" && value.length > 0, + ) ?? null; + return { + pendingEventCount: pending + queued, + activeProviderTurnId, + providerSettled: + activeProviderTurnId === null && state.ambiguousTurnStartPending !== true, + }; +} + function bridgedCodexQuestionParams( request: Record, method: string, @@ -1089,7 +1152,7 @@ function approvedRunnerArtifact(runnerBinaryPath: string): { } type BuildOwnedCliArtifact = - "acpx-runtime-sidecar.js" | "opencode-app-server-proxy.js"; + "acpx-runtime-sidecar.cjs" | "opencode-app-server-proxy.cjs"; function buildOwnedCliArtifactCandidates( artifact: BuildOwnedCliArtifact, @@ -1111,6 +1174,34 @@ function resolveBuildOwnedCliArtifact( ); } +function acpxProviderPackageAuthority(sidecarScript: string): { + root: string; + manifest: string; +} { + const cliDirectory = dirname(sidecarScript); + if ( + basename(sidecarScript) !== "acpx-runtime-sidecar.cjs" || + basename(cliDirectory) !== "cli" || + basename(dirname(cliDirectory)) !== "dist" + ) { + throw new Error( + "runner_provider_package_root_incompatible: ACPX sidecar must use the provider package dist/cli layout", + ); + } + const sidecarPackageRoot = resolve(cliDirectory, "../.."); + // A local source build consumes pnpm's workspace-owned node_modules tree. + // A deployed provider pack owns a closed node_modules tree at its own root. + return sidecarPackageRoot === packageRoot + ? { + root: resolve(packageRoot, "../.."), + manifest: resolve(packageRoot, "package.json"), + } + : { + root: sidecarPackageRoot, + manifest: resolve(sidecarPackageRoot, "package.json"), + }; +} + function acpxRunnerLaunchProfile( options: CapabilityRunnerdCodexTransportOptions, command: string, @@ -1127,7 +1218,7 @@ function acpxRunnerLaunchProfile( if (!options.runnerFilesystemRoot) { const buildCommand = process.execPath; const buildSidecarCandidates = buildOwnedCliArtifactCandidates( - "acpx-runtime-sidecar.js", + "acpx-runtime-sidecar.cjs", ); if ( options.providerNodeCommand !== undefined || @@ -1143,7 +1234,7 @@ function acpxRunnerLaunchProfile( ); } const buildSidecar = resolveBuildOwnedCliArtifact( - "acpx-runtime-sidecar.js", + "acpx-runtime-sidecar.cjs", buildSidecarCandidates, ); if (sidecarScript !== buildSidecar) { @@ -1252,6 +1343,24 @@ function authorizedToolSet( }; } +const ACPX_RESERVED_TERMINAL_TOOLS = new Set([ + "paperclip_finish", + "paperclip_block", +]); + +export function authorizedToolSetForProvider( + provider: CapabilityRunnerdCodexTransportOptions["provider"], + tools: readonly Readonly>[], +): Record { + return authorizedToolSet( + provider === "acpx" + ? tools.filter( + (tool) => !ACPX_RESERVED_TERMINAL_TOOLS.has(String(tool.name ?? "")), + ) + : tools, + ); +} + /** * Raw provider tracing is consumed by runnerd itself. The provider child still * receives the narrower allowlist enforced by Rust's `SupervisedProcess`, so @@ -1279,6 +1388,7 @@ export function createCapabilityRunnerdProviderEnvironment(input: { codexHome: string; runtimeContextPath: string; hasRuntimeContext: boolean; + acpxSidecarPath?: string; }): NodeJS.ProcessEnv { const commonIdentity = { PAPERCLIP_RUNNER_INSTANCE_ID: input.identity.runnerInstanceId, @@ -1300,12 +1410,23 @@ export function createCapabilityRunnerdProviderEnvironment(input: { }; } if (input.provider === "acpx") { + const sidecarPath = + input.acpxSidecarPath ?? + input.options.acpxSidecarPath ?? + resolve(packageRoot, "dist", "cli", "acpx-runtime-sidecar.cjs"); + const providerPackageAuthority = acpxProviderPackageAuthority(sidecarPath); return { ...createSanitizedAcpxSpawnInput( input.options.environment, input.options.acpxAgent ?? "codex", ).env, ...commonIdentity, + // The verified sidecar bundle cannot use import.meta.url while Node + // executes it through /proc/self/fd. Anchor its closed provider package + // lookups at the package that owns the already-authenticated bundle. + PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT: providerPackageAuthority.root, + PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST: + providerPackageAuthority.manifest, ...(input.options.providerRecoveryPolicy === "allow_replacement_after_governed_wait" ? { @@ -1467,6 +1588,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #providerIdentity: Record | null = null; #turnId = ""; #turnStartResponsePending = false; + #turnStartResponseEpoch = 0; #durableTurnId = ""; #authorizedTools: Record | null = null; #closed = false; @@ -1501,7 +1623,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { options.stateDirectory ?? mkdtempSync(resolve(tmpdir(), "paperclip-runner-lab-prp-")); if (options.resumeDynamicTools !== undefined) { - this.#authorizedTools = authorizedToolSet([ + this.#authorizedTools = authorizedToolSetForProvider(options.provider, [ ...options.resumeDynamicTools, ...codexSemanticToolSpecs(), ]); @@ -1796,6 +1918,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #providerDrainState(): | { pendingEventCount: number; + activeProviderTurnId: string | null; providerSettled: boolean; } | "unreadable" @@ -1812,31 +1935,67 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.options.runnerStateDirectory ?? resolve(this.#root, "runner"); const statePath = resolve(stateDirectory, filename); if (!existsSync(statePath)) { - return { pendingEventCount: 0, providerSettled: true }; + return { + pendingEventCount: 0, + activeProviderTurnId: null, + providerSettled: true, + }; } try { const state = record(JSON.parse(readFileSync(statePath, "utf8"))); - const pending = Array.isArray(state.pendingEvents) - ? state.pendingEvents.length - : 0; - const queued = Array.isArray(state.queuedEvents) - ? state.queuedEvents.length - : 0; - return { - pendingEventCount: pending + queued, - providerSettled: - !( - typeof state.activeProviderTurnId === "string" && - state.activeProviderTurnId.length > 0 - ) && state.ambiguousTurnStartPending !== true, - }; + return providerDrainStateFromSnapshot(state); } catch { return "unreadable"; } } - async #drainSettledProviderEventsBeforeSuspend(): Promise { + async #stopActiveProviderTurnBeforeSuspend(): Promise { + const state = this.#providerDrainState(); + const core = this.#core; + if ( + state === null || + state === "unreadable" || + state.activeProviderTurnId === null || + core === null + ) { + return false; + } + const commandId = `command_close_stop_${randomUUID().replaceAll("-", "")}`; + core.queueCommand( + "turn.stop", + { reason: "transport closing after durable run terminal" }, + commandId, + true, + ); const deadline = Date.now() + 1_000; + while (Date.now() < deadline) { + this.#pumpEventsSafely(); + const command = core.store.state.commands.find( + (candidate) => candidate.commandId === commandId, + ); + if (command?.status === "completed") { + this.#diagnostic( + `stopped active provider turn ${state.activeProviderTurnId} before runner suspension`, + ); + return true; + } + if (command !== undefined && command.status !== "pending") { + this.#diagnostic( + `provider turn stop ${command.status} before runner suspension`, + ); + return false; + } + if (this.#handle?.child.exitCode !== null) return false; + await new Promise((resolveWait) => setTimeout(resolveWait, 5)); + } + this.#diagnostic("provider turn stop timed out before runner suspension"); + return false; + } + + async #drainSettledProviderEventsBeforeSuspend( + timeoutMs = 1_000, + ): Promise { + const deadline = Date.now() + timeoutMs; let unreadable = false; let wakeSequence = 0; let crossedDrainBarrier = false; @@ -1898,7 +2057,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // its durable provider suffix is ACKed. Drain it before suspension so a // fresh run authority never inherits the prior run's pending events. if (this.#handle.child.exitCode === null) { - await this.#drainSettledProviderEventsBeforeSuspend(); + const stoppedActiveTurn = + await this.#stopActiveProviderTurnBeforeSuspend(); + await this.#drainSettledProviderEventsBeforeSuspend( + stoppedActiveTurn ? 5_000 : 1_000, + ); } const runnerAlreadyStopping = this.#handle.child.exitCode !== null || @@ -2057,16 +2220,16 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const opencodeProxyPath = this.options.opencodeProxyPath ?? (provider === "opencode" && !this.options.runnerFilesystemRoot - ? resolveBuildOwnedCliArtifact("opencode-app-server-proxy.js") + ? resolveBuildOwnedCliArtifact("opencode-app-server-proxy.cjs") : fileURLToPath( - new URL("../cli/opencode-app-server-proxy.js", import.meta.url), + new URL("../cli/opencode-app-server-proxy.cjs", import.meta.url), )); const acpxSidecarPath = this.options.acpxSidecarPath ?? (provider === "acpx" && !this.options.runnerFilesystemRoot - ? resolveBuildOwnedCliArtifact("acpx-runtime-sidecar.js") + ? resolveBuildOwnedCliArtifact("acpx-runtime-sidecar.cjs") : fileURLToPath( - new URL("../cli/acpx-runtime-sidecar.js", import.meta.url), + new URL("../cli/acpx-runtime-sidecar.cjs", import.meta.url), )); const providerNodeCommand = this.options.providerNodeCommand ?? process.execPath; @@ -2128,7 +2291,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); } } - this.#authorizedTools = authorizedToolSet(dynamicTools); + this.#authorizedTools = authorizedToolSetForProvider( + provider, + dynamicTools, + ); const acpxAgent = provider === "acpx" ? (this.options.acpxAgent ?? "codex") : null; const requestedModel = typeof params.model === "string" ? params.model : ""; @@ -2354,6 +2520,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { codexHome, runtimeContextPath, hasRuntimeContext: runtimeContext !== null, + acpxSidecarPath, }), this.options.environment, ), @@ -2543,16 +2710,16 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const opencodeProxyPath = this.options.opencodeProxyPath ?? (provider === "opencode" && !this.options.runnerFilesystemRoot - ? resolveBuildOwnedCliArtifact("opencode-app-server-proxy.js") + ? resolveBuildOwnedCliArtifact("opencode-app-server-proxy.cjs") : fileURLToPath( - new URL("../cli/opencode-app-server-proxy.js", import.meta.url), + new URL("../cli/opencode-app-server-proxy.cjs", import.meta.url), )); const acpxSidecarPath = this.options.acpxSidecarPath ?? (provider === "acpx" && !this.options.runnerFilesystemRoot - ? resolveBuildOwnedCliArtifact("acpx-runtime-sidecar.js") + ? resolveBuildOwnedCliArtifact("acpx-runtime-sidecar.cjs") : fileURLToPath( - new URL("../cli/acpx-runtime-sidecar.js", import.meta.url), + new URL("../cli/acpx-runtime-sidecar.cjs", import.meta.url), )); const providerNodeCommand = this.options.providerNodeCommand ?? process.execPath; @@ -2608,7 +2775,6 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { connectionLeaseTtlMs: 60 * 60 * 1_000, }); this.#core = core; - this.#eventIndex = core.store.state.committedEvents.length; if (rotatedAuthority) { core.queueCommand( "run.attach", @@ -2620,6 +2786,18 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ), ); } + const committedEvents = core.store.state.committedEvents; + const runAttachment = recoveredRunAttachment(core.store.state); + // A controller retry can open the exact authority after run.attach has + // already reached a durable outcome. Re-observe that command instead of + // silently waiting for an identity that a failed command can never emit. + // If attachment completed, replay only its latest identity event into the + // transport's in-memory evidence; session events are consumed internally + // and are not duplicated onto the provider notification stream. + this.#eventIndex = + runAttachment !== null && runAttachment.providerIdentityEventIndex >= 0 + ? runAttachment.providerIdentityEventIndex + : committedEvents.length; const registration = this.options.controlPlaneRegistration ? await this.options.controlPlaneRegistration(core) : null; @@ -2657,6 +2835,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { codexHome, runtimeContextPath, hasRuntimeContext: runtimeContext !== null, + acpxSidecarPath, }), this.options.environment, ), @@ -2677,7 +2856,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#evidence.runnerProcessGroupId = null; this.#publish(); this.#pump = setInterval(() => this.#pumpEventsSafely(), 5); - if (rotatedAuthority) await this.#waitCommand("run.attach"); + if (runAttachment) { + await this.#waitCommand("run.attach", runAttachment.commandId); + } await this.#waitForProviderIdentity(); this.#startupComplete = true; this.#diagnostic( @@ -2696,7 +2877,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { .join("\n"); const pendingTurnId = `turn_lab_${randomUUID().replaceAll("-", "")}`; this.#turnId = pendingTurnId; + const responseEpoch = ++this.#turnStartResponseEpoch; this.#turnStartResponsePending = true; + let responseReady = false; try { await this.#command("turn.start", { text: message }); // Command completion only means runnerd accepted the command. Codex assigns @@ -2713,9 +2896,24 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } if (this.#turnId === pendingTurnId) throw new Error("runnerd did not report the provider turn identity"); + responseReady = true; return { turn: { id: this.#turnId, status: "inProgress" } }; } finally { - this.#turnStartResponsePending = false; + if (!responseReady) { + if (this.#turnStartResponseEpoch === responseEpoch) + this.#turnStartResponsePending = false; + } else { + // Resolving this async method schedules the strict driver's response + // continuation as a microtask. Keep terminal frames held until the + // following task so the driver can bind and emit turn.accepted first. + // The epoch prevents a late release from clearing a newer turn fence. + const release = setTimeout(() => { + if (this.#turnStartResponseEpoch !== responseEpoch) return; + this.#turnStartResponsePending = false; + if (!this.#closed) this.#pumpEventsSafely(); + }, 0); + release.unref(); + } } } @@ -2815,7 +3013,22 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#flushPendingTraceRehydrations(); const events = this.#core?.store.state.committedEvents ?? []; while (this.#eventIndex < events.length) { - const event = events[this.#eventIndex++]; + const event = events[this.#eventIndex]!; + const eventPayload = record(event.envelope.payload).payload; + const terminalWhileTurnStartPending = + this.#turnStartResponsePending && + ([ + "turn.completed", + "turn.failed", + "turn.interrupted", + "turn.cancelled", + ].includes(event.eventType) || + (event.eventType === "provider.event" && + unwrapRunnerdProviderNotifications(eventPayload).some( + (notification) => notification.method === "turn/completed", + ))); + if (terminalWhileTurnStartPending) return; + this.#eventIndex += 1; if ( event.eventType === "harness.ready" || event.eventType === "session.started" || @@ -2943,7 +3156,6 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#bridgedRuntimeInputs.delete(requestId); continue; } - const eventPayload = record(event.envelope.payload).payload; const sessionUpdatePayload = record(eventPayload); const canonicalMethod = ( { @@ -3361,6 +3573,12 @@ export const createRunnerdCodexTransport = createCapabilityRunnerdCodexTransport; export const runnerdLaunchProfileInternals = Object.freeze({ + acpxProviderPackageAuthority, acpxRunnerLaunchProfile, resolveBuildOwnedCliArtifact, }); + +export const runnerdRecoveryInternals = Object.freeze({ + providerDrainStateFromSnapshot, + recoveredRunAttachment, +}); diff --git a/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs b/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs index 3b504d5741..abc5978a35 100644 --- a/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs +++ b/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs @@ -30,8 +30,33 @@ const claudePatch = await readFile( ), "utf8", ); +const qualifiedProfiles = await readFile( + new URL("../src/drivers/acpx/qualified-profiles.ts", import.meta.url), + "utf8", +); +const runnerdAcpxBackend = await readFile( + new URL( + "../runner/crates/runner-core/src/acpx_provider_backend.rs", + import.meta.url, + ), + "utf8", +); +const providerPackBuilder = await readFile( + new URL("../scripts/build-provider-pack.mjs", import.meta.url), + "utf8", +); +const nativeSessionExecutor = await readFile( + new URL( + "../../../server/src/services/native-runtime/native-session-executor.ts", + import.meta.url, + ), + "utf8", +); test("the runner pins every qualified ACPX production dependency", () => { + assert.equal(runnerPackage.dependencies["@openai/codex"], undefined); + assert.equal(runnerPackage.optionalDependencies, undefined); + assert.equal(runnerPackage.dependencies.node, undefined); assert.equal(runnerPackage.dependencies.acpx, "0.13.1"); assert.equal( runnerPackage.dependencies["@agentclientprotocol/codex-acp"], @@ -43,11 +68,31 @@ test("the runner pins every qualified ACPX production dependency", () => { ); }); +test("the patched Codex ACP command digest stays aligned across launch boundaries", () => { + const profileMatch = + /agent: "codex"[\s\S]*?commandDigest:\s*"(sha256:[a-f0-9]{64})"/.exec( + qualifiedProfiles, + ); + assert.ok(profileMatch, "qualified Codex ACPX profile digest"); + const digest = profileMatch[1]; + + assert.match(runnerdAcpxBackend, new RegExp(`"codex"[\\s\\S]*?${digest}`)); + assert.match( + providerPackBuilder, + new RegExp(`acpxProfileDigests:[\\s\\S]*?codex:[\\s\\S]*?${digest}`), + ); + assert.match( + nativeSessionExecutor, + new RegExp( + `REMOTE_PROVIDER_PACK_PROFILE_DIGESTS[\\s\\S]*?codex:[\\s\\S]*?${digest}`, + ), + ); +}); + test("the package exposes only the reviewed runner CLI binaries", () => { assert.deepEqual(runnerPackage.bin, { "paperclip-runner-eval-session": "./dist/cli/eval-session.js", - "paperclip-runner-codex-proxy": - "./dist/cli/codex-app-server-unix-proxy.js", + "paperclip-runner-codex-proxy": "./dist/cli/codex-app-server-unix-proxy.js", "paperclip-runner-acpx-sidecar": "./dist/cli/acpx-runtime-sidecar.js", "paperclip-runner-opencode-proxy": "./dist/cli/opencode-app-server-proxy.js", @@ -74,12 +119,19 @@ test("old and new pnpm configuration both apply the exact runtime patches", () = assert.match(workspace, /acpx@0\.13\.1: patches\/acpx@0\.13\.1\.patch/); assert.match( workspace, - /codex-acp@1\.6\.2': patches\/@agentclientprotocol__codex-acp@1\.6\.2\.patch/, + /codex-acp@1\.6\.2["']: patches\/@agentclientprotocol__codex-acp@1\.6\.2\.patch/, ); assert.match( workspace, - /claude-agent-acp@0\.70\.0': patches\/@agentclientprotocol__claude-agent-acp@0\.70\.0\.patch/, + /claude-agent-acp@0\.70\.0["']: patches\/@agentclientprotocol__claude-agent-acp@0\.70\.0\.patch/, ); + assert.equal(rootPackage.pnpm.patchedDependencies["node@24.11.0"], undefined); + assert.doesNotMatch(workspace, /node@24\.11\.0:/); + assert.match( + providerPackBuilder, + /copyFileSync\(process\.execPath, stableNodeCommand\)/, + ); + assert.match(codexPatch, /\+ "@openai\/codex": "0\.148\.0"/); }); test("the ACPX patch preserves launch-only state and verified spawning", () => { @@ -130,6 +182,13 @@ test("the Codex patch enforces isolated instructions, tools, and skills", () => } }); +test("the Codex patch keeps MCP tool approvals on the governed permission channel", () => { + assert.match( + codexPatch, + /!context\.isToolApproval && this\.shouldUseAcpElicitation\(params\)/, + ); +}); + test("the Claude patch removes ambient project and local configuration", () => { for (const token of [ "PAPERCLIP_ACPX_ISOLATED_CONTEXT", diff --git a/packages/shared/src/environment-support.test.ts b/packages/shared/src/environment-support.test.ts index 94926016ae..0b462267ba 100644 --- a/packages/shared/src/environment-support.test.ts +++ b/packages/shared/src/environment-support.test.ts @@ -7,6 +7,15 @@ import { } from "./environment-support.js"; describe("isSandboxProviderSupportedForAdapter", () => { + it("treats Paperclip Runner as a remote-managed adapter", () => { + expect(adapterSupportsRemoteManagedEnvironments("paperclip_runner")).toBe(true); + expect(supportedEnvironmentDriversForAdapter("paperclip_runner")).toEqual([ + "local", + "ssh", + "sandbox", + ]); + }); + it("accepts additional sandbox providers for remote-managed adapters", () => { expect( isSandboxProviderSupportedForAdapter("codex_local", "fake-plugin", ["fake-plugin"]), diff --git a/packages/shared/src/environment-support.ts b/packages/shared/src/environment-support.ts index 5bb5a076ea..ab10a20e17 100644 --- a/packages/shared/src/environment-support.ts +++ b/packages/shared/src/environment-support.ts @@ -67,6 +67,7 @@ export interface EnvironmentCapabilities { const REMOTE_MANAGED_ADAPTERS = new Set([ "claude_local", "codex_local", + "paperclip_runner", "cursor", "gemini_local", "grok_local", diff --git a/patches/@agentclientprotocol__codex-acp@1.6.2.patch b/patches/@agentclientprotocol__codex-acp@1.6.2.patch index f6a8c34bfe..efe631113b 100644 --- a/patches/@agentclientprotocol__codex-acp@1.6.2.patch +++ b/patches/@agentclientprotocol__codex-acp@1.6.2.patch @@ -1,6 +1,27 @@ +diff --git a/package.json b/package.json +--- a/package.json ++++ b/package.json +@@ -65,7 +65,7 @@ + }, + "dependencies": { + "@agentclientprotocol/sdk": "^1.3.0", +- "@openai/codex": "^0.148.0", ++ "@openai/codex": "0.148.0", + "diff": "^9.0.0", + "open": "^11.0.0", + "vscode-jsonrpc": "^9.0.1", diff --git a/dist/index.js b/dist/index.js --- a/dist/index.js +++ b/dist/index.js +@@ -25341,7 +25341,7 @@ + async handleElicitation(params) { + try { + const context = this.createMcpElicitationContext(params); +- if (this.shouldUseAcpElicitation(params)) { ++ if (!context.isToolApproval && this.shouldUseAcpElicitation(params)) { + const response2 = await this.connection.request( + methods.client.elicitation.create, + this.buildElicitationRequest(params, context), @@ -25563,7 +25563,7 @@ toolCall: { toolCallId: context.correlatedCallId, diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1a903feeef..9723d1c65f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -19,6 +19,6 @@ patchedDependencies: embedded-postgres@18.1.0-beta.16: patches/embedded-postgres@18.1.0-beta.16.patch acpx@0.12.0: patches/acpx@0.12.0.patch acpx@0.13.1: patches/acpx@0.13.1.patch - '@agentclientprotocol/claude-agent-acp@0.70.0': patches/@agentclientprotocol__claude-agent-acp@0.70.0.patch - '@agentclientprotocol/claude-agent-acp@0.73.0': patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch - '@agentclientprotocol/codex-acp@1.6.2': patches/@agentclientprotocol__codex-acp@1.6.2.patch + "@agentclientprotocol/claude-agent-acp@0.70.0": patches/@agentclientprotocol__claude-agent-acp@0.70.0.patch + "@agentclientprotocol/claude-agent-acp@0.73.0": patches/@agentclientprotocol__claude-agent-acp@0.73.0.patch + "@agentclientprotocol/codex-acp@1.6.2": patches/@agentclientprotocol__codex-acp@1.6.2.patch diff --git a/server/src/__tests__/heartbeat-direct-adapter-native-isolation.test.ts b/server/src/__tests__/heartbeat-direct-adapter-native-isolation.test.ts index 1c80057966..db5dcb7ea0 100644 --- a/server/src/__tests__/heartbeat-direct-adapter-native-isolation.test.ts +++ b/server/src/__tests__/heartbeat-direct-adapter-native-isolation.test.ts @@ -1,5 +1,13 @@ import { randomUUID } from "node:crypto"; -import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from "vitest"; import { sql } from "drizzle-orm"; import { agents, @@ -25,7 +33,9 @@ import { import { heartbeatService } from "../services/heartbeat.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); -const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; +const describeEmbeddedPostgres = embeddedPostgresSupport.supported + ? describe + : describe.skip; const DIRECT_ADAPTERS = [ ["codex_local", "codex"], ["claude_local", "claude"], @@ -55,11 +65,15 @@ async function waitForRunToFinish( describeEmbeddedPostgres("direct adapter native-runner isolation", () => { let db!: ReturnType; let heartbeat!: ReturnType; - let tempDb: Awaited> | null = null; + let tempDb: Awaited< + ReturnType + > | null = null; const execute = vi.fn(); beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("heartbeat-direct-adapter-isolation-"); + tempDb = await startEmbeddedPostgresTestDatabase( + "heartbeat-direct-adapter-isolation-", + ); db = createDb(tempDb.connectionString); heartbeat = heartbeatService(db); for (const [adapterType] of DIRECT_ADAPTERS) { @@ -79,13 +93,16 @@ describeEmbeddedPostgres("direct adapter native-runner isolation", () => { afterEach(async () => { await drainHeartbeatRunsToQuiescence(db, heartbeat); - const runStatuses = await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns); + const runStatuses = await db + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns); const pendingRuns = runStatuses.filter( (run) => run.status === "queued" || run.status === "running", ); expect(pendingRuns).toEqual([]); vi.clearAllMocks(); - await db.execute(sql.raw(` + await db.execute( + sql.raw(` TRUNCATE TABLE "native_run_finalizations", "status_decisions", @@ -103,11 +120,12 @@ describeEmbeddedPostgres("direct adapter native-runner isolation", () => { "agents", "companies" RESTART IDENTITY CASCADE - `)); + `), + ); }); afterAll(async () => { - await heartbeat.drainActiveRunExecutions(); + await drainHeartbeatRunsToQuiescence(db, heartbeat); for (const [adapterType] of DIRECT_ADAPTERS) { unregisterServerAdapter(adapterType); } @@ -119,7 +137,8 @@ describeEmbeddedPostgres("direct adapter native-runner isolation", () => { async (adapterType, provider) => { const companyId = randomUUID(); const agentId = randomUUID(); - const directProofJson = '{"schema":"direct-proof.v1","value":"byte-stable"}'; + const directProofJson = + '{"schema":"direct-proof.v1","value":"byte-stable"}'; execute.mockResolvedValue({ exitCode: 0, signal: null, @@ -161,7 +180,10 @@ describeEmbeddedPostgres("direct adapter native-runner isolation", () => { runtimeMode: "legacy", nativePhase: null, }); - const persistedResult = finished?.resultJson as Record | null; + const persistedResult = finished?.resultJson as Record< + string, + unknown + > | null; expect(persistedResult?.directProofJson).toBe(directProofJson); const nativeRows = await Promise.all([ diff --git a/server/src/__tests__/redaction.test.ts b/server/src/__tests__/redaction.test.ts index b42699be7c..32b24fe263 100644 --- a/server/src/__tests__/redaction.test.ts +++ b/server/src/__tests__/redaction.test.ts @@ -199,6 +199,7 @@ describe("redaction", () => { "environment.workspace.realize", "native.coordinator.claim", "runner.transport.selected", + "runner.prp.authenticate", "runner.prp.route.register", "runner.transport.connect", "runner.session.bootstrap", diff --git a/server/src/redaction.ts b/server/src/redaction.ts index e9a29c5531..8b8871ac23 100644 --- a/server/src/redaction.ts +++ b/server/src/redaction.ts @@ -318,6 +318,7 @@ const NATIVE_RUN_SPAN_NAMES = new Set([ "provider.turn.queue", "runner.artifact.discover", "runner.artifact.prepare", + "runner.prp.authenticate", "runner.prp.route.register", "runner.runtime.stage", "runner.session.bootstrap", diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index eba23196cd..e272de38bb 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -31,6 +31,7 @@ import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; type BackendFactoryOptions = { runnerInstanceId?: string; acpxRuntimeDirectory?: string; + workingDirectoryAuthority?: "local_filesystem" | "remote_runner"; codexTransportFactory?: () => unknown; dynamicToolHandler?: (call: unknown) => Promise; onSpawn?: (meta: { @@ -250,11 +251,11 @@ describe("remote provider pack manifest", () => { const opencodeCommand = "#!/bin/sh\n"; const opencodeExecutable = "opencode-binary\n"; await writeFile( - join(root, "dist", "cli", "opencode-app-server-proxy.js"), + join(root, "dist", "cli", "opencode-app-server-proxy.cjs"), proxy, ); await writeFile( - join(root, "dist", "cli", "acpx-runtime-sidecar.js"), + join(root, "dist", "cli", "acpx-runtime-sidecar.cjs"), sidecar, ); await writeFile(join(root, "node_modules", "node", "bin", "node"), node); @@ -288,7 +289,7 @@ describe("remote provider pack manifest", () => { claude: "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", codex: - "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79", + "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400", }, artifacts: { nodeCommand: { @@ -305,11 +306,11 @@ describe("remote provider pack manifest", () => { sha256: digest(opencodeExecutable), }, opencodeProxy: { - path: "dist/cli/opencode-app-server-proxy.js", + path: "dist/cli/opencode-app-server-proxy.cjs", sha256: proxySha, }, acpxSidecar: { - path: "dist/cli/acpx-runtime-sidecar.js", + path: "dist/cli/acpx-runtime-sidecar.cjs", sha256: sidecarSha, }, }, @@ -352,14 +353,14 @@ describe("remote provider pack manifest", () => { } await writeManifest(); await writeFile( - join(root, "dist", "cli", "opencode-app-server-proxy.js"), + join(root, "dist", "cli", "opencode-app-server-proxy.cjs"), "tampered\n", ); expect(() => readRemoteProviderPackManifest(root)).toThrow( "OpenCode proxy digest mismatch", ); await writeFile( - join(root, "dist", "cli", "opencode-app-server-proxy.js"), + join(root, "dist", "cli", "opencode-app-server-proxy.cjs"), proxy, ); await writeFile( @@ -3072,6 +3073,7 @@ describe("runnerd provider runtime wiring", () => { executionWorkspaceId: "run-projectless-next", }, } as NativeExecutionInputV1; + const remoteCwd = "/home/daytona/paperclip-workspace"; try { state.createBackend.mockClear(); state.createTransport.mockClear(); @@ -3134,7 +3136,37 @@ describe("runnerd provider runtime wiring", () => { execution: continuation, runnerInstanceId: "runner-new-heartbeat", useRunnerd: true, + runnerExecutionTarget: { + kind: "remote", + transport: "ssh", + remoteCwd, + spec: { + host: "runner.internal", + port: 22, + username: "runner", + remoteWorkspacePath: remoteCwd, + remoteCwd, + privateKey: null, + knownHosts: null, + strictHostKeyChecking: true, + }, + }, }); + expect(state.createBackend).toHaveBeenCalledWith( + expect.objectContaining({ + workspace: expect.objectContaining({ cwd: remoteCwd }), + }), + expect.objectContaining({ + workingDirectoryAuthority: "remote_runner", + }), + ); + expect(state.execute).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + workspace: expect.objectContaining({ cwd: remoteCwd }), + }), + }), + ); const backendOptions = state.createBackend.mock.calls[0]![1]; backendOptions.codexTransportFactory!(); expect(state.createTransport).toHaveBeenCalledWith( @@ -4601,7 +4633,9 @@ describe("runnerd provider runtime wiring", () => { expect.objectContaining({ workspace: expect.objectContaining({ cwd: remoteCwd }), }), - expect.any(Object), + expect.objectContaining({ + workingDirectoryAuthority: "remote_runner", + }), ); expect(state.execute).toHaveBeenCalledWith( expect.objectContaining({ @@ -4615,11 +4649,53 @@ describe("runnerd provider runtime wiring", () => { backendOptions.codexTransportFactory!(); expect(state.createTransport).toHaveBeenCalledWith( expect.objectContaining({ + runnerBinary: "/tmp/paperclip-runnerd", environment: expect.objectContaining({ PAPERCLIP_WORKSPACE_CWD: remoteCwd, }), }), ); + expect(state.createTransport.mock.calls[0]![0].runnerBinary).not.toBe( + `${remoteCwd}/.paperclip-runtime/paperclip-runner/bin/paperclip-runnerd`, + ); + }); + + it("binds a remote launch to the configured controller-owned runner artifact", async () => { + const remoteCwd = "/home/daytona/paperclip-workspace"; + const controllerArtifact = "/controller/artifacts/paperclip-runnerd"; + const remoteExecution = { + ...execution, + binding: { ...execution.binding, runId: "run-remote-runner-artifact" }, + workspace: { ...execution.workspace, cwd: "/host/paperclip-workspace" }, + } as NativeExecutionInputV1; + + await createRunnerdBackend({ + db: leaseDb(remoteExecution), + execution: remoteExecution, + runnerInstanceId: "runner", + runnerExecutionTarget: { + kind: "remote", + transport: "ssh", + remoteCwd, + spec: { + host: "runner.internal", + port: 22, + username: "runner", + remoteWorkspacePath: remoteCwd, + remoteCwd, + privateKey: null, + knownHosts: null, + strictHostKeyChecking: true, + }, + }, + runnerRemoteBinaryPath: controllerArtifact, + }); + + state.createTransport.mockClear(); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + expect(state.createTransport).toHaveBeenCalledWith( + expect.objectContaining({ runnerBinary: controllerArtifact }), + ); }); it.each([ diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 242d415ff0..8007bd57f6 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -3865,7 +3865,10 @@ async function executePaperclipNativeSessionWithinScope( input.useRunnerd && input.backend === undefined ? await createRunnerdBackend({ ...input, - execution: runnerExecution, + // Durable scope and prior-run verification use the controller's + // canonical workspace identity. createRunnerdBackend separately + // projects remoteCwd into the provider execution boundary. + execution: input.execution, runnerInstanceId: effectiveRunnerInstanceId, durableEnvironmentLeaseId: durableRunnerBinding?.environmentLeaseId, trace, @@ -4441,15 +4444,15 @@ const REMOTE_PROVIDER_PACK_PROFILE_DIGESTS = { claude: "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", codex: - "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79", + "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400", } as const; const REMOTE_PROVIDER_PACK_ARTIFACT_PATHS = { nodeCommand: "node_modules/node/bin/node", productionLock: "pnpm-lock.yaml", opencodeCommand: "node_modules/.bin/opencode", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe", - opencodeProxy: "dist/cli/opencode-app-server-proxy.js", - acpxSidecar: "dist/cli/acpx-runtime-sidecar.js", + opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", + acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs", } as const; type RemoteProviderPackManifest = { @@ -5320,6 +5323,14 @@ async function createRunnerdBackendWithinSessionClaim( const remoteBinary = remoteRuntimeRoot ? posix.join(remoteRuntimeRoot, "bin", "paperclip-runnerd") : null; + // The transport hashes runnerBinary on the controller before an external + // launcher starts runnerd. Keep that artifact identity in the controller's + // filesystem; the remote launcher separately owns the sandbox command path. + // When an explicit remote artifact is configured, prepareRemoteRunner stages + // these exact bytes at remoteBinary before launch. + const controllerRunnerBinary = remoteTarget + ? input.runnerRemoteBinaryPath?.trim() || resolvePaperclipRunnerBinary() + : resolvePaperclipRunnerBinary(); const explicitRemoteCodex = input.runnerRemoteCodexPath?.trim() || null; const remoteCodexNpmSpec = input.runnerRemoteCodexNpmSpec?.trim() || null; if (explicitRemoteCodex && remoteCodexNpmSpec) { @@ -6549,6 +6560,9 @@ async function createRunnerdBackendWithinSessionClaim( const backend = createNativeSessionBackend(runnerExecution, { runnerInstanceId: input.runnerInstanceId, environment: effectiveRunnerEnvironment, + workingDirectoryAuthority: remoteTarget + ? "remote_runner" + : "local_filesystem", onSpawn: input.onSpawn, dynamicTools, dynamicToolHandler: (call) => authorityEpoch.execute(call), @@ -6672,7 +6686,7 @@ async function createRunnerdBackendWithinSessionClaim( stateDirectory: remoteStateDirectory, }) : undefined, - runnerBinary: remoteBinary ?? resolvePaperclipRunnerBinary(), + runnerBinary: controllerRunnerBinary, codexCommand: remoteCodexBinary ?? undefined, sourceCodexHome: remoteTarget ? resolveSourceCodexHome(input.runnerEnvironment ?? process.env) diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index be091a341a..ad62738f60 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -82,12 +82,16 @@ workflows: 42 cells. Its cases are: `openrouter-model-breadth` (**OpenRouter Model Breadth**) is four qualified models from the tracked weekly tool-capable ranking snapshot × native OpenCode -× local, with 11 supported model/workflow cells. Xiaomi MiMo V2.5 remains +× local, with 10 supported model/workflow cells. Xiaomi MiMo V2.5 remains recorded in the immutable ranking snapshot but is excluded from paid qualification because its latency repeatedly exhausts the cell deadline. DeepSeek V4 Flash remains qualified for hello and question/resume, but its Plan cell is excluded after three successful semantic completions consistently -ignored the required exact final response. Its cases are: +ignored the required exact final response. Tencent HY3 likewise remains +qualified for hello and question/resume, but its Plan cell is excluded after +two fresh attempts completed every durable Plan and finalization operation yet +consistently replaced the required exact visible terminal marker with prose. +Its cases are: - `hello-complete`: a basic nonce response and explicit Done transition; - `question-resume-complete`: one structured question, browser selection of @@ -103,9 +107,10 @@ duplicating the final response. The second workflow restarts the isolated Paperclip server while the interaction is waiting, reloads that state, and then resumes it. The suite has no Daytona cells. -The complete catalog is 67 cells and 116 expected paid agent turns. Follow-up -steps remain ordered within their cell; all other cells are independent. -Narrow selectors are strongly recommended while developing fixtures. +The complete catalog is 66 cells (45 local and 21 Daytona) and 114 expected +paid agent turns. Follow-up steps remain ordered within their cell; all other +cells are independent. Narrow selectors are strongly recommended while +developing fixtures. `--suite`, `--group`, `--profile`, `--environment`, and `--case` are repeatable. Repeated values in one dimension use OR semantics; dimensions and repeated groups use @@ -330,7 +335,7 @@ Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped ephemeral AWS RunsOn fleet selected by `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an -integer from 1–100 on AWS (default 100); use at least 67 to run the current +integer from 1–100 on AWS (default 100); use at least 66 to run the current complete catalog in one wave. The fallback runner retains its 1–57 limit and default of 32. Multi-turn steps are sequential inside their cell while independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 1b812b225d..518e9f1a47 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -88,7 +88,7 @@ the actor gate, environment branch restriction, and protected default branch. When `RUNNER_E2E_AWS_ENABLED=true`, paid matrix cells use the exact RunsOn fleet selector `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`, matching the AWS fleet selected by `pr-trusted.yml` only after its stable numeric-ID trust gate. -Any other or missing toggle value falls back to the standard GitHub-hosted +Any other or missing toggle value falls back to the GitHub-hosted `ubuntu-latest` runner and its lower concurrency ceiling. The workflow chooses between those two reviewed literal labels; it never evaluates a configured runner label. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 05498dedf6..7c4e5620d7 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -29,10 +29,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 42, 14, 11, + 42, 14, 10, ]); - expect(validateRunnerCatalog()).toHaveLength(67); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(67); + expect(validateRunnerCatalog()).toHaveLength(66); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(66); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); @@ -45,26 +45,36 @@ describe("runner E2E catalog", () => { runnerMatrix.filter( (entry) => entry.suite.id === "openrouter-model-breadth", ), - ).toHaveLength(11); + ).toHaveLength(10); expect( runnerMatrix.reduce( (total, execution) => total + execution.task.expectedRunCount, 0, ), - ).toBe(116); + ).toBe(114); }); it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => { expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]); expect(openRouterBreadthExcludedExecutionIds).toEqual([ "openrouter-model-breadth.openrouter-deepseek-deepseek-v4-flash-0731.local.plan-approve-complete", + "openrouter-model-breadth.openrouter-tencent-hy3.local.plan-approve-complete", ]); expect( - runnerMatrix.some( - (execution) => - execution.id === openRouterBreadthExcludedExecutionIds[0], + openRouterBreadthExcludedExecutionIds.every( + (excludedExecutionId) => + !runnerMatrix.some( + (execution) => execution.id === excludedExecutionId, + ), ), - ).toBe(false); + ).toBe(true); + expect( + runnerMatrix + .filter( + (execution) => execution.profile.id === "openrouter-tencent-hy3", + ) + .map((execution) => execution.task.id), + ).toEqual(["hello-complete", "question-resume-complete"]); expect( openRouterBreadthProfiles.map((profile) => profile.ranking?.rank), ).toEqual([1, 3, 4, 5]); @@ -105,11 +115,43 @@ describe("runner E2E catalog", () => { expectedRunCount: 2, }); expect(localQuestion?.buildPrompt("nonce")).toContain("ask_user_questions"); + expect(localQuestion?.buildPrompt("nonce")).toContain( + "do not spell, quote, repeat, announce, or include PAPERCLIP_E2E_QUESTION_DONE_nonce", + ); + expect(localQuestion?.buildPrompt("nonce")).toContain( + "refer to it only as “the terminal marker.”", + ); + expect(localQuestion?.buildPrompt("nonce")).toContain( + 'API_ORIGIN="${PAPERCLIP_API_URL%/}"; API_ORIGIN="${API_ORIGIN%/api}"', + ); + expect(localQuestion?.buildPrompt("nonce")).toContain( + '"idempotencyKey":"question-nonce"', + ); + expect(localQuestion?.buildPrompt("nonce")).toContain( + 'PATCH $API_ORIGIN/api/issues/$PAPERCLIP_TASK_ID with exactly {"status":"in_review"}', + ); + expect(localQuestion?.buildPrompt("nonce")).toContain( + "Do not include `reviewInteractionId`", + ); + expect(localQuestion?.buildPrompt("nonce")).toContain( + "retry only that PATCH and never POST the interaction again", + ); + expect(localQuestion?.buildPrompt("nonce")).toContain( + "make exactly one completion write", + ); + const legacyQuestionExitInstruction = + "In a legacy runner, after those two writes succeed, end the current response and heartbeat immediately. Do not wait, sleep, poll, or fetch the interaction; `wake_assignee` will start a new heartbeat after the user answers."; + expect(localQuestion?.buildPrompt("nonce")).toContain( + legacyQuestionExitInstruction, + ); expect(restartQuestion).toMatchObject({ flow: "question_resume_completion", expectedRunCount: 2, restartServerBeforeQuestionAnswer: true, }); + expect(restartQuestion?.buildPrompt("nonce")).toContain( + legacyQuestionExitInstruction, + ); expect(plan).toMatchObject({ flow: "plan_approval_completion", expectedRunCount: 2, @@ -117,6 +159,55 @@ describe("runner E2E catalog", () => { expect(plan?.buildPrompt("nonce")).toContain("exactly two numbered steps"); }); + it("emits native terminal text after the terminal tool succeeds", () => { + const message = runnerTasks.find((task) => task.id === "message-marker"); + const ask = runnerTasks.find((task) => task.id === "ask-question"); + const plan = runnerTasks.find((task) => task.id === "plan-revise-accept"); + const question = localIntegrityTasks.find( + (task) => task.id === "structured-question-resume", + ); + const breadthTasks = openRouterBreadthTasks.map((task) => + task.buildPrompt("nonce"), + ); + + for (const prompt of [ + message?.buildPrompt("nonce"), + ask?.buildPrompt("nonce"), + plan?.buildPrompt("nonce"), + question?.buildPrompt("nonce"), + ...breadthTasks, + ]) { + expect(prompt).toContain("then emit exactly"); + expect(prompt!.indexOf("paperclip_finish exactly once")).toBeLessThan( + prompt!.indexOf("then emit exactly"), + ); + expect(prompt).toContain("Wait for that tool call to succeed"); + } + + for (const taskId of [ + "question-resume-complete", + "plan-approve-complete", + ]) { + const prompt = openRouterBreadthTasks + .find((task) => task.id === taskId) + ?.buildPrompt("nonce"); + expect(prompt).toContain( + "do not spell, quote, repeat, announce, or include", + ); + expect(prompt).toContain("refer to it only as “the terminal marker.”"); + } + + const breadthHello = openRouterBreadthTasks + .find((task) => task.id === "hello-complete") + ?.buildPrompt("nonce"); + expect(breadthHello).toContain( + "Your first response action must be the paperclip_finish tool call", + ); + expect(breadthHello).toContain( + "Do not emit any assistant text, acknowledgement, or preamble before calling it", + ); + }); + it("uses only declared secret references in generated payloads", () => { expect( runnerMatrix.every((entry) => @@ -215,6 +306,15 @@ describe("runner E2E catalog", () => { }); expect(task!.buildPrompt("nonce")).toContain("request_confirmation"); expect(task!.buildPrompt("nonce")).toContain("baseRevisionId"); + expect(task!.buildPrompt("nonce")).toContain( + "do not spell, quote, repeat, announce, or include PAPERCLIP_E2E_PLAN_DONE_nonce", + ); + expect(task!.buildPrompt("nonce")).toContain( + 'summary:"PAPERCLIP_E2E_PLAN_DONE_nonce"', + ); + expect(task!.buildPrompt("nonce")).toContain( + "one atomic issue PATCH with status `done` and that exact comment", + ); expect(task!.buildRevisionRequest?.("nonce")).toContain("baseRevisionId"); }); @@ -269,7 +369,7 @@ describe("runner E2E selectors", () => { const selected = selectRunnerExecutions( parseRunnerSelectors(["--suite", "openrouter-model-breadth"]), ); - expect(selected).toHaveLength(11); + expect(selected).toHaveLength(10); expect( selected.every( (entry) => @@ -306,9 +406,10 @@ describe("runner E2E selectors", () => { const jobs = buildMatrixJobs( selectRunnerExecutions(parseRunnerSelectors(["--all"])), ); - expect(jobs).toHaveLength(67); + expect(jobs).toHaveLength(66); expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21); - expect(new Set(jobs.map((job) => job.executionId)).size).toBe(67); + expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(45); + expect(new Set(jobs.map((job) => job.executionId)).size).toBe(66); expect( jobs.every((job) => runnerMatrix.some( diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index f1feae7f6a..28c34e99c8 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -265,6 +265,7 @@ export const runnerProfiles: readonly RunnerProfileFixture[] = [ export const openRouterBreadthExcludedModelIds = ["xiaomi/mimo-v2.5"] as const; export const openRouterBreadthExcludedExecutionIds = [ "openrouter-model-breadth.openrouter-deepseek-deepseek-v4-flash-0731.local.plan-approve-complete", + "openrouter-model-breadth.openrouter-tencent-hy3.local.plan-approve-complete", ] as const; const openRouterBreadthExcludedModelIdSet = new Set( openRouterBreadthExcludedModelIds, @@ -393,7 +394,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ "Complete this task in a single run.", `The exact marker also appears unescaped in the task title: PAPERCLIP_E2E_OK_${nonce}`, `Your final visible task-thread response must be exactly this marker: PAPERCLIP_E2E_OK_${nonce}`, - `In a native runner, first call paperclip_finish with that marker as its summary and an objective-satisfied claim for the supplied contract revision. After it succeeds, emit that exact marker once as the complete user-facing final response and do not call another tool.`, + `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_OK_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly PAPERCLIP_E2E_OK_${nonce} once as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, `In a legacy runner, make exactly one public-API write containing the marker: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"done","comment":"PAPERCLIP_E2E_OK_${nonce}"}. Do not POST to /comments, and do not include the marker in any other write.`, "The visible task-thread response is asserted; hidden reasoning or provider terminal output alone does not count.", "Use underscore characters exactly as shown and do not insert backslashes.", @@ -456,11 +457,12 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ "First, create a small canonical Plan with exactly two numbered steps and request approval; do not implement it.", `The initial Plan body must contain PAPERCLIP_E2E_PLAN_DRAFT_${nonce}.`, "If the plan is sent back, revise that same Plan document according to the revision note, publish a new revision-bound confirmation, and still do not implement.", + `Before the revised Plan is accepted, do not spell, quote, repeat, announce, or include PAPERCLIP_E2E_PLAN_DONE_${nonce} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, "Only after the revised plan is accepted, implement it by posting one final visible task-thread response containing exactly " + `PAPERCLIP_E2E_PLAN_DONE_${nonce}` + " and mark the task Done.", - "For a native runner, remain in the requested planning collaboration mode. Call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`. Do not call paperclip_finish while waiting for either Plan confirmation.", - "For a legacy runner, use the public Paperclip API. The first PUT of the `plan` issue document creates it. For every later PUT, first GET the current document and set `baseRevisionId` to its `latestRevisionId`; a 409 means you must GET again and retry with the new latest revision. Create a `request_confirmation` targeting the successful PUT response's `latestRevisionId` with `continuationPolicy: wake_assignee`, and move the issue to `in_review` while waiting.", + `For a native runner, remain in the requested planning collaboration mode. Call write_document for key \`plan\`, then call request_human_input exactly once with interactionKind \`confirmation\`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy \`wake_assignee\`. Do not call paperclip_finish while waiting for either Plan confirmation. After the revised Plan is accepted, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_PLAN_DONE_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly PAPERCLIP_E2E_PLAN_DONE_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, + `For a legacy runner, use the public Paperclip API. The first PUT of the \`plan\` issue document creates it. For every later PUT, first GET the current document and set \`baseRevisionId\` to its \`latestRevisionId\`; a 409 means you must GET again and retry with the new latest revision. Create a \`request_confirmation\` targeting the successful PUT response's \`latestRevisionId\` with \`continuationPolicy: wake_assignee\`, and move the issue to \`in_review\` while waiting. After the revised Plan is accepted, write PAPERCLIP_E2E_PLAN_DONE_${nonce} exactly once through one atomic issue PATCH with status \`done\` and that exact comment; do not POST a separate comment or perform a second write.`, "Do not create files, child tasks, or unrelated work, and do not expose credentials.", ].join("\n"), buildMatchers(nonce, execution) { @@ -510,7 +512,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ `Your final visible task-thread answer must be exactly this complete marker, including its final suffix: E2E_ASK_12_${nonce}.`, "Do not create or modify files, do not create a plan or additional work, and do not expose credentials.", "After posting the direct answer, mark this task Done.", - `In a native runner, first call paperclip_finish with E2E_ASK_12_${nonce} as the summary. After it succeeds, emit exactly E2E_ASK_12_${nonce} as the complete final response and do not call another tool.`, + `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"E2E_ASK_12_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly E2E_ASK_12_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, `In a legacy runner, post exactly E2E_ASK_12_${nonce} as the task comment body and mark the task Done through the public API.`, ].join("\n"), buildMatchers(nonce, execution) { @@ -584,11 +586,15 @@ const structuredQuestionResumeTask = { [ "Ask the user one structured question before completing this task.", "The question must be required, single-select, and offer Cobalt and Amber. Do not publish a final answer or mark the task Done while it is pending.", + `Before the answer arrives, do not spell, quote, repeat, announce, or include PAPERCLIP_E2E_QUESTION_DONE_${nonce} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, `In a native runner, call request_human_input exactly once with idempotencyKey \`question-${nonce}\`, interactionKind \`questions\`, title \`Verification word\`, prompt \`Choose the verification word\`, continuationPolicy \`wake_assignee\`, and payload {version:1,questions:[{id:\`verification-word\`,prompt:\`Choose the verification word.\`,selectionMode:\`single\`,required:true,options:[{id:\`cobalt\`,label:\`Cobalt\`},{id:\`amber\`,label:\`Amber\`}]}]}.`, - "In a legacy runner, POST /api/issues/$PAPERCLIP_TASK_ID/interactions with {kind:`ask_user_questions`,continuationPolicy:`wake_assignee`,payload:{version:1,questions:[{id:`verification-word`,prompt:`Choose the verification word.`,selectionMode:`single`,required:true,options:[{id:`cobalt`,label:`Cobalt`},{id:`amber`,label:`Amber`}]}]}} using Authorization and X-Paperclip-Run-Id, then move the issue to `in_review`.", + 'In a legacy runner, derive `API_ORIGIN` exactly once with `API_ORIGIN="${PAPERCLIP_API_URL%/}"; API_ORIGIN="${API_ORIGIN%/api}"`. Build every endpoint as `$API_ORIGIN/api/...`; never append `/api` to a base that already ends in `/api`.', + `In a legacy runner, create exactly one question interaction: POST $API_ORIGIN/api/issues/$PAPERCLIP_TASK_ID/interactions once with {"kind":"ask_user_questions","idempotencyKey":"question-${nonce}","continuationPolicy":"wake_assignee","payload":{"version":1,"questions":[{"id":"verification-word","prompt":"Choose the verification word.","selectionMode":"single","required":true,"options":[{"id":"cobalt","label":"Cobalt"},{"id":"amber","label":"Amber"}]}]}} using Authorization and X-Paperclip-Run-Id. Do not create a replacement interaction if a later write fails.`, + 'In a legacy runner, after that POST returns 2xx, PATCH $API_ORIGIN/api/issues/$PAPERCLIP_TASK_ID with exactly {"status":"in_review"}. Do not include `reviewInteractionId`: it only designates confirmation interactions, not `ask_user_questions`. If the PATCH fails, retry only that PATCH and never POST the interaction again.', + "In a legacy runner, after those two writes succeed, end the current response and heartbeat immediately. Do not wait, sleep, poll, or fetch the interaction; `wake_assignee` will start a new heartbeat after the user answers.", `After the answer arrives, if it is Cobalt, publish exactly PAPERCLIP_E2E_QUESTION_DONE_${nonce} once as the complete visible response and mark the task Done.`, - `In a native runner, first call paperclip_finish with PAPERCLIP_E2E_QUESTION_DONE_${nonce} as its summary, then emit that exact final response and do not call another tool.`, - `In a legacy runner, post exactly PAPERCLIP_E2E_QUESTION_DONE_${nonce} as the task comment body and mark the task Done through the public API.`, + `In a native runner, after the answer arrives, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_QUESTION_DONE_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly PAPERCLIP_E2E_QUESTION_DONE_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, + `In a legacy runner, make exactly one completion write: PATCH $API_ORIGIN/api/issues/$PAPERCLIP_TASK_ID with {"status":"done","comment":"PAPERCLIP_E2E_QUESTION_DONE_${nonce}"}. Do not POST a separate comment or perform a second write containing the marker.`, "Do not create files, plans, child tasks, or unrelated work, and do not expose credentials.", ].join("\n"), buildMatchers(nonce, execution) { @@ -645,8 +651,8 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ buildPrompt: (nonce) => [ "Complete this deterministic hello task in one turn.", - `Return ${breadthMarker("H", nonce)} as the complete visible response.`, - `Call paperclip_finish with ${breadthMarker("H", nonce)} as its summary and mark the task Done.`, + "Your first response action must be the paperclip_finish tool call. Do not emit any assistant text, acknowledgement, or preamble before calling it.", + `Call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("H", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly ${breadthMarker("H", nonce)} as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, "Do not create files, plans, interactions, or additional work.", ].join("\n"), buildMatchers: (nonce, execution) => @@ -671,8 +677,9 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ [ "Ask the user one structured question before completing this task.", `Call request_human_input exactly once with idempotencyKey \`question-${nonce}\`, interactionKind \`questions\`, title \`Verification word\`, prompt \`Choose the verification word\`, continuationPolicy \`wake_assignee\`, and payload {version:1,questions:[{id:\`verification-word\`,prompt:\`Choose the verification word.\`,selectionMode:\`single\`,required:true,options:[{id:\`cobalt\`,label:\`Cobalt\`},{id:\`amber\`,label:\`Amber\`}]}]}.`, + `Before the answer arrives, do not spell, quote, repeat, announce, or include ${breadthMarker("Q_C", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, "Do not call paperclip_finish while the question is pending.", - `After the answer arrives, if it is Cobalt, return ${breadthMarker("Q_C", nonce)} visibly and call paperclip_finish with that marker as the summary.`, + `After the answer arrives, if it is Cobalt, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("Q_C", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly ${breadthMarker("Q_C", nonce)} as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, "Do not create files, plans, or additional work.", ].join("\n"), buildMatchers: (nonce, execution) => @@ -698,8 +705,9 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ "Create a canonical Plan with exactly two numbered steps and request approval; do not implement before approval.", `The Plan body must contain ${breadthMarker("P_READY", nonce)}.`, "Call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`.", + `Before that exact Plan revision is accepted, do not spell, quote, repeat, announce, or include ${breadthMarker("P_OK", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, "Do not call paperclip_finish while confirmation is pending.", - `After that exact Plan revision is accepted, return ${breadthMarker("P_OK", nonce)} visibly and call paperclip_finish with that marker as the summary.`, + `After that exact Plan revision is accepted, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("P_OK", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly ${breadthMarker("P_OK", nonce)} as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, "Do not create files, child tasks, or unrelated work.", ].join("\n"), buildMatchers: (nonce, execution) => @@ -744,7 +752,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [localEnvironment], tasks: openRouterBreadthTasks, excludedExecutionIds: openRouterBreadthExcludedExecutionIds, - expectedMatrixSize: 11, + expectedMatrixSize: 10, definitionMetadata: { rankingSnapshotId: openRouterRankingSnapshot.snapshotId, rankingContentHash: openRouterRankingSnapshot.contentHash, @@ -960,8 +968,8 @@ export function validateRunnerCatalog(): MatrixExecution[] { ); } } - if (matrix.length !== 67) - throw new Error(`Expected 67 runner executions; received ${matrix.length}`); + if (matrix.length !== 66) + throw new Error(`Expected 66 runner executions; received ${matrix.length}`); return matrix; } diff --git a/tests/runner-e2e/daytona-image-content.ts b/tests/runner-e2e/daytona-image-content.ts index ab566d7bd7..d29cc3e601 100644 --- a/tests/runner-e2e/daytona-image-content.ts +++ b/tests/runner-e2e/daytona-image-content.ts @@ -6,7 +6,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); export const DAYTONA_IMAGE_CONTENT_SCHEMA = - "paperclip-daytona-runner-image-content/v3"; + "paperclip-daytona-runner-image-content/v4"; export const DAYTONA_IMAGE_PLATFORM = "linux/amd64"; export const DAYTONA_IMAGE_DOCKERFILE_PATH = "docker/daytona-runner/Dockerfile"; @@ -27,11 +27,32 @@ export const DAYTONA_IMAGE_INPUT_PATHS = [ "packages/paperclip-runner", ] as const; -const ignoredDirectoryPaths = new Set([ +const ignoredGeneratedDirectoryPaths = new Set([ "packages/paperclip-runner/dist", "packages/paperclip-runner/runner/target", ]); +// These paths do not contribute to the release runnerd binary or the +// executable/digested provider-pack runtime payload. They are also excluded +// from the real Docker build context by .dockerignore. Keep the two lists in +// lockstep: if a future build starts consuming one of these inputs, Docker must +// fail instead of publishing bytes that the content identity did not hash. +const ignoredRunnerDevelopmentDirectoryPaths = new Set([ + "packages/paperclip-runner/devtools", + "packages/paperclip-runner/docs", + "packages/paperclip-runner/examples", + "packages/paperclip-runner/test", + "packages/paperclip-runner/test-fixtures", + "packages/paperclip-runner/test-support", +]); + +const runnerDocumentationFilePattern = /\.md$/; +const runnerTestFilePattern = /\.(?:spec|test)\.(?:[cm]?[jt]sx?)$/; +const runnerRustIntegrationTestPathPattern = + /^packages\/paperclip-runner\/runner\/crates\/[^/]+\/tests(?:\/|$)/; +const runnerSmokeScriptPattern = + /^packages\/paperclip-runner\/scripts\/[^/]+-smoke\.mjs$/; + export interface DaytonaImageContentOptions { repositoryRoot?: string; inputPaths?: readonly string[]; @@ -49,10 +70,21 @@ function normalizedRelativePath(value: string): string { } function shouldIgnore(relativePath: string): boolean { - if (ignoredDirectoryPaths.has(relativePath)) return true; + if (ignoredGeneratedDirectoryPaths.has(relativePath)) return true; return relativePath.split("/").includes("node_modules"); } +function shouldIgnoreRunnerDevelopmentInput(relativePath: string): boolean { + if (!relativePath.startsWith("packages/paperclip-runner/")) return false; + if (ignoredRunnerDevelopmentDirectoryPaths.has(relativePath)) return true; + return ( + runnerDocumentationFilePattern.test(relativePath) || + runnerTestFilePattern.test(relativePath) || + runnerRustIntegrationTestPathPattern.test(relativePath) || + runnerSmokeScriptPattern.test(relativePath) + ); +} + function updateRecord( hash: ReturnType, kind: string, @@ -160,7 +192,12 @@ async function hashEntry( relativePath: string, ): Promise { const normalizedPath = normalizedRelativePath(relativePath); - if (shouldIgnore(normalizedPath)) return; + if ( + shouldIgnore(normalizedPath) || + shouldIgnoreRunnerDevelopmentInput(normalizedPath) + ) { + return; + } const absolutePath = path.resolve(root, relativePath); const relativeFromRoot = path.relative(root, absolutePath); diff --git a/tests/runner-e2e/daytona-image.test.ts b/tests/runner-e2e/daytona-image.test.ts index ec56047665..09e19b65ff 100644 --- a/tests/runner-e2e/daytona-image.test.ts +++ b/tests/runner-e2e/daytona-image.test.ts @@ -59,6 +59,20 @@ describe("runner E2E Daytona image contract", () => { expect(dockerignore).toContain("**/node_modules"); expect(dockerignore).toContain("packages/paperclip-runner/dist"); expect(dockerignore).toContain("packages/paperclip-runner/runner/target"); + for (const developmentOnlyInput of [ + "packages/paperclip-runner/devtools", + "packages/paperclip-runner/docs", + "packages/paperclip-runner/examples", + "packages/paperclip-runner/test", + "packages/paperclip-runner/test-fixtures", + "packages/paperclip-runner/test-support", + "packages/paperclip-runner/**/*.md", + "packages/paperclip-runner/**/*.test.ts", + "packages/paperclip-runner/runner/crates/*/tests", + "packages/paperclip-runner/scripts/*-smoke.mjs", + ]) { + expect(dockerignore).toContain(developmentOnlyInput); + } expect(workflow).toContain("--platform linux/amd64"); expect(workflow).toContain( "Compute Daytona image content ID with pinned bases", @@ -190,6 +204,111 @@ describe("runner E2E Daytona image contract", () => { } }); + it("reuses the image for runner-only tests and documentation", async () => { + const root = await mkdtemp( + path.join(tmpdir(), "paperclip-daytona-runner-development-inputs-"), + ); + const options = { + repositoryRoot: root, + inputPaths: ["packages/paperclip-runner"], + baseImages: [`example.test/base:1@sha256:${"a".repeat(64)}`], + frontendDigest: `sha256:${"c".repeat(64)}`, + } as const; + try { + const runnerRoot = path.join(root, "packages/paperclip-runner"); + await mkdir(path.join(runnerRoot, "src/live"), { recursive: true }); + await mkdir(path.join(runnerRoot, "docs"), { recursive: true }); + await mkdir(path.join(runnerRoot, "spec"), { recursive: true }); + await mkdir(path.join(runnerRoot, "scripts"), { recursive: true }); + await mkdir(path.join(runnerRoot, "test-fixtures"), { recursive: true }); + await mkdir(path.join(runnerRoot, "runner/crates/runner-core/src"), { + recursive: true, + }); + await mkdir(path.join(runnerRoot, "runner/crates/runner-core/tests"), { + recursive: true, + }); + await writeFile( + path.join(runnerRoot, "src/live/transport.ts"), + "export const runtime = 'one';\n", + ); + await writeFile( + path.join(runnerRoot, "runner/crates/runner-core/src/lib.rs"), + 'pub const RUNTIME: &str = "one";\n', + ); + await writeFile(path.join(runnerRoot, "README.md"), "first readme\n"); + await writeFile( + path.join(runnerRoot, "docs/local-runner.md"), + "first documentation\n", + ); + await writeFile( + path.join(runnerRoot, "spec/architecture.md"), + "first architecture note\n", + ); + await writeFile( + path.join(runnerRoot, "src/live/transport.test.ts"), + "first TypeScript test\n", + ); + await writeFile( + path.join(runnerRoot, "test-fixtures/provider.json"), + '{"fixture":"one"}\n', + ); + await writeFile( + path.join(runnerRoot, "scripts/capability-clean-room-smoke.mjs"), + "first smoke probe\n", + ); + await writeFile( + path.join(runnerRoot, "runner/crates/runner-core/tests/recovery.rs"), + "// first Rust integration test\n", + ); + + const baseline = await computeDaytonaImageContentId(options); + await writeFile(path.join(runnerRoot, "README.md"), "second readme\n"); + await writeFile( + path.join(runnerRoot, "docs/local-runner.md"), + "second documentation\n", + ); + await writeFile( + path.join(runnerRoot, "spec/architecture.md"), + "second architecture note\n", + ); + await writeFile( + path.join(runnerRoot, "src/live/transport.test.ts"), + "second TypeScript test\n", + ); + await writeFile( + path.join(runnerRoot, "test-fixtures/provider.json"), + '{"fixture":"two"}\n', + ); + await writeFile( + path.join(runnerRoot, "scripts/capability-clean-room-smoke.mjs"), + "second smoke probe\n", + ); + await writeFile( + path.join(runnerRoot, "runner/crates/runner-core/tests/recovery.rs"), + "// second Rust integration test\n", + ); + expect(await computeDaytonaImageContentId(options)).toBe(baseline); + + await writeFile( + path.join(runnerRoot, "src/live/transport.ts"), + "export const runtime = 'two';\n", + ); + expect(await computeDaytonaImageContentId(options)).not.toBe(baseline); + + await writeFile( + path.join(runnerRoot, "src/live/transport.ts"), + "export const runtime = 'one';\n", + ); + await writeFile( + path.join(runnerRoot, "runner/crates/runner-core/src/lib.rs"), + 'pub const RUNTIME: &str = "two";\n', + ); + expect(await computeDaytonaImageContentId(options)).not.toBe(baseline); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it("rejects mutable Docker base references", () => { expect(() => extractDaytonaBaseImages("FROM node:24-bookworm\n")).toThrow( "must use an immutable sha256 digest", diff --git a/tests/runner-e2e/failure-classifier.ts b/tests/runner-e2e/failure-classifier.ts index c70a9fa26b..0073d906ae 100644 --- a/tests/runner-e2e/failure-classifier.ts +++ b/tests/runner-e2e/failure-classifier.ts @@ -26,5 +26,8 @@ export function classifyFailure(error: unknown): FailureClass { } export function shouldRetryFailure(failureClass: FailureClass) { - return failureClass === "transient_infrastructure"; + return ( + failureClass === "transient_infrastructure" || + failureClass === "provider_variance" + ); } diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index 3264ba7bfa..df7596992e 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -26,8 +26,23 @@ const AMBIENT_EXTERNAL_STATE_KEYS = [ ] as const; const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)(?:_|$)/; +export function runnerE2EServerControlPaths(temporaryRoot: string) { + const controlDirectory = path.join(temporaryRoot, "control"); + return { + controlDirectory, + restartRequestPath: path.join( + controlDirectory, + "server-restart.request.json", + ), + restartAcknowledgementPath: path.join( + controlDirectory, + "server-restart.ack.json", + ), + }; +} + /** - * Local native cells use the debug binary produced by build:runner-binaries. + * Native cells use the debug binary produced once by build:runner-binaries. * Preserve an explicit override for release builds and developer workflows. */ export function resolvePaperclipRunnerBinaryForHarness( @@ -38,11 +53,7 @@ export function resolvePaperclipRunnerBinaryForHarness( ): string | undefined { if (configuredPath?.trim()) return configuredPath; if ( - !executions.some( - (execution) => - execution.environment.id === "local" && - execution.profile.generation === "native", - ) + !executions.some((execution) => execution.profile.generation === "native") ) { return undefined; } @@ -58,6 +69,48 @@ export function resolvePaperclipRunnerBinaryForHarness( ); } +/** + * Remote native cells stage the same controller-owned binary whose digest is + * authorized by the PRP control plane. Local cells launch it directly. + */ +export function resolvePaperclipRemoteRunnerBinaryForHarness( + executions: readonly MatrixExecution[], + runnerBinary: string | undefined, +): string | undefined { + if (!runnerBinary) return undefined; + return executions.some( + (execution) => + execution.profile.generation === "native" && + execution.environment.expectedExecutionTarget.kind === "remote", + ) + ? runnerBinary + : undefined; +} + +/** + * Keep fixture-only provider switches scoped to the one isolated harness that + * needs them. In particular, the pinned legacy OpenCode model is routed by the + * paid gateway and may not appear in OpenCode's public model catalog. + */ +export function buildRunnerE2EProcessEnvironment( + source: NodeJS.ProcessEnv, + executions: readonly MatrixExecution[], +): NodeJS.ProcessEnv { + const result = { ...source }; + delete result.OPENCODE_ALLOW_ALL_MODELS; + if ( + executions.length > 0 && + executions.every( + (execution) => + execution.profile.generation === "legacy" && + execution.profile.provider === "opencode", + ) + ) { + result.OPENCODE_ALLOW_ALL_MODELS = "true"; + } + return result; +} + /** * Build the environment inherited by the Paperclip server. Paid credentials * deliberately stay in the launcher/Playwright process and cross the server @@ -107,6 +160,11 @@ export function assertIsolatedServerEnvironment( "Paperclip server paths escape the isolated temporary root", ); } + if (env.XDG_CACHE_HOME !== path.join(expected.temporaryRoot, "xdg-cache")) { + throw new Error( + "Paperclip server cache does not use the allocated temporary root", + ); + } for (const key of [ ...CREDENTIAL_NAMES, ...DATABASE_KEYS, diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index 8a6cec4c7c..556d09d83a 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { runnerMatrix } from "./catalog.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; @@ -24,6 +24,7 @@ import type { MatrixExecution, RunnerE2EResult } from "./types.js"; const temporaryDirectories: string[] = []; afterEach(async () => { + vi.unstubAllEnvs(); await Promise.all( temporaryDirectories .splice(0) @@ -55,6 +56,35 @@ function result(execution: MatrixExecution, status: "passed" | "failed") { } describe("runner E2E campaign history", () => { + it("records the resolved paid target instead of the trusted workflow checkout", () => { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_SHA", "target-sha"); + vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_REF", "refs/heads/target"); + vi.stubEnv("GITHUB_SHA", "trusted-master-sha"); + vi.stubEnv("GITHUB_REF", "refs/heads/master"); + const execution = runnerMatrix[0]!; + const campaign = buildRunnerCampaign({ + campaignId: "target-provenance", + generatedAt: "2026-08-28T00:01:00.000Z", + expected: [execution.id], + results: [ + { + ...result(execution, "passed"), + source: { + sha: "retained-result-sha", + ref: "refs/heads/retained-result", + workflowRunUrl: "https://example.test/actions/runs/1", + }, + }, + ], + }); + + expect(campaign.source).toMatchObject({ + sha: "target-sha", + ref: "refs/heads/target", + workflowRunUrl: "https://example.test/actions/runs/1", + }); + }); + it("migrates v1 execution IDs and keeps partial suite runs out of overall trends", () => { expect(canonicalExecutionId("legacy-codex.local.message-marker")).toBe( "core-compatibility.legacy-codex.local.message-marker", @@ -68,16 +98,16 @@ describe("runner E2E campaign history", () => { expected: breadth.map((execution) => execution.id), results: breadth.map((execution) => result(execution, "passed")), }); - expect(campaign).toMatchObject({ complete: false, passed: 11, failed: 0 }); + expect(campaign).toMatchObject({ complete: false, passed: 10, failed: 0 }); expect(campaign.suites[0]).toMatchObject({ suiteId: "openrouter-model-breadth", complete: true, - selected: 11, + selected: 10, }); expect(campaign.billing).toMatchObject({ - llm: { inputTokens: 1_100, outputTokens: 275 }, + llm: { inputTokens: 1_000, outputTokens: 250 }, }); - expect(campaign.billing.reportedLlmCostUsd).toBeCloseTo(0.11, 10); + expect(campaign.billing.reportedLlmCostUsd).toBeCloseTo(0.1, 10); const history = mergeRunnerHistory( emptyRunnerHistory(), campaignHistoryRecord(campaign, "https://history.example/runner-e2e"), @@ -151,8 +181,8 @@ describe("runner E2E campaign history", () => { expect(index).toContain("Runner E2E campaigns"); expect(index).toContain("complete-green"); expect(index).toContain("complete-red"); - expect(index).toContain("67/67 passed"); - expect(index).toContain("66/67 passed"); + expect(index).toContain("66/66 passed"); + expect(index).toContain("65/66 passed"); expect(index).toContain("Open report →"); expect(index).toContain( "Visual evidence remains in access-controlled workflow artifacts", diff --git a/tests/runner-e2e/history.ts b/tests/runner-e2e/history.ts index 9101836ca7..bcb3871253 100644 --- a/tests/runner-e2e/history.ts +++ b/tests/runner-e2e/history.ts @@ -3,6 +3,7 @@ import { aggregateCampaignBilling, summarizeExecutionBilling, } from "./billing.js"; +import { resolveRunnerE2ESource } from "./source.js"; import type { RunnerE2ECampaign, RunnerE2EHistoryCampaign, @@ -56,15 +57,7 @@ export function buildRunnerCampaign(input: { })); const resultSource = results.find((result) => result.source)?.source; const source = { - sha: resultSource?.sha ?? process.env.GITHUB_SHA ?? null, - ref: resultSource?.ref ?? process.env.GITHUB_REF ?? null, - workflowRunUrl: - resultSource?.workflowRunUrl ?? - (process.env.GITHUB_SERVER_URL && - process.env.GITHUB_REPOSITORY && - process.env.GITHUB_RUN_ID - ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` - : null), + ...resolveRunnerE2ESource(resultSource), eventName: input.eventName ?? process.env.GITHUB_EVENT_NAME ?? null, }; const suites = runnerSuites diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 3f2d037860..ebe10543ca 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -2,7 +2,6 @@ import { randomBytes } from "node:crypto"; import { spawn } from "node:child_process"; import { createWriteStream } from "node:fs"; import { createRequire } from "node:module"; -import { createServer } from "node:net"; import os from "node:os"; import path from "node:path"; import { @@ -22,7 +21,11 @@ import { renderRunnerE2EDashboard } from "./dashboard.js"; import { packageEvidence } from "./evidence.js"; import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js"; import { buildRunnerCampaign } from "./history.js"; -import { resolvePaperclipRunnerBinaryForHarness } from "./harness-env.js"; +import { + buildRunnerE2EProcessEnvironment, + resolvePaperclipRemoteRunnerBinaryForHarness, + resolvePaperclipRunnerBinaryForHarness, +} from "./harness-env.js"; import { assertEmbeddedDatabaseIsolation } from "./instance-isolation.js"; import { assertSecretFree, @@ -37,6 +40,7 @@ import { RunnerSelectorError, selectRunnerExecutions, } from "./selectors.js"; +import { resolveRunnerE2ESource } from "./source.js"; import { CREDENTIAL_NAMES, type MatrixExecution, @@ -46,6 +50,7 @@ import { reapNewDetachedDarwinSharedMemory, snapshotDarwinSharedMemory, } from "./shared-memory.js"; +import { reserveRunnerE2EServerPort } from "./ports.js"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); const localEnvPath = path.join(repositoryRoot, ".env.runner-e2e.local"); @@ -157,21 +162,6 @@ async function loadLocalEnvironment(target: NodeJS.ProcessEnv) { } } -async function reservePort() { - const server = createServer(); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(0, "127.0.0.1", resolve); - }); - const address = server.address(); - if (!address || typeof address === "string") - throw new Error("Failed to reserve a loopback port"); - await new Promise((resolve, reject) => - server.close((error) => (error ? reject(error) : resolve())), - ); - return address.port; -} - async function prepareProviderPath( temporaryRoot: string, inheritedPath: string | undefined, @@ -292,16 +282,7 @@ function syntheticResult( executionId: execution.id, suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash, - source: { - sha: process.env.GITHUB_SHA ?? null, - ref: process.env.GITHUB_REF ?? null, - workflowRunUrl: - process.env.GITHUB_SERVER_URL && - process.env.GITHUB_REPOSITORY && - process.env.GITHUB_RUN_ID - ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` - : null, - }, + source: resolveRunnerE2ESource(), ...(execution.profile.ranking ? { rankingSnapshot: execution.profile.ranking } : {}), @@ -388,7 +369,7 @@ async function runAttempt(input: { instanceId, "config.json", ); - const port = await reservePort(); + const port = await reserveRunnerE2EServerPort(); await Promise.all([ mkdir(paperclipHome, { recursive: true }), mkdir(workspace, { recursive: true }), @@ -410,8 +391,12 @@ async function runAttempt(input: { betterAuthSecret, ]); attemptSecrets = credentials; + const runnerBinary = resolvePaperclipRunnerBinaryForHarness( + executions, + repositoryRoot, + ); const childEnv: NodeJS.ProcessEnv = { - ...process.env, + ...buildRunnerE2EProcessEnvironment(process.env, executions), PATH: providerPath, PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify( executions.map((candidate) => candidate.id), @@ -422,10 +407,9 @@ async function runAttempt(input: { PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir, PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace, PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(privateDir, "server.log"), - PAPERCLIP_RUNNER_BINARY: resolvePaperclipRunnerBinaryForHarness( - executions, - repositoryRoot, - ), + PAPERCLIP_RUNNER_BINARY: runnerBinary, + PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: + resolvePaperclipRemoteRunnerBinaryForHarness(executions, runnerBinary), // Vite's optimized dependency cache embeds revision query strings. A // private per-attempt cache prevents an earlier cell or local rebuild // from producing `504 Outdated Optimize Dep` during browser bootstrap. @@ -770,7 +754,7 @@ async function runExecutionWithRetry(input: { } if (cancelled) throw new Error("Runner E2E campaign cancelled"); console.warn( - `Retrying ${execution.id} in a fresh isolated harness after transient infrastructure failure`, + `Retrying ${execution.id} in a fresh isolated harness after ${firstResult.failureClass.replaceAll("_", " ")}`, ); const [retryResult] = await runAttempt({ executions: [execution], diff --git a/tests/runner-e2e/playwright.config.ts b/tests/runner-e2e/playwright.config.ts index 0bbc899847..4a42e7a881 100644 --- a/tests/runner-e2e/playwright.config.ts +++ b/tests/runner-e2e/playwright.config.ts @@ -14,6 +14,19 @@ const privateDir = required("PAPERCLIP_RUNNER_E2E_PRIVATE_DIR"); const paperclipHome = required("PAPERCLIP_HOME"); const configPath = required("PAPERCLIP_CONFIG"); const baseURL = `http://127.0.0.1:${port}`; +const playwrightChannel = process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL?.trim(); +const chromiumExecutable = + process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?.trim(); +if (playwrightChannel && chromiumExecutable) { + throw new Error( + "PAPERCLIP_PLAYWRIGHT_CHANNEL and PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE are mutually exclusive", + ); +} +if (chromiumExecutable && !path.isAbsolute(chromiumExecutable)) { + throw new Error( + "PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE must be an absolute path", + ); +} required("PAPERCLIP_INSTANCE_ID"); required("PAPERCLIP_AGENT_JWT_SECRET"); required("PAPERCLIP_DECISION_SIGNING_SECRET"); @@ -38,12 +51,18 @@ export default defineConfig({ use: { baseURL, browserName: "chromium", + ...(playwrightChannel ? { channel: playwrightChannel } : {}), + ...(chromiumExecutable + ? { launchOptions: { executablePath: chromiumExecutable } } + : {}), headless: true, actionTimeout: 30_000, navigationTimeout: 30_000, screenshot: "only-on-failure", trace: "retain-on-failure", - video: "retain-on-failure", + // A developer-supplied system Chromium keeps the local smoke loop + // installation-free; CI's managed browser retains failure video as usual. + video: chromiumExecutable ? "off" : "retain-on-failure", }, webServer: { // Do not put an env object here: Playwright serializes webServer config in diff --git a/tests/runner-e2e/ports.ts b/tests/runner-e2e/ports.ts new file mode 100644 index 0000000000..10d0703cfc --- /dev/null +++ b/tests/runner-e2e/ports.ts @@ -0,0 +1,92 @@ +import { createServer } from "node:net"; +import { derivePaperclipViteHmrPort } from "../../packages/shared/src/runtime-exposure/ports.js"; + +export const RUNNER_E2E_EMBEDDED_POSTGRES_PORT = 54_329; + +export interface LoopbackPortReservation { + port: number; + close(): Promise; +} + +export type OpenLoopbackPort = ( + requestedPort: number, +) => Promise; + +async function openLoopbackPort( + requestedPort: number, +): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(requestedPort, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + throw new Error("Failed to reserve a loopback port"); + } + return { + port: address.port, + close: () => + new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ), + }; +} + +export function runnerE2EServerPortConflictsWithDatabase( + serverPort: number, + databasePort = RUNNER_E2E_EMBEDDED_POSTGRES_PORT, +) { + return ( + serverPort === databasePort || + derivePaperclipViteHmrPort(serverPort) === databasePort + ); +} + +function isAddressInUse(error: unknown) { + return (error as NodeJS.ErrnoException | undefined)?.code === "EADDRINUSE"; +} + +export async function reserveRunnerE2EServerPort( + options: { + databasePort?: number; + maxAttempts?: number; + openPort?: OpenLoopbackPort; + } = {}, +) { + const databasePort = + options.databasePort ?? RUNNER_E2E_EMBEDDED_POSTGRES_PORT; + const maxAttempts = options.maxAttempts ?? 32; + const openPort = options.openPort ?? openLoopbackPort; + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const serverReservation = await openPort(0); + let hmrReservation: LoopbackPortReservation | undefined; + try { + if ( + runnerE2EServerPortConflictsWithDatabase( + serverReservation.port, + databasePort, + ) + ) { + continue; + } + const hmrPort = derivePaperclipViteHmrPort(serverReservation.port); + try { + hmrReservation = await openPort(hmrPort); + } catch (error) { + if (isAddressInUse(error)) continue; + throw error; + } + return serverReservation.port; + } finally { + await hmrReservation?.close(); + await serverReservation.close(); + } + } + + throw new Error( + `Failed to reserve a conflict-free Paperclip/Vite port pair after ${maxAttempts} attempts`, + ); +} diff --git a/tests/runner-e2e/redaction.ts b/tests/runner-e2e/redaction.ts index 5c42e529c6..e07adb402c 100644 --- a/tests/runner-e2e/redaction.ts +++ b/tests/runner-e2e/redaction.ts @@ -30,6 +30,9 @@ export function isEphemeralCodexRuntimeAuthFile( ) || /^instances\/[^/]+\/runtime\/paperclip-runner\/durable-sessions\/[^/]+\/codex-home\/auth\.json$/.test( relative, + ) || + /^instances\/[^/]+\/runtime\/paperclip-runner\/acpx\/acpx\/[^/]+\/codex-home\/auth\.json$/.test( + relative, ) ); } diff --git a/tests/runner-e2e/run-observations.ts b/tests/runner-e2e/run-observations.ts index 4a99b12146..74938a3ac7 100644 --- a/tests/runner-e2e/run-observations.ts +++ b/tests/runner-e2e/run-observations.ts @@ -3,6 +3,78 @@ export interface ObservableRunState { errorCode?: string | null; } +export interface ObservableRunEvent { + eventType?: string; + payload?: Record | null; +} + +export interface ObservableProviderSessionRun { + id: string; + sessionIdBefore?: string | null; + sessionIdAfter?: string | null; + contextSnapshot?: Record | null; +} + +export interface ObservableMatcherResult { + matcher: { + kind?: string; + expected?: unknown; + count?: unknown; + }; + passed: boolean; +} + +export interface OpenRouterHelloTerminalVarianceObservation { + suiteId: string; + profileId: string; + taskId: string; + expectedMarker: string; + finalRunMessage: string; + allAgentMessages: string; + semanticSummary: unknown; + issueStatus: string; + runStatuses: readonly string[]; + matcherResults: readonly ObservableMatcherResult[]; + invariantFailures: readonly string[]; +} + +export function acceptedPlanSessionResetFailures( + provider: "codex" | "opencode" | "acpx", + previousSessionId: string | null | undefined, + current: ObservableProviderSessionRun, +): string[] | null { + const context = record(current.contextSnapshot); + const acceptedPlanReset = + context.forceFreshSession === true && + context.workspaceRefreshReason === "accepted_plan_confirmation" && + context.source === "issue.interaction.accept" && + context.interactionStatus === "accepted"; + if (!acceptedPlanReset) return null; + + const failures: string[] = []; + if (current.sessionIdBefore) { + failures.push( + `expected accepted Plan run ${current.id} to start without a prior provider session`, + ); + } + if ( + previousSessionId && + current.sessionIdAfter && + current.sessionIdAfter === previousSessionId + ) { + failures.push( + `expected accepted Plan run ${current.id} to rotate the ${provider} provider session`, + ); + } + return failures; +} + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + export function isNonExecutingReviewFenceRun(run: ObservableRunState) { return ( run.status === "cancelled" && @@ -10,6 +82,155 @@ export function isNonExecutingReviewFenceRun(run: ObservableRunState) { ); } +function normalizeMessage(value: string) { + return value + .replace(/\r\n/g, "\n") + .replace(/\\_/g, "_") + .replace(/[ \t]+/g, " ") + .trim(); +} + +function countOccurrences(value: string, expected: string) { + if (!expected) return 0; + let count = 0; + let cursor = 0; + while (cursor <= value.length - expected.length) { + const index = value.indexOf(expected, cursor); + if (index < 0) break; + count += 1; + cursor = index + expected.length; + } + return count; +} + +export function isOpenRouterDeepSeekHelloTerminalVariance( + observation: OpenRouterHelloTerminalVarianceObservation, +) { + const marker = normalizeMessage(observation.expectedMarker); + const finalRunMessage = normalizeMessage(observation.finalRunMessage); + const allAgentMessages = normalizeMessage(observation.allAgentMessages); + const failedMatchers = observation.matcherResults.filter( + (result) => !result.passed, + ); + const hasExpectedExactFailure = failedMatchers.some( + (result) => + result.matcher.kind === "message_exact" && + result.matcher.expected === observation.expectedMarker, + ); + const hasExpectedOccurrenceFailure = failedMatchers.some( + (result) => + result.matcher.kind === "message_occurrences" && + result.matcher.expected === observation.expectedMarker && + result.matcher.count === 1, + ); + + return ( + observation.suiteId === "openrouter-model-breadth" && + observation.profileId === "openrouter-deepseek-deepseek-v4-flash-0731" && + observation.taskId === "hello-complete" && + observation.issueStatus === "done" && + observation.runStatuses.length === 1 && + observation.runStatuses[0] === "succeeded" && + observation.semanticSummary === observation.expectedMarker && + finalRunMessage.length > 0 && + countOccurrences(finalRunMessage, marker) === 0 && + countOccurrences(allAgentMessages, marker) === 0 && + observation.invariantFailures.length === 0 && + failedMatchers.length === 2 && + hasExpectedExactFailure && + hasExpectedOccurrenceFailure + ); +} + +export function isControlPlaneGovernedResponseWait( + events: readonly ObservableRunEvent[], +) { + const accepted = events.filter( + (event) => event.eventType === "run.result.accepted", + ); + if (accepted.length !== 1) return false; + const envelope = record(accepted[0]?.payload?.prpEvent); + const result = record(record(envelope.payload).result); + const continuation = record(result.continuation); + const idempotencyKey = continuation.idempotencyKey; + if ( + typeof idempotencyKey !== "string" || + !idempotencyKey.startsWith("interaction-response:") + ) { + return false; + } + const interactionId = idempotencyKey.slice("interaction-response:".length); + if (!interactionId) return false; + const interactionRef = `interaction:${interactionId}`; + const hasEvidence = + Array.isArray(result.evidence) && + result.evidence.some((value) => record(value).ref === interactionRef); + const hasInteractionArtifact = + Array.isArray(result.artifacts) && + result.artifacts.some((value) => { + const artifact = record(value); + return ( + artifact.kind === "issue_thread_interaction" && + artifact.ref === interactionRef + ); + }); + return ( + envelope.schema === "paperclip.prp.event.v1" && + envelope.eventType === "run.result.accepted" && + envelope.sourceKind === "control_plane" && + result.schema === "paperclip.run_result.v1" && + result.reportedWorkDisposition === "yielded" && + continuation.kind === "response_wake" && + hasEvidence && + hasInteractionArtifact + ); +} + +export function providerSessionContinuityFailures( + provider: "codex" | "opencode", + runs: readonly ObservableProviderSessionRun[], +): string[] { + const failures: string[] = []; + for (let index = 0; index < runs.length; index += 1) { + const current = runs[index]!; + const currentSessionId = current.sessionIdAfter; + if (!currentSessionId) { + failures.push( + `expected ${provider} run ${current.id} to record provider session identity`, + ); + continue; + } + if (index === 0) continue; + + const previousSessionId = runs[index - 1]?.sessionIdAfter; + const acceptedPlanResetFailures = acceptedPlanSessionResetFailures( + provider, + previousSessionId, + current, + ); + if (acceptedPlanResetFailures) { + failures.push(...acceptedPlanResetFailures); + continue; + } + + if (!previousSessionId || currentSessionId !== previousSessionId) { + failures.push( + `expected ${provider} to preserve its provider session for run ${current.id}`, + ); + continue; + } + if ( + current.sessionIdBefore && + current.sessionIdBefore !== previousSessionId + ) { + failures.push( + `expected ${provider} run ${current.id} to resume provider session ${previousSessionId}`, + ); + } + } + return failures; +} + export function numberedPlanStepCount(body: string | null | undefined) { return (body ?? "").split(/\r?\n/).filter((line) => { const normalized = line diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 5a0ad01cbe..d85ca19939 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -6,12 +6,18 @@ import { RunnerApi, pollUntil } from "./api.js"; import { buildRuntimeUsage, summarizeExecutionBilling } from "./billing.js"; import { runnerExecutionById } from "./catalog.js"; import { classifyFailure } from "./failure-classifier.js"; +import { runnerE2EServerControlPaths } from "./harness-env.js"; import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js"; import { evaluateMatcher, type MatcherResult } from "./matchers.js"; import { + acceptedPlanSessionResetFailures, + isControlPlaneGovernedResponseWait, isNonExecutingReviewFenceRun, + isOpenRouterDeepSeekHelloTerminalVariance, numberedPlanStepCount, + providerSessionContinuityFailures, } from "./run-observations.js"; +import { resolveRunnerE2ESource } from "./source.js"; import { assertSecretFree, findSecretLeakInJsonValues, @@ -166,15 +172,11 @@ async function restartIsolatedPaperclipServer(input: { requestId: string; deadlineAt: number; }): Promise { - const controlDirectory = path.join(privateRoot!, "control"); - const requestPath = path.join( + const { controlDirectory, - "server-restart.request.json", - ); - const acknowledgementPath = path.join( - controlDirectory, - "server-restart.ack.json", - ); + restartRequestPath: requestPath, + restartAcknowledgementPath: acknowledgementPath, + } = runnerE2EServerControlPaths(temporaryRoot!); await mkdir(controlDirectory, { recursive: true }); const temporaryRequestPath = `${requestPath}.${process.pid}.${input.requestId}.tmp`; await writeFile( @@ -237,10 +239,11 @@ const executionIds = (() => { })(); const executions = executionIds.map(runnerExecutionById); const attempt = Number(process.env.PAPERCLIP_RUNNER_E2E_ATTEMPT ?? "1"); +const temporaryRoot = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; const privateRoot = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR; const workspacePath = process.env.PAPERCLIP_RUNNER_E2E_WORKSPACE; -if (!privateRoot || !workspacePath) - throw new Error("Runner E2E private/workspace paths are required"); +if (!temporaryRoot || !privateRoot || !workspacePath) + throw new Error("Runner E2E temporary/private/workspace paths are required"); function record(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) @@ -454,9 +457,12 @@ function nativeRunEventIntegrityFailures( } } + const runnerResultCount = runnerEventTypes.filter( + (value) => value === "run.result.proposed", + ).length; if ( - runnerEventTypes.filter((value) => value === "run.result.proposed") - .length !== 1 + runnerResultCount !== 1 && + !(runnerResultCount === 0 && isControlPlaneGovernedResponseWait(events)) ) { failures.push( `run ${run.id} must persist exactly one runner semantic result`, @@ -1125,7 +1131,8 @@ for (const execution of executions) { planLifecycleEvidence = { interaction, plan }; } - const terminal = await pollUntil({ + const taskMatchers = execution.task.buildMatchers(nonce, execution); + let terminal = await pollUntil({ label: `issue ${issue.id} and heartbeat run terminal state`, deadlineAt, load: loadTaskState, @@ -1136,6 +1143,32 @@ for (const execution of executions) { reject: ({ taskRuns }) => definitiveRunFailure(taskRuns), }); + // Finalization commits the issue/run decision before the derived agent + // comment is guaranteed to be visible through the comments endpoint. + // Give that projection a short consistency window so a successful + // terminal response is not misclassified as an empty provider reply. + // If no comment arrives, retain the original terminal observation and + // let the ordinary message matcher report the product failure. + if (taskMatchers.some((matcher) => matcher.kind.startsWith("message_"))) { + terminal = await pollUntil({ + label: `final agent comment for issue ${issue.id}`, + deadlineAt: Math.min(deadlineAt, Date.now() + 30_000), + load: loadTaskState, + accept: ({ taskRuns, comments }) => { + if (taskRuns.length !== execution.task.expectedRunCount) { + return false; + } + const finalRun = sortRunsChronologically(taskRuns).at(-1); + return comments.some( + (comment) => + comment.createdByRunId === finalRun?.id && + comment.authorAgentId === fixtures!.agent.id, + ); + }, + reject: ({ taskRuns }) => definitiveRunFailure(taskRuns), + }).catch(() => terminal); + } + issue = terminal.currentIssue; selectedRuns = terminal.taskRuns; if (selectedRuns.length !== execution.task.expectedRunCount) { @@ -1383,7 +1416,7 @@ for (const execution of executions) { }, }; matcherResults = await Promise.all( - execution.task.buildMatchers(nonce, execution).map((matcher) => + taskMatchers.map((matcher) => evaluateMatcher(matcher, { ...matcherObservation, // Multi-run tasks intentionally retain earlier waiting/revision @@ -1396,6 +1429,45 @@ for (const execution of executions) { ), ); const failedMatchers = matcherResults.filter((result) => !result.passed); + const exactMessageMatcher = taskMatchers.find( + (matcher) => matcher.kind === "message_exact", + ); + if ( + execution.profile.id === "runner-opencode" && + execution.task.id === "structured-question-restart-resume" && + exactMessageMatcher?.kind === "message_exact" && + finalRunMessage === exactMessageMatcher.expected.replace(/-\d+$/, "") && + record(finalRun.resultJson).summary === exactMessageMatcher.expected + ) { + // OpenCode can occasionally copy the complete marker into the + // accepted semantic result while dropping only the synthetic attempt + // suffix from its visible answer. Keep exact matching strict, but let + // the campaign retry this narrowly proven provider variance once in a + // fresh harness. A repeated near miss remains a failed cell. + failureClassOverride = "provider_variance"; + } + const retryInteractiveTerminalProviderVariance = + (execution.suite.id === "openrouter-model-breadth" && + (execution.task.id === "question-resume-complete" || + execution.task.id === "plan-approve-complete")) || + (execution.suite.id === "core-compatibility" && + execution.profile.generation === "native" && + execution.task.id === "plan-revise-accept"); + if ( + retryInteractiveTerminalProviderVariance && + exactMessageMatcher?.kind === "message_exact" && + selectedRuns.every((candidate) => candidate.status === "succeeded") && + issue.status === "done" && + finalRunMessage !== exactMessageMatcher.expected && + record(finalRun.resultJson).summary === exactMessageMatcher.expected + ) { + // An interactive breadth model can occasionally satisfy the durable + // terminal contract while paraphrasing the visible final response. + // Preserve the exact persisted-message and DOM assertions, but + // classify this narrowly proven provider variance for one + // fresh-harness retry. + failureClassOverride = "provider_variance"; + } const observedEnvironmentId = environmentContext.id ?? (execution.environment.id === "local" @@ -1424,8 +1496,7 @@ for (const execution of executions) { )?.[1] ?? /Using fallback workspace "([^"]+)"/.exec(runLogContent)?.[1]; const cwd = String(workspaceContext.cwd ?? fallbackWorkspace ?? ""); - const isolatedRoot = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? ""; - if (!isolatedRoot || !cwd.startsWith(`${isolatedRoot}/`)) { + if (!cwd.startsWith(`${temporaryRoot}/`)) { invariantFailures.push( `local run workspace escaped the isolated root: ${cwd}`, ); @@ -1461,17 +1532,12 @@ for (const execution of executions) { execution.profile.provider === "opencode") && selectedRuns.length > 1 ) { - const providerSessions = selectedRuns.map( - (candidate) => candidate.sessionIdAfter, + invariantFailures.push( + ...providerSessionContinuityFailures( + execution.profile.provider, + selectedRuns, + ), ); - if ( - providerSessions.some((sessionId) => !sessionId) || - new Set(providerSessions).size !== 1 - ) { - invariantFailures.push( - `expected ${execution.profile.provider} to preserve one provider session across all heartbeat runs`, - ); - } } else if ( execution.profile.provider === "acpx" && selectedRuns.length > 1 @@ -1486,6 +1552,15 @@ for (const execution of executions) { ); continue; } + const acceptedPlanResetFailures = acceptedPlanSessionResetFailures( + "acpx", + previousSessionId, + current, + ); + if (acceptedPlanResetFailures) { + invariantFailures.push(...acceptedPlanResetFailures); + continue; + } if (previousSessionId === currentSessionId) continue; const currentEvents = runEventsByRun.find((captured) => captured.runId === current.id) @@ -1572,6 +1647,29 @@ for (const execution of executions) { secrets, ); + if ( + exactMessageMatcher?.kind === "message_exact" && + isOpenRouterDeepSeekHelloTerminalVariance({ + suiteId: execution.suite.id, + profileId: execution.profile.id, + taskId: execution.task.id, + expectedMarker: exactMessageMatcher.expected, + finalRunMessage, + allAgentMessages: message, + semanticSummary: record(finalRun.resultJson).summary, + issueStatus: issue.status, + runStatuses: selectedRuns.map((candidate) => candidate.status), + matcherResults, + invariantFailures, + }) + ) { + // DeepSeek can occasionally complete the semantic finish correctly but + // expose its pre-tool acknowledgement as the visible final answer. Keep + // exact persisted-message, global occurrence, and DOM checks strict, + // while allowing one fresh-harness retry only for the zero-marker form. + failureClassOverride = "provider_variance"; + } + // The backend polling above can observe a terminal transition before a // websocket invalidation reaches the already-open task page. Reload the // canonical task route so the screenshot and UI assertions prove the @@ -1731,16 +1829,7 @@ for (const execution of executions) { executionId: execution.id, suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash, - source: { - sha: process.env.GITHUB_SHA ?? null, - ref: process.env.GITHUB_REF ?? null, - workflowRunUrl: - process.env.GITHUB_SERVER_URL && - process.env.GITHUB_REPOSITORY && - process.env.GITHUB_RUN_ID - ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` - : null, - }, + source: resolveRunnerE2ESource(), ...(execution.profile.ranking ? { rankingSnapshot: execution.profile.ranking } : {}), diff --git a/tests/runner-e2e/select-rerun-artifacts.test.ts b/tests/runner-e2e/select-rerun-artifacts.test.ts new file mode 100644 index 0000000000..baba8e0d82 --- /dev/null +++ b/tests/runner-e2e/select-rerun-artifacts.test.ts @@ -0,0 +1,297 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { selectRerunArtifacts } from "./select-rerun-artifacts.js"; +import type { RunnerE2EResult } from "./types.js"; + +const RUN_ID = "33843305626"; +const SOURCE_SHA = "0123456789abcdef0123456789abcdef01234567"; +const SOURCE_REF = "refs/heads/fix/runner-paid-matrix-integrity-v2"; +const WORKFLOW_RUN_URL = + "https://github.com/paperclipai/paperclip/actions/runs/33843305626"; +const RETAINED = "core-compatibility.legacy-codex.local.message-marker"; +const RERUN = "core-compatibility.runner-codex.local.plan-revise-accept"; +const cleanupDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + cleanupDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +function result(executionId: string, status: "passed" | "failed") { + const [suiteId, profileId, environmentId, caseId] = executionId.split("."); + return { + schema: "paperclip.runner-e2e.result/v2", + executionId, + suiteId, + source: { + sha: SOURCE_SHA, + ref: SOURCE_REF, + workflowRunUrl: WORKFLOW_RUN_URL, + }, + attempt: 1, + status, + ...(status === "failed" + ? { failureClass: "candidate_failure" as const, error: "failed" } + : {}), + profileId: profileId!, + environmentId: environmentId as RunnerE2EResult["environmentId"], + caseId: caseId!, + provider: "codex", + model: "fixture-model", + runtimeMode: "native", + startedAt: "2026-09-04T00:00:00.000Z", + finishedAt: "2026-09-04T00:00:01.000Z", + durationMs: 1_000, + cleanup: status === "passed" ? "passed" : "not_started", + } satisfies RunnerE2EResult; +} + +async function addArtifact(input: { + root: string; + executionId: string; + workflowAttempt: number; + status: "passed" | "failed"; + sourceSha?: string; + campaignName?: string; +}) { + const artifactName = `runner-e2e-${RUN_ID}-${input.workflowAttempt}-${input.executionId}`; + const campaignName = + input.campaignName ?? + `gha-${RUN_ID}-${input.workflowAttempt}-${input.executionId}`; + const directory = path.join( + input.root, + artifactName, + campaignName, + "results", + "attempt-1", + ); + await mkdir(directory, { recursive: true }); + const value = result(input.executionId, input.status); + await writeFile( + path.join(directory, "result.json"), + JSON.stringify({ + ...value, + source: { ...value.source, sha: input.sourceSha ?? value.source.sha }, + }), + ); + return { artifactName, campaignName }; +} + +async function fixture() { + const root = await mkdtemp( + path.join(os.tmpdir(), "runner-e2e-rerun-artifacts-"), + ); + cleanupDirectories.push(root); + return { + root, + artifactRoot: path.join(root, "downloaded"), + selectedRoot: path.join(root, "selected"), + }; +} + +function selectionInput(paths: Awaited>) { + return { + ...paths, + jobs: { + jobs: [ + { + name: RETAINED, + run_attempt: 1, + started_at: "2026-09-04T00:00:01.000Z", + }, + { + name: RERUN, + run_attempt: 1, + started_at: "2026-09-04T00:00:02.000Z", + }, + // filter=all synthesizes this attempt-2 row even though GitHub retained + // the successful attempt-1 job. Its original start time exposes it. + { + name: RETAINED, + run_attempt: 2, + started_at: "2026-09-04T00:00:01.000Z", + }, + { + name: RERUN, + run_attempt: 2, + started_at: "2026-09-04T01:00:01.000Z", + }, + ], + attempts: [ + { + run_attempt: 1, + run_started_at: "2026-09-04T00:00:00.000Z", + }, + { + run_attempt: 2, + run_started_at: "2026-09-04T01:00:00.000Z", + }, + ], + }, + expectedExecutionIds: [RETAINED, RERUN], + workflowRunId: RUN_ID, + workflowRunAttempt: 2, + sourceSha: SOURCE_SHA, + sourceRef: SOURCE_REF, + workflowRunUrl: WORKFLOW_RUN_URL, + }; +} + +describe("runner E2E workflow rerun artifact selection", () => { + it("combines retained successes with the latest rerun artifact", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 1, + status: "failed", + }); + const latest = await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 2, + status: "passed", + }); + + const selected = await selectRerunArtifacts(selectionInput(paths)); + + expect(selected).toEqual([ + { + executionId: RETAINED, + workflowAttempt: 1, + artifactName: `runner-e2e-${RUN_ID}-1-${RETAINED}`, + }, + { + executionId: RERUN, + workflowAttempt: 2, + artifactName: latest.artifactName, + }, + ]); + const selectedResult = JSON.parse( + await readFile( + path.join( + paths.selectedRoot, + latest.artifactName, + latest.campaignName, + "results", + "attempt-1", + "result.json", + ), + "utf8", + ), + ); + expect(selectedResult.status).toBe("passed"); + }); + + it("never falls back when the latest workflow attempt has no artifact", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 1, + status: "passed", + }); + + const selected = await selectRerunArtifacts(selectionInput(paths)); + + expect(selected.map((entry) => entry.executionId)).toEqual([RETAINED]); + }); + + it("does not let an older pass mask a latest failed artifact", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 1, + status: "passed", + }); + const latest = await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 2, + status: "failed", + }); + + await selectRerunArtifacts(selectionInput(paths)); + + const selectedResult = JSON.parse( + await readFile( + path.join( + paths.selectedRoot, + latest.artifactName, + latest.campaignName, + "results", + "attempt-1", + "result.json", + ), + "utf8", + ), + ); + expect(selectedResult.status).toBe("failed"); + }); + + it("rejects artifacts from another source", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + sourceSha: "ffffffffffffffffffffffffffffffffffffffff", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 2, + status: "passed", + }); + + await expect(selectRerunArtifacts(selectionInput(paths))).rejects.toThrow( + "contains result from another source", + ); + }); + + it("rejects an artifact carrying another campaign", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 2, + status: "passed", + campaignName: `gha-another-run-2-${RERUN}`, + }); + + await expect(selectRerunArtifacts(selectionInput(paths))).rejects.toThrow( + /must contain only its exact campaign/u, + ); + }); +}); diff --git a/tests/runner-e2e/select-rerun-artifacts.ts b/tests/runner-e2e/select-rerun-artifacts.ts new file mode 100644 index 0000000000..af5294fd09 --- /dev/null +++ b/tests/runner-e2e/select-rerun-artifacts.ts @@ -0,0 +1,304 @@ +import { cp, mkdir, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import type { RunnerE2EResult } from "./types.js"; + +interface WorkflowJob { + name: string; + run_attempt: number; + started_at: string; +} + +interface WorkflowJobsResponse { + jobs: WorkflowJob[]; + attempts: Array<{ + run_attempt: number; + run_started_at: string; + }>; +} + +export interface SelectRerunArtifactsInput { + artifactRoot: string; + selectedRoot: string; + jobs: WorkflowJobsResponse; + expectedExecutionIds: readonly string[]; + workflowRunId: string; + workflowRunAttempt: number; + sourceSha: string; + sourceRef: string; + workflowRunUrl: string; +} + +function safeIdentifier(value: string, label: string) { + if (!value || !/^[A-Za-z0-9_.-]+$/u.test(value)) { + throw new Error( + `${label} contains unsafe characters: ${JSON.stringify(value)}`, + ); + } + return value; +} + +function positiveInteger(value: unknown, label: string) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1) { + throw new Error(`${label} must be a positive integer`); + } + return value; +} + +function timestamp(value: unknown, label: string) { + if (typeof value !== "string" || !Number.isFinite(Date.parse(value))) { + throw new Error(`${label} must be an ISO timestamp`); + } + return Date.parse(value); +} + +async function walk(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + const full = path.join(root, entry.name); + if (entry.isDirectory()) files.push(...(await walk(full))); + else if (entry.isFile()) files.push(full); + } + return files; +} + +/** + * Selects the artifact produced by the latest workflow job for each execution. + * GitHub reruns retain earlier-attempt artifacts, so validity or result + * timestamps must never be used to choose between workflow attempts. + */ +export async function selectRerunArtifacts(input: SelectRerunArtifactsInput) { + const runId = safeIdentifier(input.workflowRunId, "workflow run ID"); + const currentAttempt = positiveInteger( + input.workflowRunAttempt, + "workflow run attempt", + ); + const expected = input.expectedExecutionIds.map((executionId) => + safeIdentifier(executionId, "execution ID"), + ); + if (expected.length === 0 || new Set(expected).size !== expected.length) { + throw new Error("expected execution IDs must be non-empty and unique"); + } + const preexistingSelections = await readdir(input.selectedRoot).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }, + ); + if (preexistingSelections.length > 0) { + throw new Error("selected artifact root must start empty"); + } + const expectedSet = new Set(expected); + const attemptStartedAt = new Map(); + for (const attemptMetadata of input.jobs.attempts) { + const attempt = positiveInteger( + attemptMetadata.run_attempt, + "workflow metadata attempt", + ); + if (attempt > currentAttempt || attemptStartedAt.has(attempt)) { + throw new Error(`invalid workflow metadata for attempt ${attempt}`); + } + attemptStartedAt.set( + attempt, + timestamp( + attemptMetadata.run_started_at, + `workflow attempt ${attempt} start`, + ), + ); + } + for (let attempt = 1; attempt <= currentAttempt; attempt += 1) { + if (!attemptStartedAt.has(attempt)) { + throw new Error(`workflow metadata omitted attempt ${attempt}`); + } + } + const attemptsByExecution = new Map>(); + for (const candidate of input.jobs.jobs) { + if (!expectedSet.has(candidate.name)) continue; + const attempt = positiveInteger(candidate.run_attempt, "job run attempt"); + if (attempt > currentAttempt) { + throw new Error( + `job ${candidate.name} claims future workflow attempt ${attempt}`, + ); + } + const jobStartedAt = timestamp( + candidate.started_at, + `job ${candidate.name} start`, + ); + // GitHub's filter=all response synthesizes current-attempt rows for jobs + // retained from an earlier attempt. Their started_at remains before the + // current attempt's trusted run_started_at, so they are not real reruns. + if (jobStartedAt < attemptStartedAt.get(attempt)!) continue; + const attempts = attemptsByExecution.get(candidate.name) ?? new Map(); + if (attempts.has(attempt)) { + throw new Error( + `workflow attempt ${attempt} contains duplicate job ${candidate.name}`, + ); + } + attempts.set(attempt, candidate); + attemptsByExecution.set(candidate.name, attempts); + } + + const latestAttemptByExecution = new Map(); + for (const executionId of expected) { + const attempts = [...(attemptsByExecution.get(executionId)?.keys() ?? [])]; + if (attempts.length === 0) { + throw new Error( + `workflow job history omitted expected execution ${executionId}`, + ); + } + latestAttemptByExecution.set(executionId, Math.max(...attempts)); + } + + const recognizedArtifactNames = new Map< + string, + { executionId: string; workflowAttempt: number } + >(); + for (const executionId of expected) { + for (const workflowAttempt of attemptsByExecution + .get(executionId)! + .keys()) { + recognizedArtifactNames.set( + `runner-e2e-${runId}-${workflowAttempt}-${executionId}`, + { executionId, workflowAttempt }, + ); + } + } + + const artifactEntries = await readdir(input.artifactRoot, { + withFileTypes: true, + }).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }); + const artifactDirectories = new Map(); + for (const entry of artifactEntries) { + const identity = recognizedArtifactNames.get(entry.name); + if (!identity || !entry.isDirectory()) { + throw new Error(`downloaded unexpected runner artifact ${entry.name}`); + } + artifactDirectories.set( + entry.name, + path.join(input.artifactRoot, entry.name), + ); + } + + const selections: Array<{ + executionId: string; + workflowAttempt: number; + artifactName: string; + }> = []; + for (const executionId of expected) { + const workflowAttempt = latestAttemptByExecution.get(executionId)!; + const artifactName = `runner-e2e-${runId}-${workflowAttempt}-${executionId}`; + const artifactDirectory = artifactDirectories.get(artifactName); + // A latest job without an artifact must remain missing. Falling back to an + // older successful artifact would mask an infrastructure/upload failure. + if (!artifactDirectory) continue; + + const campaignName = `gha-${runId}-${workflowAttempt}-${executionId}`; + const topLevelEntries = await readdir(artifactDirectory, { + withFileTypes: true, + }); + if ( + topLevelEntries.length !== 1 || + topLevelEntries[0]?.name !== campaignName || + !topLevelEntries[0].isDirectory() + ) { + throw new Error( + `${artifactName} must contain only its exact campaign ${campaignName}`, + ); + } + const campaignDirectory = path.join(artifactDirectory, campaignName); + const resultFiles = (await walk(campaignDirectory)).filter( + (file) => path.basename(file) === "result.json", + ); + if (resultFiles.length === 0) { + throw new Error(`${artifactName} contains no normalized result`); + } + for (const resultFile of resultFiles) { + const result = JSON.parse( + await readFile(resultFile, "utf8"), + ) as RunnerE2EResult; + if (result.executionId !== executionId) { + throw new Error( + `${artifactName} contains result for ${String(result.executionId)}`, + ); + } + const source = result.source; + if ( + !source || + source.sha !== input.sourceSha || + source.ref !== input.sourceRef || + source.workflowRunUrl !== input.workflowRunUrl + ) { + throw new Error(`${artifactName} contains result from another source`); + } + } + const destination = path.join( + input.selectedRoot, + artifactName, + campaignName, + ); + await mkdir(path.dirname(destination), { recursive: true }); + await cp(campaignDirectory, destination, { + recursive: true, + force: false, + errorOnExist: true, + }); + selections.push({ executionId, workflowAttempt, artifactName }); + } + return selections; +} + +async function main() { + const required = (name: string) => { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; + }; + const expected = JSON.parse( + required("PAPERCLIP_RUNNER_E2E_EXPECTED_IDS"), + ) as unknown; + if ( + !Array.isArray(expected) || + expected.some((value) => typeof value !== "string") + ) { + throw new Error( + "PAPERCLIP_RUNNER_E2E_EXPECTED_IDS must be a JSON string array", + ); + } + const jobs = JSON.parse( + await readFile(required("PAPERCLIP_RUNNER_E2E_JOBS_JSON"), "utf8"), + ) as WorkflowJobsResponse; + if (!jobs || !Array.isArray(jobs.jobs) || !Array.isArray(jobs.attempts)) { + throw new Error("workflow jobs JSON must contain jobs and attempts arrays"); + } + const runId = required("GITHUB_RUN_ID"); + const serverUrl = required("GITHUB_SERVER_URL"); + const repository = required("GITHUB_REPOSITORY"); + const selections = await selectRerunArtifacts({ + artifactRoot: required("PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT"), + selectedRoot: required("PAPERCLIP_RUNNER_E2E_SELECTED_ROOT"), + jobs, + expectedExecutionIds: expected, + workflowRunId: runId, + workflowRunAttempt: Number(required("GITHUB_RUN_ATTEMPT")), + sourceSha: required("PAPERCLIP_RUNNER_E2E_SOURCE_SHA"), + sourceRef: required("PAPERCLIP_RUNNER_E2E_SOURCE_REF"), + workflowRunUrl: `${serverUrl}/${repository}/actions/runs/${runId}`, + }); + console.log( + `Selected ${selections.length}/${expected.length} latest cell artifacts`, + ); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href +) { + await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index bdf286b771..55e84e4b6e 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { assertIsolatedServerEnvironment, buildPaperclipServerEnvironment, + runnerE2EServerControlPaths, } from "./harness-env.js"; function required(name: string) { @@ -21,17 +22,19 @@ const port = required("PAPERCLIP_RUNNER_E2E_PORT"); const repositoryRoot = path.resolve(import.meta.dirname, "../.."); const tsxCli = path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"); const paperclipCli = path.join(repositoryRoot, "cli/src/index.ts"); -const controlDirectory = path.join(temporaryRoot, "control"); -const restartRequestPath = path.join( +const { controlDirectory, - "server-restart.request.json", -); -const restartAckPath = path.join(controlDirectory, "server-restart.ack.json"); + restartRequestPath, + restartAcknowledgementPath: restartAckPath, +} = runnerE2EServerControlPaths(temporaryRoot); const restartTimeoutMs = 180_000; const gracefulStopTimeoutMs = 30_000; const serverEnvironment = buildPaperclipServerEnvironment(process.env, { NODE_ENV: "test", PORT: port, + // Keep provider caches attempt-private without changing Playwright's browser + // cache lookup in the parent process. + XDG_CACHE_HOME: path.join(temporaryRoot, "xdg-cache"), PAPERCLIP_HOME: paperclipHome, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: required("PAPERCLIP_INSTANCE_ID"), diff --git a/tests/runner-e2e/source.test.ts b/tests/runner-e2e/source.test.ts new file mode 100644 index 0000000000..969a5e4bbc --- /dev/null +++ b/tests/runner-e2e/source.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { resolveRunnerE2ESource } from "./source.js"; + +describe("runner E2E source provenance", () => { + it("prefers the resolved target over result and workflow revision contexts", () => { + expect( + resolveRunnerE2ESource( + { + sha: "result-sha", + ref: "refs/heads/result", + workflowRunUrl: "https://example.test/result-run", + }, + { + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "target-sha", + PAPERCLIP_RUNNER_E2E_SOURCE_REF: "refs/heads/target", + GITHUB_SHA: "workflow-sha", + GITHUB_REF: "refs/heads/master", + GITHUB_SERVER_URL: "https://github.com", + GITHUB_REPOSITORY: "paperclipai/paperclip", + GITHUB_RUN_ID: "123", + }, + ), + ).toEqual({ + sha: "target-sha", + ref: "refs/heads/target", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/123", + }); + }); + + it("falls back through retained result provenance, workflow context, and null", () => { + expect( + resolveRunnerE2ESource( + { + sha: "result-sha", + ref: null, + workflowRunUrl: null, + }, + { + GITHUB_SHA: "workflow-sha", + GITHUB_REF: "refs/heads/master", + GITHUB_SERVER_URL: "https://github.com", + GITHUB_REPOSITORY: "paperclipai/paperclip", + GITHUB_RUN_ID: "456", + }, + ), + ).toEqual({ + sha: "result-sha", + ref: "refs/heads/master", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/456", + }); + expect(resolveRunnerE2ESource(null, {})).toEqual({ + sha: null, + ref: null, + workflowRunUrl: null, + }); + }); +}); diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts index 015a7c83c7..c1eb609a11 100644 --- a/tests/runner-e2e/support.test.ts +++ b/tests/runner-e2e/support.test.ts @@ -10,9 +10,12 @@ import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js"; import { assertIsolatedServerEnvironment, buildPaperclipServerEnvironment, + buildRunnerE2EProcessEnvironment, + resolvePaperclipRemoteRunnerBinaryForHarness, resolvePaperclipRunnerBinaryForHarness, + runnerE2EServerControlPaths, } from "./harness-env.js"; -import { runnerExecutionById } from "./catalog.js"; +import { runnerExecutionById, runnerMatrix } from "./catalog.js"; import { assertEmbeddedDatabaseIsolation } from "./instance-isolation.js"; import { evaluateMatchers } from "./matchers.js"; import { @@ -26,8 +29,17 @@ import { } from "./redaction.js"; import { parseDarwinSharedMemory } from "./shared-memory.js"; import { + reserveRunnerE2EServerPort, + runnerE2EServerPortConflictsWithDatabase, + type LoopbackPortReservation, +} from "./ports.js"; +import { + acceptedPlanSessionResetFailures, + isControlPlaneGovernedResponseWait, isNonExecutingReviewFenceRun, + isOpenRouterDeepSeekHelloTerminalVariance, numberedPlanStepCount, + providerSessionContinuityFailures, } from "./run-observations.js"; import { runnerE2EWebServerCommand } from "./web-server-command.js"; @@ -46,6 +58,9 @@ describe("runner E2E local binary resolution", () => { const localNativeExecution = runnerExecutionById( "core-compatibility.runner-codex.local.message-marker", ); + const remoteNativeExecution = runnerExecutionById( + "core-compatibility.runner-acpx-claude.daytona.message-marker", + ); it("uses the debug runner binary built by the E2E workflow", () => { expect( @@ -73,6 +88,98 @@ describe("runner E2E local binary resolution", () => { ), ).toBe("/custom/paperclip-runnerd"); }); + + it("uses and stages the same build-once binary for remote native cells", () => { + const runnerBinary = resolvePaperclipRunnerBinaryForHarness( + [remoteNativeExecution], + "/repository", + undefined, + "linux", + ); + expect(runnerBinary).toBe( + path.join( + "/repository", + "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", + ), + ); + expect( + resolvePaperclipRemoteRunnerBinaryForHarness( + [remoteNativeExecution], + runnerBinary, + ), + ).toBe(runnerBinary); + expect( + resolvePaperclipRemoteRunnerBinaryForHarness( + [localNativeExecution], + runnerBinary, + ), + ).toBeUndefined(); + }); +}); + +describe("runner E2E provider environment", () => { + const legacyLocal = runnerExecutionById( + "core-compatibility.legacy-opencode.local.message-marker", + ); + const legacyDaytona = runnerExecutionById( + "core-compatibility.legacy-opencode.daytona.message-marker", + ); + const nativeOpenCode = runnerExecutionById( + "core-compatibility.runner-opencode.local.message-marker", + ); + const breadthOpenCode = runnerMatrix.find( + (execution) => execution.suite.id === "openrouter-model-breadth", + )!; + + it("allows the pinned model only for isolated legacy OpenCode harnesses", () => { + for (const execution of [legacyLocal, legacyDaytona]) { + expect( + buildRunnerE2EProcessEnvironment( + { KEEP_ME: "yes", OPENCODE_ALLOW_ALL_MODELS: "ambient" }, + [execution], + ), + ).toEqual({ KEEP_ME: "yes", OPENCODE_ALLOW_ALL_MODELS: "true" }); + } + + for (const execution of [nativeOpenCode, breadthOpenCode]) { + expect( + buildRunnerE2EProcessEnvironment( + { KEEP_ME: "yes", OPENCODE_ALLOW_ALL_MODELS: "ambient" }, + [execution], + ), + ).toEqual({ KEEP_ME: "yes" }); + } + }); +}); + +describe("runner E2E server port allocation", () => { + it("rejects direct and derived embedded-Postgres collisions", () => { + expect(runnerE2EServerPortConflictsWithDatabase(44_329)).toBe(true); + expect(runnerE2EServerPortConflictsWithDatabase(54_329)).toBe(true); + expect(runnerE2EServerPortConflictsWithDatabase(64_329)).toBe(true); + expect(runnerE2EServerPortConflictsWithDatabase(43_123)).toBe(false); + }); + + it("retries a derived collision while closing every reservation", async () => { + const closed: number[] = []; + const ports = [44_329, 43_123, 53_123]; + const openPort = vi.fn(async (requestedPort: number) => { + const port = requestedPort === 0 ? ports.shift() : requestedPort; + if (port === undefined) throw new Error("Missing fake port"); + return { + port, + close: async () => { + closed.push(port); + }, + } satisfies LoopbackPortReservation; + }); + + await expect(reserveRunnerE2EServerPort({ openPort })).resolves.toBe( + 43_123, + ); + expect(openPort.mock.calls.map(([port]) => port)).toEqual([0, 0, 53_123]); + expect(closed).toEqual([44_329, 53_123, 43_123]); + }); }); describe("runner E2E sensitive API boundary", () => { @@ -266,6 +373,71 @@ describe("runner E2E matchers", () => { }); describe("runner E2E run observations", () => { + it("retries only the zero-marker DeepSeek hello terminal emission variance", () => { + const expectedMarker = "PC_H_nonce-1"; + const observation = { + suiteId: "openrouter-model-breadth", + profileId: "openrouter-deepseek-deepseek-v4-flash-0731", + taskId: "hello-complete", + expectedMarker, + finalRunMessage: + "I'll complete this deterministic hello task by calling paperclip_finish once.", + allAgentMessages: + "I'll complete this deterministic hello task by calling paperclip_finish once.", + semanticSummary: expectedMarker, + issueStatus: "done", + runStatuses: ["succeeded"], + matcherResults: [ + { + matcher: { kind: "message_exact", expected: expectedMarker }, + passed: false, + }, + { + matcher: { + kind: "message_occurrences", + expected: expectedMarker, + count: 1, + }, + passed: false, + }, + { matcher: { kind: "issue_status", expected: "done" }, passed: true }, + ], + invariantFailures: [], + }; + + expect(isOpenRouterDeepSeekHelloTerminalVariance(observation)).toBe(true); + expect( + isOpenRouterDeepSeekHelloTerminalVariance({ + ...observation, + allAgentMessages: `${expectedMarker}\n${expectedMarker}`, + }), + ).toBe(false); + expect( + isOpenRouterDeepSeekHelloTerminalVariance({ + ...observation, + semanticSummary: "different-summary", + }), + ).toBe(false); + expect( + isOpenRouterDeepSeekHelloTerminalVariance({ + ...observation, + invariantFailures: ["missing native terminal event"], + }), + ).toBe(false); + expect( + isOpenRouterDeepSeekHelloTerminalVariance({ + ...observation, + matcherResults: [ + ...observation.matcherResults, + { + matcher: { kind: "environment", expected: "local" }, + passed: false, + }, + ], + }), + ).toBe(false); + }); + it("counts provider-equivalent numbered Plan step formats", () => { expect(numberedPlanStepCount("1. First\n2) Second")).toBe(2); expect( @@ -298,6 +470,139 @@ describe("runner E2E run observations", () => { }), ).toBe(false); }); + + it("recognizes only authoritative control-plane governed response waits", () => { + const event = { + eventType: "run.result.accepted", + payload: { + prpEvent: { + schema: "paperclip.prp.event.v1", + eventType: "run.result.accepted", + sourceKind: "control_plane", + payload: { + result: { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + evidence: [{ ref: "interaction:pending" }], + artifacts: [ + { + kind: "issue_thread_interaction", + ref: "interaction:pending", + }, + ], + continuation: { + kind: "response_wake", + idempotencyKey: "interaction-response:pending", + }, + }, + }, + }, + }, + }; + expect(isControlPlaneGovernedResponseWait([event])).toBe(true); + expect( + isControlPlaneGovernedResponseWait([ + { + ...event, + payload: { + prpEvent: { + ...event.payload.prpEvent, + sourceKind: "runner", + }, + }, + }, + ]), + ).toBe(false); + expect( + isControlPlaneGovernedResponseWait([ + { + ...event, + payload: { + prpEvent: { + ...event.payload.prpEvent, + payload: { + result: { + ...event.payload.prpEvent.payload.result, + artifacts: [], + }, + }, + }, + }, + }, + ]), + ).toBe(false); + expect( + isControlPlaneGovernedResponseWait([ + event, + { ...event, payload: structuredClone(event.payload) }, + ]), + ).toBe(false); + }); + + it("requires continuity except for an explicit accepted-Plan reset", () => { + const initial = { + id: "initial", + sessionIdBefore: null, + sessionIdAfter: "session-one", + }; + const resumed = { + id: "resumed", + sessionIdBefore: "session-one", + sessionIdAfter: "session-one", + }; + const acceptedPlan = { + id: "accepted-plan", + sessionIdBefore: null, + sessionIdAfter: "session-two", + contextSnapshot: { + forceFreshSession: true, + workspaceRefreshReason: "accepted_plan_confirmation", + source: "issue.interaction.accept", + interactionStatus: "accepted", + }, + }; + expect( + providerSessionContinuityFailures("codex", [ + initial, + resumed, + acceptedPlan, + ]), + ).toEqual([]); + expect( + providerSessionContinuityFailures("codex", [ + initial, + { ...acceptedPlan, sessionIdAfter: "session-one" }, + ]), + ).toEqual([ + "expected accepted Plan run accepted-plan to rotate the codex provider session", + ]); + expect( + providerSessionContinuityFailures("codex", [ + initial, + { ...resumed, sessionIdAfter: "session-two" }, + ]), + ).toEqual([ + "expected codex to preserve its provider session for run resumed", + ]); + expect( + acceptedPlanSessionResetFailures( + "acpx", + initial.sessionIdAfter, + acceptedPlan, + ), + ).toEqual([]); + expect( + acceptedPlanSessionResetFailures("acpx", initial.sessionIdAfter, { + ...acceptedPlan, + sessionIdBefore: initial.sessionIdAfter, + }), + ).toEqual([ + "expected accepted Plan run accepted-plan to start without a prior provider session", + ]); + expect( + acceptedPlanSessionResetFailures("acpx", initial.sessionIdAfter, resumed), + ).toBeNull(); + }); }); describe("runner E2E failure policy", () => { @@ -320,6 +625,7 @@ describe("runner E2E failure policy", () => { expect( shouldRetryFailure(classifyFailure(new Error("marker matcher failed"))), ).toBe(false); + expect(shouldRetryFailure("provider_variance")).toBe(true); expect( classifyFailure( new Error( @@ -354,6 +660,22 @@ describe("runner E2E failure policy", () => { }); describe("runner E2E server isolation", () => { + it("shares restart control files beneath the isolated temporary root", () => { + expect(runnerE2EServerControlPaths("/tmp/cell")).toEqual({ + controlDirectory: path.join("/tmp/cell", "control"), + restartRequestPath: path.join( + "/tmp/cell", + "control", + "server-restart.request.json", + ), + restartAcknowledgementPath: path.join( + "/tmp/cell", + "control", + "server-restart.ack.json", + ), + }); + }); + it("strips database and paid-provider credentials from the Paperclip process", () => { const env = buildPaperclipServerEnvironment( { @@ -376,6 +698,7 @@ describe("runner E2E server isolation", () => { { PAPERCLIP_HOME: "/tmp/cell/paperclip-home", PAPERCLIP_CONFIG: "/tmp/cell/paperclip-home/instances/e2e/config.json", + XDG_CACHE_HOME: "/tmp/cell/xdg-cache", PAPERCLIP_AGENT_JWT_SECRET: "generated-agent-jwt", PAPERCLIP_DECISION_SIGNING_SECRET: "generated-decision-key", PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: "generated-tool-key", @@ -388,6 +711,7 @@ describe("runner E2E server isolation", () => { expect(env.OPENAI_ORG_ID).toBeUndefined(); expect(env.PAPERCLIP_API_KEY).toBeUndefined(); expect(env.PAPERCLIP_AGENT_API_KEY).toBeUndefined(); + expect(env.XDG_CACHE_HOME).toBe("/tmp/cell/xdg-cache"); expect(env.PAPERCLIP_AGENT_JWT_SECRET).toBe("generated-agent-jwt"); expect(env.PAPERCLIP_TASK_BRIDGE_TOKEN).toBeUndefined(); expect(env.PAPERCLIP_SETUP_TOKEN).toBeUndefined(); @@ -543,7 +867,7 @@ describe("runner E2E evidence redaction", () => { ).resolves.toMatchObject({ file: forbiddenConfig }); }); - it("recognizes both managed and durable-session Codex runtime auth files", () => { + it("recognizes managed and durable Codex runtime auth files", () => { const root = path.join(os.tmpdir(), "paperclip-home"); expect( isEphemeralCodexRuntimeAuthFile( @@ -554,6 +878,15 @@ describe("runner E2E evidence redaction", () => { ), ), ).toBe(true); + expect( + isEphemeralCodexRuntimeAuthFile( + root, + path.join( + root, + "instances/instance-1/runtime/paperclip-runner/acpx/acpx/session-1/codex-home/auth.json", + ), + ), + ).toBe(true); expect( isEphemeralCodexRuntimeAuthFile( root, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index ffc3e671bd..3191832682 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -174,6 +174,7 @@ export interface MatrixJob { export type FailureClass = | "candidate_failure" + | "provider_variance" | "transient_infrastructure" | "permanent_infrastructure" | "secret_leak" diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index bac160ea97..ee46454dd1 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -186,6 +186,10 @@ describe("public repository paid workflow security", () => { fullStack.indexOf(" target_lock:"), fullStack.indexOf(" catalog:"), ); + const daytonaImageJob = fullStack.slice( + fullStack.indexOf(" daytona_image:"), + fullStack.indexOf(" build_runner_artifacts:"), + ); expect(authorizeJob).toContain( "aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'", ); @@ -193,6 +197,11 @@ describe("public repository paid workflow security", () => { expect(authorizeJob).toContain( "AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}", ); + expect(authorizeJob).toContain( + "playwright_channel: ${{ steps.runner.outputs.playwright_channel }}", + ); + expect(authorizeJob).toContain('echo "playwright_channel=chrome"'); + expect(authorizeJob).toContain('echo "playwright_channel="'); expect(authorizeJob).toContain( "Resolve requested repository branch to an immutable commit", ); @@ -234,6 +243,23 @@ describe("public repository paid workflow security", () => { expect(paidJob).toMatch( /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/, ); + expect(paidJob).toContain( + "pnpm exec playwright install --with-deps --only-shell chromium", + ); + expect(paidJob).toContain("name: Qualify preinstalled Chrome"); + expect(paidJob).toContain( + "if: needs.authorize.outputs.playwright_channel == 'chrome'", + ); + expect(paidJob).toContain("google-chrome --version"); + expect(paidJob).toContain( + "if: needs.authorize.outputs.playwright_channel != 'chrome'", + ); + expect(paidJob).toContain( + "PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }}", + ); + expect(paidJob).not.toContain( + "pnpm exec playwright install --with-deps chromium", + ); const paidInstall = paidJob.indexOf( "pnpm install --frozen-lockfile --ignore-scripts", ); @@ -282,11 +308,30 @@ describe("public repository paid workflow security", () => { '[ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]', ); expect(fullStack).toContain( - "group: runner-full-stack-e2e-${{ inputs.target_branch || github.event.repository.default_branch }}", + "group: runner-full-stack-e2e-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch && format('development-{0}', inputs.target_branch) || format('protected-{0}', github.run_id) }}", ); expect(fullStack).toContain( "cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}", ); + expect(daytonaImageJob).toMatch( + /- if: needs\.catalog\.outputs\.needs_daytona == 'true'\n\s+uses: actions\/checkout@[0-9a-f]{40}/u, + ); + for (const stepName of [ + "Download resolved target lockfile", + "Restore resolved target lockfile", + ]) { + expect(daytonaImageJob).toMatch( + new RegExp( + `- name: ${stepName}\\n\\s+if: needs\\.catalog\\.outputs\\.needs_daytona == 'true'`, + "u", + ), + ); + } + expect(daytonaImageJob).toMatch( + /- name: No Daytona image needed\n\s+id: local_only\n\s+if: needs\.catalog\.outputs\.needs_daytona != 'true'/u, + ); + expect(daytonaImageJob).toContain('echo "source_revision="'); + expect(daytonaImageJob).toContain('echo "content_id="'); const targetCodeJobs = [ fullStack.slice( fullStack.indexOf(" catalog:"), @@ -364,6 +409,9 @@ describe("public repository paid workflow security", () => { "ref: ${{ needs.authorize.outputs.target_sha }}", ); expect(historyJob).not.toContain("Download resolved target lockfile"); + expect(fullStack).toContain( + "if: always() && !cancelled() && needs.catalog.result == 'success'", + ); for (const targetProvenanceJob of [paidJob, reportJob]) { expect(targetProvenanceJob).toContain( "PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}", @@ -487,6 +535,11 @@ describe("public repository paid workflow security", () => { expect(testJob).toMatch(fullStackTestNeeds); expect(testJob).toContain("Download immutable campaign outputs"); + expect( + testJob.match( + /if: startsWith\(matrix\.profileId, 'runner-'\) \|\| matrix\.suiteId == 'openrouter-model-breadth'/gu, + ), + ).toHaveLength(2); expect(testJob).toContain("Download immutable remote provider pack"); expect(testJob).toContain( "needs.build_runner_artifacts.outputs.build_artifact_name", @@ -519,6 +572,70 @@ describe("public repository paid workflow security", () => { expect(testJob).not.toContain("build-provider-pack.mjs"); }); + it("uses the reviewed AWS Chrome channel without weakening the local executable override", async () => { + const config = await readFile( + path.join(repositoryRoot, "tests/runner-e2e/playwright.config.ts"), + "utf8", + ); + + expect(config).toContain( + "process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL?.trim()", + ); + expect(config).toContain("{ channel: playwrightChannel }"); + expect(config).toContain( + "process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?.trim()", + ); + expect(config).toContain( + "PAPERCLIP_PLAYWRIGHT_CHANNEL and PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE are mutually exclusive", + ); + }); + + it("binds rerun evidence and Pages artifacts to the exact workflow attempt", async () => { + const workflow = await readFile( + path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), + "utf8", + ); + const reportStart = workflow.indexOf(" report:"); + const publisherStart = workflow.indexOf(" publish_history:", reportStart); + const report = workflow.slice(reportStart, publisherStart); + const publisher = workflow.slice(publisherStart); + + expect(report).toContain("actions: read"); + expect(report).toContain( + '"repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?filter=all&per_page=100"', + ); + expect(report).toContain("gh api --paginate --slurp"); + expect(report).toContain( + '"repos/$REPOSITORY/actions/runs/$RUN_ID/attempts/$attempt"', + ); + expect(report).toContain("--jq '{run_attempt, run_started_at}'"); + expect(report).toContain("pattern: runner-e2e-${{ github.run_id }}-*-*"); + expect(report).not.toContain( + "pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*", + ); + expect(report.match(/merge-multiple: false/g)).toHaveLength(2); + expect(report).toContain("Select latest workflow attempt per cell"); + expect(report).toContain("tests/runner-e2e/select-rerun-artifacts.ts"); + expect(report).toContain( + "PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e", + ); + expect( + report.indexOf("Select latest workflow attempt per cell"), + ).toBeLessThan(report.indexOf("Collect blob reports")); + expect(publisher).toContain( + 'echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}"', + ); + expect(publisher).toContain( + "pages_artifact_name: ${{ steps.pages_artifact_name.outputs.name }}", + ); + expect(publisher).toContain( + "name: ${{ steps.pages_artifact_name.outputs.name }}", + ); + expect(publisher).toContain( + "artifact_name: ${{ needs.publish_history.outputs.pages_artifact_name }}", + ); + }); + it("uses environment-scoped OIDC for a no-delete history publisher", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), diff --git a/ui/src/components/transcript/native-run-events.test.ts b/ui/src/components/transcript/native-run-events.test.ts index fa366f84c7..4e7455ee2c 100644 --- a/ui/src/components/transcript/native-run-events.test.ts +++ b/ui/src/components/transcript/native-run-events.test.ts @@ -178,6 +178,144 @@ describe("nativeRunEventsToTranscript", () => { ]); }); + it("coalesces sparse Codex tool lifecycle events at the named write boundary", () => { + const transcript = nativeRunEventsToTranscript([ + event(1, "tool.execution.started", { + schema: "paperclip.tool.execution.v1", + executionId: "exec-write-plan", + transport: "dynamic", + operation: "unknown", + name: null, + status: "running", + output: null, + }), + itemEvent(2, "item.started", "exec-write-plan", { + kind: "dynamicToolCall", + item: { id: "exec-write-plan" }, + }), + itemEvent(3, "item.started", "exec-write-plan", { + kind: "dynamicToolCall", + item: { + type: "tool_use", + id: "exec-write-plan", + name: "write_document", + input: { + key: "plan", + title: "Plan", + body: "Ship the durable runner.", + }, + }, + }), + itemEvent(4, "item.completed", "exec-write-plan", { + kind: "dynamicToolCall", + item: { + type: "tool_result", + id: "exec-write-plan", + tool_use_id: "exec-write-plan", + result: { + commandKind: "write_document", + revisionId: "revision-1", + }, + }, + }), + event(5, "tool.execution.completed", { + schema: "paperclip.tool.execution.v1", + executionId: "exec-write-plan", + transport: "dynamic", + operation: "unknown", + name: null, + status: "completed", + output: null, + }), + itemEvent(6, "item.completed", "exec-write-plan", { + kind: "dynamicToolCall", + item: { id: "exec-write-plan", status: "completed" }, + }), + ]); + + expect(transcript).toEqual([ + expect.objectContaining({ + kind: "tool_call", + name: "write_document", + toolUseId: "exec-write-plan", + input: { + key: "plan", + title: "Plan", + body: "Ship the durable runner.", + }, + }), + expect.objectContaining({ + kind: "tool_result", + toolUseId: "exec-write-plan", + toolName: "write_document", + content: JSON.stringify({ + commandKind: "write_document", + revisionId: "revision-1", + }), + isError: false, + }), + ]); + }); + + it("projects ACPX item-only tool lifecycles at the named write boundary", () => { + const transcript = nativeRunEventsToTranscript([ + itemEvent(1, "item.started", "2", { + kind: "dynamicToolCall", + item: { + type: "tool_use", + id: "2", + name: "write_document", + input: { + key: "plan", + title: "Plan", + body: "Ship the durable runner.", + }, + }, + }), + itemEvent(2, "item.completed", "2", { + kind: "dynamicToolCall", + item: { + type: "tool_result", + id: "2", + tool_use_id: "2", + result: { + disposition: "applied", + document: { + key: "plan", + latestRevisionId: "revision-1", + }, + }, + }, + }), + ]); + + expect(transcript).toEqual([ + expect.objectContaining({ + kind: "tool_call", + name: "write_document", + toolUseId: "2", + input: { + key: "plan", + title: "Plan", + body: "Ship the durable runner.", + }, + }), + expect.objectContaining({ + kind: "tool_result", + toolUseId: "2", + toolName: "write_document", + content: JSON.stringify({ + disposition: "applied", + document: { + key: "plan", + latestRevisionId: "revision-1", + }, + }), + isError: false, + }), + ]); + }); + it("streams deltas until a loss-resistant completed item is available", () => { expect(nativeRunEventsToTranscript([ event(1, "item.delta", { itemId: "message-1", kind: "agentMessage", text: "Still " }), diff --git a/ui/src/components/transcript/native-run-events.ts b/ui/src/components/transcript/native-run-events.ts index 67d3056fa9..2796d5bcfe 100644 --- a/ui/src/components/transcript/native-run-events.ts +++ b/ui/src/components/transcript/native-run-events.ts @@ -503,10 +503,58 @@ function timestamp(event: HeartbeatRunEvent, envelope: Record): return Number.isNaN(Date.parse(createdAt)) ? new Date(0).toISOString() : createdAt; } -function toolPresentation(payload: Record): { name: string; input: unknown } { +interface NativeToolItemDetails { + name: string | null; + input?: unknown; + result?: unknown; + isError: boolean; +} + +function nativeToolItemDetails( + payload: Record, +): { + id: string | null; + kind: "tooluse" | "toolresult"; + details: NativeToolItemDetails; +} | null { + const item = normalizedItem(payload); + const kind = (text(item.type) ?? "").replaceAll("_", "").toLowerCase(); + if (kind !== "tooluse" && kind !== "toolresult") return null; + const id = kind === "toolresult" + ? text(item.tool_use_id) ?? text(item.id) + : text(item.id); + return { + id, + kind, + details: { + name: text(item.name), + ...(Object.prototype.hasOwnProperty.call(item, "input") + ? { input: item.input } + : {}), + ...(Object.prototype.hasOwnProperty.call(item, "result") + ? { result: item.result } + : {}), + isError: item.isError === true || item.is_error === true, + }, + }; +} + +function serializedNativeToolResult(item: NativeToolItemDetails): string { + if (item.result === undefined) return ""; + try { + return JSON.stringify(item.result) ?? ""; + } catch { + return "Tool result could not be serialized"; + } +} + +function toolPresentation( + payload: Record, + item?: NativeToolItemDetails, +): { name: string; input: unknown } { const transport = text(payload.transport); const operation = text(payload.operation); - const reportedName = text(payload.name); + const reportedName = text(payload.name) ?? item?.name ?? null; if (transport === "process") { return { name: "Bash", @@ -515,7 +563,7 @@ function toolPresentation(payload: Record): { name: string; inp } return { name: reportedName ?? operation ?? "Tool", - input: { + input: item?.input ?? { ...(operation ? { operation } : {}), ...(text(payload.namespace) ? { namespace: text(payload.namespace) } : {}), ...(text(payload.target) ? { target: text(payload.target) } : {}), @@ -531,6 +579,7 @@ function toolPresentation(payload: Record): { name: string; inp export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[]): TranscriptEntry[] { const entries: TranscriptEntry[] = []; const startedToolIds = new Set(); + const completedToolIds = new Set(); let hasFinalAssistantMessage = false; let usageSummary: { ts: string; @@ -569,6 +618,7 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] const completedAgentMessageIds = new Set(); const completedReasoningIds = new Set(); const completionItemIdentityById = new Map(); + const nativeToolItemsById = new Map(); for (const event of orderedEvents) { if (!isItemIdentityEvent(event.eventType)) continue; const envelope = record(event.payload?.prpEvent); @@ -583,6 +633,25 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] if (!payload) continue; const itemId = normalizedItemId(envelope, payload); if (!itemId) continue; + const toolItem = nativeToolItemDetails(payload); + if (toolItem) { + const toolId = toolItem.id ?? itemId; + const previous = nativeToolItemsById.get(toolId); + nativeToolItemsById.set(toolId, { + name: toolItem.details.name ?? previous?.name ?? null, + ...(toolItem.details.input !== undefined + ? { input: toolItem.details.input } + : previous?.input !== undefined + ? { input: previous.input } + : {}), + ...(toolItem.details.result !== undefined + ? { result: toolItem.details.result } + : previous?.result !== undefined + ? { result: previous.result } + : {}), + isError: toolItem.details.isError || previous?.isError === true, + }); + } const identity = resolveItemIdentity( payload, completionItemIdentityById.get(itemId), @@ -671,6 +740,50 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] continue; } + // Some provider transports expose their complete dynamic-tool lifecycle + // directly as item.started/item.completed events and do not emit the + // parallel tool.execution.* activity stream. Project those canonical + // tool items at their own stable item boundary so saved documents can be + // embedded beside the write that created them instead of falling back to + // the end of the run timeline. + const nativeToolEvent = isItemIdentityEvent(event.eventType) + ? nativeToolItemDetails(payload) + : null; + if (nativeToolEvent) { + const toolId = nativeToolEvent.id ?? itemId; + if (!toolId) continue; + const nativeToolItem = nativeToolItemsById.get(toolId) + ?? nativeToolEvent.details; + const presentation = toolPresentation({}, nativeToolItem); + if (!startedToolIds.has(toolId)) { + startedToolIds.add(toolId); + entries.push({ + kind: "tool_call", + ts, + name: presentation.name, + input: presentation.input, + toolUseId: toolId, + }); + } + if ( + event.eventType === "item.completed" + && (nativeToolEvent.kind === "toolresult" + || nativeToolEvent.details.result !== undefined) + && !completedToolIds.has(toolId) + ) { + completedToolIds.add(toolId); + entries.push({ + kind: "tool_result", + ts, + toolUseId: toolId, + toolName: presentation.name, + content: serializedNativeToolResult(nativeToolItem), + isError: nativeToolItem.isError, + }); + } + continue; + } + const providerActivity = providerActivityPresentation(event, payload); if (providerActivity) { if (!startedToolIds.has(providerActivity.id)) { @@ -700,7 +813,8 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] if (payload.schema !== TOOL_EXECUTION_SCHEMA) continue; const executionId = text(payload.executionId); if (!executionId) continue; - const presentation = toolPresentation(payload); + const nativeToolItem = nativeToolItemsById.get(executionId); + const presentation = toolPresentation(payload, nativeToolItem); if (!startedToolIds.has(executionId)) { startedToolIds.add(executionId); entries.push({ @@ -711,14 +825,19 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] toolUseId: executionId, }); } - if (event.eventType === "tool.execution.completed") { + if (event.eventType === "tool.execution.completed" && !completedToolIds.has(executionId)) { + completedToolIds.add(executionId); + const output = text(payload.output); + const content = output ?? (nativeToolItem + ? serializedNativeToolResult(nativeToolItem) + : ""); entries.push({ kind: "tool_result", ts, toolUseId: executionId, toolName: presentation.name, - content: text(payload.output) ?? "", - isError: payload.status === "failed", + content, + isError: payload.status === "failed" || nativeToolItem?.isError === true, }); } continue;