diff --git a/.github/actions/setup-ci-dependencies/action.yml b/.github/actions/setup-ci-dependencies/action.yml index 3367001b4b5..0a78e542c91 100644 --- a/.github/actions/setup-ci-dependencies/action.yml +++ b/.github/actions/setup-ci-dependencies/action.yml @@ -65,6 +65,14 @@ runs: pnpm install --frozen-lockfile fi + - name: Replay workspace install hooks after restore + if: steps.installed.outputs.cache-hit == 'true' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + pnpm -r --if-present run postinstall + pnpm run postinstall + - name: Restore certified shared build outputs (read only) if: inputs.restore-cache == 'true' && steps.inputs.outputs.cacheable == 'true' id: builds diff --git a/.github/scripts/tests/ci-dependency-cache.test.mjs b/.github/scripts/tests/ci-dependency-cache.test.mjs index 1b79b00c401..36195ad32e9 100644 --- a/.github/scripts/tests/ci-dependency-cache.test.mjs +++ b/.github/scripts/tests/ci-dependency-cache.test.mjs @@ -71,6 +71,36 @@ test('unmodelled workspace lifecycle scripts disable materialized reuse', () => } }); +test('known Copilot hooks permit caching, track their helpers, and still run on restore', () => { + const files = { ...entries, + 'server/package.json': JSON.stringify({ scripts: { postinstall: 'node scripts/install-copilot-assets.mjs' } }), + 'packages/paperclip-runner/package.json': JSON.stringify({ scripts: { postinstall: 'node scripts/install-copilot-assets.mjs' } }), + 'server/scripts/install-copilot-assets.mjs': 'server bootstrap', + 'packages/paperclip-runner/scripts/install-copilot-assets.mjs': 'runner bootstrap', + 'packages/paperclip-runner/scripts/materialize-copilot-binary.mjs': 'native materializer', + 'packages/paperclip-runner/scripts/copilot-inner-distribution.mjs': 'closure pins', + }; + const compute = (input) => cachePlan({ files: Object.keys(input), + read: file => input[file] == null ? null : Buffer.from(input[file]), scope, + packageDirs: ['.', 'server', 'packages/paperclip-runner'], + }); + const original = compute(files); + assert.equal(original.cacheable, true); + for (const file of Object.keys(files).filter(file => /^(server|packages\/paperclip-runner)\/scripts\//.test(file))) { + for (const replacement of ['changed', null]) { + assert.notEqual(compute({ ...files, [file]: replacement }).dependencyKey, original.dependencyKey, file); + } + } + assert.equal(cachePlan({ files: ['elsewhere/package.json'], + read: () => Buffer.from(files['server/package.json']), scope, packageDirs: ['elsewhere'], + }).cacheable, false, 'the command is allowed only in its reviewed owners'); + const action = readFileSync(new URL('../../actions/setup-ci-dependencies/action.yml', import.meta.url), 'utf8'); + const replay = action.split(' - name: Replay workspace install hooks after restore')[1].split(' - name:')[0]; + assert.match(replay, /if: steps\.installed\.outputs\.cache-hit == 'true'/); + assert.match(replay, /pnpm -r --if-present run postinstall\s+pnpm run postinstall/); + assert.ok(action.indexOf('Replay workspace install hooks') < action.indexOf('Validate workspace links')); +}); + test('cache restores are exact and only a frozen canonical master push can publish entries', () => { const action = readFileSync(new URL('../../actions/setup-ci-dependencies/action.yml', import.meta.url), 'utf8'); const installed = action.slice(action.indexOf(' - name: Restore installed'), action.indexOf(' - name: Locate pnpm')); @@ -97,6 +127,15 @@ test('paired test timings remain available when lifecycle hooks disable material assert.match(step(name), /if: steps.qualification.outputs.cacheable == 'true'/); } assert.match(step('Explain disabled materialized reuse'), /cacheable != 'true'/); - assert.doesNotMatch(step('Prepare unchanged baseline'), /if:/); - assert.doesNotMatch(step('Compare complete commands and executed case inventories'), /if:/); + for (const name of ['Prepare unchanged baseline', 'Compare complete commands and executed case inventories']) { + assert.match(step(name), /if: steps\.test_scope\.outputs\.run == 'true'/); + assert.doesNotMatch(step(name), /if:.*cacheable/); + } + const scope = step('Select changed server test work'); + assert.match(scope, /git diff --name-only HEAD\^1 HEAD --/); + for (const file of ['scripts/benchmark-ci-work.mjs', 'scripts/run-vitest-stable.mjs', + 'server/src/__tests__/setup-supertest.ts', 'server/src/__tests__/workspace-runtime.test.ts', + 'server/src/services/workspace-git-snapshot-streaming.test.ts']) assert.ok(scope.includes(file)); + assert.doesNotMatch(scope, /scripts\/ci-dependency-cache|\.github\/actions\/setup-ci-dependencies/); + assert.match(step('Exercise restored dependencies for cache-only changes'), /pnpm exec vitest run --project @paperclipai\/shared/); }); diff --git a/.github/scripts/tests/post-merge-runner-routing.test.mjs b/.github/scripts/tests/post-merge-runner-routing.test.mjs index 7c46aa47931..0e905635214 100644 --- a/.github/scripts/tests/post-merge-runner-routing.test.mjs +++ b/.github/scripts/tests/post-merge-runner-routing.test.mjs @@ -10,7 +10,8 @@ const base = { ref: "refs/heads/master", event_name: "push", sha, run_attempt: 1, }; const expectedJobs = { - "cloud-readiness.yml": [], + "docker.yml": ["source_verification", "publication_verification"], + "cloud-readiness.yml": ["admission", "source_verified"], "cloud-migrator-artifacts.yml": [], "release-verify.yml": ["typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"], "runner-chaos-evals.yml": ["chaos_and_recovery"], @@ -59,9 +60,9 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) { cases.push({ name: "stable release", inputs: { channel: "stable" } }); } cases.push({ name: "first attempt as a string", github: { run_attempt: "1" }, expected: fleet }); - cases.push({ name: "rerun after runner interruption", github: { run_attempt: 2 } }); - cases.push({ name: "later rerun", github: { run_attempt: 3 } }); - cases.push({ name: "rerun attempt as a string", github: { run_attempt: "2" } }); + cases.push({ name: "admitted retry", github: { run_attempt: 2 }, expected: release ? "ubuntu-latest" : fleet }); + cases.push({ name: "later retry", github: { run_attempt: 3 }, expected: release ? "ubuntu-latest" : fleet }); + cases.push({ name: "retry attempt as a string", github: { run_attempt: "2" }, expected: release ? "ubuntu-latest" : fleet }); cases.push({ name: "missing attempt", github: { run_attempt: undefined } }); for (const { name, github = {}, inputs: overrides = {}, enabled = "true", expected = "ubuntu-latest" } of cases) { test(`${file}/${job}: ${name}`, () => { @@ -88,7 +89,7 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) { } -test("Cloud readiness bookkeeping never waits for the AWS verification fleet", () => { +test("Cloud source proof keeps source-only gating on protected verification capacity", () => { const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8"); const bodies = new Map(); for (const [name, needs] of [ @@ -97,7 +98,7 @@ test("Cloud readiness bookkeeping never waits for the AWS verification fleet", ( const body = workflow.match(new RegExp(`^ ${name}:\\n([\\s\\S]*?)(?=^ [a-z_]+:|(?![\\s\\S]))`, "m"))?.[1]; assert.ok(body, `missing ${name} job`); bodies.set(name, body); - assert.match(body, /^ runs-on: ubuntu-latest$/m); + assert.match(body, /runs-on:.*paperclip-post-merge-x64/); assert.doesNotMatch(body, /^ +continue-on-error:|^ +if:.*always\(\)/m); assert.match(body, /^ if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'$/m); assert.match(body, /^ +SOURCE_SHA: \$\{\{ github.sha \}\}$/m); diff --git a/.github/scripts/tests/pr-runner-rust-cache.test.mjs b/.github/scripts/tests/pr-runner-rust-cache.test.mjs index 6089bcec638..1136e5cf150 100644 --- a/.github/scripts/tests/pr-runner-rust-cache.test.mjs +++ b/.github/scripts/tests/pr-runner-rust-cache.test.mjs @@ -48,11 +48,27 @@ test("every rust-cache restore in the PR workflow belongs to a guarded reader jo const keyInputs = [ /uses: Swatinem\/rust-cache@([0-9a-f]{40}) # v[0-9.]+/, /workspaces: (\$\{\{ steps\.runner_rust_workspace\.outputs\.path \}\} -> target)/, - /shared-key: (release-runner-v2)/, + /shared-key: (release-runner-v2-.+)/, /cache-workspace-crates: (false)/, /cache-bin: (false)/, ]; +test("virtual workspace profiles and ancestor Cargo configuration partition every dependency cache", () => { + for (const [name, workflow] of [["PR", prWorkflow], ["release", releaseWorkflow]]) { + const keys = [...workflow.matchAll(/^\s+shared-key: (.+)$/gm)].map(([, key]) => key); + assert.ok(keys.length > 0); + for (const key of keys) { + assert.ok(key.includes("hashFiles('packages/paperclip-runner/runner/Cargo.toml'"), name); + for (const directory of ['.', 'packages', 'packages/paperclip-runner', 'packages/paperclip-runner/runner']) { + const prefix = directory === '.' ? '' : `${directory}/`; + for (const file of ['config', 'config.toml']) { + assert.ok(key.includes(`'${prefix}.cargo/${file}'`), `${name}: ${prefix}.cargo/${file}`); + } + } + } + } +}); + test("every PR reader restores the Rust cache under the same key the master push writes", () => { for (const [name, pr] of readers) { for (const pattern of keyInputs) { diff --git a/.github/workflows/ci-work-benchmark.yml b/.github/workflows/ci-work-benchmark.yml index fe7ad35782d..96dffab12d5 100644 --- a/.github/workflows/ci-work-benchmark.yml +++ b/.github/workflows/ci-work-benchmark.yml @@ -7,6 +7,9 @@ on: - .github/actions/setup-ci-dependencies/** - .github/workflows/ci-work-benchmark.yml - scripts/ci-dependency-cache.mjs + - .github/workflows/pr-trusted.yml + - .github/workflows/release-verify.yml + - scripts/runner-debug-crate-cache.mjs - scripts/benchmark-ci-work.mjs - scripts/run-vitest-stable.mjs - server/src/__tests__/setup-supertest.ts @@ -39,6 +42,21 @@ jobs: fetch-depth: 2 persist-credentials: false + - name: Select changed server test work + id: test_scope + run: | + changed=$(git diff --name-only HEAD^1 HEAD -- \ + scripts/benchmark-ci-work.mjs \ + scripts/run-vitest-stable.mjs \ + server/src/__tests__/setup-supertest.ts \ + server/src/__tests__/workspace-runtime.test.ts \ + server/src/services/workspace-git-snapshot-streaming.test.ts) + if [ -n "$changed" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi + - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: @@ -123,6 +141,7 @@ jobs: echo "Materialized dependency reuse is disabled by an unmodelled install hook. Retaining cold setup and comparing test commands only." >> "$GITHUB_STEP_SUMMARY" - name: Prepare unchanged baseline + if: steps.test_scope.outputs.run == 'true' run: | base_sha=$(git rev-parse HEAD^1) git worktree add --detach ../baseline "$base_sha" @@ -135,12 +154,17 @@ jobs: pnpm --filter @paperclipai/plugin-sdk ensure-build-deps - name: Compare complete commands and executed case inventories + if: steps.test_scope.outputs.run == 'true' env: BENCHMARK_OUTPUT: ${{ runner.temp }}/ci-work-benchmark run: | node scripts/benchmark-ci-work.mjs ../baseline "$BENCHMARK_OUTPUT" cat "$BENCHMARK_OUTPUT/summary.md" >> "$GITHUB_STEP_SUMMARY" + - name: Exercise restored dependencies for cache-only changes + if: steps.test_scope.outputs.run != 'true' && steps.qualification.outputs.cacheable == 'true' + run: pnpm exec vitest run --project @paperclipai/shared + - name: Retain timings, inventories and failure logs if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a @@ -148,3 +172,120 @@ jobs: name: ci-work-benchmark path: ${{ runner.temp }}/ci-work-benchmark retention-days: 14 + + rust_prime: + if: >- + contains(github.event.pull_request.labels.*.name, 'profile-ci') && + (github.event.action != 'labeled' || github.event.label.name == 'profile-ci') + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - name: Select the pinned compiler and workspace path + id: workspace + working-directory: packages/paperclip-runner + run: | + toolchain=$(rustup show active-toolchain | awk '{print $1}') + echo "RUSTUP_TOOLCHAIN=$toolchain" >> "$GITHUB_ENV" + rustup toolchain list | awk '{print $1}' | grep -vx "$toolchain" | xargs -r -n1 rustup toolchain uninstall || true + pinned="$HOME/paperclip-runner-rust" + ln -s "$GITHUB_WORKSPACE/packages/paperclip-runner/runner" "$pinned" + echo "path=$pinned" >> "$GITHUB_OUTPUT" + - name: Qualify the dependency cache + id: dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ${{ steps.workspace.outputs.path }} -> target + shared-key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} + cache-workspace-crates: false + cache-bin: false + save-if: 'true' + - name: Build the exact debug units + working-directory: packages/paperclip-runner + run: cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins + - name: Compute workspace artifact identity + id: debug + run: node scripts/runner-debug-crate-cache.mjs plan + - name: Save workspace artifacts in this PR only + id: artifacts + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.debug.outputs.key }} + path: ${{ steps.debug.outputs.paths }} + + rust_restore: + needs: rust_prime + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - name: Select the pinned compiler and workspace path + id: workspace + working-directory: packages/paperclip-runner + run: | + toolchain=$(rustup show active-toolchain | awk '{print $1}') + echo "RUSTUP_TOOLCHAIN=$toolchain" >> "$GITHUB_ENV" + rustup toolchain list | awk '{print $1}' | grep -vx "$toolchain" | xargs -r -n1 rustup toolchain uninstall || true + pinned="$HOME/paperclip-runner-rust" + ln -s "$GITHUB_WORKSPACE/packages/paperclip-runner/runner" "$pinned" + echo "path=$pinned" >> "$GITHUB_OUTPUT" + - name: Qualify the dependency cache + id: dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ${{ steps.workspace.outputs.path }} -> target + shared-key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} + cache-workspace-crates: false + cache-bin: false + save-if: 'false' + - name: Compute workspace artifact identity + id: debug + run: node scripts/runner-debug-crate-cache.mjs plan + - name: Restore workspace artifacts in this PR only + id: artifacts + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.debug.outputs.key }} + path: ${{ steps.debug.outputs.paths }} + - name: Verify fresh-host cache reuse + env: + DEPENDENCIES_HIT: ${{ steps.dependencies.outputs.cache-hit }} + ARTIFACTS_HIT: ${{ steps.artifacts.outputs.cache-hit }} + run: | + test "$DEPENDENCIES_HIT" = true + test "$ARTIFACTS_HIT" = true + node scripts/runner-debug-crate-cache.mjs mark-current + cd packages/paperclip-runner + started=$(date +%s) + cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins --message-format=json > "$RUNNER_TEMP/rust-restored-build.jsonl" + echo "Restored Rust build: $(($(date +%s) - started)) seconds." >> "$GITHUB_STEP_SUMMARY" + node --input-type=module <<'NODE' + import { readFileSync } from 'node:fs'; + const artifacts = readFileSync(`${process.env.RUNNER_TEMP}/rust-restored-build.jsonl`, 'utf8') + .trim().split('\n').map(line => JSON.parse(line)).filter(row => row.reason === 'compiler-artifact'); + if (!artifacts.length || artifacts.some(row => !row.fresh)) { + throw new Error(`Cache hit rebuilt units: ${JSON.stringify(artifacts.filter(row => !row.fresh).map(row => row.package_id))}`); + } + console.log(`All ${artifacts.length} Cargo units were fresh on a new host.`); + NODE + - name: Retain Cargo reuse evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: rust-cache-qualification + path: ${{ runner.temp }}/rust-restored-build.jsonl + retention-days: 14 diff --git a/.github/workflows/cloud-migrator-artifacts.yml b/.github/workflows/cloud-migrator-artifacts.yml index 5c698230938..e60b7a54456 100644 --- a/.github/workflows/cloud-migrator-artifacts.yml +++ b/.github/workflows/cloud-migrator-artifacts.yml @@ -12,9 +12,33 @@ concurrency: cancel-in-progress: false jobs: - build: + admission: + name: Cloud image candidate admission v1 if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }} + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + admitted: ${{ steps.admission.outputs.admitted }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + sparse-checkout: scripts + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - id: admission + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/cloud-release-admission.mjs image-only + + build: + needs: admission + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true' + runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }} timeout-minutes: 10 permissions: contents: read @@ -52,7 +76,7 @@ jobs: publish: needs: build if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }} timeout-minutes: 5 permissions: contents: read diff --git a/.github/workflows/cloud-readiness.yml b/.github/workflows/cloud-readiness.yml index 6c709ade0ed..b9ee017ddea 100644 --- a/.github/workflows/cloud-readiness.yml +++ b/.github/workflows/cloud-readiness.yml @@ -8,14 +8,38 @@ on: permissions: {} -# Source verification must start outside the full npm release's queue. +# Source verification stays independent of npm publication and image builds. concurrency: group: cloud-readiness-${{ github.sha }} cancel-in-progress: false jobs: + admission: + name: Cloud candidate admission v1 + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + admitted: ${{ steps.admission.outputs.admitted }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + sparse-checkout: scripts + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - id: admission + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/cloud-release-admission.mjs + verify: - if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' + needs: admission + if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true' permissions: contents: read uses: ./.github/workflows/release-verify.yml @@ -29,8 +53,9 @@ jobs: name: Cloud source verified v1 needs: [verify] if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' - # Bookkeeping must not wait for the AWS builders it observes. - runs-on: ubuntu-latest + # Keep the final proof off the shared hosted queue too. The protected + # fleet has a separate job ceiling and launches Spot workers on demand. + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 permissions: contents: read diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ec7c71000e5..6048c277e41 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,29 @@ concurrency: cancel-in-progress: false jobs: + source_verification: + name: Watch exact-source verification + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: 44 + permissions: + contents: read + actions: write + outputs: + verified: ${{ steps.watch.outputs.verified }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Stop images when source checks fail and report runner waits + id: watch + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/watch-release-source.mjs + reuse_canary_image: if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v') runs-on: ubuntu-latest @@ -100,11 +123,10 @@ jobs: - platform: linux/arm64 runner: ubuntu-24.04-arm arch: arm64 - runs-on: ${{ matrix.runner }} - # Native builds land well inside this; it is a backstop, not a budget. - # (The interim fix while this PR landed raised the single QEMU job's cap - # to 120 minutes; native per-arch legs make that headroom unnecessary.) - timeout-minutes: 60 + runs-on: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && format('runs-on/fleet=paperclip-core-image-{0}/env=public-ci', matrix.arch == 'amd64' && 'x64' || 'arm64') || matrix.runner }} + # Stop before the AWS worker's 45-minute maximum lifetime. Hosted tag + # builds retain their existing limit. + timeout-minutes: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && 40 || 60 }} permissions: contents: read packages: write @@ -115,6 +137,9 @@ jobs: # Full history and tags so `git describe` below can compute the # release version to stamp into the image. fetch-depth: 0 + # Keep the version's full commit graph without downloading blobs + # from every historical revision of this large repository. + filter: blob:none # `.git` is dockerignored, so a running image cannot derive its own # version and otherwise reports the source package.json placeholder in @@ -183,6 +208,7 @@ jobs: echo "Using refreshed pnpm-lock.yaml in the Docker build context." - name: Free runner disk + if: ${{ !(vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master') }} run: | set -euo pipefail echo "Disk before cleanup:" @@ -304,16 +330,40 @@ jobs: if-no-files-found: error retention-days: 1 + # Wait for proof on ephemeral verification capacity. Publication runners do + # only registry/signing work. Each Spot observer stays below its 45m lifetime. + publication_verification: + name: Require exact-source verification + if: ${{ !cancelled() && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.build-and-push.result == 'success' && needs.source_verification.result == 'success' && needs.source_verification.outputs.verified != 'true' }} + needs: [build-and-push, source_verification] + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: 44 + permissions: + contents: read + actions: write # restart only exact-source producers deferred before admission + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Require source proof and migrator admission before publication + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/watch-release-source.mjs --require-proof + # Names the per-architecture digests as one manifest list under the real # tags. Nothing is publicly tagged until this runs, so a half-published # multi-arch image is not a state anything can pull. merge-and-push: - if: ${{ !cancelled() && needs.build-and-push.result == 'success' }} - needs: build-and-push - runs-on: ubuntu-latest + if: ${{ !cancelled() && needs.build-and-push.result == 'success' && (needs.source_verification.result == 'success' || needs.source_verification.result == 'skipped') && (needs.publication_verification.result == 'success' || (needs.publication_verification.result == 'skipped' && (github.repository != 'paperclipai/paperclip' || github.repository_id != '1170821064' || github.ref != 'refs/heads/master' || needs.source_verification.outputs.verified == 'true'))) }} + needs: [build-and-push, source_verification, publication_verification] + runs-on: ${{ github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32') || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read + actions: read packages: write id-token: write attestations: write @@ -322,8 +372,6 @@ jobs: # script the verification step pipes in. - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - name: Download digests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -375,6 +423,19 @@ jobs: io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }} io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }} + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + + # Revalidate the latest attempt after runner assignment without waiting + # here. A concurrent source retry must invalidate the earlier proof. + - name: Revalidate exact-source proof before publication + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/watch-release-source.mjs --check-proof + - name: Create manifest list and push working-directory: /tmp/digests run: | diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 455512ee2de..420f1d98259 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -370,7 +370,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false @@ -784,7 +784,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false @@ -1035,7 +1035,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false @@ -1376,7 +1376,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index e36bdbccbd2..b0319840ec1 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -34,7 +34,7 @@ jobs: typecheck: if: ${{ !inputs.runner_binaries_only }} name: Typecheck - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -92,7 +92,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-typecheck-v2 + shared-key: release-typecheck-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Rebuild workspace code and rerun every check. Cache only compiled # dependencies; never restore installed executables from cargo/bin. cache-workspace-crates: false @@ -120,7 +120,7 @@ jobs: general_tests: if: ${{ !inputs.runner_binaries_only }} name: General tests (${{ matrix.group_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -232,7 +232,7 @@ jobs: serialized_tests: if: ${{ !inputs.runner_binaries_only }} name: Serialized tests (${{ matrix.shard_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -282,7 +282,7 @@ jobs: runner_workflow_evals: if: ${{ !inputs.runner_binaries_only }} name: Runner workflow eval scorer contract - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 10 permissions: contents: read @@ -313,7 +313,7 @@ jobs: verify_paperclip_runner: if: ${{ !inputs.runner_binaries_only }} name: Verify Paperclip Runner (${{ matrix.lane }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -398,7 +398,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Rebuild workspace code and rerun every check. Cache only compiled # dependencies; never restore installed executables from cargo/bin. cache-workspace-crates: false @@ -523,7 +523,7 @@ jobs: build: if: ${{ !inputs.runner_binaries_only }} name: Build - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -680,7 +680,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 with: workspaces: ${{ steps.native_rust_workspace.outputs.path }} -> target - shared-key: release-native-v2-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }} + shared-key: release-native-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }}-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }} cache-workspace-crates: false cache-bin: false cache-on-failure: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7b310f4ee1..d9bdc1824f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -320,12 +320,12 @@ jobs: verify_canary: name: Reuse exact-source verification - if: github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' runs-on: ubuntu-latest - timeout-minutes: 50 + timeout-minutes: 70 permissions: contents: read - actions: read + actions: write # restart only an exact-source push deferred before admission steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml index 810d28ec2e7..fbb60238de3 100644 --- a/.github/workflows/runner-chaos-evals.yml +++ b/.github/workflows/runner-chaos-evals.yml @@ -20,8 +20,8 @@ concurrency: jobs: chaos_and_recovery: name: Restart, replay, trace, and recovery faults - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} - timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} permissions: contents: read diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 8976eccb990..ec278667695 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1095,6 +1095,22 @@ verifying Vite/HMR behavior. For normal issue work, start with the smallest targeted check that proves the change. Reserve repo-wide typecheck/build/test runs for PR-ready handoff or changes broad enough that narrow checks do not cover the risk. +The Runner transport suite keeps attachment recovery, warm attachment, live +adoption, and suspension in separate test files. Vitest can schedule these +independent fixtures on its existing file workers; cases within each file stay +serial. The package sequencer starts the ten files measured above 50 seconds in +the November 4, 2026 AWS run first, ordered by approximate duration. This includes +transport deadlines, settlement, provider launch, recovery, and streaming waits, +so fresh CI runners do not have to infer duration from file size. Package CI uses +up to four workers, bounded by available CPUs; local runs retain Vitest defaults. +Vitest's file selection and shard membership are unchanged. Real lease/deadline +waits and assertions are unchanged. Run the complete +family, including the separate deadline and settlement regressions, with: + +```sh +pnpm --filter @paperclipai/paperclip-runner exec vitest run runnerd-codex-transport +``` + #### Embedded Postgres template cache To run the suite with one owned PostgreSQL server and a separate disposable @@ -3187,7 +3203,7 @@ Run the credential-free real-process restart suite with: ```sh pnpm --filter @paperclipai/paperclip-runner build:runner-binaries pnpm exec vitest run server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts -pnpm --filter @paperclipai/paperclip-runner exec vitest run src/live/runnerd-codex-transport.test.ts -t 'adopts a live runner' +pnpm --filter @paperclipai/paperclip-runner exec vitest run src/live/runnerd-codex-transport-adoption.test.ts -t 'adopts a live runner' ``` The suite uses isolated PostgreSQL state, isolated `PAPERCLIP_HOME` roots, real diff --git a/doc/RELEASE-AUTOMATION-SETUP.md b/doc/RELEASE-AUTOMATION-SETUP.md index 324a6adb07e..021188f2e53 100644 --- a/doc/RELEASE-AUTOMATION-SETUP.md +++ b/doc/RELEASE-AUTOMATION-SETUP.md @@ -368,7 +368,14 @@ readiness can succeed. A failed lane does not cancel the other lane. Both lanes restore Cargo dependencies with the pinned Rust Cache action. The compiler comes from the Runner package's `rust-toolchain.toml` before the action computes its key. Compiler and Cargo metadata changes select a new cache. The -`release-runner-v2` shared key avoids separate copies for these lanes. +`release-runner-v2` shared key avoids separate copies for these lanes. Its suffix +hashes the virtual workspace's `runner/Cargo.toml` and ancestor Cargo configuration. +Rust Cache discovers member manifests through Cargo metadata, which omits this +virtual manifest. Without the suffix, changing `[profile.dev]` can restore an +exact but obsolete dependency cache on every run. GitHub caches are immutable, +so the successful rebuild cannot replace that old entry. The profile/configuration +hash also partitions fallback restores. Every reader and writer uses the same +inputs, including the separate typecheck and cross-target caches. Only the Rust lane saves this cache. After verification it also runs `build:rust` to warm the debug dependencies used by the protocol lane; its own tests already warm release dependencies. The cache writer is shorter than the protocol lane. @@ -385,13 +392,36 @@ paths and the PR lanes can restore master's entry. The guard tests under Rust cache step in both workflows. Workspace crates and installed Cargo binaries are excluded from this entry. -Every run runs all assigned checks, including on a cache hit. Only an -own-repository master-push run verifying that push's exact SHA can restore the -cache, and only a successful Rust lane saves it. PR, tag, and manual candidate +Every run runs all assigned checks, including on a cache hit. In release +verification, only an own-repository master-push run verifying that push's exact +SHA can restore the cache, and only a successful Rust lane saves it. The PR +workflow restores master's cache read only. Tag and manual candidate release verification compile without this cache. A miss or eviction costs compilation time but does not change the checks. To discard old dependency caches, increment the shared-key version and let the next successful master verification warm it. +### Installed JavaScript dependencies + +`setup-ci-dependencies` can restore exact `node_modules` trees and separately +certified shared/SDK build outputs. `scripts/ci-dependency-cache.mjs` hashes +manifests, resolution, patches, install scripts, and the runtime/checkout layout. +The root workspace-link hook and the server/Runner Copilot asset hooks are known +postinstall commands. An exact dependency restore replays these hooks before +validating workspace links and build outputs. Copilot defers installation in an +unbuilt source checkout; its normal build installs and verifies the assets. +Provider assets are not included in the dependency cache. Any unrecognized +workspace lifecycle hook still disables materialized reuse. Only a frozen install +from the canonical master push can publish the shared entries. + +For CI cache changes, the `profile-ci` label runs the paired setup benchmark +and exercises restored dependencies with the shared-package tests. The longer +server test comparison runs when its measured scripts or fixtures change; +cache-only changes do not require unchanged test commands to run faster. +It also builds Rust units on one GitHub-hosted runner and restores the dependency +and workspace caches on a second runner with a fresh checkout. Both restores +must be exact, and Cargo must report every unit as fresh. These qualification +caches are scoped to the PR and run; they cannot populate master's caches. + ### Runner release crate cache The dependency entry leaves the Runner's own crate out, and a fresh checkout diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md index 18611b833b0..df1a595bf56 100644 --- a/doc/cloud-build-readiness.md +++ b/doc/cloud-build-readiness.md @@ -1,30 +1,42 @@ # Cloud build readiness -The `Cloud readiness` workflow starts for every master push and retains the -versioned `Cloud source verified v1` job. It calls the full `Release Verify` -workflow for that exact commit, including typecheck, builds, general and -serialized tests, and Runner verification. The source proof depends on every -source check and fails closed if verification fails, is cancelled, or is skipped. +The `Cloud readiness` workflow starts a small admission check for each master +push. Full verification starts only when the exact source has a running master +Docker or automatic npm release workflow. Superseded pending sources avoid the +full test matrix. The migrator producer similarly waits for image admission. + +A selected consumer restarts a previously deferred push run, preserving its +original source and push identity. This restarts only a successful admission +job followed entirely by skipped jobs. A failed check, cancelled job, incomplete +listing, or identity mismatch cannot trigger this automatic retry. Explicit +manual runs and whole-workflow retries request full verification directly. + +Admitted source runs retain the versioned `Cloud source verified v1` job. They +call the full `Release Verify` workflow for that exact commit, including +typecheck, builds, general and serialized tests, and Runner verification. The +proof depends on every source check and fails closed if verification fails, is +cancelled, or is skipped. Consumers accept only the latest attempt's proof. The recurring public `-cloud` publisher and its `Cloud deployable v1` gate are retired. The workflow no longer builds a legacy image or waits for one. Keep its filename and source-proof job name stable: npm canary publication and downstream image composers consume that exact contract. -Standard images still publish independently through `docker.yml`. The +Standard images build in parallel through `docker.yml`. Master image publication +waits for the exact-source proof; a failed source check cancels its image build +without waiting for the remaining test shards. The `cloud-migrator-artifacts.yml` workflow still publishes signed exact-source migrators independently. A downstream composer must verify those artifacts, build and test its own image, and record separate deployment readiness. -Verification runs outside the full npm release's concurrency group. Different -commits have independent groups. The npm canary release reuses the source proof +Verification runs outside the full npm release's concurrency group. Each source retains its independent verification run. The npm canary release reuses the source proof for its exact master push instead of starting another `Release Verify` run. Stable releases and candidate-branch betas still run full verification. The canary consumer requires the expected workflow ID and path, upstream source repository, master push event, full SHA, and a successful job in the latest run attempt. It checks the run again after reading the jobs to reject a concurrent -rerun. Missing proof waits for up to 45 minutes; failed, skipped, cancelled, +rerun. Missing proof waits for up to 60 minutes; failed, skipped, cancelled, ambiguous, or mismatched proof cannot authorize publication. API failures fail closed. If a source check fails, fix it and rerun Cloud readiness before retrying the release. Use **Re-run all jobs** when a later attempt did not rerun the source @@ -120,15 +132,16 @@ excluded or sleeping targets, retries, and failures with the fleet result. Recor runner queue conditions and cache state; one warm or cold run is a sample, not a latency guarantee. -## Reserved AWS verification capacity +## Dedicated AWS verification capacity `AWS_POST_MERGE_CI_ENABLED=true` routes cloud source verification and exact-master migrator preparation to the `paperclip-post-merge` runner group. The separate Fleet label is -`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. Its 36 reserved slots use -the same four-vCPU, 16-GiB machines as approved PR jobs. PR capacity is reduced -to 64; the separately provisioned image capacity is unchanged by this retirement. -This keeps PR bursts from consuming every post-merge verification slot. +`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. The protected fleet uses +separate 8-vCPU, 32-GiB Spot workers that launch only when jobs arrive. Its +job ceiling and public PR capacity are managed independently. There are no +hot or stopped pools. Quota ceilings do not reserve hardware; measure cold +launch and runner assignment times as well as job duration. Every selector checks the canonical repository name and ID, master ref, and a push or manual event. Reusable verification also requires `inputs.ref` to equal @@ -142,7 +155,8 @@ its trusted-publisher identity. Before enabling the switch, deploy the separate Fleet and restrict its GitHub runner group to repository ID `1170821064` and these workflows at `refs/heads/master`: `cloud-readiness.yml`, -`release-verify.yml`, `runner-chaos-evals.yml`, and `release.yml`. Do not authorize +`release-verify.yml`, `runner-chaos-evals.yml`, `release.yml`, and `docker.yml`. +The Docker workflow uses this group for its exact-source monitor and proof wait. Do not authorize PR-controlled workflow versions. The direct migrator producer always uses GitHub-hosted runners and needs no AWS runner-group authorization. PR placement retains its independent pinned workflow and six-account author/actor allowlist. @@ -158,6 +172,43 @@ route a public image job after this retirement. This source change does not delete runner groups, Fleets, credentials, registry images, or cache tags. Review shared infrastructure ownership separately before removing those resources. +## Protected standard-image builders + +`CORE_RELEASE_AWS_ENABLED=true` routes canonical master image jobs to native +`paperclip-core-image-x64` and `paperclip-core-image-arm64` fleets. Each has two +32-vCPU / 128-GiB Spot slots, for four large image jobs in total. Workers +launch for jobs and terminate afterward; no idle pool is maintained. The `paperclip-core-release` runner group permits only +Core's `docker.yml@refs/heads/master`. PRs, branches, and tag releases keep their +existing GitHub-hosted placement. Disable the switch and rerun the whole image +workflow to restore hosted placement. + +Image checkout retains the complete commit graph for version stamping but filters +out historical blobs. The manifest job needs only a shallow source checkout. +Prepared AWS image disks skip the standard hosted runner's disk cleanup. Native +architecture builds, digest publication, the orphan-reaping smoke test, and +attestation remain required. The source monitor has Actions write permission to cancel its own canonical +master Docker run and failed exact-source verification siblings. It never +automatically retries a failed source check. Its guarded retry permission can +restart only an exact-source producer deferred before full admission. It checks run, attempt, SHA, +repository, ref, and workflow identity before cancellation. Its bounded +observation window does not cancel healthy pending checks. A successful observer +records proof for publication immediately. If healthy checks remain pending after +40 minutes, a separate required proof job waits for up to another 40 minutes on +the verification fleet. Both jobs fit within the Spot worker lifetime. Neither +wait occupies the four-slot publication pool. The hosted manifest job requires +successful source proof and admitted migrator work before it starts; a queued +migrator job alone is not evidence of admission. It retains its 20-minute budget +for manifest verification and signing. It revalidates the latest source attempt +once before publishing; changed or missing proof fails immediately without +starting another wait on the publication runner. +Image jobs retain +their existing package permissions. + +The monitor emits an Actions warning and step summary when an image or source job waits +more than 60 seconds for a runner. This measures runner assignment, not image +execution. Compare automatic merge-to-ready and merge-to-healthy-target timings; +a manual dispatch alone does not prove deployment latency. + ### Typecheck Rust dependency cache Source verification's typecheck job builds the native Runner binary through the @@ -236,3 +287,10 @@ fi A subsequent master install can populate the missing entry. Check the saved archive size and package reuse in install logs; a cache hit alone does not prove that the entry contains dependencies. + +Image manifest verification and attestation, plus exact-source migrator build +and publication, use the protected four-slot GitHub-hosted `cloud-release-32` +pool. `CORE_PUBLICATION_RUNNER=ubuntu-latest` restores standard hosted placement. +The signing jobs remain GitHub-hosted because downstream provenance explicitly +requires that certificate claim. Their master-only group also serves the private +Cloud publisher and harness deployment; it excludes PRs and branch benchmarks. diff --git a/doc/connections/GITHUB.md b/doc/connections/GITHUB.md index 0f93999ab86..96826dfddfb 100644 --- a/doc/connections/GITHUB.md +++ b/doc/connections/GITHUB.md @@ -77,6 +77,12 @@ repeat an existing instance approval. ## Identity resolution +Fresh setup does not reuse an archived personal connection found by name and +provider. It creates a separate application and connection with an available +name, such as `GitHub (2)`. The archived identity, credentials, grants, and +history stay unchanged. An explicit reconnect or application-targeted setup +continues the existing identity and requires its personal owner. + Every MCP call, `gh` invocation, native Git operation, checkout, health check, and webhook binding uses the same order: diff --git a/doc/evals.md b/doc/evals.md index 7d62a82f9f5..f59035c2843 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -1,5 +1,7 @@ # Paperclip evaluation guide +The explicit deployment-targeted [saved agent instructions journey](../tests/runner-e2e/product-journeys/AGENT-SETTINGS.md) checks an owned agent’s ordinary UI save, persisted instruction bytes, one current-run output, and restoration. The task request omits its instruction-only reference. It reuses the existing Product E2E target and report path; local controls do not establish a staging pass. + The explicit-only [verified API artifact transfer](../tests/runner-e2e/README.md#verified-api-artifact-transfer) cell checks a native Codex agent on Daytona installing an authorized saved API response and delivering unchanged bytes. The oracle binds the source capture, @@ -111,14 +113,20 @@ reply with the late requirement, and persistence after reload. Its attached and managed targets share the same story and permit one steered run or one normal successor. Registration and calibration are separate from a live staging pass. +The explicit [dependency continuation journey](../tests/runner-e2e/product-journeys/task-dependency-resume.md) checks a finite blocked wait and one automatic native continuation after a browser-completed blocker. Local evidence controls are separate from a deployed QA pass. + [Reusable Product journeys](../tests/runner-e2e/PRODUCT-JOURNEYS.md) add explicit deployment targets and starting-state recipes. The shared task-properties journey and task-document create/edit/reopen journey use no model; Cloud onboarding supplies reusable verified state. The document journey checks exact -saved content and both revision records through independent API reads. Cloud -fleet operations and trusted CI remain private. Local eval gateway qualification +saved content and both revision records through independent API reads. The explicit `chat-continuity` selection verifies browser reload/reopen, prompt-hidden recall, +ordinary `/new` reset, fresh native context and retained history against an attached target. +It requires a fresh owned agent with no prior work runs and uses three provider turns plus one synthetic reset row; it does not restart a shared stack. +Cloud fleet operations and trusted CI remain private. Local eval gateway qualification does not qualify Cloud routes, and these selections stay outside `--all`. +The deployment-targeted [saved skill journey](../tests/runner-e2e/product-journeys/saved-skill-use.md) checks one company skill assigned by key, its exact admitted immutable version, a bound native source read/load, and a run-authored saved result from skill-only inputs. It uses the reusable Product journey target and remains unqualified until a real target run passes. + The explicit-only [case handoff workflow](../tests/runner-e2e/README.md#persistent-case-handoffs-explicit-only) contains arithmetic and receipt smoke cells for a same-task round trip among three native agents using real case/task APIs, saved documents, handoff receipts, and browser readback. Receipt smoke has diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index bf481923610..95d333ac571 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -937,6 +937,13 @@ identity, unknown failures, active leases, and incomplete cleanup still require their normal reconciliation and termination proof. Unknown historical setup errors remain outside this policy, and generic recovery cannot renew its exhausted budget. +When an admitted continuation reuses a sandbox after verified cleanup, an +unpublished preparation uses a directory bound to its writer epoch and lease. +Earlier unpublished directories and their receipts stay intact. Repeating the +same preparation retains its directory and receipt check. A published workspace +on the same allocation retains its recorded directory. Automatic bootstrap +timeout retries still require a fresh allocation under the policy above. + ### Runner authentication and ACP startup handshake bounds The native Runner waits up to the ten-second connection setup budget for the @@ -1258,6 +1265,18 @@ checks coordinator ownership in its terminal update, so a result recorded after the process check is also protected. Terminal task status remains authoritative; exhausted finalization retries do not keep an otherwise orphaned run alive. +A native failure that schedules a same-run resume marks the run `failed` but +keeps the task's execution and checkout locks for that resume. The sweep leaves +those locks in place and clears them only when the task is done or cancelled. +If a scheduled resume finds its execution lock free, restart recovery reclaims +it, as the local resume claim does, unless another run for the task is queued, +running, or scheduled. A lock held by another run still blocks the resume. +Another task can hold the same routine execution slot even when this task's +lock is free. If that unique claim fails, restart recovery rolls back the whole +candidate claim and waits through the existing recovery deadline. It preserves +the current routine owner, checkpoint, and provider attempt, and continues the +startup batch. Other database failures still fail the recovery pass. + Automatic productivity reviews are retired. Run counts, missing comments, and elapsed task time do not create review tasks or impose continuation holds. Bounded continuation, provider recovery, budget limits, explicit blockers, and normal review/approval stages remain in force. Existing productivity-review tasks, comments, assignments, and dependencies remain unchanged and readable; their historical origins still identify them as recovery work for recursion suppression. ### Issue-thread interaction resolution @@ -1834,9 +1853,10 @@ An interrupted conversation does not permanently block its task. For local conve Shutdown, process loss, and provider failure use the existing durable failure retry counter and delay. Ordinary failure recovery permits at most two automatic retries in a failure chain. Accepted-interaction infrastructure recovery retains its existing bounded policy. Repeated scheduler visits reuse the same successor; restarting the server does not reset the counter. After exhaustion, automatic attempts stop. A new explicit user message can start a fresh run and failure budget. Productive max-turn continuation and confirmed workspace waits keep their separate existing semantics. -If shutdown interrupts Paperclip Runner preparation before native execution was -selected, recovery verifies that no provider, process launch, native result or -coordinator exists. The previous controller must have settled or lost its lease. +If shutdown or a recorded unexpected system interruption stops Paperclip Runner +preparation before native execution was selected, recovery verifies that no +provider, process launch, native result or coordinator exists. The previous +controller must have settled or lost its lease. Required workspace recovery must have an exact saved-file receipt and stopped leases before the task can continue. Once these checks pass, the server retires its own unchanged bootstrap-failure block and schedules one bounded retry. @@ -1845,7 +1865,8 @@ restart. A pending Retry or saved message owns its continuation instead of starting a competing automatic attempt. Retry uses the same preparation proof and preserves the recovered workspace. Current task ownership, human blocks, pauses, approvals, budgets and the existing retry limit remain effective. -Operator Stops retain their explicit continuation requirements. +Operator Stops retain their explicit continuation requirements. Other ambiguous +interruptions without a server system-cancellation record remain held. Graceful shutdown interrupts an owned legacy adapter through its cancellation control, then waits within one shared 30-second window for provider stop and @@ -2612,6 +2633,26 @@ configuration, and controller-owned credential or instruction resources use verified suspension and saved-history resume. This recovery can restart a provider; it does not suspend every healthy warm turn. +A different task can reuse the same physical sandbox after its prior task +finishes. Provider ingress has one listener per allocation. Before a new logical +session starts, Runner retires a locally retained idle session that still owns +that listener. The new run must hold the current native claim, task assignment, +environment lease and remote workspace writer. Company, agent, provider, +allocation, workspace and directory must match. The prior run must be committed +and accounted, with its original controller generation and a clean released or +expired lease. Retirement saves the prior session through its own checkpoint and +file-collection callbacks and waits for close before the new listener binds. +After close, admission rechecks the canonical native claim, task, workspace and +leases before staging files or invoking the new backend. Ownership loss, Stop +or a changed writer during retirement cannot authorize the new execution. +The new task keeps its own logical session and authority. + +Active owners, a changed controller or writer, missing canonical proof and failed +close block admission. A retained entry stays available for verified retirement; +no process is selected by its port number. This rule covers current-controller +warm entries with recorded physical identity. A historical process outside that +registry still needs the existing exact-owner recovery proof. + A fresh native session admitted for the current task may replace a terminal predecessor's goal source epoch when the saved goal is absent or complete. It must hold the task's execution lock and assignee. Its accepted terminal @@ -2732,6 +2773,23 @@ replay or access: the saved input and current principal, repository, conversatio runtime fence and execution containment still pass normal admission. Explicit conflicting App bindings remain denials. +When comments join a queued run with an unclaimed queued wake, admission saves +the accepted batch in both the run context and its authoritative primary wake in +the same transaction as the coalesced receipt. Claim rebuilds input from that primary wake and filters deleted +comments. It does not reconstruct input from historical coalesced receipts. +Admission locks and rechecks the run and its same-company, same-agent primary +wake. A changed status, mismatched binding, or mismatched durable-receipt actor preserves the incoming +input as deferred work instead of consuming it into the old source. Legacy wakes +without an authoritative batch retain their existing context-based claim path. +An already claimed wake keeps its source unchanged for its existing recovery +and replay path; this change does not grant it a new claim. +Ordinary mixed-author comments keep their existing coalescing behavior. The +primary wake retains its original actor, and each new receipt records its own +actual actor. Coalescing updates the queued batch while preserving the original +trigger comment fields, so recovery still verifies the original source actor. +Same-named agents with different IDs retain separate authority; +their new input is deferred without changing the other agent's queued work. + ### Queued input after a native Stop A run-only Stop ends the current response. It does not discard queued user diff --git a/doc/observability.md b/doc/observability.md index 3c0ba327c29..300743ae057 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -210,7 +210,8 @@ diagnostic line on boot and continues without tracing — your server stays up. ## Scope The OpenTelemetry export carries **traces only**. Metrics and log exporters -are out of scope and intentionally not configured here. Auto-instrumentations +are out of scope. The SDK receives explicit empty metric-reader and log-processor +lists so its default OTLP exporters cannot emit unrequested metrics or logs. Auto-instrumentations for `fs`, `dns`, and `net` are disabled by default because they are too chatty for this workload; everything else from `@opentelemetry/auto-instrumentations-node` is on (HTTP, Express, PG, etc.). diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 0f028d6a037..f4150f62fc9 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -102,3 +102,26 @@ Controller leases, in-flight recovery, active writers, and uncertain workspace operations remain blockers. Tests must prove migration leaves failure state intact, rejects stale copyback even if the old folder exists, and preserves recovery source validation and company boundaries. + +## Queued turns during offline enrollment + +The task drain stops new executions. It does not consume queued turns. After +Cloud has stopped every application deployment, offline maintenance preserves +never-started queued run and wake rows unchanged. Workspace, instruction-file, +and historical-log migration use the same execution predicate at this boundary. +The replacement uses normal queue admission against the migrated authority. + +Running executions still block migration. A queued row also blocks if it has +started, retains process/session/controller evidence, or has a coordinator or +environment lease. Task checkout and execution bindings remain separate blockers. +Live migration retains its original queued-run exclusion. The offline flag alone +is not evidence that old deployments have stopped; Cloud owns that provider fence. + +PostgreSQL/S3 tests cover queued turns and wake payloads surviving migration and +loss of the predecessor filesystem, plus queued execution evidence that must +remain blocked. The packaged maintenance entry must also retain an unstarted +queued turn while it archives historical run output. The assembled heartbeat +test resumes that same queued run after deleting the old disk, verifies saved +workspace bytes and instructions at provider dispatch, and checks one result and +one wake consumption even after a second queue sweep. Its sandbox transport and +provider session are scripted; database, S3, admission and restoration are real. diff --git a/doc/sleep-qualification.md b/doc/sleep-qualification.md new file mode 100644 index 00000000000..4bbacc05d00 --- /dev/null +++ b/doc/sleep-qualification.md @@ -0,0 +1,244 @@ +# Sleep configuration qualification + +This opt-in harness boots the actual server against an explicitly supplied fresh +test database. It measures local SQL, connection and network quiet. A local pass +does not establish Railway or Neon suspension. Keep a separate verdict for each +layer: quiet, eligibility, database suspension, host suspension, backup persistence, +inbound wake, timed wake and return to idle. + +## Local configured services + +Use a reviewed integrated source containing the incoming-wake and domain-deadline +contracts. Provision and migrate a fresh dedicated PostgreSQL database ending in +`_test` through the normal test operator. The suite does not allocate a database, +start embedded PostgreSQL or delete a database. It preserves its fixture rows. +Never supply a live tenant database. + +The configured exporter needs the exact optional peer versions already declared +in `server/package.json`. With Node 24 and npm available, explicitly install the +fixture peers into a new temporary prefix and link only missing packages in this +checkout: + +```sh +node scripts/sleep-qualification/setup-otel.mjs +``` + +The setup leaves application manifests and lockfiles unchanged. Retain the printed +setup receipt and temporary package lock. A version mismatch fails without +replacing an installed package. An opt-in CI job must run the same setup command; +setting an OTLP endpoint alone does not qualify an exporter. + +The helper records its exact symlink targets in +`server/node_modules/.paperclip-sleep-otel-links.json`. If an owned temporary prefix +disappears, rerunning repairs only those matching recorded symlinks. Existing +matching packages are left alone. An unrecorded dangling link or other entry is +preserved with a recovery error; its pathname is not evidence of ownership. +`node scripts/sleep-qualification/setup-otel.mjs --cleanup` unlinks only matching +owned links. It preserves foreign replacements and keeps temporary install +prefixes for setup evidence. Remove those owned prefixes separately when their +receipts are no longer needed. + +Set `PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE=1`, +`PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL` to the owned migrated fixture and +`PAPERCLIP_SLEEP_CLOUD_SHA` to the exact reviewed Cloud source SHA. Set +`PAPERCLIP_SLEEP_CLOUD_CHECKOUT` to that built private checkout; the suite verifies +its HEAD and imports its production broker adapter without copying private source. +Retain artifacts +with `PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS=1`. Run the two scoped Vitest files with +one worker and no file parallelism: + +```sh +pnpm exec vitest run server/src/__tests__/sleep-otel-export.test.ts \ + server/src/__tests__/sleep-configuration-acceptance.test.ts \ + --maxWorkers=1 --no-file-parallelism +``` + +The first case requires a successful named span at the real local OTLP collector +and no metric or log export during 90 real seconds, beyond the SDK's default +60-second metric cadence. It fails if the production SDK falls back to no-op. +The assembled case keeps +normal automatic backups, announcements, first-party telemetry and OTel enabled. +It creates an agent, useful completed process execution, API credential and future +routine through normal APIs. It waits the real ten-minute authenticated-use +cooldown, then measures at least 90 real seconds without tenant HTTP/SQL observer +polling. A passive listener on the actual app HTTP server counts incoming requests +and WebSocket upgrades, including direct clients outside the harness API helper. +It requires the app's existing request and upgrade handlers before installing; +it cannot become the sole upgrade listener and change unsupported-upgrade handling. +IPC snapshots record its capture identity, installation/validity, cumulative safe-integer +counts and observation times. Both snapshots must enclose the complete quiet interval, +remain valid and yield an exact zero delta. No URL, headers or body are recorded; +snapshot reads themselves make no HTTP requests or SQL queries. Missing or detached +capture, malformed counts and the legacy empty `observerRequests` list cannot prove +quiet. The passive protocol proxy separately measures database effects. It obtains +two owned signed safety reports after the window and preserves fixture history. +After ordinary API reactivation it waits another full real ten-minute cooldown, +measures another 90 seconds of quiet and requests fresh owned eligibility proof. +The helper keeps its legacy defaults for existing maintenance tests. + +Announcements are enabled and the actual route/feed service validates and caches +the known fixture. An owned test-child module wrapper injects the existing feed +transport seam for the exact default HTTPS URL; unmatched requests fail. Startup +fails if the wrapper did not intercept the actual app constructor. The receipt +requires one named fetch and no subsequent calls during quiet. Browser mode also +requires the fixture copy to render. The production DNS/HTTPS address guard stays +unchanged and is not exercised in this combined case; its owning tests cover that +boundary separately. Active user activity can legitimately keep an instance awake. + +The assembled scenario includes real tool approval rejection/delivery retirement +and signed chat-webhook completion followed by ordinary endpoint removal. That +removed-endpoint subcase is separate from the retained enabled GitHub connection: +the latter uses actual encrypted user credentials, refresh/continuity owners, +signed broker bindings and the production event delivery worker. The host wake +capability comes from the production connector store. Local in-memory broker +storage and synthetic GitHub HTTP responses remain explicit external boundaries; +this does not establish deployed broker durability or host suspension. These +assembled paths require a successful run against the reviewed integration before +their receipts can qualify them; source presence alone is not evidence. +Unknown plugins, persistent provider sessions, unfinished delivery/accounting, +uncertain writes and cleanup remain real blockers. Every inventory family needs +an explicit disposition before claiming all configurations qualified. + +## Evaluate a receipt + +```sh +pnpm exec vitest run server/scripts/sleep-qualification.test.mjs --maxWorkers=1 +node scripts/sleep-qualification/evidence.mjs receipt.json expected.json report.json +``` + +`expected.json` binds `appSha`, `cloudSha`, `target` and a SHA-256 `configDigest`. +The digest uses `logicalSleepConfiguration(configuration)`: explicit backup interval +and checkpoint policy, announcements, telemetry activation, OTLP activation/protocol, +and UI activation in a fixed order. It excludes ephemeral loopback ports. Compute +the expected logical contract independently before starting the fixture; the +evaluator recomputes the receipt digest. Keep actual `collectorOrigins.telemetry` +and `.otel` separately, matching the actual configuration and optional expected +origin bindings. + +Configured coverage must also have recorded observations. Announcements require +actual interception, a request for the expected feed, and no unmatched URL. +`signalObservation` records telemetry requests, trace requests, actual OTLP span +records and the named qualification span separately. Metric, log, malformed-trace +and unexpected request counts must be explicitly zero. The assembled case freezes +these observations after each quiet window; feature booleans are derived from +them and the observed backup checkpoint. Missing or malformed counts fail local +qualification even when a feature flag claims success. + +Before the normal cooldown, a signed diagnostic-only inventory saves both raw +reports. It explicitly expects the active API cooldown aggregate and the skipped +backup diagnostic; every other reported blocker must be absent. Aggregate local +probes cannot identify which probe is present, so this early inventory never +proves eligibility. Its hold is released before the full unchanged cooldown. +After quiet, raw ordinary reports are saved before assertions; a refusal also saves +a diagnostic report under that hold. + +Automatic backup coverage requires a new archive from the normal idle checkpoint, +not an enabled flag. After quiet counters freeze, the harness reads every new +owned gzip through its trailer, hashes compressed bytes, checks nonempty SQL for +the fixture identifier without retaining contents, and observes the wake marker +and absence of backup failure markers. This local evidence carries no provider +volume or restore claim. The owning backup suite separately qualifies restoration. +Archive provenance uses a positive finite opened-file birthtime, falling back to +positive finite content mtime only when the filesystem reports birthtime as zero. +The selected timestamp and its source are recorded. Existing names, old positive +birthtimes, malformed timestamps and future chronology remain disqualifying. +Original receipts are retained as produced; the new evidence fields are not +retroactively added to earlier runs. +The historical Full Y at `487e450b` and short composition at `57744f4e` wrote an +unmeasured empty observer-request list. Their recorded SQL, connection, outbound, +browser and producer observations remain pinned to those runs, but that list does +not meet the corrected inbound-observation contract. A new combined qualification +requires both real cooldowns and quiet windows with the installed incoming counter; +a short injected-poll regression cannot retroactively establish observer absence. +Receipt finalization follows all observations and runs the same evaluator before +writing the final receipt. A return-to-idle claim also needs explicit in-run quiet +start/end timestamps spanning at least 90 real seconds after the latest wake; +a duration field alone cannot prove it. +The evaluator rejects mismatched identities and chronology. Missing evidence stays +`INCONCLUSIVE`; a measured failed condition is `NOT VERIFIED`, including an +identity-verified active Neon endpoint or exact Railway service with running deployments. It does not produce +provider proof from local counters, fake time, a registry sleep flag or a successful +control response. Preserve original failures and their raw logs after a retry. + +## Deployed boundary + +No deployment target is authorized by this document. The private Cloud companion +provides a gated read-only provider observer. An operator must first approve one +exact owned disposable staging target, revisions, configuration, provider refs, +fixture operations and finite observation deadline. A named connector or shared +QA workspace is not that authorization. + +Capture app/Cloud identity and effective policy while awake. Use only provider and +central control-plane metadata during sleep; tenant health/API/SQL probes can wake +the target. Confirm the exact Neon endpoint is idle and all owned Railway services +have stopped deployments. Observe ordinary authorized API, broker and deadline +wake cycles separately. Verify useful persisted results and return to idle. +Automatic backup persistence additionally requires an exact attached volume +covering the configured home/backup directory and a fsynced sentinel/archive +continuity receipt across the owned stop/wake. A writable local fallback directory +is insufficient. Missing live access leaves these layers `INCONCLUSIVE`. + +For actual foreground/background browser coverage, first build the shipped UI into +`server/ui-dist` in the same reviewed checkout. Install the normal Playwright +Chromium runtime and run on a display-capable worker. Set +`PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE=1` for the configured suite. The harness loads +the actual UI, records an ordinary active reload, opens another real foreground +tab and requires Chromium to report the app tab hidden. It observes 90 real +seconds without app API traffic, then closes the browser before the server quiet +window. Failure to establish real visibility fails that mode; it never substitutes +a fake visibility event or frozen page. Without this explicit mode, the receipt +claims closed-browser coverage only. + +The browser uses a fresh owned Chromium profile and loopback debugging endpoint. +It attaches with `connectOverCDP({ noDefaults: true })` to the native default +context. Normal Playwright contexts install focus emulation that kept real +background tabs visible in the measured failure. This path avoids that override. +The receipt includes native window IDs, initial/final visibility and passive +visibility events for the entire 90-second interval. An intervening activation +fails the observation. Cleanup disconnects, terminates only the owned process and +removes its temporary profile. It never attaches to a user's existing browser. + + +## Managed fixture preflight + +Configured qualification uses the supported managed authenticated profile. The +actual app runs on loopback, with a lifetime-stable fixture vault key and the +normal managed configuration. A fixture Cloud front door supplies trusted tenant +headers to invoke the real primary-company/user/membership provisioning path; +ordinary company creation remains forbidden. Bearer-key and signed-control +requests omit tenant headers. Native browser headers can reach only the owned +app origin; foreign HTTP/WebSocket transports are blocked and fail qualification. + +Managed application storage requires S3. A bounded loopback protocol fixture +supplies the remote object service for actual SDK requests; unsupported operations, +framing and conditional requests fail closed. This establishes neither provider +IAM nor durable S3 storage. The instance home and database archives remain real +local filesystem paths. Normal managed migration, historical cleanup and routine +owners remain enabled. The logical digest binds managed activation, auth mode and +storage provider; actual ephemeral transport origins are recorded separately. + +Before another long campaign, run the opt-in `sleep-managed-preflight.test.ts` on +its own root-leased fresh external database using +`PAPERCLIP_SLEEP_MANAGED_PREFLIGHT=1` and an explicit dedicated +`PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_TEST_DATABASE_URL`. Retain artifacts with +`PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS=1`. It uses the same profile and negotiated +broker receiver, provisions real authority, admits and completes one process run, +and proves real key activity through signed diagnostics. It performs no browser, +ten-minute wait or quiet-window claim. No embedded allocation or automatic drop. +Loopback execution requires the host tool's explicit network/runtime permission. + +Setup and successful browser receipts are written immediately before later +assertions. Early diagnostics first use the existing owner-settled contract; they +reject durable blockers immediately and retry only observed local in-flight / +generation inspection incompleteness, up to three rounds. Every attempt is saved. +Both accepted reports must be strict, complete, structurally matching and under +the exact live hold, with each internal `inspection.generation=none` and no active +work before/after. Real HTTP authentication itself changes the global work +counter, so equality across deliberate observer requests is not asserted. + +A fresh successful opaque-key request anchors the later full real cooldown. +Neither setup stabilization nor the fast preflight substitutes for both real +600100ms cooldowns, both untouched 90000ms windows and useful reactivation. S3 +observations are saved before and after owned app cleanup, including failures; +cleanup preserves the original error. Full provider sleep remains a separate +unqualified layer until an exact deployed target is authorized and observed. diff --git a/doc/testing-native-remote-continuity.md b/doc/testing-native-remote-continuity.md index bd4fb833711..810c16e9060 100644 --- a/doc/testing-native-remote-continuity.md +++ b/doc/testing-native-remote-continuity.md @@ -540,3 +540,26 @@ explicit incomplete usage report before failure; this preserves known budget exposure without settling ambiguous usage. That required report and its receipt retain the accounting measurements and matching saved prior identity, so an optional diagnostic acknowledgement cannot discard reconciliation evidence. + +## Shared allocation ingress retirement + +The `retires the completed child listener` cases in +`native-session-executor.test.ts` use the actual executor and warm registry, +PostgreSQL ownership rows, and real local TCP listeners. The provider process +boundary is supplied by the fixture. The listeners use an ephemeral test port; +production provider ingress keeps its fixed port. A completed child's listener +must close before a distinct parent session binds in that allocation. The child's +accepted result stays unchanged. + +```sh +pnpm --filter @paperclipai/server exec vitest run \ + src/services/native-runtime/native-session-executor.test.ts \ + -t 'retires the completed child listener' +``` + +Controls cover foreign companies, allocations and workspaces; live or unaccounted +owners; changed controllers, reassignment and competing leases; workspace-lock +and cancellation races; unconfirmed close; and controller, Stop and writer +changes inside awaited close. The latter must prevent parent backend invocation +even when the old listener has closed. These local checks do not identify +a hosted conflicting PID or prove a hosted provider Stop or recovery. diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index a7e77abe79b..13c354a89c0 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -726,6 +726,29 @@ actual shell discovery, successful durable activity recording, and denial of unr Runner state, descriptor writes and ambient secrets. The normal suite covers bounded framing, concurrent publishers, fixed-path cleanup and attachment fencing. +Codex receives write access to Runner's private activity-channel directory, with +the lock file explicitly read-only. The FIFO itself must not be a writable +filesystem root: Codex adds protected metadata children to writable roots, which +cannot exist beneath a FIFO. The helper and discovery descriptor remain read-only; +the activity grant does not expose Runner state or change workspace/network policy. + +To check restrictive Plan mode without credentials or a model call, run the +actual channel/policy producer against the pinned Linux Codex sandbox: + +```sh +CODEX_NOTIFICATION_SANDBOX_COMMAND=/absolute/path/to/pinned/codex \ +CODEX_NOTIFICATION_SANDBOX_RESOURCES=/absolute/path/to/pinned/executable/resources \ +cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --lib \ + read_only_codex_opens_with_runtime_activity -- --ignored +``` + +This checks instruction reads, FIFO writes, and denial of workspace writes, +unrelated Runner-state reads, descriptor writes, and lock writes/removal. An +isolated non-Linux host can instead set `CODEX_NOTIFICATION_FIXTURE_EXPORT` to a +JSON file and omit `CODEX_NOTIFICATION_SANDBOX_COMMAND`; that only exports a +synthetic producer fixture for a separately controlled Linux sandbox. Exporting +the fixture does not execute or verify the sandbox. + ## Direct live protocol qualification The canonical direct live protocol suite lives in the separate diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 3205943c454..d1e0be1a1af 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -70,6 +70,35 @@ a scripted harness, removes the controller's launch files, then recovers the same process and provider session and completes another turn. This exercises the real transport; it does not qualify a hosted OpenCode or ACPX deployment. +## Settled ACPX sessions across a Runner upgrade + +Saved ACPX provider state records the digest of the launch profile that wrote +it: the provider-pack authority, the sidecar command and arguments, each launch +artifact's SHA-256, and any candidate admission. A deploy that ships a new +provider pack or sidecar build changes the digest. The state stores only the +digest, so restore cannot tell which part changed. + +Restore re-binds a saved session to the current launch profile, and keeps the +conversation, only when all of these are true: + +- The session is suspended. It has no active turn, pending runtime request, + undelivered provider event, or unconfirmed provider exit. +- The startup has no candidate admission. +- The saved descriptor names this startup's sidecar command and arguments. + +Every other binding stays exact. The saved descriptor must match this build's +qualified agent profile (ACPX version, agent server package and version, and +command digest). Session start verifies each artifact and the restored provider +identity. Candidate grants and run bindings keep their own checks. + +Any other launch-profile mismatch, and a saved descriptor that names a release +this build does not qualify, fails restore with the closed command-result +classification `provider_state_incompatible`. The controller reads the classification from the +authenticated result, not from the message. If the run allows a replacement +after a failed resume, it opens a fresh provider session as before. Otherwise the +run fails with `native_provider_state_incompatible` and is not retried. A board +Retry then continues the task in a fresh provider session in the same workspace. + ## Profile identity Every ACPX provider's `commandDigest` attests only its own upstream release. diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index 1e445b261d1..bcc1625dc88 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -26,7 +26,7 @@ use crate::durable::QualifiedLaunchArtifact; use crate::durable::{ create_private_temporary_file, open_private_regular_file, verify_private_directory, AcpxLaunchProfile, Command, CommandExecution, CommandExecutor, DurableRunnerConfig, - DurableRunnerError, EventPriority, PolledEvent, + DurableRunnerError, EventPriority, PolledEvent, PROVIDER_STATE_INCOMPATIBLE, }; use crate::generated_acpx_sidecar_contract::GeneratedAcpxSidecarCommand; use crate::process_supervisor::{VerifiedProcessArgument, VerifiedProcessLaunch}; @@ -279,6 +279,19 @@ fn only_registered_asset_text_changed( } impl AcpxProviderDescriptor { + // Whether this build qualifies the release that the descriptor names. + fn names_qualified_release(&self) -> bool { + acpx_release_profile(&self.agent).is_some_and(|expected| { + self.provider_version == QUALIFIED_ACPX_VERSION + && self.acpx_version == QUALIFIED_ACPX_VERSION + && self.agent_server_package == expected.agent_server_package + && self.agent_server_version == expected.agent_server_version + && self.agent_runtime_package.as_deref() == expected.agent_runtime_package + && self.agent_runtime_version.as_deref() == expected.agent_runtime_version + && self.command_digest == expected.command_digest + }) + } + fn validate_session( &self, context: &AcpxEventProjectionContext, @@ -622,11 +635,21 @@ impl AcpxDurableState { // so its persisted descriptor legitimately carries the prior run ID // until run.attach rotates authority. Fresh command descriptors still // use validate(), which binds them to the current run. - self.descriptor.validate_session(context)?; + // A deploy that changes the agent's qualified release also leaves + // saved state that this build cannot restore. Keep the same check and + // only name its class. + self.descriptor.validate_session(context).map_err(|error| { + if self.descriptor.names_qualified_release() { + error + } else { + error.with_classification(PROVIDER_STATE_INCOMPATIBLE) + } + })?; if self.launch_profile_digest != expected_launch_profile_digest { return Err(DurableRunnerError::invalid( "ACPX durable launch profile digest does not match runner startup", - )); + ) + .with_classification(PROVIDER_STATE_INCOMPATIBLE)); } validate_pending_runtime_requests(&self.pending_runtime_requests)?; let mut ids = HashSet::new(); @@ -960,6 +983,34 @@ impl AcpxCommandExecutor { prior.canonical_digest() } + // A deploy changes the provider-pack authority and the sidecar and Node + // artifact digests, so a saved digest from the previous build never + // matches. The digest is a hash, so restore cannot tell which part + // changed. Re-bind only when no provider work is in flight: + // - The session is suspended. No turn, request, undelivered event, or + // unconfirmed provider exit remains. Nothing from the old provider + // process continues; the next launch only loads the conversation. + // - The startup is ordinary. Candidate admission carries company, model + // and expiry authority in the digest, so candidate sessions stay + // exactly bound. + // - The saved descriptor names this startup's sidecar command and + // arguments. Only the qualified content at those paths changed. + // validate() still binds the descriptor to this build's qualified agent + // profile (ACPX, agent server and command digest), and session start + // still verifies every artifact and the restored provider identity. + fn rebinds_settled_session(&self, state: &AcpxDurableState) -> bool { + self.launch_profile.as_ref().is_some_and(|profile| { + profile.candidate_admission.is_none() + && state.descriptor.sidecar_command == profile.command + && state.descriptor.sidecar_args == profile.args + }) && state.lifecycle == "suspended" + && state.identity.is_some() + && state.active_turn_id.is_none() + && !state.provider_exit_unconfirmed + && state.pending_events.is_empty() + && state.pending_runtime_requests.is_empty() + } + fn restore(&mut self) -> Result<(), DurableRunnerError> { self.ensure_persistence_healthy()?; if self.restore_checked { @@ -1016,10 +1067,15 @@ impl AcpxCommandExecutor { file.read_to_end(&mut bytes).map_err(|error| { DurableRunnerError::invalid(format!("failed to read ACPX provider state: {error}")) })?; - let state: AcpxDurableState = serde_json::from_slice(&bytes).map_err(|error| { + let mut state: AcpxDurableState = serde_json::from_slice(&bytes).map_err(|error| { DurableRunnerError::invalid(format!("ACPX provider state is malformed: {error}")) })?; let launch_profile_digest = self.restore_launch_profile_digest(&state)?; + if state.launch_profile_digest != launch_profile_digest + && self.rebinds_settled_session(&state) + { + state.launch_profile_digest = launch_profile_digest.clone(); + } state.validate(&self.context, &launch_profile_digest)?; self.state = Some(state); self.restore_session_if_needed() @@ -5298,14 +5354,11 @@ mod tests { "grant_run", "injected_candidate", "context_run", - "authority", "command", "args", - "artifact", "missing_profile", - "saved_digest", ] { - let (directory, mut config, payload, before, sentinel, candidate) = + let (directory, mut config, payload, _, sentinel, candidate) = kimi_cold_restore_fixture("kimi-released-cold-reject"); assert!(config .acpx_launch_profile @@ -5316,26 +5369,11 @@ mod tests { match field { "context_run" => config.run_id = "foreign-run".into(), "missing_profile" => config.acpx_launch_profile = None, - "saved_digest" => { - let mut state: AcpxDurableState = serde_json::from_slice(&before).unwrap(); - state.launch_profile_digest = format!("sha256:{}", "0".repeat(64)); - fs::write( - directory.join("acpx-provider-state.json"), - serde_json::to_vec(&state).unwrap(), - ) - .unwrap(); - } _ => { let profile = config.acpx_launch_profile.as_mut().unwrap(); match field { - "authority" => { - profile.authority_digest = format!("sha256:{}", "0".repeat(64)) - } "command" => profile.command = directory.join("foreign-command"), "args" => profile.args.push("foreign-argument".into()), - "artifact" => { - profile.artifacts[0].sha256 = format!("sha256:{}", "0".repeat(64)) - } _ => { let mut grant = candidate; match field { @@ -5382,6 +5420,60 @@ mod tests { } } + #[test] + #[cfg(any( + all(target_os = "macos", target_arch = "aarch64"), + all(target_os = "linux", target_arch = "x86_64") + ))] + fn released_kimi_cold_restore_rebinds_a_settled_session_to_new_launch_content() { + // An ordinary startup re-binds a settled session after a provider-pack + // or artifact change at the same launch paths. The checks above for + // candidate grants, command, arguments and run binding still apply. + for field in ["authority", "artifact", "saved_digest"] { + let (directory, mut config, payload, before, sentinel, _) = + kimi_cold_restore_fixture("kimi-released-cold-rebind"); + let profile = config.acpx_launch_profile.as_mut().unwrap(); + match field { + "authority" => profile.authority_digest = format!("sha256:{}", "0".repeat(64)), + "artifact" => { + let command = profile.command.clone(); + write_artifact(&command, b"#!/bin/sh\nexit 2\n", true); + profile.artifacts[0] = artifact(&command); + } + "saved_digest" => { + let mut state: AcpxDurableState = serde_json::from_slice(&before).unwrap(); + state.launch_profile_digest = format!("sha256:{}", "0".repeat(64)); + fs::write( + directory.join(ACPX_PROVIDER_STATE_FILE), + serde_json::to_vec(&state).unwrap(), + ) + .unwrap(); + } + _ => unreachable!(), + } + config.validate().unwrap(); + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + recovered.restore().expect(field); + recovered.attach_run(&payload).expect(field); + let saved: AcpxDurableState = + serde_json::from_slice(&fs::read(recovered.state_path()).unwrap()).unwrap(); + assert_eq!(saved.descriptor.run_id, "run-2", "{field}"); + assert_eq!( + saved.launch_profile_digest, + config + .acpx_launch_profile + .as_ref() + .unwrap() + .canonical_digest() + .unwrap(), + "{field}" + ); + assert!(recovered.session.is_none(), "{field}"); + assert!(!sentinel.exists(), "{field}"); + fs::remove_dir_all(directory).unwrap(); + } + } + #[test] #[cfg(any( all(target_os = "macos", target_arch = "aarch64"), @@ -5498,4 +5590,217 @@ mod tests { assert!(!sentinel.exists()); fs::remove_dir_all(directory).unwrap(); } + + // A settled Claude session saved by one runner build. The next deploy + // ships a new sidecar build and provider-pack authority at the same + // qualified launch paths, and the next run cold-attaches in a new runner. + #[cfg(unix)] + fn settled_session_after_runner_upgrade( + label: &str, + ) -> (PathBuf, DurableRunnerConfig, Value, Vec) { + let directory = temporary_directory(label); + let command = directory.join("node"); + let sidecar = directory.join("acpx-runtime-sidecar.cjs"); + write_artifact(&command, b"#!/bin/sh\nexit 1\n", true); + write_artifact(&sidecar, b"// sidecar build 1\n", false); + let profile = AcpxLaunchProfile { + candidate_admission: None, + authority_digest: format!("sha256:{}", "d".repeat(64)), + command: command.clone(), + args: vec![sidecar.to_string_lossy().into_owned()], + artifacts: vec![artifact(&command), artifact(&sidecar)], + }; + let runtime = directory.join("runtime"); + let workspace = directory.join("workspace"); + fs::create_dir_all(&runtime).unwrap(); + fs::create_dir_all(&workspace).unwrap(); + fs::set_permissions(&runtime, fs::Permissions::from_mode(0o700)).unwrap(); + fs::set_permissions(&workspace, fs::Permissions::from_mode(0o700)).unwrap(); + let mut value = descriptor("claude"); + value["runtimeDirectory"] = json!(runtime); + value["cwd"] = json!(workspace); + value["sidecarCommand"] = json!(command); + value["sidecarArgs"] = json!([sidecar]); + let desc: AcpxProviderDescriptor = serde_json::from_value(value.clone()).unwrap(); + let identity = AcpxProviderSessionIdentity { + kind: "acpx".to_owned(), + normalized_session_id: "session-1".to_owned(), + acpx_record_id: "record-1".to_owned(), + backend_session_id: "backend-1".to_owned(), + agent_session_id: "agent-1".to_owned(), + profile_digest: desc.command_digest.clone(), + workspace_digest: format!("sha256:{}", "a".repeat(64)), + requested_model: desc.model.clone(), + effective_model: desc.model.clone(), + permission_mode: Some(desc.permission_mode), + mode: None, + pi_thinking_level: None, + claude_effort: None, + grok_effort: None, + grok_disable_web_search: None, + provider_lifetime_fence_candidates: [60_001, 60_002, 60_003], + }; + let operations = Vec::new(); + let tools = AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }; + let mut state = AcpxDurableState::new(desc, tools, profile.canonical_digest().unwrap()); + state.lifecycle = "suspended".to_owned(); + state.identity = Some(identity); + let config = test_config(&directory, Some(profile)); + let mut executor = AcpxCommandExecutor::with_runner_config(&directory, &config); + executor.state = Some(state); + executor.save_state().unwrap(); + let saved = fs::read(executor.state_path()).unwrap(); + + write_artifact(&sidecar, b"// sidecar build 2\n", false); + let mut next_config = config; + next_config.run_id = "run-2".to_owned(); + next_config.turn_id = "turn-2".to_owned(); + next_config.item_id = "item-2".to_owned(); + let next_profile = next_config.acpx_launch_profile.as_mut().unwrap(); + next_profile.authority_digest = format!("sha256:{}", "e".repeat(64)); + next_profile.artifacts = vec![artifact(&command), artifact(&sidecar)]; + next_config.validate().unwrap(); + let mut provider = value; + provider["runId"] = json!("run-2"); + ( + directory, + next_config, + json!({ "provider": provider }), + saved, + ) + } + + #[cfg(unix)] + #[test] + fn settled_session_rebinds_to_the_upgraded_runner_launch_profile() { + let (directory, config, payload, saved) = + settled_session_after_runner_upgrade("upgrade-rebind"); + let prior: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); + let current_digest = config + .acpx_launch_profile + .as_ref() + .unwrap() + .canonical_digest() + .unwrap(); + assert_ne!(prior.launch_profile_digest, current_digest); + + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + recovered + .restore() + .expect("a settled session survives a new qualified sidecar build"); + assert!(recovered.session.is_none()); + assert!(recovered.poll_events().unwrap().is_empty()); + assert_eq!(fs::read(recovered.state_path()).unwrap(), saved); + recovered + .attach_run(&payload) + .expect("the next run cold-attaches the same conversation"); + assert!(recovered.session.is_none()); + + let attached: AcpxDurableState = + serde_json::from_slice(&fs::read(recovered.state_path()).unwrap()).unwrap(); + assert_eq!(attached.launch_profile_digest, current_digest); + assert_eq!(attached.descriptor.run_id, "run-2"); + assert_eq!(attached.lifecycle, "suspended"); + assert_eq!(attached.identity, prior.identity); + assert_eq!( + attached.descriptor.command_digest, + prior.descriptor.command_digest + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[cfg(unix)] + #[test] + fn unrestorable_saved_sessions_keep_strict_bindings_and_name_their_class() { + for field in [ + "active_turn", + "pending_request", + "pending_event", + "session_open", + "exit_unconfirmed", + "no_identity", + "command_moved", + "args_changed", + "agent_server_release", + "acpx_release", + ] { + let (directory, mut config, _, saved) = + settled_session_after_runner_upgrade("upgrade-strict"); + let state_path = directory.join(ACPX_PROVIDER_STATE_FILE); + let mut state: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); + let profile = config.acpx_launch_profile.as_mut().unwrap(); + match field { + "active_turn" => { + state.lifecycle = "turn_active".into(); + state.active_turn_id = Some("turn-1".into()); + } + "pending_request" => { + state.lifecycle = "turn_active".into(); + state.active_turn_id = Some("turn-1".into()); + state + .push(NormalizedProviderEvent { + event_type: "runtime_request.created".into(), + priority: EventPriority::P0, + payload: json!({"request":pending_input_request("input-1")}), + }) + .unwrap(); + state.pending_events.clear(); + } + "pending_event" => state + .push(NormalizedProviderEvent { + event_type: "session.updated".into(), + priority: EventPriority::P1, + payload: json!({}), + }) + .unwrap(), + "session_open" => state.lifecycle = "session_open".into(), + "exit_unconfirmed" => { + state.lifecycle = "closed".into(); + state.provider_exit_unconfirmed = true; + } + "no_identity" => { + state.lifecycle = "prepared".into(); + state.identity = None; + } + "command_moved" => { + let moved = directory.join("relocated-node"); + write_artifact(&moved, b"#!/bin/sh\nexit 1\n", true); + profile.command = moved.clone(); + profile.artifacts[0] = artifact(&moved); + } + "args_changed" => profile.args.push("--changed".into()), + "agent_server_release" => state.descriptor.agent_server_version = "0.0.1".into(), + "acpx_release" => { + state.descriptor.acpx_version = "0.0.1".into(); + state.descriptor.provider_version = "0.0.1".into(); + } + _ => unreachable!(), + } + config.validate().unwrap(); + fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); + let retained = fs::read(&state_path).unwrap(); + + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + let error = recovered.restore().unwrap_err(); + let expected = if field.ends_with("_release") { + "does not match a qualified immutable profile" + } else { + "launch profile digest does not match runner startup" + }; + assert!(error.to_string().contains(expected), "{field}: {error}"); + assert_eq!( + error.classification(), + Some(crate::durable::PROVIDER_STATE_INCOMPATIBLE), + "{field}" + ); + assert!(recovered.session.is_none(), "{field}"); + assert_eq!(fs::read(&state_path).unwrap(), retained, "{field}"); + fs::remove_dir_all(directory).unwrap(); + } + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index fefedb0501a..1687b1ad23a 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -61,8 +61,13 @@ pub(crate) fn runtime_notification_codex_args( let Some(filesystem) = config["filesystem"].as_inline_table_mut() else { continue; }; - for (path, mode) in paths.iter().zip(["read", "read", "read", "write"]) { - filesystem.insert(path.to_string_lossy().as_ref(), mode.into()); + let Some(channel_directory) = paths[3].parent().filter(|path| path.parent().is_some()) + else { + continue; + }; + filesystem.insert(channel_directory.to_string_lossy().as_ref(), "write".into()); + for path in &paths[..3] { + filesystem.insert(path.to_string_lossy().as_ref(), "read".into()); } extra.extend(["-c".into(), format!("{prefix}{filesystem}")]); } @@ -4522,8 +4527,8 @@ mod tests { let paths = [ PathBuf::from("/runtime/runnerd"), PathBuf::from("/state/private/runtime-notifications.json"), - PathBuf::from("/state/private/runtime-activity.lock"), - PathBuf::from("/tmp/prn-channel"), + PathBuf::from("/tmp/prn-channel/activity.lock"), + PathBuf::from("/tmp/prn-channel/activity.fifo"), ]; let extra = runtime_notification_codex_args(&args, "paperclip-runner-workspace-only", &paths); @@ -4562,7 +4567,7 @@ mod tests { ("/state/private", "none"), ("/runtime/runnerd", "read"), ("/state/private/runtime-notifications.json", "read"), - ("/state/private/runtime-activity.lock", "read"), + ("/tmp/prn-channel/activity.lock", "read"), ("/tmp/prn-channel", "write"), ] { assert_eq!( @@ -4570,6 +4575,7 @@ mod tests { Some(mode) ); } + assert!(filesystem.get("/tmp/prn-channel/activity.fifo").is_none()); } assert!(!serialized.contains("network.")); assert!(!serialized.contains("PAPERCLIP_RUN_ID")); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs index fc33db5cc3c..4b026797f66 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs @@ -37,12 +37,17 @@ pub fn redact_diagnostic_text(input: &str) -> String { state::redact_text(input) } +/// Saved provider state was written under a launch or provider profile that +/// this runner cannot adopt. The outcome is deterministic for this runner +/// build: retrying the same restore cannot succeed. +pub const PROVIDER_STATE_INCOMPATIBLE: &str = "provider_state_incompatible"; + #[derive(Clone, Debug, PartialEq, Eq)] -pub struct DurableRunnerError(String, Option); +pub struct DurableRunnerError(String, Option, Option<&'static str>); impl DurableRunnerError { pub fn invalid(message: impl Into) -> Self { - Self(message.into(), None) + Self(message.into(), None, None) } pub fn with_provider_failure(mut self, value: Option) -> Self { @@ -53,6 +58,17 @@ impl DurableRunnerError { pub fn provider_failure(&self) -> Option { self.1.clone() } + + /// Adds a closed failure class that the controller reads from the command + /// result instead of parsing the message. + pub fn with_classification(mut self, classification: &'static str) -> Self { + self.2 = Some(classification); + self + } + + pub fn classification(&self) -> Option<&'static str> { + self.2 + } } impl Display for DurableRunnerError { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index ddfd4034c94..97bf9644745 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -1930,6 +1930,9 @@ fn process_command( "code": "command_execution_failed", "message": message, }); + if let Some(classification) = error.classification() { + failure["classification"] = json!(classification); + } if let Some(provider_failure) = error.provider_failure() { failure["providerFailure"] = provider_failure; } @@ -2489,6 +2492,17 @@ mod tests { } } + struct IncompatibleStateExecutor; + + impl CommandExecutor for IncompatibleStateExecutor { + fn execute(&mut self, _command: &Command) -> Result { + Err( + DurableRunnerError::invalid("saved provider state cannot be adopted") + .with_classification(crate::durable::PROVIDER_STATE_INCOMPATIBLE), + ) + } + } + impl CommandExecutor for ShutdownFailingExecutor { fn execute(&mut self, _command: &Command) -> Result { Ok(CommandExecution::result(json!({"status": "completed"}))) @@ -4865,6 +4879,46 @@ mod tests { .diagnostics .iter() .all(|diagnostic| !diagnostic.contains("test-secret"))); + assert!(failed.result.get("classification").is_none()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn classified_executor_failure_keeps_its_closed_class_in_the_durable_result() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-command-classified-failure-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let command = command("run.attach"); + let (failed, _) = process_command( + &mut state, + &store, + &config, + &mut IncompatibleStateExecutor, + &command, + ) + .unwrap(); + let (mut recovered, _) = store.load_or_create(&config).unwrap(); + let replay = process_command( + &mut recovered, + &store, + &config, + &mut IncompatibleStateExecutor, + &command, + ) + .unwrap() + .0; + assert_eq!(failed.status, "failed"); + assert_eq!(failed.result["code"], "command_execution_failed"); + assert_eq!( + failed.result["classification"], + "provider_state_incompatible" + ); + assert_eq!(failed, replay); fs::remove_dir_all(directory).unwrap(); } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications.rs b/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications.rs index d1722519fb1..5b05061290c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications.rs @@ -666,6 +666,8 @@ mod tests { assert_eq!(fs::read(nested.join("protected")).unwrap(), b"unchanged"); fs::remove_dir_all(pipe.parent().unwrap()).unwrap(); } + include!("runtime_notifications/tests/codex_sandbox.rs"); + #[test] #[ignore = "requires CODEX_NOTIFICATION_SANDBOX_COMMAND and PAPERCLIP_RUNTIME_NOTIFICATION_HELPER"] fn publisher_crosses_real_codex_tool_boundary_without_runner_state_access() { @@ -709,7 +711,8 @@ mod tests { if cat "$1" >/dev/null 2>&1; then exit 71; fi if /bin/sh -c ': >> "$PAPERCLIP_RUNTIME_NOTIFICATION_CONTEXT"' 2>/dev/null; then exit 72; fi if touch "$2/probe" 2>/dev/null; then exit 75; fi -if touch "$3/probe" 2>/dev/null; then exit 76; fi +touch "$3/probe" || exit 76 +rm "$3/probe" || exit 76 if /bin/sh -c ': >> "$1"' ignored "$4" 2>/dev/null; then exit 77; fi context=$("$PAPERCLIP_RUNTIME_NOTIFICATION_HELPER" notification-context) || exit 73 "$PAPERCLIP_RUNTIME_NOTIFICATION_HELPER" record-activity "$context" example.runtime desktop.used || exit 74 diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications/tests/codex_sandbox.rs b/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications/tests/codex_sandbox.rs new file mode 100644 index 00000000000..1638b7d84c5 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/runtime_notifications/tests/codex_sandbox.rs @@ -0,0 +1,157 @@ +#[test] +fn codex_activity_grant_uses_only_its_own_private_channel_directory() { + let (_directory, config, _store, _state, server) = setup(); + let (_other_directory, _other_config, _other_store, _other_state, other) = setup(); + let channel = server.pipe.0.parent().unwrap(); + let other_channel = other.pipe.0.parent().unwrap(); + assert_ne!(channel, other_channel); + verify_private_directory(channel).unwrap(); + assert!(fs::symlink_metadata(&server.pipe.0) + .unwrap() + .file_type() + .is_fifo()); + let paths = [ + PathBuf::from("/runtime/runnerd"), + context_path(&config.state_dir), + server.pipe.0.with_extension("lock"), + server.pipe.0.clone(), + ]; + let profile = "paperclip-runner-workspace-read-only"; + let args = vec![ + "shell_environment_policy.include_only=[]".into(), + format!("permissions.{profile}.filesystem={{\":root\"=\"none\"}}"), + ]; + let projected = crate::codex_provider::runtime_notification_codex_args(&args, profile, &paths); + let prefix = format!("permissions.{profile}.filesystem="); + let value = projected + .iter() + .find_map(|arg| arg.strip_prefix(&prefix)) + .unwrap(); + let config = format!("filesystem={value}") + .parse::() + .unwrap(); + let filesystem = config["filesystem"].as_inline_table().unwrap(); + assert_eq!( + filesystem + .get(channel.to_str().unwrap()) + .and_then(toml_edit::Value::as_str), + Some("write") + ); + assert!(filesystem.get(other_channel.to_str().unwrap()).is_none()); + assert!(filesystem + .get(paths[1].parent().unwrap().to_str().unwrap()) + .is_none()); +} + +#[test] +#[ignore = "requires source-pinned Linux CODEX_NOTIFICATION_SANDBOX_COMMAND or CODEX_NOTIFICATION_FIXTURE_EXPORT"] +fn read_only_codex_opens_with_runtime_activity_without_widening_file_access() { + use std::process::Command; + + let directory = TestDirectory::create(&std::env::temp_dir()).unwrap(); + let config = config(directory.directory.join("state")); + fs::DirBuilder::new() + .mode(0o700) + .create(&config.state_dir) + .unwrap(); + let server = RuntimeNotifications::bind(&config.state_dir).unwrap(); + let workspace = directory.directory.join("workspace"); + let home = directory.directory.join("home"); + fs::create_dir(&workspace).unwrap(); + fs::create_dir(&home).unwrap(); + fs::write( + home.join("credential-fixture"), + b"synthetic private credential", + ) + .unwrap(); + fs::write( + workspace.join("AGENTS.md"), + b"Read-only fixture instructions.\n", + ) + .unwrap(); + fs::write( + config.state_dir.join("private-state"), + b"must remain inaccessible", + ) + .unwrap(); + let helper = config.state_dir.join("helper"); + fs::write(&helper, b"synthetic executable resource\n").unwrap(); + let paths = [ + helper, + context_path(&config.state_dir), + server.pipe.0.with_extension("lock"), + server.pipe.0.clone(), + ]; + let profile = "paperclip-runner-workspace-read-only"; + let resources = std::env::var("CODEX_NOTIFICATION_SANDBOX_RESOURCES") + .expect("set the source-pinned Codex executable resource directory"); + let resources = serde_json::to_string(&resources).unwrap(); + let args = vec![ + "-c".into(), + "shell_environment_policy.include_only=[\"PATH\"]".into(), + "-c".into(), + format!("permissions.{profile}.filesystem={{\":root\"=\"none\",\":minimal\"=\"read\",\":tmpdir\"=\"none\",{resources}=\"read\",\"{}\"=\"none\",\"{}\"=\"read\"}}", home.display(), workspace.display()), + "-c".into(), + format!("permissions.{profile}.network.enabled=false"), + ]; + let extra = crate::codex_provider::runtime_notification_codex_args(&args, profile, &paths); + let script = r#" +set -eu +test -z "${UNRELATED_SECRET:-}" +test "$(cat "$4/AGENTS.md")" = 'Read-only fixture instructions.' +if touch "$4/forbidden" 2>/dev/null; then exit 71; fi +if cat "$1/private-state" >/dev/null 2>&1; then exit 72; fi +if sh -c ': >> "$1"' ignored "$2" 2>/dev/null; then exit 73; fi +if sh -c ': >> "$1"' ignored "$3" 2>/dev/null; then exit 74; fi +if rm "$3" 2>/dev/null; then exit 75; fi +if cat "$6/credential-fixture" >/dev/null 2>&1; then exit 76; fi +cat "$2" >/dev/null +cat "$3" >/dev/null +printf 'synthetic notification frame\n' > "$5" +"#; + let mut args = args; + args.extend(extra); + args.extend(["sandbox", "-P", profile, "-C"].map(str::to_owned)); + args.push(workspace.to_string_lossy().into_owned()); + args.extend(["--", "/bin/sh", "-c", script, "notification-sandbox"].map(str::to_owned)); + args.extend( + [ + &config.state_dir, + &paths[1], + &paths[2], + &workspace, + &paths[3], + &home, + ] + .map(|path| path.to_string_lossy().into_owned()), + ); + if let Some(path) = std::env::var_os("CODEX_NOTIFICATION_FIXTURE_EXPORT") { + fs::write( + path, + serde_json::to_vec_pretty(&json!({ + "args": args, "stateDirectory": config.state_dir, + "pipe": server.pipe.0, "home": home, "workspace": workspace, + "descriptor": fs::read_to_string(&paths[1]).unwrap(), + })) + .unwrap(), + ) + .unwrap(); + return; + } + let command = std::env::var_os("CODEX_NOTIFICATION_SANDBOX_COMMAND").unwrap(); + let output = Command::new(command) + .args(args) + .current_dir(&workspace) + .env_clear() + .env("PATH", std::env::var_os("PATH").unwrap()) + .env("HOME", &home) + .env("CODEX_HOME", &home) + .env("UNRELATED_SECRET", "must-not-reach-command") + .output() + .unwrap(); + assert!( + output.status.success(), + "source-pinned sandbox failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/packages/paperclip-runner/scripts/vitest-sequencer.mjs b/packages/paperclip-runner/scripts/vitest-sequencer.mjs new file mode 100644 index 00000000000..c8069693070 --- /dev/null +++ b/packages/paperclip-runner/scripts/vitest-sequencer.mjs @@ -0,0 +1,27 @@ +import { BaseSequencer } from "vitest/node"; + +const slowFiles = [ + "runnerd-codex-transport.test.ts", + "runnerd-codex-transport-settlement.test.ts", + "cursor-cloud-runner-launch.test.ts", + "runnerd-codex-transport-deadlines.test.ts", + "opencode-server-driver.test.ts", + "acpx-controller-recovery.test.ts", + "runnerd-codex-transport-warm-attachment.test.ts", + "runnerd-opencode-streaming-tools.test.ts", + "live-session.test.ts", + "runnerd-codex-transport-adoption.test.ts", +]; + +const priority = (file) => { + const name = file.moduleId.replaceAll("\\", "/").split("/").at(-1); + const index = slowFiles.indexOf(name); + return index < 0 ? slowFiles.length : index; +}; + +export class RunnerSequencer extends BaseSequencer { + async sort(files) { + const sorted = await super.sort(files); + return sorted.sort((a, b) => priority(a) - priority(b)); + } +} diff --git a/packages/paperclip-runner/scripts/vitest-sequencer.test.mjs b/packages/paperclip-runner/scripts/vitest-sequencer.test.mjs new file mode 100644 index 00000000000..781fe81d8d1 --- /dev/null +++ b/packages/paperclip-runner/scripts/vitest-sequencer.test.mjs @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { test } from "vitest"; +import { BaseSequencer } from "vitest/node"; +import { RunnerSequencer } from "./vitest-sequencer.mjs"; + +const project = { name: "runner", config: { isolate: true, sequence: { groupOrder: 0 } } }; +const files = [ + "large.test.ts", + "runnerd-codex-transport.test.ts", + "small.test.ts", + "runnerd-codex-transport-settlement.test.ts", + "runnerd-codex-transport-deadlines.test.ts", +].map((name) => ({ moduleId: `/repo/src/live/${name}`, project })); +const ctx = { + config: { root: "/repo", shard: { index: 1, count: 2 } }, + cache: { + getFileTestResults: () => undefined, + getFileStats: (key) => ({ size: 100 - files.findIndex((file) => key.endsWith(file.moduleId.slice(6))) }), + }, +}; + +test("starts real-wait transport files first without changing test membership", async () => { + const sorted = await new RunnerSequencer(ctx).sort(files); + assert.deepEqual(sorted, [files[1], files[3], files[4], files[0], files[2]]); + assert.equal(new Set(sorted).size, files.length); + assert.deepEqual(await new BaseSequencer(ctx).sort(files), files); +}); + +test("selected subsets stay selected and retain default ordering", async () => { + const selected = [files[2], files[0]]; + assert.deepEqual(await new RunnerSequencer(ctx).sort(selected), await new BaseSequencer(ctx).sort(selected)); + assert.deepEqual(await new RunnerSequencer(ctx).sort([files[4]]), [files[4]]); + assert.deepEqual(await new RunnerSequencer(ctx).sort([]), []); +}); + +test("keeps Vitest shard membership and schedules waits within each shard", async () => { + const all = []; + for (const index of [1, 2]) { + const shardCtx = { ...ctx, config: { ...ctx.config, shard: { index, count: 2 } } }; + const sequencer = new RunnerSequencer(shardCtx); + const shard = await sequencer.shard(files); + assert.deepEqual(shard, await new BaseSequencer(shardCtx).shard(files)); + const sorted = await sequencer.sort(shard); + assert.deepEqual(new Set(sorted), new Set(shard)); + all.push(...sorted); + } + assert.equal(all.length, files.length); + assert.deepEqual(new Set(all), new Set(files)); +}); + +test("recognizes Windows paths", async () => { + const deadline = { ...files[4], moduleId: "C:\\repo\\src\\live\\runnerd-codex-transport-deadlines.test.ts" }; + assert.deepEqual(await new RunnerSequencer(ctx).sort([files[0], deadline]), [deadline, files[0]]); +}); + +test("starts measured slow provider files before short files without adding work", async () => { + const streaming = { ...files[0], moduleId: "/repo/src/live/runnerd-opencode-streaming-tools.test.ts" }; + const launch = { ...files[0], moduleId: "/repo/src/control-plane/cursor-cloud-runner-launch.test.ts" }; + assert.deepEqual(await new RunnerSequencer(ctx).sort([files[0], streaming, files[4], launch]), + [launch, files[4], streaming, files[0]]); +}); diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index 094362e0fa6..6791372425a 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; +import { PROVIDER_STATE_INCOMPATIBLE, runnerRecoveryFailureClass } from "../contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; import { HarnessDriverBackend } from "./harness-driver-backend.js"; @@ -323,6 +324,36 @@ describe("HarnessDriverBackend", () => { expect(recoveredDispositionAllowance).toBe(true); }); + it("keeps the runner's closed class of a failed recovery", async () => { + const commandFailure = Object.defineProperty(new Error("PRP command run.attach failed"), "runnerCommandResult", { + value: { status: "failed", commandType: "run.attach", + result: { code: "command_execution_failed", message: "saved launch profile changed", + classification: PROVIDER_STATE_INCOMPATIBLE } }, + enumerable: false, + }); + expect(runnerRecoveryFailureClass(commandFailure)).toBe(PROVIDER_STATE_INCOMPATIBLE); + expect(runnerRecoveryFailureClass(new Error("wrapped", { cause: commandFailure }))).toBe(PROVIDER_STATE_INCOMPATIBLE); + expect(runnerRecoveryFailureClass(new Error(`text only: ${PROVIDER_STATE_INCOMPATIBLE}`))).toBeUndefined(); + const unclassified = Object.defineProperty(new Error("PRP command run.attach failed"), "runnerCommandResult", { + value: { status: "failed", result: { code: "command_execution_failed", classification: "other" } }, + enumerable: false, + }); + expect(runnerRecoveryFailureClass(unclassified)).toBeUndefined(); + + const backend = new HarnessDriverBackend({ + ...driver, + recoverSession: async () => ({ recovered: false, reason: String(commandFailure), + failureClass: runnerRecoveryFailureClass(commandFailure) }), + }); + await expect(backend.recoverSession({ + backendKind: "runner", driverKind: "fake", sessionId: "driver-1", providerSessionId: "provider-1", + providerIdentity, + identity: { runId: "run-2", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + }, { signal: new AbortController().signal })).resolves.toEqual({ + recovered: false, reason: "Error: PRP command run.attach failed", failureClass: PROVIDER_STATE_INCOMPATIBLE, + }); + }); + it("preserves a consumed disposition allowance in native snapshots", async () => { class ConsumedRecoverySession extends FakeHarnessSession { override async snapshot(): Promise { diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index 1339830d047..56caad7ecab 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -162,7 +162,11 @@ export class HarnessDriverBackend implements NativeSessionBackend { recoveryOptions, ); if (!recovered.recovered || recovered.session === undefined) { - return { recovered: false, reason: recovered.reason }; + return { + recovered: false, + reason: recovered.reason, + ...(recovered.failureClass ? { failureClass: recovered.failureClass } : {}), + }; } try { assertProviderSessionIdentity( diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index ede129c07b7..adccad8d49b 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -496,10 +496,29 @@ export interface PersistedHarnessSession { | "allow_replacement_after_resume_failure"; } +/** Runner proof that saved provider state cannot be restored by this runner + * build. Restoring the same state again fails the same way. */ +export const PROVIDER_STATE_INCOMPATIBLE = "provider_state_incompatible"; + export interface HarnessSessionRecoveryResult { recovered: boolean; session?: HarnessSession; reason?: string; + /** Closed runner class of a deterministic recovery failure. */ + failureClass?: typeof PROVIDER_STATE_INCOMPATIBLE; +} + +/** Reads the closed class of an authenticated runner command failure. */ +export function runnerRecoveryFailureClass( + error: unknown, + depth = 0, +): typeof PROVIDER_STATE_INCOMPATIBLE | undefined { + if (!error || typeof error !== "object" || depth > 5) return undefined; + const value = error as Record; + const result = value.runnerCommandResult as Record | undefined; + const detail = result?.result as Record | undefined; + if (detail?.classification === PROVIDER_STATE_INCOMPATIBLE) return PROVIDER_STATE_INCOMPATIBLE; + return runnerRecoveryFailureClass(value.cause, depth + 1); } export interface HarnessSession { diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 9761858e790..07b829c0fd5 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -15,6 +15,7 @@ import type { HarnessRuntimeRequestHandoff, HarnessRuntimeRequestResolution, HarnessGoalOperation, + HarnessSessionRecoveryResult, HarnessThreadGoal, HarnessThreadLineageEntry, NativeRuntimeContextCapabilities, @@ -82,6 +83,7 @@ export interface NativeSessionRecoveryResult { recovered: boolean; session?: NativeSession; reason?: string; + failureClass?: HarnessSessionRecoveryResult["failureClass"]; } /** Runner-owned cause: the process was restored, but its active turn was lost. */ @@ -278,6 +280,17 @@ export interface NativeSessionBackend { ): Promise; } +/** Same-session recovery is forbidden here, and the runner proved that the + * saved provider session cannot be restored by its build. Retrying the same + * run cannot succeed. */ +export class NativeProviderStateIncompatibleError extends Error { + readonly code = "native_provider_state_incompatible"; + constructor(reason: string) { + super(`native_session_recovery_failed: ${reason}`); + this.name = "NativeProviderStateIncompatibleError"; + } +} + /** A provider failed terminal is not a missing completion proposal. */ export class NativeProviderTerminalFailure extends Error { readonly code = "native_provider_terminal_failed"; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 995424acc63..c3ea63d6b03 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -20,7 +20,7 @@ import type { PersistedHarnessTurnTerminal, } from "../../contracts/harness-driver.js"; import { NativeSessionProtocolIntegrityError, nativeRestartInterruptedTurnId } from "../../contracts/native-session-backend.js"; -import { HarnessReconciliationError } from "../../contracts/harness-driver.js"; +import { HarnessReconciliationError, runnerRecoveryFailureClass } from "../../contracts/harness-driver.js"; import { CODEX_CODEX_PROTOCOL_VERSION, CODEX_SKILLLESS_BASE_INSTRUCTIONS, @@ -677,7 +677,12 @@ export class CodexAppServerDriver implements HarnessDriver { await cancellation.close().catch(() => {}); if (error instanceof NativeSessionProtocolIntegrityError) throw error; if (options.signal.aborted) options.signal.throwIfAborted(); - return { recovered: false, reason: redactCodexDiagnostic(String(error)) }; + const failureClass = runnerRecoveryFailureClass(error); + return { + recovered: false, + reason: redactCodexDiagnostic(String(error)), + ...(failureClass ? { failureClass } : {}), + }; } finally { cancellation.detach(); } diff --git a/packages/paperclip-runner/src/drivers/cursor-cloud/cursor-cloud-driver.test.ts b/packages/paperclip-runner/src/drivers/cursor-cloud/cursor-cloud-driver.test.ts index 532faaf17b6..219f45cd95d 100644 --- a/packages/paperclip-runner/src/drivers/cursor-cloud/cursor-cloud-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/cursor-cloud/cursor-cloud-driver.test.ts @@ -994,21 +994,32 @@ describe("Cursor Cloud native driver", () => { try { const turn = await f.session.startTurn({ message: { role: "user", text: rejection === "unsent_payload" ? "\u0001".repeat(400_000) : "Rejected task" } }); await entered.promise; + const observeHeldCheckpoint = async () => { + const checkpointPath = join(f.root, (await readdir(f.root)).find(name => /^checkpoint-.*\.json$/.test(name))!); + expect(JSON.parse(await readFile(checkpointPath, "utf8")).phase).not.toBe("idle"); + return checkpointPath; + }; + let checkpointPath: string; if (overlap === "stop_write") { stop(); await stopHeld.promise; rejectCreate.resolve(); await new Promise(accept => setImmediate(accept)); + // The Stop checkpoint gate still holds its cancellation clock here. + checkpointPath = await observeHeldCheckpoint(); releaseStop.resolve(); - await waitFor(() => pausedRejection || stopDone); + await Promise.race([rejectionHeld.promise, stopped!]); expect(pausedRejection, "the definite rejection must reach durable settlement even while stopping").toBe(true); } else { rejectCreate.resolve(); await rejectionHeld.promise; + // File observations must not consume Stop's 300ms settlement budget. + checkpointPath = await observeHeldCheckpoint(); stop(); await new Promise(accept => setImmediate(accept)); } expect(stopDone, "Stop must wait for the rejected-create checkpoint").toBe(false); expect(await f.session.read()).toMatchObject({ phase: "stopping" }); await expect(f.session.startTurn({ message: { role: "user", text: "Too early" } })).rejects.toThrow("unresolved remote work"); - const checkpointPath = join(f.root, (await readdir(f.root)).find(name => /^checkpoint-.*\.json$/.test(name))!); - expect(JSON.parse(await readFile(checkpointPath, "utf8")).phase).not.toBe("idle"); + expect(peer.getAgent).toHaveBeenCalledTimes(before.getAgent); + expect(peer.getRun).toHaveBeenCalledTimes(before.getRun); + expect(peer.cancelRun).toHaveBeenCalledTimes(before.cancel); releaseRejection.resolve(); expect(await stopped).toEqual({ error: null }); await waitFor(() => f.events.some(event => event.turnId === turn.turnId && event.eventType === "turn.failed" && event.payload.remoteWorkAccepted === false)); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts new file mode 100644 index 00000000000..424db66444b --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts @@ -0,0 +1,556 @@ +import { + createRunnerProviderLaunch, + runnerLaunchRequirementsDigest, + type RunnerProviderLaunchRequirements, +} from "./runner-provider-launch.js"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { spawn } from "node:child_process"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +async function verifyLiveRunnerAdoption( + mismatchedCheckpoint: boolean, + mismatchedArtifact = false, + goalMidTurn = false, + detachBeforeCleanup = false, + startWithoutCheckpoint = false, + preparedTask = false, + adoptionAuthorization: "allow" | "deny" | "legacy" | "legacy-missing" | "missing-binding" = "allow", + toolHandlerOrder?: "before" | "after" | "completion-before" | "completion-after", + uncertainAtClose = false, + semanticRecoveryMode?: "denied-before-cancel" | "cleanup-only", + stopExitProofFault?: "missing" | "false", +) { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-live-adopt-")); + const pendingCompletion = toolHandlerOrder?.startsWith("completion-") === true; + const completionPath = join(stateDirectory, "pending-completion.json"); + if (pendingCompletion) await writeFile(completionPath, JSON.stringify({ + schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed the synthetic work", + completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] }, + evidence: [], verification: [], attentionRequests: [], artifacts: [], + })); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null; + const checkpoint = vi.fn(async () => undefined); + server.on("upgrade", (request, socket, head) => { + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, "/runner", head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Expected adoption test listener"); + const registration = vi.fn(async (next: DurablePrpControlPlane) => { + authority = next; + return { + connectUrl: `ws://127.0.0.1:${address.port}/runner`, + ...(mismatchedArtifact || uncertainAtClose || semanticRecoveryMode === "cleanup-only" ? { checkpoint } : {}), + release: async () => { + if (authority === next) authority = null; + }, + }; + }); + const identity = { + runnerInstanceId: "runner-live-adopt", + environmentLeaseId: "lease-live-adopt", + runId: "run-live-adopt", + normalizedSessionId: "session-live-adopt", + turnId: "turn-live-adopt", + itemId: "item-live-adopt", + }; + const sharedOptions = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, ...(pendingCompletion ? ["--terminal-result-file", completionPath] : toolHandlerOrder ? ["--emit-tool-call"] : []), ...(goalMidTurn ? ["--goal-autostart", "--goal-item-trigger", join(stateDirectory, "emit-goal-item")] : [])), + stateDirectory, + prpIdentity: identity, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + controlPlaneRegistration: registration, + }; + const prepareLaunch = vi.fn(async (requirements: RunnerProviderLaunchRequirements) => ({ + configurationId: "adopted-preparation", requirementsDigest: runnerLaunchRequirementsDigest(requirements), + provider: createRunnerProviderLaunch(requirements, { options: { codexArgs: sharedOptions.codexArgs }, + root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), + codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "" }), + })); + const options = { ...sharedOptions, ...(preparedTask ? { + prepareLaunch, runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), + runnerProcessLauncher: (spec: durableControlPlane.RunnerProcessLaunchSpec) => { + const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: { ...process.env, ...spec.environment }, + detached: process.platform !== "win32", stdio: "ignore" }); + return { child, processGroupId: child.pid ?? null, completion: new Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }>((resolveExit, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" })); + }) }; + }, + } : {}) }; + const first = createCapabilityRunnerdCodexTransport(options); + const pendingTool = vi.fn(() => new Promise>(() => undefined)); + if (toolHandlerOrder) first.transport.setServerRequestHandler(pendingTool); + let releaseRecovery!: () => void; + const recoveryGate = new Promise(resolve => { releaseRecovery = resolve; }); + const recoveryDenied = new Error("execution authority revoked after Stop"); + const recoveredTool = vi.fn(async () => { + if (semanticRecoveryMode) { + // Model server authority denying replay after Stop cleared the task lock. + // The gate holds that denial until reattachment has returned its session, + // before the caller invokes the eventual onSession cancellation callback. + await recoveryGate; + throw recoveryDenied; + } + return { result: { receipt: "known-outcome" }, isError: false }; + }); + const redispatchedTool = vi.fn(async () => ({ success: true, contentItems: [] })); + let runnerPid: number | null = null; + let runnerStartedAt: string | null = null; + let adopted: ReturnType | null = + null; + let restoreIntervalSpies: () => void = () => undefined; + try { + let opened: Record; + try { + opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: toolHandlerOrder && !pendingCompletion ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : codexSemanticToolSpecs(), + }); + } catch (error) { + const stderr = await readFile( + join(stateDirectory, "diagnostics", "runnerd.stderr.log"), + "utf8", + ).catch(() => ""); + throw new Error( + `${String(error)}\n${JSON.stringify(first.evidence())}${stderr ? `\n${stderr}` : ""}`, + ); + } + runnerPid = first.evidence().runnerPid; + runnerStartedAt = first.transport.processInfo().startedAt; + expect(runnerPid).toEqual(expect.any(Number)); + + if (goalMidTurn) { + await first.transport.request("thread/goal/set", { objective: "Recover a live goal", status: "active" }); + for await (const event of first.transport.notifications()) { + if (event.method === "turn/started") break; + } + } + + if (toolHandlerOrder) { + await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); + await vi.waitFor(() => expect(pendingTool).toHaveBeenCalledOnce(), { timeout: 5_000 }); + } + await first.detachControllerForRestart(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + if (goalMidTurn) { + await writeFile(join(stateDirectory, "emit-goal-item"), "emit"); + // runnerd need not poll the provider into its PRP outbox while disconnected. + // Wait for flushed provider output, not a platform-dependent final poll + // racing the disconnect. Adoption must still bind that buffered item. + await vi.waitFor(async () => { + expect(await readFile(join(stateDirectory, "emit-goal-item.sent"), "utf8")).toBe("sent"); + }, { timeout: 5_000 }); + } + + const controlPlaneStatePath = join( + stateDirectory, + "control-plane", + "control-plane-state.json", + ); + const compactProviderIdentityEvents = async () => { + const controlPlaneState = JSON.parse( + await readFile(controlPlaneStatePath, "utf8"), + ) as { committedEvents: Array<{ eventType: string }> }; + controlPlaneState.committedEvents = + controlPlaneState.committedEvents.filter( + (event) => + event.eventType !== "harness.ready" && + event.eventType !== "session.started" && + event.eventType !== "session.resumed", + ); + await writeFile( + controlPlaneStatePath, + `${JSON.stringify(controlPlaneState, null, 2)}\n`, + { mode: 0o600 }, + ); + }; + if (!startWithoutCheckpoint) await compactProviderIdentityEvents(); + + if (preparedTask) { + if (adoptionAuthorization !== "legacy") await rm(join(stateDirectory, "launch-preparation.json")); + prepareLaunch.mockClear(); + registration.mockClear(); + if (adoptionAuthorization === "missing-binding") { + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + delete retained.preparedLaunchConfigurations; + await writeFile(controlPlaneStatePath, JSON.stringify(retained), { mode: 0o600 }); + } + } + const authorizeRunnerAdoption = vi.fn(async () => { + expect(registration).not.toHaveBeenCalled(); + if (adoptionAuthorization === "deny") throw new Error("project grant revoked"); + }); + const duplicateLauncher = vi.fn(() => { + throw new Error("duplicate runner spawn attempted"); + }); + const openedThread = opened.thread as Record; + const signal = vi.fn(() => true); + const probeError = new Error("fixture remote process identity is unavailable"); + let rejectLiveness = false; + const intervals = uncertainAtClose ? vi.spyOn(globalThis, "setInterval") : null; + const clearedIntervals = uncertainAtClose ? vi.spyOn(globalThis, "clearInterval") : null; + restoreIntervalSpies = () => { intervals?.mockRestore(); clearedIntervals?.mockRestore(); }; + adopted = createCapabilityRunnerdCodexTransport({ + ...options, + ...(preparedTask && !adoptionAuthorization.startsWith("legacy") ? { authorizeRunnerAdoption } : {}), + ...(uncertainAtClose ? { closeGraceMs: 2_000 } : {}), + ...(semanticRecoveryMode === "cleanup-only" ? { recoveryPurpose: "stop_cleanup" as const } : {}), + // Hash different stable bytes without replacing the real runner artifact + // used by concurrent tests. Adoption must never execute this path. + ...(mismatchedArtifact + ? { + runnerBinary: resolve(import.meta.dirname, "../../package.json"), + runnerReconnectGraceMs: 150, + } + : {}), + ...(toolHandlerOrder && !pendingCompletion ? { recoverSemanticToolInput: recoveredTool } : {}), + resumeDynamicTools: pendingCompletion ? codexSemanticToolSpecs() : toolHandlerOrder ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : [], + resumeProviderSession: startWithoutCheckpoint ? undefined : { + driverSessionId: String(openedThread.id), + providerSessionId: mismatchedCheckpoint + ? "wrong-provider-session" + : String(openedThread.sessionId), + }, + runnerProcessLauncher: duplicateLauncher, + adoptExistingRunner: { + pid: runnerPid!, + processGroupId: runnerPid, + startedAt: runnerStartedAt, + signal, + isAlive: () => { + if (rejectLiveness) return Promise.reject(probeError); + try { + process.kill(runnerPid!, 0); + return true; + } catch { + return false; + } + }, + }, + }); + if (preparedTask && ["deny", "legacy-missing", "missing-binding"].includes(adoptionAuthorization)) { + await expect(adopted.transport.request("thread/read", {})).rejects.toThrow( + adoptionAuthorization === "deny" ? "project grant revoked" : adoptionAuthorization === "legacy-missing" + ? "runner_launch_recovery_requirements_unavailable" : "runner_launch_recovery_configuration_unavailable"); + expect(registration).not.toHaveBeenCalled(); + expect(prepareLaunch).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(signal).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + expect(authorizeRunnerAdoption).toHaveBeenCalledTimes(adoptionAuthorization === "deny" ? 1 : 0); + return; + } + if (mismatchedArtifact) { + await expect( + adopted.transport.request("thread/read", {}), + ).rejects.toThrow("native_adopted_runner_authentication_timeout"); + expect(authority?.activeRunnerConnectionCount()).toBe(0); + await expect( + adopted.transport.request("turn/start", { + input: [{ type: "text", text: "must not be dispatched" }], + }), + ).rejects.toThrow("native_adopted_runner_authentication_timeout"); + await adopted.transport.close(); + expect(signal).not.toHaveBeenCalled(); + expect(checkpoint).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + const retained = JSON.parse( + await readFile(controlPlaneStatePath, "utf8"), + ) as { + identity: unknown; + commands: Array<{ type: string; status: string }>; + }; + expect(retained.identity).toEqual(identity); + expect( + retained.commands.some( + (command) => + command.type === "runner.drain" && command.status === "pending", + ), + ).toBe(true); + expect(retained.commands.map((command) => command.type)).not.toEqual( + expect.arrayContaining(["runner.suspend"]), + ); + expect(retained.commands.map((command) => command.type)).not.toEqual( + expect.arrayContaining(["turn.stop"]), + ); + return; + } + if (mismatchedCheckpoint) { + await expect( + adopted.transport.request("thread/read", {}), + ).rejects.toThrow("native_adopted_provider_identity_mismatch"); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + return; + } + if (toolHandlerOrder?.endsWith("before")) adopted.transport.setServerRequestHandler(redispatchedTool); + await expect(adopted.transport.request(startWithoutCheckpoint ? "thread/start" : "thread/read", {})).resolves.toEqual( + expect.objectContaining({ + thread: expect.objectContaining({ id: "codex-thread-1" }), + }), + ); + if (toolHandlerOrder?.endsWith("after")) adopted.transport.setServerRequestHandler(redispatchedTool); + if (semanticRecoveryMode) { + const retained = authority as DurablePrpControlPlane | null; + expect(retained).not.toBeNull(); + expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); + expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); + expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); + if (semanticRecoveryMode === "denied-before-cancel") { + await vi.waitFor(() => expect(recoveredTool).toHaveBeenCalledOnce()); + // This is a real Core replay callback, not an artificial transport error. + // Deliver its denial before giving the caller its cancellation turn. + releaseRecovery(); + await vi.waitFor(async () => { + await expect(adopted!.transport.request("thread/read", {})).rejects.toBe(recoveryDenied); + }); + await expect(adopted.transport.request("turn/interrupt", {})).rejects.toBe(recoveryDenied); + expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); + } else { + // Let all startup/handler microtasks and a real reconciliation read run. + // Cleanup must not enter replay even while its authority would deny it. + await adopted.transport.request("thread/read", {}); + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).not.toHaveBeenCalled(); + releaseRecovery(); + await expect(adopted.transport.request("turn/interrupt", {})).resolves.toEqual({}); + expect(retained!.store.state.commands.filter(command => command.type === "turn.interrupt")).toMatchObject([{ status: "completed" }]); + } + expect(redispatchedTool).not.toHaveBeenCalled(); + expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); + expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); + expect(duplicateLauncher).not.toHaveBeenCalled(); + if (semanticRecoveryMode === "cleanup-only") { + const faultedStopProof = vi.fn(); + if (stopExitProofFault) { + const queue = retained!.queueCommand.bind(retained!); + vi.spyOn(retained!, "queueCommand").mockImplementation((type, ...args) => { + const command = queue(type, ...args); + if (type === "turn.stop") { + let result = command.result; + Object.defineProperty(command, "result", { + enumerable: true, + configurable: true, + get: () => result, + set: value => { + const receipt = value as { result?: Record } | null; + if (receipt?.result && typeof receipt.result === "object") { + faultedStopProof(receipt.result.providerExitConfirmed); + const stopped = { ...receipt.result }; + if (stopExitProofFault === "missing") delete stopped.providerExitConfirmed; + else stopped.providerExitConfirmed = false; + result = { ...receipt, result: stopped }; + } else result = value; + }, + }); + } + return command; + }); + await expect(adopted.transport.close()).rejects.toBeInstanceOf(NativeSessionCloseUnrecoverableError); + expect(faultedStopProof).toHaveBeenCalledWith(true); + expect(checkpoint).toHaveBeenCalledExactlyOnceWith("unsettled"); + } else { + await expect(adopted.transport.close()).resolves.toBeUndefined(); + expect(checkpoint).toHaveBeenCalledExactlyOnceWith("settled"); + expect(retained!.store.state.commands.filter(command => command.type === "turn.stop")).toMatchObject([{ + status: "completed", result: { result: { providerExitConfirmed: true } }, + }]); + } + expect(retained!.semanticToolResultsSettled()).toBe(false); + expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); + expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); + if (!stopExitProofFault) { + expect(retained!.store.state.commands.filter(command => command.type === "runner.suspend")).toMatchObject([{ status: "completed" }]); + const runnerState = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(runnerState).toMatchObject({ ...identity, lifecycle: "suspended" }); + } + const providerState = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); + expect(providerState).toMatchObject({ activeProviderTurnId: null, pendingEvents: [], queuedEvents: [] }); + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).not.toHaveBeenCalled(); + } + return; + } + if (toolHandlerOrder) { + await vi.waitFor(() => { + expect(pendingCompletion ? redispatchedTool : recoveredTool).toHaveBeenCalledOnce(); + const result = authority!.store.state.commands.find(command => command.type === "semantic_tool.result"); + expect(result?.status).toBe("completed"); + }, { timeout: 5_000 }); + if (pendingCompletion) { + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).toHaveBeenCalledWith(expect.objectContaining({ params: expect.objectContaining({ tool: "paperclip_finish" }) })); + } else expect(redispatchedTool).not.toHaveBeenCalled(); + } + expect(adopted.evidence().runnerPid).toBe(runnerPid); + expect(adopted.transport.processInfo().startedAt).toBe(runnerStartedAt); + if (startWithoutCheckpoint) { + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + for (const type of ["run.prepare", "session.open"]) { + expect(retained.commands.filter((command: { type: string }) => command.type === type)).toHaveLength(1); + } + } + if (goalMidTurn) { + const observed = await Promise.race([ + (async () => { + for await (const notification of adopted!.transport.notifications()) { + if (notification.method === "item/started") return notification; + } + throw new Error("recovered goal item was lost"); + })(), + new Promise((_, reject) => setTimeout(() => reject(new Error("recovered item timed out")), 5_000)), + ]); + expect(observed.params).toMatchObject({ threadId: "codex-thread-1", turnId: "provider-goal-turn-1" }); + } + expect(duplicateLauncher).not.toHaveBeenCalled(); + if (preparedTask) { + if (adoptionAuthorization === "legacy") { + expect(prepareLaunch).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ identity }), expect.objectContaining({ configurationId: "adopted-preparation" })); + expect(authorizeRunnerAdoption).not.toHaveBeenCalled(); + } else { + expect(prepareLaunch).not.toHaveBeenCalled(); + expect(authorizeRunnerAdoption).toHaveBeenCalledExactlyOnceWith("adopted-preparation", identity); + } + expect((await adopted.transport.request("thread/read", {})).thread).toMatchObject({ cwd: tmpdir() }); + } + if (uncertainAtClose) { + const before = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + const retainedAuthority = authority as DurablePrpControlPlane | null; + rejectLiveness = true; + await expect(adopted.transport.close()).rejects.toBe(probeError); + expect(retainedAuthority?.activeRunnerConnectionCount()).toBe(0); + expect(intervals!.mock.results.length).toBeGreaterThan(0); + for (const interval of intervals!.mock.results) if (interval.type === "return") expect(clearedIntervals).toHaveBeenCalledWith(interval.value); + const notificationsClosed = (async () => { for await (const _event of adopted!.transport.notifications()) {} return true; })(); + let queueDeadline: ReturnType | undefined; + try { + expect(await Promise.race([notificationsClosed, new Promise(resolve => { queueDeadline = setTimeout(() => resolve(false), 250); })])).toBe(true); + } finally { clearTimeout(queueDeadline); } + await expect(adopted.transport.close()).rejects.toBe(probeError); + expect(adopted.transport.processInfo()).toMatchObject({ exited: false }); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + expect(signal).not.toHaveBeenCalled(); + expect(checkpoint).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + expect(retained.identity).toEqual(before.identity); + expect(retained.commands).toEqual(before.commands); + return; + } + expect(adopted.evidence().diagnostics).toContain( + `adopted runner ${runnerPid} authenticated to its durable PRP authority`, + ); + if (!startWithoutCheckpoint) { + expect(adopted.evidence().diagnostics).toContain( + "restored adopted provider identity from the exact durable checkpoint after PRP event compaction; awaiting live confirmation", + ); + expect(adopted.evidence().diagnostics).toContain( + "confirmed adopted provider identity against authenticated recovery session.snapshot", + ); + } + if (detachBeforeCleanup) { + await adopted.detachControllerForRestart(); + await adopted.transport.close("old controller finalizer"); + expect(signal).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + const commandTypes = retained.commands.map((command: { type: string }) => command.type); + expect(commandTypes).not.toContain("turn.stop"); + expect(commandTypes).not.toContain("runner.suspend"); + } + } finally { + releaseRecovery(); + await adopted?.transport.close().catch(() => undefined); + if (uncertainAtClose) { + const providerPid = adopted?.evidence().codexPid; + if (providerPid) { + try { process.kill(providerPid, "SIGKILL"); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } + } + await (authority as DurablePrpControlPlane | null)?.stop(); + } + restoreIntervalSpies(); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // The adopted runner normally exits after its durable suspend command. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +} + +it( + "adopts a live runner on the same durable authority without spawning a duplicate", + () => verifyLiveRunnerAdoption(false), + 30_000, +); + +it("adopts a prepared runner after losing its controller launch cache without preparing resources again", + () => verifyLiveRunnerAdoption(false, false, false, false, false, true), 30_000); + +it.each(["deny", "legacy", "legacy-missing", "missing-binding"] as const)("preserves adoption authorization with %s", + mode => verifyLiveRunnerAdoption(false, false, false, false, false, true, mode), 30_000); + +it("retains authenticated adopted-runner ownership when a close probe rejects", () => verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", undefined, true), 30_000); + +it( + "blocks adopted runner artifact drift without duplicate launch, checkpoint replacement, or process signals", + () => verifyLiveRunnerAdoption(false, true), + 15_000, +); + +it( + "rejects a live runner whose provider identity mismatches the compacted checkpoint", + () => verifyLiveRunnerAdoption(true), + 30_000, +); + +it("binds buffered mid-goal items only after the authenticated recovery snapshot", () => verifyLiveRunnerAdoption(false, false, true), 30_000); + +it("keeps an adopted runner alive when the detached controller finalizer closes", () => verifyLiveRunnerAdoption(false, false, true, true), 30_000); + +it("adopts an opening session without a checkpoint instead of bootstrapping a duplicate provider", () => verifyLiveRunnerAdoption(false, false, false, false, true), 30_000); + +it.each(["before", "after"] as const)("settles a saved tool outcome with the handler installed %s recovery startup", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); + +it.each(["completion-before", "completion-after"] as const)("recovers a pending finish without an ordinary-tool lookup (%s)", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); + +it("exposes semantic recovery before cancellation when the authority has been revoked", () => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "denied-before-cancel"), 30_000); + +it.each(["before", "after", "completion-before", "completion-after"] as const)("allows cleanup-only adoption to cancel without replaying a pending semantic input (%s)", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order, false, "cleanup-only"), 30_000); + +it.each(["missing", "false"] as const)("refuses cleanup settlement when the exact stop receipt has %s provider exit proof", fault => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "cleanup-only", fault), 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts new file mode 100644 index 00000000000..72b95a8206c --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts @@ -0,0 +1,1077 @@ +import { + createRunnerProviderLaunch, + runnerLaunchRequirementsDigest, + type PreparedRunnerProviderLaunch, + type RunnerProviderLaunchRequirements, +} from "./runner-provider-launch.js"; +import { cp, lstat, mkdtemp, readFile, readdir, readlink, rm, writeFile } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +it.each([ + ...[ + "before-result", + "after-result", + "after-activation", + "before-confirmation", + "after-confirmation", + ].flatMap((lossPoint) => + [false, true].map((routed) => ({ + lossPoint, + routed, + recoveryFault: "none", + })), + ), + ...["endpoint", "missing-capability", "listen", "malformed-core"].map( + (recoveryFault) => ({ + lossPoint: "after-result", + routed: true, + recoveryFault, + }), + ), + ...["before_bootstrap", "before_spawn", "before_authentication"].map( + (stage) => ({ + lossPoint: "after-result", + routed: true, + recoveryFault: `authorize_${stage}`, + }), + ), + ...[ + "attach-wait", + ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER + ? ["attach-capability"] + : []), + ].map((recoveryFault) => ({ + lossPoint: "before-result", + routed: true, + recoveryFault, + })), + ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER + ? [ + { + lossPoint: "after-result", + routed: false, + recoveryFault: "legacy-parser", + }, + ] + : []), + ...["before-result", "after-result", "after-activation", "after-confirmation"].map(lossPoint => ({ + lossPoint, routed: true, recoveryFault: "none", executionPrepared: true, + })), + { + lossPoint: "after-result", + routed: true, + recoveryFault: "none", + externalArtifact: true, + }, + { + lossPoint: "after-confirmation", + routed: true, + recoveryFault: "none", + ordinaryFollowup: true, + }, + ...[ + "missing_snapshot", + "rejected_snapshot", + "wrong_thread", + "callback_failure", + "callback_async_failure", + ].map((ordinaryFollowup) => ({ + lossPoint: "after-confirmation", + routed: true, + recoveryFault: "none", + ordinaryFollowup, + })), +])( + "recovers a warm attachment with a fresh controller and runner ($lossPoint, routed=$routed, fault=$recoveryFault, followup=$ordinaryFollowup, external=$externalArtifact, prepared=$executionPrepared)", + async (testCase) => { + const { lossPoint, routed, recoveryFault } = testCase; + const externalArtifact = "externalArtifact" in testCase && testCase.externalArtifact; + const executionPrepared = "executionPrepared" in testCase && testCase.executionPrepared; + const configurations = new Map(); + const ordinaryFollowup = + "ordinaryFollowup" in testCase && testCase.ordinaryFollowup; + const snapshotFault = + typeof ordinaryFollowup === "string" && + !ordinaryFollowup.startsWith("callback_"); + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-attach-restart-"), + ); + const callsPath = join(stateDirectory, "calls.log"); + const cores: DurablePrpControlPlane[] = []; + const handles: ReturnType[] = []; + const routes = new Map< + string, + { core: DurablePrpControlPlane; generation: symbol } + >(); + const routeCalls: string[] = []; + let recovering = false; + let recoveryClaimCurrent = true; + let recoveryFenceActive = true; + const completionSnapshotIds: string[] = []; + let rejectHeldCompletion: ((error: Error) => void) | undefined; + let markHeldCompletionEntered!: () => void; + const heldCompletionEntered = new Promise((resolve) => { + markHeldCompletionEntered = resolve; + }); + let snapshotObserverSpy: ReturnType | undefined; + const routeServer = createServer((_request, response) => + response.writeHead(404).end(), + ); + routeServer.on("upgrade", (request, socket, head) => { + const entry = routes.get(request.url ?? ""); + if (!entry) { + socket.destroy(); + return; + } + entry.core.handleUpgrade(request, socket, request.url!, head); + }); + if (routed) + await new Promise((resolveListen) => + routeServer.listen(0, "127.0.0.1", resolveListen), + ); + const routeAddress = routeServer.address(); + const routePort = + routeAddress && typeof routeAddress === "object" ? routeAddress.port : 0; + const registration = async ( + core: DurablePrpControlPlane, + identity = core.store.state.identity, + ) => { + if ( + recovering && + ordinaryFollowup && + recoveryFenceActive && + !recoveryClaimCurrent + ) { + throw new Error("fixture old recovery claim is no longer current"); + } + const path = `/api/runner/v1/connect/${identity.runId}`; + const generation = Symbol(); + routes.set(path, { core, generation }); + routeCalls.push(identity.runId); + return { + connection: + recovering && recoveryFault === "listen" + ? { + mode: "listen" as const, + listenAddress: "0.0.0.0" as const, + listenPort: routePort, + listenPath: path, + } + : { + mode: "connect" as const, + connectUrl: `ws://127.0.0.1:${routePort}${path}${recovering && recoveryFault === "endpoint" ? "/changed" : ""}`, + }, + release: () => { + if (routes.get(path)?.generation === generation) routes.delete(path); + }, + }; + }; + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi + .spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function ( + options: ConstructorParameters[0], + ) { + const core = new OriginalCore(options); + cores.push(core); + if (recovering && snapshotFault) { + const getCommand = core.getCommand.bind(core); + snapshotObserverSpy = vi + .spyOn(core, "getCommand") + .mockImplementation((id) => { + const command = getCommand(id); + if ( + command?.type !== "session.snapshot" || + command.status !== "completed" + ) + return command; + const observed = structuredClone(command); + if (ordinaryFollowup === "rejected_snapshot") { + observed.status = "failed"; + observed.result = { + result: { message: "fixture snapshot observation rejected" }, + }; + } else if (ordinaryFollowup === "wrong_thread") { + observed.result = { + ...observed.result, + result: { + ...(observed.result?.result as Record), + driverSessionId: "foreign-thread", + providerSessionId: "foreign-thread", + }, + }; + } else observed.result = { result: { status: "session_open" } }; + return observed; + }); + } + return core; + } as unknown as typeof OriginalCore); + const launch = durableControlPlane.spawnRunner; + const launchSpy = vi + .spyOn(durableControlPlane, "spawnRunner") + .mockImplementation((options) => { + const handle = launch({ ...options, + ...(executionPrepared ? { processLauncher: undefined } : {}), + ...(externalArtifact && recovering ? { runnerBinaryPath: defaultCapabilityRunnerdBinary() } : {}), + }); + handles.push(handle); + return handle; + }); + const dead = (pid: number) => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }; + const stopOwnedProvider = async (pid: number) => { + // Exact runner completion can race the OS reaping its already-signalled + // provider. Absence, not the outcome of a redundant signal, is required. + try { + await vi.waitFor(() => expect(dead(-pid)).toBe(true), { + timeout: 500, + interval: 10, + }); + } catch { + try { + process.kill(-pid, "SIGKILL"); + } catch (error) { + if ( + !["ESRCH", "EPERM"].includes( + String((error as NodeJS.ErrnoException).code), + ) + ) + throw error; + } + await vi.waitFor(() => expect(dead(-pid)).toBe(true), { + timeout: 2_000, + interval: 10, + }); + } + expect(dead(pid)).toBe(true); + }; + const fingerprintTree = async (root: string): Promise => { + const rows: unknown[] = []; + const visit = async (path: string, relative: string) => { + const metadata = await lstat(path); + rows.push({ + path: relative, + inode: metadata.ino, + mode: metadata.mode, + mtimeMs: metadata.mtimeMs, + digest: metadata.isFile() + ? createHash("sha256") + .update(await readFile(path)) + .digest("hex") + : metadata.isSymbolicLink() + ? createHash("sha256") + .update(await readlink(path)) + .digest("hex") + : null, + }); + if (metadata.isDirectory()) + for (const child of (await readdir(path)).sort()) + await visit(join(path, child), `${relative}/${child}`); + }; + await visit(root, "."); + return rows; + }; + const within = async (promise: Promise, timeout = 5_000) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error("warm restart fixture timed out")), + timeout, + ); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + }; + const options = { + runnerBinary: + recoveryFault === "attach-capability" + ? process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER! + : (process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? + defaultCapabilityRunnerdBinary()), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--call-log", + callsPath, + "--record-process-start", + ), + stateDirectory, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 2_000, + prpIdentity: (() => { + const runId = randomUUID(); + return { + runnerInstanceId: randomUUID(), + environmentLeaseId: + process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_SCOPE === + "transient" + ? runId + : randomUUID(), + runId, + normalizedSessionId: randomUUID(), + turnId: `turn-${runId}`, + itemId: `item-${runId}`, + }; + })(), + ...(routed + ? { + controlPlaneRegistration: registration, + warmTransitionRegistrationMode: "routed_connect" as const, + } + : {}), + ...(executionPrepared ? { + runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), + readRunnerState: async () => JSON.parse(await readFile(join(stateDirectory, "runner/runner-state.json"), "utf8")), + // The fixture's spawn spy supplies the real local process owner. + runnerProcessLauncher: () => { throw new Error("fixture process owner was bypassed"); }, + prepareLaunch: async (requirements: RunnerProviderLaunchRequirements, recovery?: PreparedRunnerProviderLaunch) => { + const existing = configurations.get(requirements.identity.runId); + if (existing) { + expect(requirements).toEqual(existing.requirements); + if (recovering) expect(recovery).toEqual(existing.prepared); + return existing.prepared; + } + const prepared = { configurationId: `prepared-${requirements.identity.runId}`, + requirementsDigest: runnerLaunchRequirementsDigest(requirements), + provider: createRunnerProviderLaunch(requirements, { + options: { codexArgs: fakeCodexArgs(stateDirectory, "--call-log", callsPath, "--record-process-start") }, + root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), + codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "", + }), + }; + configurations.set(requirements.identity.runId, { requirements: structuredClone(requirements), prepared }); + return prepared; + }, + } : {}), + }; + const first = createCapabilityRunnerdCodexTransport(options); + let resumed: + ReturnType | undefined; + const providerPids = new Set(); + let commitSpy: ReturnType | undefined; + let commandObserverSpy: ReturnType | undefined; + let detached: Promise | undefined; + let cleanupProven = false; + const legacyProbeDirectories: string[] = []; + try { + const opened = await within( + first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }), + ); + const openedThread = opened.thread as { id: string; sessionId: string }; + const thread = { id: openedThread.id, sessionId: openedThread.sessionId }; + const firstThreadSnapshot = await within( + first.transport.request("thread/read", {}), + ); + const firstEvidence = first.evidence(); + const cleanRunnerStateBytes = await readFile( + join(stateDirectory, "runner/runner-state.json"), + ); + const runnerProcessStartedAt = + process.platform === "darwin" + ? new Date( + execFileSync( + "ps", + ["-o", "lstart=", "-p", String(handles[0]!.child.pid)], + { encoding: "utf8", timeout: 1_500 }, + ).trim(), + ).toISOString() + : null; + if ((first.evidence().codexPid ?? 0) > 0) + providerPids.add(first.evidence().codexPid!); + const core = cores[0]!; + const oldIdentity = structuredClone(core.store.state.identity); + const nextRunId = randomUUID(); + const desired = { + ...oldIdentity, + runId: nextRunId, + turnId: `turn-${nextRunId}`, + itemId: `item-${nextRunId}`, + }; + if ( + recoveryFault === "attach-capability" || + recoveryFault === "attach-wait" + ) { + if (recoveryFault === "attach-wait") { + const getCommand = core.getCommand.bind(core); + commandObserverSpy = vi + .spyOn(core, "getCommand") + .mockImplementation((id) => { + if ( + core.store.state.commands.find( + (entry) => entry.commandId === id, + )?.type === "run.attach" + ) { + throw new Error("fixture result observer unavailable"); + } + return getCommand(id); + }); + } + const bootstrapCount = core.store.state.freshBootstraps; + await expect( + first.transport.attachRun!({ + runId: desired.runId, + turnId: desired.turnId, + itemId: desired.itemId, + }), + ).rejects.toThrow( + recoveryFault === "attach-capability" + ? "capability is required" + : "result observer unavailable", + ); + expect(routeCalls).toEqual([oldIdentity.runId, desired.runId]); + expect([...routes.keys()]).toEqual([ + `/api/runner/v1/connect/${oldIdentity.runId}`, + ]); + expect(core.store.state.freshBootstraps).toBe(bootstrapCount); + expect(handles).toHaveLength(1); + if (recoveryFault === "attach-capability") + expect( + core.store.state.commands.some( + (entry) => entry.type === "run.attach", + ), + ).toBe(false); + return; + } + const commit = core.store.commit.bind(core.store); + let lossObserved = false; + let signalLoss!: () => void; + const loss = new Promise((resolveLoss) => { + signalLoss = resolveLoss; + }); + commitSpy = vi + .spyOn(core.store, "commit") + .mockImplementation((candidate) => { + // The selected process-crash boundary stays unavailable until the + // exact owned runner is joined. A later replay must not silently + // settle this fixture before its retained pair is exported. + if (lossObserved) + throw new Error( + "fixture controller persistence unavailable after loss", + ); + const phase = candidate.warmTransition?.phase; + const target = + lossPoint === "after-activation" ? "activated" : "prepared"; + const atBoundary = lossPoint.includes("confirmation") + ? candidate.completedWarmTransition !== undefined && + candidate.warmTransition === undefined + : phase === target; + if (!lossObserved && atBoundary) { + lossObserved = true; + if ( + lossPoint !== "before-result" && + lossPoint !== "before-confirmation" + ) + commit(candidate); + // A dead controller cannot accept a reconnect through the second + // route installed for the in-flight authority rotation. Detaching + // only the current route leaves that overlapping fixture listener + // alive; completed-receipt replay needs no further store commit. + for (const [path, entry] of routes) { + if (entry.core === core) routes.delete(path); + } + core.disconnectActiveRunner(); + detached = first.transport.detachControllerForRestart!(); + signalLoss(); + // A committed confirmation is still before its ACK. Returning from + // this hook lets the current frame handler send that ACK even after + // close begins, collapsing the intended crash window on fast peers. + throw new Error("fixture interrupted exact result commit"); + } + commit(candidate); + }); + const attachment = first.transport.attachRun!({ + runId: desired.runId, + turnId: desired.turnId, + itemId: desired.itemId, + }); + void attachment.catch(() => undefined); + await within(loss); + await within(detached!); + expect(lossObserved).toBe(true); + const runnerPath = join(stateDirectory, "runner/runner-state.json"); + const runner = JSON.parse(await readFile(runnerPath, "utf8")); + expect(runner.warmTransition.phase).toBe( + lossPoint === "after-activation" || lossPoint.includes("confirmation") + ? "activating" + : "prepared", + ); + const persisted = JSON.parse(await readFile(core.store.path, "utf8")); + expect( + persisted.commands.find( + (entry: { type: string }) => entry.type === "run.attach", + )?.status, + ).toBe( + lossPoint === "before-result" + ? "pending" + : lossPoint === "after-result" + ? "completed" + : undefined, + ); + // Stop only this fixture's exact owned process handles. No stored receipt + // or PID absence is used as authority to terminate an unknown owner. + await durableControlPlane + .waitForProcess(handles[0]!, 100) + .catch(() => undefined); + await within(handles[0]!.completion); + for (const pid of providerPids) { + await stopOwnedProvider(pid); + } + await within(attachment.catch(() => undefined)); + const joinedRunner = JSON.parse(await readFile(runnerPath, "utf8")); + const joinedCore = JSON.parse(await readFile(core.store.path, "utf8")); + expect(joinedRunner.warmTransition).toEqual(runner.warmTransition); + expect(joinedCore.schema).toBe(persisted.schema); + expect(joinedCore.warmTransition).toEqual(persisted.warmTransition); + expect(joinedCore.completedWarmTransition).toEqual( + persisted.completedWarmTransition, + ); + commitSpy.mockRestore(); + const fixtureOutput = + process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_DIRECTORY; + if (fixtureOutput && recoveryFault === "none") { + const retainedArtifact = join(fixtureOutput, "paperclip-runnerd"); + await cp(options.runnerBinary, retainedArtifact, { force: false }); + expect( + createHash("sha256") + .update(await readFile(retainedArtifact)) + .digest("hex"), + ).toBe( + createHash("sha256") + .update(await readFile(options.runnerBinary)) + .digest("hex"), + ); + const retained = join( + fixtureOutput, + `${lossPoint}-${routed ? "routed" : "local"}`, + ); + await cp(stateDirectory, retained, { + recursive: true, + errorOnExist: true, + force: false, + }); + await writeFile( + join(retained, "transition-fixture-metadata.json"), + JSON.stringify( + { + schema: "paperclip.test.warm-transition-fixture.v1", + oldIdentity, + newIdentity: desired, + lossPoint, + routed, + runner: { + pid: handles[0]!.child.pid, + processGroupId: handles[0]!.processGroupId, + startedAt: runnerProcessStartedAt, + spawnObservedAt: handles[0]!.startedAt, + completion: await handles[0]!.completion, + processAbsent: dead(handles[0]!.child.pid!), + groupAbsent: dead(-handles[0]!.processGroupId!), + }, + providers: [...providerPids].map((pid) => ({ + pid, + processGroupId: pid, + startedAt: firstEvidence.providerProcessStartedAt, + processAbsent: dead(pid), + groupAbsent: dead(-pid), + })), + artifact: { + path: retainedArtifact, + version: runner.warmTransition.receipt.runnerVersion, + digest: runner.warmTransition.receipt.runnerDigest, + }, + thread, + firstThreadSnapshot, + firstEvidence, + }, + null, + 2, + ), + { mode: 0o600 }, + ); + } + routes.clear(); + const routeCallCount = routeCalls.length; + const beforeCalls = (await readFile(callsPath, "utf8")) + .trim() + .split(/\r?\n/); + expect( + beforeCalls.filter((call) => call === "process-start"), + ).toHaveLength(1); + if (recoveryFault === "legacy-parser") { + const legacyBinary = + process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER!; + const legacyDigest = `sha256:${createHash("sha256") + .update(await readFile(legacyBinary)) + .digest("hex")}`; + for (const [mode, bytes] of [ + ["pending", await readFile(runnerPath)], + ["clean", cleanRunnerStateBytes], + ] as const) { + const probe = await mkdtemp( + join(tmpdir(), "runnerd-legacy-schema-probe-"), + ); + legacyProbeDirectories.push(probe); + await writeFile(join(probe, "runner-state.json"), bytes, { + mode: 0o600, + }); + const handle = durableControlPlane.spawnRunner({ + connection: runner.warmTransition.receipt.connection, + stateDirectory: probe, + identity: oldIdentity, + runnerBinaryPath: legacyBinary, + runnerVersion: "0.3.0", + runnerDigest: legacyDigest, + ticket: "bootstrap_legacy_parser_probe", + maxOutboxBytes: runner.maxOutboxBytes, + p0ReserveBytes: runner.p0ReserveBytes, + maxRuntimeMs: 200, + reconnectGraceMs: 200, + }); + const result = await within(handle.completion, 5_000); + expect(result.code).not.toBe(0); + if (mode === "pending") { + expect(result.stderr).toContain( + "durable state binding does not match", + ); + expect(await readFile(join(probe, "runner-state.json"))).toEqual( + bytes, + ); + } else { + expect(result.stderr).not.toContain( + "durable state binding does not match", + ); + expect( + JSON.parse( + await readFile(join(probe, "runner-state.json"), "utf8"), + ).nextSourceSeq, + ).toBeGreaterThan(JSON.parse(bytes.toString("utf8")).nextSourceSeq); + } + } + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual(beforeCalls); + return; + } + recovering = true; + if (recoveryFault === "malformed-core") + await writeFile(core.store.path, "{", { mode: 0o600 }); + const beforeRecoveryBytes = await Promise.all( + [core.store.path, runnerPath].map((path) => readFile(path)), + ); + const beforeRecoveryTree = + recoveryFault === "none" ? null : await fingerprintTree(stateDirectory); + const authorizationStages: string[] = []; + let releaseHeldAuthorization: (() => void) | undefined; + let markHeldAuthorizationEntered!: () => void; + const heldAuthorizationEntered = new Promise((resolve) => { + markHeldAuthorizationEntered = resolve; + }); + const authorizationFault = recoveryFault.startsWith("authorize_"); + const failedAuthorizationStage = recoveryFault.slice("authorize_".length); + const beforeProviderBytes = await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + ); + resumed = createCapabilityRunnerdCodexTransport({ + ...options, + ...(externalArtifact ? { + runnerBinary: "/controller-missing-runner", + runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), + } : {}), + ...(ordinaryFollowup + ? { + authorizeWarmTransitionRecovery: async () => { + if (!recoveryClaimCurrent) + throw new Error( + "fixture old recovery claim is no longer current", + ); + }, + onWarmTransitionRecoveryCompleted: (completion: { + transitionId: string; + }) => { + expect(completion.transitionId).toBe( + runner.warmTransition.receipt.transitionId, + ); + const completedSnapshot = cores[1]!.store.state.commands + .filter((command) => command.type === "session.snapshot") + .at(-1)!; + expect(completedSnapshot.status).toBe("completed"); + expect(cores[1]!.store.state.identity).toEqual(desired); + expect(cores[1]!.store.state.warmTransition).toBeUndefined(); + completionSnapshotIds.push(completedSnapshot.commandId); + if ( + ordinaryFollowup === "callback_failure" && + completionSnapshotIds.length === 1 + ) { + throw new Error( + "fixture recovery completion callback failed", + ); + } + if ( + ordinaryFollowup === "callback_async_failure" && + completionSnapshotIds.length === 1 + ) { + return new Promise((_resolve, reject) => { + rejectHeldCompletion = reject; + markHeldCompletionEntered(); + }); + } + recoveryFenceActive = false; + }, + } + : {}), + ...(authorizationFault + ? { + authorizeWarmTransitionRecovery: async (stage: string) => { + authorizationStages.push(stage); + if ( + stage === "before_bootstrap" && + failedAuthorizationStage === "before_bootstrap" + ) { + await new Promise((resolveGate) => { + releaseHeldAuthorization = resolveGate; + markHeldAuthorizationEntered(); + }); + } + if (stage === failedAuthorizationStage) + throw new Error("fixture recovery authority revoked"); + }, + } + : {}), + ...(recoveryFault === "missing-capability" + ? { warmTransitionRegistrationMode: undefined } + : {}), + ...(recoveryFault !== "none" + ? { + environment: { + CODEX_API_KEY: "synthetic-refused-route-credential", + }, + } + : {}), + prpIdentity: desired, + resumeProviderSession: { + driverSessionId: thread.id, + providerSessionId: thread.sessionId, + }, + }); + if (recoveryFault !== "none") { + const recoveryRequest = within( + resumed.transport.request("thread/read", {}), + ); + void recoveryRequest.catch(() => undefined); + if ( + authorizationFault && + failedAuthorizationStage === "before_bootstrap" + ) { + await within(Promise.race([ + heldAuthorizationEntered, + recoveryRequest.then(() => { + throw new Error("warm restart authorization callback was not entered"); + }), + ])); + expect(releaseHeldAuthorization).toBeTypeOf("function"); + const queuedBefore = cores[1]!.store.state.commands.map( + (command) => command.commandId, + ); + try { + await expect( + resumed.transport.request("turn/start", { + input: [{ text: "must not queue before bootstrap" }], + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.attachRun!({ + runId: "must-not-attach", + turnId: "must-not-attach", + itemId: "must-not-attach", + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.resolveRuntimeRequest!({ + requestId: "must-not-resolve", + turnId: desired.turnId, + resolution: { action: "cancel" }, + }), + ).rejects.toThrow("warm_transition_completion_pending"); + expect( + cores[1]!.store.state.commands.map( + (command) => command.commandId, + ), + ).toEqual(queuedBefore); + expect(handles).toHaveLength(1); + } finally { + releaseHeldAuthorization!(); + } + } + const failedRecovery = expect(recoveryRequest).rejects; + if (authorizationFault) { + await failedRecovery.toThrow( + failedAuthorizationStage === "before_authentication" + ? "native_runner_warm_transition_recovery_pending" + : "fixture recovery authority revoked", + ); + const expectedStages = [ + "before_bootstrap", + "before_spawn", + "before_authentication", + ]; + expect([...new Set(authorizationStages)]).toEqual( + expectedStages.slice( + 0, + expectedStages.indexOf(failedAuthorizationStage) + 1, + ), + ); + expect(handles).toHaveLength( + failedAuthorizationStage === "before_authentication" ? 2 : 1, + ); + expect( + await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + ), + ).toEqual(beforeProviderBytes); + const refusedRunner = JSON.parse(await readFile(runnerPath, "utf8")); + expect(refusedRunner.warmTransition).toEqual(runner.warmTransition); + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual(beforeCalls); + if (failedAuthorizationStage !== "before_authentication") + expect(await readFile(runnerPath)).toEqual(beforeRecoveryBytes[1]); + if (failedAuthorizationStage === "before_bootstrap") + expect(await readFile(core.store.path)).toEqual( + beforeRecoveryBytes[0], + ); + await within(resumed.transport.close()).catch(() => undefined); + expect([...routes.keys()]).toEqual([]); + return; + } + if (recoveryFault === "malformed-core") await failedRecovery.toThrow(); + else + await failedRecovery.toThrow( + recoveryFault === "missing-capability" + ? "requires_exact_owned_endpoint" + : "registered_endpoint_mismatch", + ); + expect(handles).toHaveLength(1); + expect( + await Promise.all( + [core.store.path, runnerPath].map((path) => readFile(path)), + ), + ).toEqual(beforeRecoveryBytes); + expect(await fingerprintTree(stateDirectory)).toEqual( + beforeRecoveryTree, + ); + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual(beforeCalls); + expect([...routes.keys()]).toEqual([]); + return; + } + if ( + snapshotFault || + ordinaryFollowup === "callback_failure" || + ordinaryFollowup === "callback_async_failure" + ) { + const firstRead = within(resumed.transport.request("thread/read", {})); + void firstRead.catch(() => undefined); + if (ordinaryFollowup === "callback_async_failure") { + await within(Promise.race([ + heldCompletionEntered, + firstRead.then(() => { + throw new Error("warm restart completion callback was not entered"); + }), + ])); + expect(rejectHeldCompletion).toBeTypeOf("function"); + try { + expect(recoveryFenceActive).toBe(true); + const queuedBefore = cores[1]!.store.state.commands.map( + (command) => command.commandId, + ); + await expect( + resumed.transport.request("turn/start", { + input: [{ text: "must not pass held completion" }], + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.request("thread/resume", {}), + ).rejects.toThrow("warm_transition_completion_pending"); + expect( + cores[1]!.store.state.commands.map( + (command) => command.commandId, + ), + ).toEqual(queuedBefore); + } finally { + rejectHeldCompletion!( + new Error("fixture recovery completion callback failed"), + ); + } + } + await expect(firstRead).rejects.toThrow( + snapshotFault + ? ordinaryFollowup === "rejected_snapshot" + ? "snapshot observation rejected" + : "completion_unproven" + : "recovery completion callback failed", + ); + expect(recoveryFenceActive).toBe(true); + expect(completionSnapshotIds).toHaveLength(snapshotFault ? 0 : 1); + const beforeDeniedWork = cores[1]!.store.state.commands.map( + (command) => command.commandId, + ); + const beforeDeniedProviderCalls = await readFile(callsPath, "utf8"); + await expect( + resumed.transport.request("turn/start", { + input: [{ text: "must remain fenced" }], + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.request("thread/resume", {}), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.attachRun!({ + runId: "must-not-attach", + turnId: "must-not-attach", + itemId: "must-not-attach", + }), + ).rejects.toThrow("warm_transition_completion_pending"); + expect( + cores[1]!.store.state.commands.map((command) => command.commandId), + ).toEqual(beforeDeniedWork); + expect(await readFile(callsPath, "utf8")).toBe( + beforeDeniedProviderCalls, + ); + snapshotObserverSpy?.mockRestore(); + const providerCallsBeforeRetry = await readFile(callsPath, "utf8"); + expect( + (await within(resumed.transport.request("thread/read", {}))).thread, + ).toMatchObject(thread); + expect(recoveryFenceActive).toBe(false); + expect(completionSnapshotIds).toHaveLength(snapshotFault ? 1 : 2); + expect(new Set(completionSnapshotIds).size).toBe( + completionSnapshotIds.length, + ); + expect(await readFile(callsPath, "utf8")).toBe( + providerCallsBeforeRetry, + ); + return; + } + const read = await within( + resumed.transport.request("thread/read", {}), + 10_000, + ); + if ((resumed.evidence().codexPid ?? 0) > 0) + providerPids.add(resumed.evidence().codexPid!); + expect(read.thread).toMatchObject(thread); + expect(cores).toHaveLength(2); + expect(cores[1]!.store.state.identity).toEqual(desired); + expect(cores[1]!.store.state.warmTransition).toBeUndefined(); + expect( + cores[1]!.store.state.completedWarmTransition?.receipt.transitionId, + ).toBe(runner.warmTransition.receipt.transitionId); + if (routed) { + expect(routeCalls.slice(routeCallCount)).toEqual([ + oldIdentity.runId, + desired.runId, + ]); + expect([...routes.keys()]).toEqual([ + `/api/runner/v1/connect/${desired.runId}`, + ]); + } + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter((call) => call === "process-start")).toHaveLength(2); + expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); + expect(calls.filter((call) => call === "thread/resume")).toHaveLength(1); + expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); + if (ordinaryFollowup) { + recoveryClaimCurrent = false; + const resumedCore = cores[1]!; + resumedCore.disconnectActiveRunner(); + await vi.waitFor( + () => expect(resumedCore.activeRunnerConnectionCount()).toBe(1), + { timeout: 2_000 }, + ); + expect(recoveryFenceActive).toBe(false); + expect( + (await within(resumed.transport.request("thread/read", {}))).thread, + ).toMatchObject(thread); + const thirdRunId = randomUUID(); + await within( + resumed.transport.attachRun!({ + runId: thirdRunId, + turnId: `turn-${thirdRunId}`, + itemId: `item-${thirdRunId}`, + }), + ); + expect(resumedCore.store.state.identity.runId).toBe(thirdRunId); + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual([...calls, "thread/goal/get"]); + } + } finally { + snapshotObserverSpy?.mockRestore(); + commitSpy?.mockRestore(); + commandObserverSpy?.mockRestore(); + try { + await within(resumed?.transport.close() ?? Promise.resolve()).catch( + () => undefined, + ); + await within(first.transport.detachControllerForRestart!()).catch( + () => undefined, + ); + for (const handle of handles) { + await durableControlPlane + .waitForProcess(handle, 250) + .catch(() => undefined); + await within(handle.completion); + if (handle.processGroupId && !dead(-handle.processGroupId)) + process.kill(-handle.processGroupId, "SIGKILL"); + await vi.waitFor(() => { + expect(handle.child.pid && dead(handle.child.pid)).toBe(true); + expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( + true, + ); + }); + } + for (const pid of providerPids) { + await stopOwnedProvider(pid); + } + cleanupProven = true; + } finally { + for (const core of cores) await core.stop().catch(() => undefined); + if (routed) + await new Promise((resolveClose) => + routeServer.close(() => resolveClose()), + ); + coreSpy.mockRestore(); + launchSpy.mockRestore(); + if (cleanupProven) + await rm(stateDirectory, { recursive: true, force: true }); + if (cleanupProven) + for (const directory of legacyProbeDirectories) + await rm(directory, { recursive: true, force: true }); + } + } + }, + 30_000, +); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts new file mode 100644 index 00000000000..dba8ec470b3 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts @@ -0,0 +1,769 @@ +import { mkdtemp, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import type { ControlPlanePort } from "../contracts/control-plane-port.js"; +import type { NativeExecutionInputV1 } from "../contracts/native-execution.js"; +import type { NativeSession, NativeSessionBackend } from "../contracts/native-session-backend.js"; +import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; +import { executeNativeSession } from "../native-session-runtime.js"; +import { createCodexTaskEnvelope } from "../contracts/codex.js"; +import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver.js"; +import { createCapabilityRunnerdCodexTransport } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, expectTurnStarted, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +it("rejects active work and buffered tools from a resumed stopped checkpoint", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-stopped-resume-active-"), + ); + const identity = { + runnerInstanceId: "runner-stopped-active", + environmentLeaseId: "lease-stopped-active", + runId: "run-stopped-first", + normalizedSessionId: "session-stopped-active", + turnId: "turn-stopped-first", + itemId: "item-stopped-first", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--linger-after-turn-start", + "--resume-unowned-turn-when-marked", + "--emit-tool-call-on-resume", + ), + stateDirectory, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + }; + const first = createCapabilityRunnerdCodexTransport({ + ...options, + prpIdentity: identity, + }); + let second: + ReturnType | undefined; + const semanticHandler = vi.fn(async () => ({ + success: true, + contentItems: [], + })); + first.transport.setServerRequestHandler(semanticHandler); + try { + const opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [ + { + name: "get_task_context", + description: "Read the current task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ], + }); + await first.transport.request("turn/start", { + input: [{ type: "text", text: "Wait for another instruction." }], + }); + await expectTurnStarted( + first.transport.notifications()[Symbol.asyncIterator](), + ); + await first.transport.close(); + const providerPath = join( + stateDirectory, + "runner", + "codex-provider-state.json", + ); + expect(JSON.parse(await readFile(providerPath, "utf8"))).toMatchObject({ + lifecycle: "prepared", + activeProviderTurnId: null, + }); + await writeFile(join(stateDirectory, "resume-unowned-turn"), "armed"); + const secondIdentity = { + ...identity, + runId: "run-stopped-second", + turnId: "turn-stopped-second", + itemId: "item-stopped-second", + }; + second = createCapabilityRunnerdCodexTransport({ + ...options, + prpIdentity: secondIdentity, + }); + second.transport.setServerRequestHandler(semanticHandler); + await expect(second.transport.request("thread/read", {})).rejects.toThrow( + "prepared provider checkpoint resumed unexpected active work", + ); + // Exercise the normal durable transfer before inspecting the rejection: + // provider pendingEvents is an acknowledged queue, not an event journal. + await vi.waitFor( + async () => { + const provider = JSON.parse(await readFile(providerPath, "utf8")); + expect(provider.pendingEvents).toEqual([]); + const control = JSON.parse( + await readFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + "utf8", + ), + ) as { + committedEvents: Array<{ + eventType: string; + envelope: { payload: { payload: { code?: string } } }; + }>; + }; + const rejections = control.committedEvents.filter( + (event) => + event.eventType === "harness.diagnostic" && + event.envelope.payload.payload.code === + "prepared_provider_checkpoint_has_active_work", + ); + expect(rejections).toEqual([ + expect.objectContaining({ + logicalEffectCount: 1, + envelope: expect.objectContaining({ + ...secondIdentity, + payload: expect.objectContaining({ + payload: expect.objectContaining({ + code: "prepared_provider_checkpoint_has_active_work", + paperclipAccepted: false, + providerReportedActive: true, + }), + }), + }), + }), + ]); + }, + { timeout: 5_000 }, + ); + const closed = JSON.parse(await readFile(providerPath, "utf8")); + expect(closed).toMatchObject({ + lifecycle: "closed", + threadId: (opened.thread as Record).id, + activeProviderTurnId: null, + }); + expect(semanticHandler).not.toHaveBeenCalled(); + expect(closed.pendingEvents).toEqual([]); + } finally { + await Promise.allSettled([ + first.transport.close(), + second?.transport.close(), + ]); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("does not retry a real memoized transport close whose suspension proof is unavailable", async () => { + const identity = { + runId: "run-recovery", + sessionId: "session-recovery", + companyId: "company-recovery", + issueId: "issue-recovery", + agentId: "agent-recovery", + }; + + const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Recovered native work completed.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: true, + criteria: [ + { criterionId: "objective", status: "satisfied", evidenceRefs: [] }, + ], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "recovery", status: "passed" }], + attentionRequests: [], + artifacts: [], + }; + + const terminal: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", + }; + + const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { + companyId: identity.companyId, + runId: identity.runId, + issueId: identity.issueId, + agentId: identity.agentId, + executionWorkspaceId: "workspace-recovery", + }, + task: { + identifier: "PAP-RECOVERY", + title: "Recover native work", + description: null, + prompt: "# PAP-RECOVERY: Recover native work", + workMode: "standard", + }, + workspace: { + cwd: "/workspace", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: identity.sessionId, + driverKind: "codex_app_server", + protocolVersion: 1, + }, + provider: { kind: "codex", model: null }, + completionContract: { + id: "contract-recovery", + sha256: "contract-recovery-sha", + schemaVersion: "paperclip.completion-contract.v1", + contract: { + revision: "1", + objective: "Recover native work", + criteria: [{ id: "objective", requirement: "Complete after recovery" }], + }, + }, + interactionResponses: [], + credentialBindings: [], + }; + + function runnerEvent( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record = {}, + ): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-recovery", + sourceKind: "runner", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload, + }; + } + const stateDirectory = await mkdtemp( + join(tmpdir(), "native-close-quarantine-"), + ); + const readRunnerState = vi.fn(async () => ({ + schema: "paperclip.runner.durable.state.v1", + runnerInstanceId: "runner-close-quarantine", + environmentLeaseId: "lease-close-quarantine", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-close-quarantine", + itemId: "item-close-quarantine", + lifecycle: "ready", + })); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: resolve( + import.meta.dirname, + "../../runner/target/debug/fake-codex-app-server", + ), + codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")], + stateDirectory, + closeGraceMs: 400, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: await readRunnerState(), + readRunnerState, + // A checkpoint owner requires durable suspension proof. A local transport + // without a checkpoint can simply terminate its process on close. + controlPlaneRegistration: async (authority) => { + await authority.start(); + return { + connectUrl: authority.connectUrl, + checkpoint: async () => {}, + release: async () => {}, + }; + }, + }); + try { + await bundle.transport.request("thread/start", { + cwd: stateDirectory, + dynamicTools: [], + }); + const failedClose = bundle.transport.close(); + const failure = await failedClose.catch((error: unknown) => error); + expect(failure).toMatchObject({ + code: "native_session_close_unrecoverable", + }); + expect(bundle.transport.close()).toBe(failedClose); + + vi.useFakeTimers(); + const close = vi.fn(({ reason }: { reason: string }) => + bundle.transport.close(reason), + ); + const capabilities = { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return capabilities; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "real-memoized-close-quarantine", + version: "1", + capabilities, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + requireSessionCloseBeforeReturn: true, + }); + await expect(execute()).rejects.toBe(failure); + const readsAfterClose = readRunnerState.mock.calls.length; + await expect(execute()).rejects.toMatchObject({ + code: "native_session_cleanup_quarantined", + recovery: "operator_required", + }); + await vi.advanceTimersByTimeAsync(600_000); + await expect(execute()).rejects.toMatchObject({ + code: "native_session_cleanup_quarantined", + }); + expect(openSession).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(readRunnerState).toHaveBeenCalledTimes(readsAfterClose); + } finally { + vi.useRealTimers(); + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 10_000); + +it.each(["not_suspended", "wrong_identity"] as const)( + "does not report local runner close healthy with %s durable evidence", + async (mode) => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-local-close-unproven-"), + ); + const identity = { + runnerInstanceId: "runner-close-unproven", + environmentLeaseId: "lease-close-unproven", + runId: "run-close-unproven", + normalizedSessionId: "session-close-unproven", + turnId: "turn-close-unproven", + itemId: "item-close-unproven", + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + closeGraceMs: 400, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: identity, + readRunnerState: async () => ({ + schema: "paperclip.runner.durable.state.v1", + ...identity, + ...(mode === "wrong_identity" ? { runId: "some-other-run" } : {}), + lifecycle: mode === "not_suspended" ? "ready" : "suspended", + }), + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + await expect(bundle.transport.close()).rejects.toThrow( + "runner did not durably suspend before checkpoint", + ); + const control = JSON.parse( + await readFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + "utf8", + ), + ); + expect(control.identity).toEqual(identity); + expect(await readdir(stateDirectory)).toContain("runner"); + } finally { + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } + }, + 10_000, +); + +it.each([ + { + label: "interrupted", + args: ["--hold-turn"], + terminal: "turn.interrupted", + disposition: "needs_review", + }, + { + label: "failed", + args: ["--fail-turn-immediately"], + terminal: "turn.failed", + disposition: "needs_review", + }, + { + label: "completed", + args: [], + terminal: "turn.completed", + disposition: "done", + }, +])("retains a $label runner's result before closing its provider turn", async ({ + label, args, terminal, disposition, +}) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-stop-result-")); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, ...args), + stateDirectory, + }); + const driver = new CodexAppServerDriver({ + taskEnvelope: createCodexTaskEnvelope({ + objective: "Stop and retain unfinished work.", + }), + environment: { + PATH: process.env.PATH, + HOME: join(tmpdir(), "runnerd-stop-host-home"), + PAPERCLIP_WORKSPACE_CWD: stateDirectory, + }, + approvalPolicy: "never", + transportFactory: () => bundle.transport, + }); + const session = await driver.openSession({ + runId: "run-stop-result", + normalizedSessionId: "session-stop-result", + workingDirectory: stateDirectory, + }); + try { + const turn = await session.startTurn({ + message: { role: "user", text: "Keep working until interrupted." }, + }); + if (label === "interrupted") { + await session.interrupt({ + turnId: turn.turnId, + reason: "Stopped by the user", + }); + } + const events: PrpEvent[] = []; + for await (const event of session.events()) { + events.push(event); + if (event.eventType === "session.failed" || event.eventType === terminal) { + break; + } + } + // Let already committed durable suffix events reach the facade as well. + await bundle.transport.request("thread/read", {}); + const snapshot = await session.snapshot(); + expect(events.some((event) => event.eventType === "session.failed")).toBe(false); + expect(events.map((event) => event.eventType)).toContain("run.result.proposed"); + expect(events.at(-1)?.eventType).toBe(terminal); + expect(snapshot).toMatchObject({ + activeTurnId: null, + semanticResult: { + turnId: turn.turnId, + result: { reportedWorkDisposition: disposition }, + }, + }); + } finally { + await session.close().catch(() => undefined); + await rm(stateDirectory, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 25, + }); + } +}, 30_000); + +it.each([0, 193 * 1024 * 1024])( + "probes an exact-authority resume with %i extra journal bytes and confirms its live provider identity", + async (extraJournalBytes) => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-exact-authority-resume-"), + ); + const identity = { + runnerInstanceId: "runner-exact-resume", + environmentLeaseId: "lease-exact-resume", + runId: "run-exact-resume", + normalizedSessionId: "session-exact-resume", + turnId: "turn-exact-resume", + itemId: "item-exact-resume", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--durable-turn-ids"), + stateDirectory, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + prpIdentity: identity, + }; + const first = createCapabilityRunnerdCodexTransport(options); + first.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let providerThread: { id: string; sessionId: string } | null = null; + try { + const opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + const thread = opened.thread as Record; + providerThread = { + id: String(thread.id), + sessionId: String(thread.sessionId), + }; + } finally { + await first.transport.close(); + } + if (providerThread === null) { + throw new Error( + "exact-authority fixture did not return a provider thread", + ); + } + + const statePath = join( + stateDirectory, + "control-plane", + "control-plane-state.json", + ); + if (extraJournalBytes > 0) { + const padding = Buffer.alloc(1024 * 1024, 0x20); + const stateHandle = await open(statePath, "a"); + try { + for (let remaining = extraJournalBytes; remaining > 0;) { + const bytesToWrite = Math.min(remaining, padding.length); + await stateHandle.write(padding, 0, bytesToWrite); + remaining -= bytesToWrite; + } + } finally { + await stateHandle.close(); + } + expect((await stat(statePath)).size).toBeGreaterThan( + 64 * 1024 * 1024, + ); + } + const beforeResume = JSON.parse(await readFile(statePath, "utf8")) as { + commands: Array<{ type: string }>; + committedEvents: Array<{ eventType: string }>; + }; + expect( + beforeResume.commands.some((command) => command.type === "run.attach"), + ).toBe(false); + const priorResumeEvents = beforeResume.committedEvents.filter( + (event) => event.eventType === "session.resumed", + ).length; + const priorSnapshots = beforeResume.commands.filter( + (command) => command.type === "session.snapshot", + ).length; + + const resumed = createCapabilityRunnerdCodexTransport({ + ...options, + resumeProviderSession: { + driverSessionId: providerThread.id, + providerSessionId: providerThread.sessionId, + }, + }); + resumed.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await resumed.transport.request("thread/read", {}); + expect(read.thread).toMatchObject(providerThread); + const afterResume = JSON.parse(await readFile(statePath, "utf8")) as { + commands: Array<{ commandId: string; type: string; status: string }>; + committedEvents: Array<{ eventType: string }>; + }; + expect(afterResume.commands).toContainEqual( + expect.objectContaining({ + commandId: expect.stringMatching(/^command_resume_probe_/), + type: "runner.drain", + status: "completed", + }), + ); + expect(afterResume.commands).toContainEqual( + expect.objectContaining({ + type: "session.snapshot", + status: "completed", + }), + ); + expect( + afterResume.commands.filter( + (command) => command.type === "session.snapshot", + ), + ).toHaveLength(priorSnapshots + 2); + // The authenticated snapshot above proves the live provider identity. + // Control-first dispatch may deliver that command before the independent + // session event is ingested. Still require exactly one durable event; + // don't mistake an immediate file read for an event-delivery barrier. + await vi.waitFor( + async () => { + const delivered = JSON.parse(await readFile(statePath, "utf8")) as { + committedEvents: Array<{ eventType: string }>; + }; + expect( + delivered.committedEvents.filter( + (event) => event.eventType === "session.resumed", + ), + ).toHaveLength(priorResumeEvents + 1); + }, + { timeout: 3_000, interval: 25 }, + ); + } finally { + await resumed.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } + }, + 30_000, +); + +it("still fails closed when a real close grace period cannot fit a durable suspension round trip", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-close-grace-too-small-"), + ); + const identity = { + runnerInstanceId: "runner-close-grace-too-small", + environmentLeaseId: "lease-close-grace-too-small", + runId: "run-close-grace-too-small", + normalizedSessionId: "session-close-grace-too-small", + turnId: "turn-close-grace-too-small", + itemId: "item-close-grace-too-small", + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + // No real durable command round trip can complete this fast. A wider + // budget for the provider-drain proof must not turn this barrier into + // one that always passes; it still needs the actual proof to arrive. + closeGraceMs: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: identity, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + const closed = bundle.transport.close(); + await expect(closed).rejects.toMatchObject({ + code: "native_session_close_unrecoverable", + semanticToolsSettled: true, + }); + // Retirement preserves the original rejected promise and its proof. + expect(bundle.transport.close()).toBe(closed); + } finally { + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("persists an active provider as settled before bounded suspension", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-active-suspension-"), + ); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--linger-after-turn-start"), + stateDirectory, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("initialize", {}); + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + await bundle.transport.request("turn/start", { + input: [{ type: "text", text: "Wait for another instruction." }], + }); + const notifications = bundle.transport + .notifications() + [Symbol.asyncIterator](); + await expectTurnStarted(notifications); + await bundle.transport.close(); + + const providerState = JSON.parse( + await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + "utf8", + ), + ) as Record; + expect(providerState).toMatchObject({ + lifecycle: "prepared", + activeProviderTurnId: null, + }); + } finally { + await bundle.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts new file mode 100644 index 00000000000..7739d4e2851 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts @@ -0,0 +1,1649 @@ +import { mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import type { PrpEvent } from "../protocol/replay-contract.js"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; +import { releaseMaterializedNativeRuntimeSkills } from "../drivers/runtime-context-materializer.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; +import { + defaultCapabilityRunnerdBinary, + fakeCodex, + fakeCodexArgs, + assignedRuntimeContext, +} from "../../test/support/runnerd-transport.js"; + +it.each([ + ["held-ack", "normal"], + ["lost-ack", "normal"], + ["rejected-attach", "normal"], + ["lost-ack", "after-activation"], +] as const)( + "preserves old warm-attach authority and event ownership across %s (%s observer)", + async (mode, observer) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-ack-")); + const callsPath = join(stateDirectory, "calls.log"); + const cores: DurablePrpControlPlane[] = []; + const effects = new Map(); + const handles: ReturnType[] = []; + let armed = false; + let heldEvent: PrpEvent | null = null; + let releaseCommit!: () => void; + let enteredCommit!: () => void; + const commitGate = new Promise((resolveCommit) => { + releaseCommit = resolveCommit; + }); + const commitEntered = new Promise((resolveEntered) => { + enteredCommit = resolveEntered; + }); + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi + .spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function ( + options: ConstructorParameters[0], + ) { + const core = new OriginalCore({ + ...options, + onCommittedEvent: async (event) => { + await options.onCommittedEvent?.(event); + const prior = effects.get(event.sourceEventId); + if (prior) { + expect(event).toEqual(prior.event); + prior.deliveries += 1; + } else { + effects.set(event.sourceEventId, { + event: structuredClone(event), + deliveries: 1, + }); + } + if ( + armed && + mode !== "rejected-attach" && + heldEvent === null && + event.eventType === "run.attached" + ) { + heldEvent = structuredClone(event); + enteredCommit(); + await commitGate; + if (mode === "lost-ack") { + // The external durable effect exists, but this connection + // disappears before its local cursor/ACK can be published. + throw new Error( + "fixture lost the old authority ACK after commit", + ); + } + } + }, + }); + cores.push(core); + return core; + } as unknown as typeof OriginalCore); + const launch = durableControlPlane.spawnRunner; + const launchSpy = vi + .spyOn(durableControlPlane, "spawnRunner") + .mockImplementation((options) => { + const handle = launch(options); + handles.push(handle); + return handle; + }); + const within = async ( + label: string, + promise: Promise, + timeout = 5_000, + ) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolveWait, rejectWait) => { + timer = setTimeout( + () => rejectWait(new Error(`${label} timeout`)), + timeout, + ); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + }; + const dead = (pid: number) => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--call-log", + callsPath, + "--record-process-start", + ), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 5_000, + }); + const readRunner = async () => + JSON.parse( + await readFile( + join(stateDirectory, "runner/runner-state.json"), + "utf8", + ), + ) as { + runId: string; + ackedSourceSeq: number; + processedCommands: Record; + outbox: { envelope: { payload: PrpEvent } }[]; + }; + let providerPid: number | null = null; + let primaryError: unknown; + let cleanupProven = false; + try { + const opened = await within( + "initial thread", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }), + ) as { thread: { id: string } }; + const core = cores[0]!; + expect(cores).toHaveLength(1); + const oldIdentity = structuredClone(core.store.state.identity); + const runnerPid = bundle.evidence().runnerPid; + providerPid = bundle.evidence().codexPid; + const rotations: (typeof core.store.state)[] = []; + const preparedStates: (typeof core.store.state)[] = []; + const commit = core.store.commit.bind(core.store); + vi.spyOn(core.store, "commit").mockImplementation((candidate) => { + commit(candidate); + // Capture the durable old epoch at publication. The authenticated + // successor may activate before attachRun observes the completed command. + if ( + candidate.identity.runId === oldIdentity.runId && + candidate.warmTransition?.phase === "prepared" + ) { + preparedStates.push(structuredClone(core.store.state)); + } + }); + if (observer === "after-activation") { + const getCommand = core.getCommand.bind(core); + vi.spyOn(core, "getCommand").mockImplementation((commandId) => { + const command = getCommand(commandId); + if ( + command?.type === "run.attach" && + command.status === "completed" && + core.store.state.completedWarmTransition === undefined + ) { + // Deterministically exercise an observer that misses the old epoch. + return { ...command, status: "pending", result: null }; + } + return command; + }); + } + const rotate = core.rotateRunIdentity.bind(core); + vi.spyOn(core, "rotateRunIdentity").mockImplementation( + (identity, template) => { + rotations.push(structuredClone(core.store.state)); + return rotate(identity, template); + }, + ); + if (mode === "rejected-attach") { + const queue = core.queueCommand.bind(core); + vi.spyOn(core, "queueCommand").mockImplementation( + (type, payload = {}, id, immediate) => + queue( + type, + type === "run.attach" + ? { + ...payload, + provider: { + ...(payload.provider as Record), + model: "foreign-profile", + }, + } + : payload, + id, + immediate, + ), + ); + } + armed = true; + const attachment = bundle.transport.attachRun!({ + runId: "run-warm-ack-next", + turnId: "turn-warm-ack-next", + itemId: "item-warm-ack-next", + }); + void attachment.catch(() => undefined); + if (mode === "rejected-attach") { + await expect(within("rejected attach", attachment)).rejects.toThrow( + "run.attach cannot change the durable Codex provider profile", + ); + expect(rotations).toHaveLength(0); + expect(core.store.state.identity).toEqual(oldIdentity); + expect((await readRunner()).runId).toBe(oldIdentity.runId); + const read = await within( + "read under unchanged authority", + bundle.transport.request("thread/read", {}), + ); + expect((read.thread as { id: string }).id).toBe(opened.thread.id); + expect( + core.store.state.commands.find((entry) => entry.type === "run.attach") + ?.status, + ).toBe("failed"); + } else { + await within("old authority commit barrier", commitEntered); + const retained = await readRunner(); + expect(retained.runId).toBe(oldIdentity.runId); + expect( + Object.values(retained.processedCommands).find((entry) => entry.commandType === "run.attach") + ?.status, + ).toBe("completed"); + expect( + retained.outbox.some( + (entry) => + entry.envelope.payload.sourceEventId === heldEvent!.sourceEventId, + ), + ).toBe(true); + expect( + core.store.state.commands.find((entry) => entry.type === "run.attach") + ?.status, + ).toBe("pending"); + expect(rotations).toHaveLength(0); + if (mode === "lost-ack") core.disconnectActiveRunner(); + releaseCommit(); + await within("warm attach after old ACK", attachment, 10_000); + expect(rotations).toHaveLength(1); + expect(preparedStates.length).toBeGreaterThan(0); + const retired = preparedStates[0]!; + if (observer === "after-activation") { + expect(rotations[0]!.identity.runId).toBe("run-warm-ack-next"); + expect( + rotations[0]!.committedEvents.some( + (entry) => entry.sourceEventId === heldEvent!.sourceEventId, + ), + ).toBe(false); + } + const attachedEvent = retired.committedEvents.find( + (entry) => entry.sourceEventId === heldEvent!.sourceEventId, + )!; + expect(attachedEvent.logicalEffectCount).toBe(1); + expect(retired.ackedSourceSeq).toBeGreaterThanOrEqual( + attachedEvent.sourceSeq, + ); + expect( + retired.committedEvents.slice(-4).map((entry) => entry.eventType), + ).toEqual([ + "session.resumed", + "session.capabilities.updated", + "session.goal.snapshot", + "run.attached", + ]); + expect( + retired.committedEvents.every( + (entry) => entry.envelope.runId === oldIdentity.runId, + ), + ).toBe(true); + if (mode === "lost-ack") { + expect(retired.connectionCount).toBeGreaterThanOrEqual(2); + expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(2); + } else { + expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(1); + } + await vi.waitFor(async () => + expect((await readRunner()).runId).toBe("run-warm-ack-next"), + ); + const read = await within( + "read under new authority", + bundle.transport.request("thread/read", {}), + ); + expect((read.thread as { id: string }).id).toBe(opened.thread.id); + } + expect(bundle.evidence()).toMatchObject({ + runnerPid, + codexPid: providerPid, + runnerExited: false, + }); + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter((call) => call === "process-start")).toHaveLength(1); + expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); + expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); + } catch (error) { + primaryError = error; + throw error; + } finally { + releaseCommit(); + try { + await within( + "warm fixture close", + bundle.transport.close(), + 10_000, + ).catch(() => undefined); + for (const handle of handles) { + await durableControlPlane + .waitForProcess(handle, 250) + .catch(() => undefined); + await within("exact warm fixture runner exit", handle.completion); + if (handle.processGroupId && !dead(-handle.processGroupId)) + process.kill(-handle.processGroupId, "SIGKILL"); + await vi.waitFor(() => { + expect(handle.child.pid && dead(handle.child.pid)).toBe(true); + expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( + true, + ); + }); + } + if (providerPid && !dead(-providerPid)) + process.kill(-providerPid, "SIGKILL"); + if (providerPid) + await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); + cleanupProven = true; + } catch (error) { + if (primaryError === undefined) throw error; + console.error( + "Warm ACK fixture cleanup unproven; retaining its private state directory.", + ); + } finally { + try { + for (const core of cores) await core.stop().catch(() => undefined); + } finally { + launchSpy.mockRestore(); + coreSpy.mockRestore(); + if (cleanupProven) + await rm(stateDirectory, { recursive: true, force: true }); + } + } + } + }, + 30_000, +); + +it.each([false, true])( + "retains warm attach authority when its result is lost before controller persistence (held observer=%s)", + async (holdObserver) => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-attach-result-loss-"), + ); + const callsPath = join(stateDirectory, "calls.log"); + const cores: DurablePrpControlPlane[] = []; + const handles: ReturnType[] = []; + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi + .spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function ( + options: ConstructorParameters[0], + ) { + const core = new OriginalCore(options); + cores.push(core); + return core; + } as unknown as typeof OriginalCore); + const launch = durableControlPlane.spawnRunner; + const launchSpy = vi + .spyOn(durableControlPlane, "spawnRunner") + .mockImplementation((options) => { + const handle = launch(options); + handles.push(handle); + return handle; + }); + const within = async ( + label: string, + promise: Promise, + timeout = 5_000, + ) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} timeout`)), + timeout, + ); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + }; + const dead = (pid: number) => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: + process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? + defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--call-log", + callsPath, + "--record-process-start", + ), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 2_000, + }); + let providerPid: number | null = null; + let cleanupProven = false; + let saveSpy: ReturnType | undefined; + let observerSpy: { mockRestore(): void } | undefined; + let attachment: Promise | undefined; + try { + await within( + "initial thread", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }), + ); + expect(cores).toHaveLength(1); + const core = cores[0]!; + const oldIdentity = structuredClone(core.store.state.identity); + if (holdObserver) { + const getCommand = core.getCommand.bind(core); + observerSpy = vi + .spyOn(core, "getCommand") + .mockImplementation((commandId) => { + const command = getCommand(commandId); + if ( + command?.type === "run.attach" && + core.store.state.completedWarmTransition?.command.commandId !== + commandId + ) { + return { ...command, status: "pending", result: null }; + } + return command; + }); + } + providerPid = bundle.evidence().codexPid; + const store = core.store as typeof core.store & { + commit(candidate: typeof core.store.state): void; + }; + const commit = store.commit.bind(store); + let lostResult = false; + let observeLoss!: () => void; + const loss = new Promise((resolveLoss) => { + observeLoss = resolveLoss; + }); + saveSpy = vi.spyOn(store, "commit").mockImplementation((candidate) => { + const attach = candidate.commands.find( + (entry) => entry.type === "run.attach", + ); + if (!lostResult && attach?.status === "completed") { + lostResult = true; + // The authenticated result reached the receiver, but the durable write + // did not. The clone has not been exposed in memory or on disk. + const persisted = JSON.parse(readFileSync(core.store.path, "utf8")); + expect(persisted.identity).toEqual(oldIdentity); + expect( + persisted.commands.find( + (entry: { type: string }) => entry.type === "run.attach", + ).status, + ).toBe("pending"); + core.disconnectActiveRunner(); + observeLoss(); + throw new Error( + "fixture lost attach result before durable controller commit", + ); + } + commit(candidate); + }); + attachment = bundle.transport.attachRun!({ + runId: "run-result-loss-next", + turnId: "turn-result-loss-next", + itemId: "item-result-loss-next", + }); + void attachment.catch(() => undefined); + await within("lost attach result", loss); + expect(lostResult).toBe(true); + expect(core.store.state.identity).toEqual(oldIdentity); + expect( + core.store.state.commands.find((entry) => entry.type === "run.attach") + ?.status, + ).toBe("pending"); + const outcome = await within( + "exact result replay after reconnect", + attachment.then( + () => ({ status: "completed" as const }), + (error: unknown) => ({ + status: "failed" as const, + message: String(error), + }), + ), + 10_000, + ); + const runner = JSON.parse( + await readFile( + join(stateDirectory, "runner/runner-state.json"), + "utf8", + ), + ); + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter((call) => call === "process-start")).toHaveLength(1); + expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); + expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); + expect( + { + outcome, + runnerRunId: runner.runId, + controllerRunId: core.store.state.identity.runId, + }, + "lost attach result must replay without leaving the two durable authorities split", + ).toEqual({ + outcome: { status: "completed" }, + runnerRunId: "run-result-loss-next", + controllerRunId: "run-result-loss-next", + }); + if (holdObserver) + expect( + core.store.state.completedWarmTransition?.receipt.newIdentity.runId, + ).toBe("run-result-loss-next"); + } finally { + saveSpy?.mockRestore(); + observerSpy?.mockRestore(); + try { + await within( + "result-loss fixture close", + bundle.transport.close(), + 5_000, + ).catch(() => undefined); + for (const handle of handles) { + await durableControlPlane + .waitForProcess(handle, 250) + .catch(() => undefined); + await within("exact result-loss runner exit", handle.completion); + if (handle.processGroupId && !dead(-handle.processGroupId)) + process.kill(-handle.processGroupId, "SIGKILL"); + await vi.waitFor(() => { + expect(handle.child.pid && dead(handle.child.pid)).toBe(true); + expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( + true, + ); + }); + } + if (providerPid && !dead(-providerPid)) + process.kill(-providerPid, "SIGKILL"); + if (providerPid) + await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); + cleanupProven = true; + } finally { + for (const core of cores) await core.stop().catch(() => undefined); + launchSpy.mockRestore(); + coreSpy.mockRestore(); + if (cleanupProven) + await rm(stateDirectory, { recursive: true, force: true }); + } + } + }, + 30_000, +); + +it("rotates PRP authority in place for a warm cross-run attachment", async () => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-")); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + const within = async (label: string, promise: Promise) => + await Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`${label} timeout`)), 5_000), + ), + ]); + try { + await within("initialize", bundle.transport.request("initialize", {})); + await within( + "thread start", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [ + { + name: "get_task_context", + description: "Read the active task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ], + completionContract: { + revision: "sha256:warm-three-turn-contract", + criterionIds: ["objective"], + }, + }), + ); + const runnerPid = bundle.evidence().runnerPid; + const providerPid = bundle.evidence().codexPid; + const notifications = bundle.transport + .notifications() + [Symbol.asyncIterator](); + const waitForCompletion = async (label: string) => { + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + const next = await Promise.race([ + notifications.next(), + new Promise((_, reject) => + setTimeout( + () => reject(new Error(`${label} notification timeout`)), + 1_000, + ), + ), + ]); + if (next.value?.method === "turn/completed") return; + } + throw new Error(`${label} completion timeout`); + }; + await within( + "first turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "first run" }], + }), + ); + await waitForCompletion("first run"); + + await within( + "warm attach", + bundle.transport.attachRun!({ + runId: "run-warm-second", + turnId: "turn-warm-second", + itemId: "item-warm-second", + }), + ); + await within( + "second turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "second run" }], + }), + ); + await waitForCompletion("second run"); + + await within( + "second warm attach", + bundle.transport.attachRun!({ + runId: "run-warm-third", + turnId: "turn-warm-third", + itemId: "item-warm-third", + }), + ); + await within( + "third turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "third run" }], + }), + ); + await waitForCompletion("third run"); + + expect(bundle.evidence()).toMatchObject({ + runnerPid, + codexPid: providerPid, + runnerExited: false, + }); + } finally { + await bundle.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("recovers an active second warm business run after deleting its controller context assets", async () => { + const root = await mkdtemp(join(tmpdir(), "runnerd-warm-second-active-")); + const firstRoot = join(root, "first-controller"), nextRoot = join(root, "next-controller"); + const assetRoot = join(root, "retired-controller-assets"); + const skillRoot = join(assetRoot, "skill"), instructionRoot = join(assetRoot, "instructions"); + await mkdir(skillRoot, { recursive: true }); await mkdir(instructionRoot, { recursive: true }); + await writeFile(join(skillRoot, "SKILL.md"), "# Pinned skill\n"); + await writeFile(join(instructionRoot, "AGENTS.md"), "Pinned instructions\n"); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null, generation = 0; + let checkpoint: durableControlPlane.DurablePrpCheckpoint | undefined; + const registered = new WeakSet(); + server.on("upgrade", (request, socket, head) => authority + ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy()); + await new Promise(done => server.listen(0, "127.0.0.1", done)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing owned Runner route"); + const registration = async (core: DurablePrpControlPlane) => { + if (!registered.has(core)) { + core.setCheckpointPersistence(async value => { checkpoint = value; }); + registered.add(core); + } + authority = core; const current = ++generation; + return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: async () => { if (generation === current) authority = null; } }; + }; + const initialIdentity = { runnerInstanceId: "runner-warm-second-active", environmentLeaseId: "lease-warm-second-active", + runId: "run-warm-first-healthy", normalizedSessionId: "session-warm-second-active", turnId: "turn-first", itemId: "item-first" }; + const options = { runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(root, "--emit-tool-call", "--durable-tool-ids"), stateDirectory: firstRoot, + runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), + prpIdentity: initialIdentity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, controlPlaneRegistration: registration }; + const first = createCapabilityRunnerdCodexTransport(options); + const within = async (label: string, promise: Promise) => { + let timer: ReturnType | undefined; + try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`warm active recovery stage timed out: ${label}`)), 5_000); })]); } + finally { clearTimeout(timer); } + }; + const waitComplete = async (transport: typeof first.transport) => { + for await (const notification of transport.notifications()) if (notification.method === "turn/completed") return; + }; + let calls = 0, runnerPid: number | null = null, providerPid: number | null = null; + let recovered: ReturnType | undefined; + const result = { ok: true, task: { id: "task-1" } }; + first.transport.setServerRequestHandler(async () => ++calls === 1 + ? { success: true, contentItems: [{ type: "inputText", text: JSON.stringify(result) }] } + : new Promise>(() => undefined)); + try { + const opened = await within("first thread/start", first.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [{ name: "get_task_context", description: "Read current task", + inputSchema: { type: "object", properties: {}, additionalProperties: false } }] })); + runnerPid = first.evidence().runnerPid; + providerPid = first.evidence().codexPid; + expect(providerPid).toEqual(expect.any(Number)); + const providerStart = execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim(); + const startedAt = first.transport.processInfo().startedAt; + const firstCompleted = waitComplete(first.transport); + await within("first turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "first healthy business run" }] })); + await within("healthy first turn completion", firstCompleted); + const attachment = { runId: "run-warm-second-active", turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; + await within("warm second business attachment", first.transport.attachRun!(attachment)); + const active = await within("second active turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "second business run held at tool" }] })); + await vi.waitFor(() => expect(calls).toBe(2), { timeout: 5_000 }); + const activeCore = authority!; + const savedIdentity = { ...activeCore.store.state.identity }; + expect(savedIdentity).toEqual({ ...initialIdentity, ...attachment }); + await activeCore.flushCheckpoint(); + await within("controller detach", first.detachControllerForRestart()); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + await rm(assetRoot, { recursive: true, force: true }); + expect(existsSync(assetRoot)).toBe(false); + checkpoint!.restoreSessionDirectory(nextRoot); + const noSpawn = vi.fn(() => { throw new Error("Duplicate Runner launch"); }); + const openedThread = opened.thread as Record; + const activeTurn = active.turn as Record; + expect(openedThread.id).toEqual(expect.any(String)); + expect(activeTurn.id).toEqual(expect.any(String)); + const shared = { ...options, stateDirectory: nextRoot, + resumeProviderSession: { driverSessionId: String(openedThread.id), providerSessionId: String(openedThread.sessionId) }, + resumeActiveTurnId: String(activeTurn.id), runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), + runnerStateDirectory: join(firstRoot, "runner"), readRunnerState: async () => JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8")), + prepareExternalRunnerState: async () => {}, archiveExternalRunnerState: async () => { throw new Error("Active authority must not rotate"); }, + runnerProcessLauncher: noSpawn, adoptExistingRunner: { pid: runnerPid!, processGroupId: runnerPid, startedAt, + isAlive: () => { try { process.kill(runnerPid!, 0); return true; } catch { return false; } }, + signal: (signal: NodeJS.Signals) => { try { process.kill(runnerPid!, signal); return true; } catch { return false; } } } }; + // The old server's synthetic pair must continue failing the strict Runner guard. + const wrong = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: { ...savedIdentity, + turnId: `turn-${attachment.runId}`, itemId: `item-${attachment.runId}` } }); + await expect(wrong.transport.request("thread/read", {})).rejects.toThrow("PRP recovery identity does not match the durable session binding"); + await wrong.detachControllerForRestart(); + // Leave the pending tool outcome unknown. Reattachment must authenticate + // and preserve it without fabricating a receipt or redispatching its handler. + recovered = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: savedIdentity }); + await within("replacement thread/read", recovered.transport.request("thread/read", {})); + expect(recovered.evidence()).toMatchObject({ runnerPid, runnerExited: false }); + expect(authority!.activeRunnerConnectionCount()).toBe(1); + expect(() => process.kill(providerPid!, 0)).not.toThrow(); + expect(execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim()).toBe(providerStart); + expect(calls).toBe(2); + expect(authority!.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(0); + expect(recovered.transport.processInfo().startedAt).toBe(startedAt); + expect(authority!.store.state.identity).toEqual(savedIdentity); + expect(authority!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); + expect(noSpawn).not.toHaveBeenCalled(); + } catch (error) { + const saved = JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8").catch(() => "{}")); + const stderr = await readFile(join(firstRoot, "diagnostics/runnerd.stderr.log"), "utf8").catch(() => ""); + throw new Error(`${String(error)}; owned protocol evidence ${JSON.stringify({ + transport: recovered?.evidence(), connections: authority?.activeRunnerConnectionCount(), + identity: authority?.store.state.identity, commands: authority?.store.state.commands.map(c => ({type:c.type,status:c.status})), + events: authority?.store.state.committedEvents.map(e => e.eventType).slice(-8), + runner: { lifecycle:saved.lifecycle, runId:saved.runId, turnId:saved.turnId, itemId:saved.itemId, + outbox:saved.outbox?.length }, stderr:stderr.slice(-1500), + })}`); + } finally { + await (recovered ? within("recovered cleanup", recovered.transport.close()).catch(() => undefined) : undefined); + await within("first cleanup", first.transport.close()).catch(() => undefined); + if (providerPid) { try { process.kill(providerPid, "SIGKILL"); } catch {} } + await (authority as DurablePrpControlPlane | null)?.stop(); + if (runnerPid) { try { process.kill(-runnerPid, "SIGKILL"); } catch {} } + server.closeAllConnections(); + if (server.listening) await new Promise(done => server.close(() => done())); + for (const controller of [firstRoot, nextRoot]) await releaseMaterializedNativeRuntimeSkills(join(controller, "codex-home", "skills")); + await rm(root, { recursive: true, force: true }); + } +}, 30_000); + +it("waits for a warm runner to re-authenticate before probing attachment readiness", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-warm-reattach-before-probe-"), + ); + const server = createServer(); + const authorities = new Map(); + let blockFirstAuthorityReconnect = false; + let resolveRejectedReconnect!: () => void; + const rejectedReconnect = new Promise((resolvePromise) => { + resolveRejectedReconnect = resolvePromise; + }); + server.on("upgrade", (request, socket, head) => { + const route = request.url ?? ""; + if (route === "/runner-1" && blockFirstAuthorityReconnect) { + resolveRejectedReconnect(); + socket.destroy(); + return; + } + const authority = authorities.get(route); + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, route, head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected warm reconnect test listener"); + } + let registrationCount = 0; + const diagnostics: string[] = []; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 5_000, + onDiagnostic: (message) => diagnostics.push(message), + controlPlaneRegistration: async (authority) => { + registrationCount += 1; + const route = `/runner-${registrationCount}`; + authorities.set(route, authority); + return { + connectUrl: `ws://127.0.0.1:${address.port}${route}`, + release: () => { + if (authorities.get(route) === authority) authorities.delete(route); + }, + }; + }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let runnerPid: number | null = null; + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: codexSemanticToolSpecs(), + }); + runnerPid = bundle.evidence().runnerPid; + const firstAuthority = authorities.get("/runner-1"); + if (!firstAuthority) throw new Error("Missing first warm authority"); + const priorSnapshotCount = firstAuthority.store.state.commands.filter( + (command) => command.type === "session.snapshot", + ).length; + + blockFirstAuthorityReconnect = true; + firstAuthority.disconnectActiveRunner(); + const attachment = bundle.transport.attachRun!({ + runId: "run-warm-after-reconnect", + turnId: "turn-warm-after-reconnect", + itemId: "item-warm-after-reconnect", + }); + await Promise.race([ + rejectedReconnect, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("runner did not attempt to reconnect")), + 5_000, + ), + ), + ]); + + // No command may be queued while its sole authenticated consumer is + // absent. The generic 30-second command timeout used to turn this state + // into same-run recovery and replace the healthy warm runner process. + expect( + firstAuthority.store.state.commands.filter( + (command) => command.type === "session.snapshot", + ), + ).toHaveLength(priorSnapshotCount); + + blockFirstAuthorityReconnect = false; + await Promise.race([ + attachment, + new Promise((_, reject) => + setTimeout(() => reject(new Error("warm attachment timeout")), 10_000), + ), + ]); + expect(bundle.evidence()).toMatchObject({ + runnerPid, + runnerExited: false, + }); + expect(diagnostics).toContain( + "warm runner connection interrupted; waiting for re-authentication before authority rotation", + ); + expect(diagnostics).toContain( + "warm runner re-authenticated before authority rotation", + ); + } finally { + await bundle.transport.close().catch(() => undefined); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // A successful durable close already stopped the runner process group. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("releases both PRP authorities when warm rotation activation fails", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-warm-attach-activation-failure-"), + ); + const server = createServer(); + const authorities = new Map(); + const released: string[] = []; + server.on("upgrade", (request, socket, head) => { + const route = request.url ?? ""; + const authority = authorities.get(route); + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, route, head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected warm activation failure test listener"); + } + let registrationCount = 0; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + controlPlaneRegistration: async (authority) => { + registrationCount += 1; + const route = `/runner-${registrationCount}`; + authorities.set(route, authority); + return { + connectUrl: `ws://127.0.0.1:${address.port}${route}`, + ...(registrationCount === 1 + ? {} + : { + activate: () => { + throw new Error("rotation activation failed"); + }, + }), + release: () => { + released.push(route); + if (authorities.get(route) === authority) authorities.delete(route); + }, + }; + }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let runnerPid: number | null = null; + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: codexSemanticToolSpecs(), + }); + runnerPid = bundle.evidence().runnerPid; + + await expect( + bundle.transport.attachRun!({ + runId: "run-warm-activation-failure", + turnId: "turn-warm-activation-failure", + itemId: "item-warm-activation-failure", + }), + ).rejects.toThrow("rotation activation failed"); + expect(new Set(released)).toEqual(new Set(["/runner-1", "/runner-2"])); + expect(authorities.size).toBe(0); + await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( + "rotation activation failed", + ); + } finally { + await bundle.transport.close().catch(() => undefined); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // A successful durable close already stopped the runner process group. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it.each([ + { + binding: "runner instance", + priorRunnerInstanceId: "runner-other", + priorEnvironmentLeaseId: "lease-current", + }, + { + binding: "environment lease", + priorRunnerInstanceId: "runner-current", + priorEnvironmentLeaseId: "lease-other", + }, +])( + "quarantines a mismatched $binding instead of reusing its provider session", + async ({ priorRunnerInstanceId, priorEnvironmentLeaseId }) => { + const container = await mkdtemp( + join(tmpdir(), "runnerd-mismatched-authority-"), + ); + const stateDirectory = join(container, "state"); + await mkdir(join(stateDirectory, "control-plane"), { recursive: true }); + await writeFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + JSON.stringify({ + identity: { + runnerInstanceId: priorRunnerInstanceId, + environmentLeaseId: priorEnvironmentLeaseId, + runId: "run-old", + normalizedSessionId: "session-current", + turnId: "turn-old", + itemId: "item-old", + }, + }), + { mode: 0o600 }, + ); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + stateDirectory, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: { + runnerInstanceId: "runner-current", + environmentLeaseId: "lease-current", + runId: "run-new", + normalizedSessionId: "session-current", + turnId: "turn-new", + itemId: "item-new", + }, + }); + try { + await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( + "native_runner_state_quarantined", + ); + expect(await readdir(stateDirectory)).toEqual([]); + expect( + (await readdir(container)).some((entry) => + entry.startsWith("state.quarantine-"), + ), + ).toBe(true); + } finally { + await bundle.transport.close(); + await rm(container, { recursive: true, force: true }); + } + }, +); + +it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accounting-rejected", "credentials-cleaned"] as const)("cold-restores a suspended provider session under its durable run binding (pending tail: %s)", async (pendingTail) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-cold-attach-")); + const tracePath = join(stateDirectory, "provider-trace.ndjson"); + const skillRoot = join(stateDirectory, "runtime-skill"); + const instructionRoot = join(stateDirectory, "runtime-instructions"); + await Promise.all([mkdir(skillRoot), mkdir(instructionRoot)]); + await writeFile(join(skillRoot, "SKILL.md"), "# Assigned runtime skill\n"); + await writeFile(join(instructionRoot, "AGENTS.md"), "Runtime instructions\n"); + const credentialsCleaned = pendingTail === "credentials-cleaned"; + const sourceCodexHome = join(stateDirectory, "selected-account-first"); + if (credentialsCleaned) { + await mkdir(sourceCodexHome); + await writeFile(join(sourceCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); + } + const baseIdentity = { + runnerInstanceId: "runner-cold-attach", + environmentLeaseId: "lease-cold-attach", + runId: "run-cold-first", + normalizedSessionId: "session-cold-attach", + turnId: "turn-cold-first", + itemId: "item-cold-first", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--include-skill-instructions", + "--durable-turn-ids", + ...(credentialsCleaned ? ["--require-codex-home-auth"] : []), + "-c", + 'shell_environment_policy.set={PATH="/run/A"}', + ), + stateDirectory, + ...(credentialsCleaned ? { sourceCodexHome } : {}), + environment: { + PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-A-capability", + PAPERCLIP_PROVIDER_TRACE_PATH: tracePath, + PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024), + ...(credentialsCleaned ? { + PAPERCLIP_NATIVE_MCP_NAME: "assigned", + PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/first", + PAPERCLIP_NATIVE_MCP_TOKEN: "first-assigned-gateway-credential", + } : {}), + }, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), + }; + const dynamicTools = [ + { + name: "get_task_context", + description: "Read the active task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ]; + const first = createCapabilityRunnerdCodexTransport({ + ...options, + prpIdentity: baseIdentity, + }); + first.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let firstProviderThread: { id: string; sessionId: string } | null = null; + try { + const started = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [...dynamicTools, ...codexSemanticToolSpecs()], + completionContract: { + revision: "contract-first", + criterionIds: ["criterion-first"], + }, + }); + const startedThread = started.thread as Record; + firstProviderThread = { + id: String(startedThread.id), + sessionId: String(startedThread.sessionId), + }; + expect( + (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) + .mode & 0o222, + ).toBe(0); + expect( + ( + await stat( + join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), + ) + ).mode & 0o222, + ).toBe(0); + await first.transport.request("turn/start", { + input: [{ type: "text", text: "$assigned first process" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], + }); + for await (const event of first.transport.notifications()) { + if (event.method === "turn/completed") break; + } + } finally { + await first.transport.close(); + } + if (!firstProviderThread) { + throw new Error("cold attach fixture did not return a provider thread"); + } + const successorCodexHome = join(stateDirectory, "selected-account-next"); + if (credentialsCleaned) { + await rm(join(stateDirectory, "codex-home", "auth.json")); + await rm(sourceCodexHome, { recursive: true }); + await mkdir(successorCodexHome); + await writeFile(join(successorCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); + } + + // Model a retained provider-owned suffix: rotation must deliver and durably + // accept it before attempting run.attach, rather than dropping or ACKing it. + const retainedDelta = "cold-recovery-retained-tail"; + if (pendingTail) { + const providerFile = join(stateDirectory, "runner", "codex-provider-state.json"); + const provider = JSON.parse(await readFile(providerFile, "utf8")); + expect(provider.pendingEvents).toEqual([]); + provider.pendingEvents.push({ + executorEventId: `codex_provider_${String(provider.nextProviderEventSeq++).padStart(16, "0")}`, + eventType: pendingTail === "unknown-semantic" ? "semantic_tool.input" : "usage.reported", + priority: "p0", + payload: pendingTail === "unknown-semantic" ? { semantic_tool: { + schema: "paperclip.prp.semantic_tool.v1", schemaVersion: 1, phase: "input", + callId: "unknown-source-call", operationId: "get_task_context", input: {}, idempotencyKey: null, + correlation: { runId: baseIdentity.runId, normalizedSessionId: baseIdentity.normalizedSessionId, + turnId: baseIdentity.turnId, itemId: baseIdentity.itemId }, + content: { digest: `sha256:${createHash("sha256").update("{}").digest("hex")}`, redactionDisposition: "digest_only", references: [] }, + } } : { provider: "codex", kind: "usage", delta: retainedDelta, + providerTurnId: provider.completed_provider_turn_id, runDelta: { inputTokens: 20, outputTokens: 2 }, + runDeltaAvailable: true, runDeltaBasis: "run" }, + }); + await writeFile(providerFile, JSON.stringify(provider)); + } + const retainedEvents: PrpEvent[] = []; + let releaseRetained!: () => void, retainedEntered!: () => void; + const retainedAccepted = new Promise(done => { releaseRetained = done; }); + const retainedAdmission = new Promise(done => { retainedEntered = done; }); + const secondIdentity = { + ...baseIdentity, + runId: "run-cold-second", + turnId: "turn-cold-second", + itemId: "item-cold-second", + }; + let stagedCheckpoint: durableControlPlane.DurablePrpCheckpoint | undefined; + let rejectStagedCommit = pendingTail === "restart-before-bootstrap"; + const rotatedOptions = { + ...options, + ...(credentialsCleaned ? { sourceCodexHome: successorCodexHome } : {}), + onStagedRunAttachment: pendingTail === "restart-before-bootstrap" ? async (authority: DurablePrpControlPlane) => { + authority.setCheckpointPersistence(async checkpoint => { + stagedCheckpoint = checkpoint; + if (rejectStagedCommit) throw new Error("fixture lost controller after durable intent"); + }); + } : undefined, + onRecoveryAccountingEvent: pendingTail ? async (event: PrpEvent) => { + if (event.payload.delta === retainedDelta) { + retainedEvents.push(structuredClone(event)); + retainedEntered(); + if (pendingTail === "accounting-rejected") throw new Error("source accounting receipt unavailable"); + await retainedAccepted; + } + } : undefined, + environment: { ...options.environment, PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-B-capability", + ...(credentialsCleaned ? { + PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/current", + PAPERCLIP_NATIVE_MCP_TOKEN: "current-assigned-gateway-credential", + } : {}), + }, + codexArgs: options.codexArgs.map((arg) => arg.replace('/run/A', '/run/B')), + resumeDynamicTools: dynamicTools, + resumeCompletionContract: { + revision: "contract-second", + criterionIds: ["criterion-second"], + }, + prpIdentity: secondIdentity, + }; + let rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); + if (pendingTail === "restart-before-bootstrap") { + await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("fixture lost controller after durable intent"); + expect(rotated.evidence().runnerPid).toBeNull(); + expect(stagedCheckpoint?.runId).toBe(secondIdentity.runId); + expect([...stagedCheckpoint!.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value).type)).not.toContain("run.attach"); + await rotated.detachControllerForRestart().catch(() => undefined); + rejectStagedCommit = false; + await rm(join(stateDirectory, "control-plane"), { recursive: true, force: true }); + stagedCheckpoint!.restoreSessionDirectory(stateDirectory); + rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); + } + const currentHandler = vi.fn(async () => ({ success: true, contentItems: [] })); + rotated.transport.setServerRequestHandler(currentHandler); + if (pendingTail === "accounting-rejected") { + const reading = rotated.transport.request("thread/read", {}); + void reading.catch(() => undefined); + try { + await Promise.race([retainedAdmission, reading]); + const source = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(source.identity).toEqual(baseIdentity); + expect(source.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); + expect(source.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); + const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => + entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); + expect(currentHandler).not.toHaveBeenCalled(); + } finally { + await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); + await expect(reading).rejects.toThrow(); + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + } + return; + } + if (pendingTail === "unknown-semantic") { + try { + await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("native_runner_source_semantic_reconciliation_required"); + expect(currentHandler).not.toHaveBeenCalled(); + const retained = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(retained.identity).toEqual(baseIdentity); + expect(retained.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); + expect(retained.commands.some((command: { type: string }) => ["run.attach", "semantic_tool.result"].includes(command.type))).toBe(false); + expect(retained.committedEvents.some((event: PrpEvent) => event.eventType === "semantic_tool.input")).toBe(true); + } finally { + await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + } + return; + } + let recoveryError: unknown; + try { + const reading = rotated.transport.request("thread/read", {}); + void reading.catch(() => undefined); + if (pendingTail) { + await Promise.race([retainedAdmission, reading]); + const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => + entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); + const retainedCore = stagedCheckpoint ? { + identity: JSON.parse(stagedCheckpoint.records.get("header")!).state.identity, + commands: [...stagedCheckpoint.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value)), + } : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(retainedCore.identity).toEqual(baseIdentity); + expect(retainedCore.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); + if (credentialsCleaned) { + const config = await readFile(join(stateDirectory, "codex-home", "config.toml"), "utf8"); + expect(config).toContain("https://paperclip.example/mcp/current"); + expect(config).toContain("current-assigned-gateway-credential"); + expect(config).not.toContain("first-assigned-gateway-credential"); + expect(currentHandler).not.toHaveBeenCalled(); + } + } + releaseRetained(); + const read = await reading; + expect(retainedEvents).toHaveLength(pendingTail ? 1 : 0); + if (pendingTail) expect(retainedEvents[0]).toMatchObject({ + runId: baseIdentity.runId, turnId: baseIdentity.turnId, itemId: baseIdentity.itemId, + }); + const persistedProvider = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); + expect(persistedProvider.pendingEvents).toEqual([]); + expect(persistedProvider.config.args.join("\n")).toContain('/run/B'); + expect(JSON.stringify(persistedProvider)).not.toContain("test-run-B-capability"); + expect(read.thread).toMatchObject({ + id: firstProviderThread.id, + sessionId: firstProviderThread.sessionId, + cwd: tmpdir(), + }); + await rotated.transport.request("turn/start", { + input: [{ type: "text", text: "$assigned second authority epoch" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], + }); + for await (const event of rotated.transport.notifications()) { + if (event.method === "paperclip/runResult") break; + } + expect(rotated.evidence().diagnostics).toContain( + "runnerd attached the durable provider session to a fresh PRP run authority", + ); + if (pendingTail) { + const target = stagedCheckpoint ? JSON.parse(stagedCheckpoint.records.get("header")!).state + : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(target.stagedRunAttachment).toBeUndefined(); + expect(target.completedWarmTransition.receipt).toMatchObject({ oldIdentity: baseIdentity, newIdentity: secondIdentity }); + } else expect(await stat(join(stateDirectory, "authority-epochs"))).toBeDefined(); + } catch (error) { + recoveryError = error; + throw error; + } finally { + releaseRetained(); + await rotated.transport.close().catch(error => { + if (recoveryError) throw new AggregateError([recoveryError, error], "Provider recovery and cleanup failed"); + throw error; + }); + } + const requestFrames = (await readFile(tracePath, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record) + .filter( + (entry) => + entry.kind === "frame" && entry.direction === "client_to_provider", + ) + .map( + (entry) => + JSON.parse( + Buffer.from(String(entry.rawBase64), "base64").toString("utf8"), + ) as Record, + ) + ; + const resumeFrames = requestFrames.filter((frame) => frame.method === "thread/resume"); + for (const frame of requestFrames.filter((f) => ["thread/start", "thread/resume"].includes(String(f.method)))) { + expect(frame.params).toMatchObject({ config: { "skills.include_instructions": true } }); + } + const turnFrames = requestFrames.filter((f) => f.method === "turn/start"); + expect(turnFrames).toHaveLength(2); + for (const frame of turnFrames) { + expect(frame.params).toMatchObject({ input: expect.arrayContaining([{ type: "skill", name: "assigned", path: join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md") }]) }); + } + expect(resumeFrames.length).toBeGreaterThanOrEqual(1); + for (const frame of resumeFrames) { + expect(frame).toEqual( + expect.objectContaining({ + method: "thread/resume", + params: expect.objectContaining({ threadId: firstProviderThread.id }), + }), + ); + } + + if (pendingTail) { + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + return; + } + + // A remote process owner keeps runner-state outside the controller's local + // session root. Resume must defer to its explicit state reader instead of + // rejecting recovery before the remote checkpoint can be made available. + const externallyOwnedRunnerStateDirectory = join( + stateDirectory, + "externally-owned-runner", + ); + await rename( + join(stateDirectory, "runner"), + externallyOwnedRunnerStateDirectory, + ); + const readRunnerState = async () => + JSON.parse( + await readFile( + join(externallyOwnedRunnerStateDirectory, "runner-state.json"), + "utf8", + ), + ) as Record; + const mismatched = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + resumeDynamicTools: dynamicTools, + prpIdentity: { + ...secondIdentity, + runId: "run-cold-other", + turnId: "turn-cold-other", + itemId: "item-cold-other", + }, + }); + await expect(mismatched.transport.request("thread/read", {})).rejects.toThrow( + "native_runner_prp_run_rotation_unavailable", + ); + await mismatched.transport.close(); + + const externalIdentity = { + ...secondIdentity, + runId: "run-cold-external", + turnId: "turn-cold-external", + itemId: "item-cold-external", + }; + const rejectedExternalRotationSteps: string[] = []; + let externalArchiveDirectory: string | null = null; + const rejectedExternalRotation = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + prepareExternalRunnerState: async () => { + rejectedExternalRotationSteps.push("prepared"); + }, + archiveExternalRunnerState: async ({ archiveKey }) => { + await expect( + stat(join(stateDirectory, "control-plane")), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect( + await stat( + join( + stateDirectory, + "authority-epochs", + `epoch-${archiveKey}`, + "control-plane", + ), + ), + ).toBeDefined(); + externalArchiveDirectory = join( + stateDirectory, + "external-authority-epochs", + archiveKey, + ); + await mkdir(externalArchiveDirectory, { recursive: true }); + await rename( + join(externallyOwnedRunnerStateDirectory, "runner-state.json"), + join(externalArchiveDirectory, "runner-state.json"), + ); + rejectedExternalRotationSteps.push("remote-archived"); + throw new Error("controller crashed after external archive"); + }, + resumeDynamicTools: dynamicTools, + prpIdentity: externalIdentity, + }); + await expect( + rejectedExternalRotation.transport.request("thread/read", {}), + ).rejects.toThrow("controller crashed after external archive"); + await rejectedExternalRotation.transport.close(); + expect(rejectedExternalRotationSteps).toEqual([ + "prepared", + "remote-archived", + ]); + await expect(stat(join(stateDirectory, "control-plane"))).rejects.toThrow(); + await expect( + stat(join(externallyOwnedRunnerStateDirectory, "runner-state.json")), + ).rejects.toThrow(); + + const externalRotationSteps: string[] = []; + const externallyRotated = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + prepareExternalRunnerState: async () => { + throw new Error("retry must not prepare a new external runner"); + }, + archiveExternalRunnerState: async ({ archiveKey }) => { + expect(externalArchiveDirectory).toBe( + join(stateDirectory, "external-authority-epochs", archiveKey), + ); + externalRotationSteps.push("archived"); + return JSON.parse( + await readFile( + join(externalArchiveDirectory!, "runner-state.json"), + "utf8", + ), + ) as Record; + }, + resumeDynamicTools: dynamicTools, + resumeCompletionContract: { + revision: "contract-external", + criterionIds: ["criterion-external"], + }, + prpIdentity: externalIdentity, + }); + externallyRotated.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await externallyRotated.transport.request("thread/read", {}); + expect(read.thread).toMatchObject({ + id: firstProviderThread.id, + sessionId: firstProviderThread.sessionId, + cwd: tmpdir(), + }); + expect(externalRotationSteps).toEqual(["archived"]); + } finally { + await externallyRotated.transport.close(); + } + + const restored = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + resumeDynamicTools: dynamicTools, + prpIdentity: externalIdentity, + }); + restored.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await restored.transport.request("thread/read", {}); + expect(read.thread).toMatchObject({ + id: "codex-thread-1", + cwd: tmpdir(), + }); + expect( + (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) + .mode & 0o222, + ).toBe(0); + expect( + ( + await stat( + join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), + ) + ).mode & 0o222, + ).toBe(0); + const providerState = JSON.parse( + await readFile( + join(externallyOwnedRunnerStateDirectory, "codex-provider-state.json"), + "utf8", + ), + ) as { toolBridge?: { authorized?: Record } }; + expect(Object.keys(providerState.toolBridge?.authorized ?? {})).toEqual([ + "get_task_context", + "paperclip_block", + "paperclip_finish", + ]); + await restored.transport.request("turn/start", { + input: [{ type: "text", text: "restored process" }], + }); + for await (const event of restored.transport.notifications()) { + if (event.method === "turn/completed") break; + } + expect(restored.evidence()).toMatchObject({ + runnerExited: false, + codexPid: expect.any(Number), + }); + } finally { + await restored.transport.close(); + await releaseMaterializedNativeRuntimeSkills( + join(stateDirectory, "codex-home", "skills"), + ); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index da352598f0c..9a3f342ef43 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1,80 +1,41 @@ -import { bindRunnerLaunchResources, createRunnerProviderLaunch, runnerLaunchRequirementsDigest, type PreparedRunnerProviderLaunch, type RunnerProviderLaunchRequirements } from "./runner-provider-launch.js"; +import { + bindRunnerLaunchResources, + createRunnerProviderLaunch, + runnerLaunchRequirementsDigest, + type RunnerProviderLaunchRequirements, +} from "./runner-provider-launch.js"; import { coldAdmissionTimeoutMs, waitForRunnerCommand } from "./runnerd-codex-transport.js"; import { configuredEnvironmentProjection } from "../configured-environment.js"; -import { - chmod, - cp, - mkdir, - lstat, - mkdtemp, - open, - readFile, - readdir, - readlink, - realpath, - rename, - rm, - stat, - truncate, - writeFile, -} from "node:fs/promises"; -import { createHash, randomUUID } from "node:crypto"; +import { chmod, cp, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, stat, truncate, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { execFileSync, spawn } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { bindAcpxAgentFiles } from "../drivers/acpx/agent-files-binding.js"; import { resolveRunnerProviderAssetsRoot } from "../drivers/acpx/provider-assets-root.js"; import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; - import { describe, expect, it, vi } from "vitest"; -import type { ControlPlanePort } from "../contracts/control-plane-port.js"; -import type { NativeExecutionInputV1 } from "../contracts/native-execution.js"; -import type { - NativeSession, - NativeSessionBackend, -} from "../contracts/native-session-backend.js"; -import type { - PrpEvent, - PrpStructuredRunResult, - PrpTerminalState, -} from "../protocol/replay-contract.js"; +import type { PrpEvent } from "../protocol/replay-contract.js"; import { PRP_PROTOCOL_VERSION } from "../protocol/replay-contract.js"; -import { executeNativeSession } from "../native-session-runtime.js"; import { redactCapabilityEvidenceData } from "./evidence-redaction.js"; import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; import { parsePaperclipQuestionSet } from "../contracts/question-set.js"; -import { ACPX_CREDENTIAL_BINDING_ENV, createAcpxCredentialBinding, createAcpxSidecarHostEnvironment } from "../drivers/acpx/environment.js"; +import { + ACPX_CREDENTIAL_BINDING_ENV, + createAcpxCredentialBinding, + createAcpxSidecarHostEnvironment, +} from "../drivers/acpx/environment.js"; import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; import * as codexCommandRuntime from "../drivers/codex/codex-command.js"; - -import { - NATIVE_RUNTIME_ASSET_SCHEMA, - PAPERCLIP_EXECUTION_PROMPT, - PAPERCLIP_EXECUTION_PROMPT_REVISION, - canonicalNativeRuntimeContextDigest, - composeNativeSystemInstructions, - nativeRuntimePromptDigest, - type NativeRuntimeContextSnapshot, -} from "../contracts/runtime-context.js"; -import { - CODEX_SKILLLESS_BASE_INSTRUCTIONS, - createCodexTaskEnvelope, -} from "../contracts/codex.js"; -import { - PRP_BLOCK_TOOL_DESCRIPTION, - PRP_COMPLETION_TOOL_DESCRIPTION, -} from "../contracts/completion-result.js"; -import { - CodexAppServerDriver, - codexSemanticToolSpecs, -} from "../drivers/codex/codex-app-server-driver.js"; -import { releaseMaterializedNativeRuntimeSkills } from "../drivers/runtime-context-materializer.js"; +import { composeNativeSystemInstructions, type NativeRuntimeContextSnapshot } from "../contracts/runtime-context.js"; +import { CODEX_SKILLLESS_BASE_INSTRUCTIONS, createCodexTaskEnvelope } from "../contracts/codex.js"; +import { PRP_BLOCK_TOOL_DESCRIPTION, PRP_COMPLETION_TOOL_DESCRIPTION } from "../contracts/completion-result.js"; +import { CodexAppServerDriver, codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; import * as runtimeMaterializer from "../drivers/runtime-context-materializer.js"; import { RUNNERD_CANONICAL_ITEM } from "../drivers/codex/codex-driver-values.js"; - import { probeNativeRunnerEnvironment, NativeEnvironmentProbeCleanupError, @@ -83,7 +44,6 @@ import { createCapabilityRunnerdCodexTransport, createCapabilityRunnerdProviderEnvironment, createRunnerdCodexAppServerArgs, - defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary, readRunnerdArtifactBinding, expandRunnerdCanonicalNotifications, latestRunnerdSessionReadiness, @@ -109,11 +69,13 @@ import { resolveRunnerdCodexSkillInputs, withCodexCollaborationRuntimeInstructions, } from "./runnerd-codex-transport.js"; - -// Explicit private-artifact test lane; production/default dist is never changed. -const defaultCapabilityRunnerdBinary = () => - process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - qualifiedCapabilityRunnerdBinary(); +import { + defaultCapabilityRunnerdBinary, + expectTurnStarted, + fakeCodex, + fakeCodexArgs, + assignedRuntimeContext, +} from "../../test/support/runnerd-transport.js"; it("attributes opened candidate sessions to the harness biller instead of the model vendor", () => { const provider = runnerdLaunchProfileInternals.openedThreadModelProvider; @@ -130,21 +92,6 @@ it("attributes opened candidate sessions to the harness biller instead of the mo expect(provider("aws_agentcore", "codex", "global.anthropic.claude-sonnet-4-6")).toBe("aws"); }); -async function expectTurnStarted( - notifications: AsyncIterator<{ method: string }>, -) { - for (let index = 0; index < 32; index += 1) { - const next = await notifications.next(); - expect(next.done).not.toBe(true); - if (next.value.method === "turn/started") return; - // Startup ownership and capability facts can precede the active turn. - expect(next.value.method).toBe("paperclip/canonicalProviderEvent"); - } - throw new Error( - "provider turn did not start within the bounded notification prefix", - ); -} - it("replaces an owned v1 runner with fresh v2 authorization before warm attachment", async () => { const directory = await mkdtemp(join(tmpdir(), "runnerd-v1-v2-replacement-")); const handles: durableControlPlane.RunnerProcessHandle[] = []; @@ -458,12 +405,6 @@ it("preserves the cause when a replacement runner's health cannot be verified", } }, 15_000); -// The retained-settlement maintenance family ("settles only retained control -// authority without starting another provider turn") lives in -// runnerd-codex-transport-settlement.test.ts so vitest can run its ~160s of -// sequential cases on a separate worker instead of serializing them behind -// this file's tests. - it.each(["alive", "pending_liveness", "pending_registration"] as const)( "bounds adopted runner authentication at its exact deadline with %s evidence", async (mode) => { @@ -2055,6 +1996,7 @@ it.each([false, true])("binds a vendored source server to its pnpm dependency ro await rm(workspace, { recursive: true, force: true }); } }); + it.each((["copilot", "pi", "antigravity"] as const).flatMap(agent => ["source", "scoped-npm", "deployment"].map(layout => ({ agent, layout }))))( "admits the controller-selected $agent package authority in a $layout layout", @@ -3498,11 +3440,6 @@ it("recovers provider readiness from an already-committed journal without replay ).toEqual(persistedReady); }); -const fakeCodex = resolve( - import.meta.dirname, - "../../runner/target/debug/fake-codex-app-server", -); - it("keeps the Copilot authentication launch requirements free of Codex search policy", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-copilot-probe-policy-")); let captured: RunnerProviderLaunchRequirements | undefined; @@ -3641,14 +3578,6 @@ it.each(["stuck", "late-release", "late-release-error", "stuck-release", "cooper }, ); -function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { - return [ - "--state-file", - join(stateDirectory, "fake-codex-state.json"), - ...args, - ]; -} - it("joins owned credential preparation before close returns and never launches after close", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-owned-startup-")); const artifact = join(root, "unexecuted-runner-artifact"); @@ -3894,51 +3823,6 @@ it.each(["admission-only", "admission-only-mismatched-model", "complete", "api-k } finally { await rm(root, { recursive: true, force: true }); } }, 20_000); -function assignedRuntimeContext( - skillRoot: string, - instructionRoot: string, -): NativeRuntimeContextSnapshot { - const digest = "0".repeat(64); - const value = { - prompt: { - revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, - text: PAPERCLIP_EXECUTION_PROMPT, - digest: nativeRuntimePromptDigest(), - }, - instructions: { - entryPath: "AGENTS.md", - bundle: { - schema: NATIVE_RUNTIME_ASSET_SCHEMA, - digest, - manifestDigest: digest, - rootPath: instructionRoot, - fileCount: 1, - totalBytes: 1, - }, - }, - skills: [ - { - key: "company/assigned", - runtimeName: "assigned", - versionId: "version-1", - bundle: { - schema: NATIVE_RUNTIME_ASSET_SCHEMA, - digest, - manifestDigest: digest, - rootPath: skillRoot, - fileCount: 1, - totalBytes: 1, - }, - }, - ], - mcp: { assignmentSetId: "assigned", digest, bindingId: "binding" }, - } satisfies Omit; - return { - ...value, - aggregateDigest: canonicalNativeRuntimeContextDigest(value), - }; -} - it("unwraps a coalesced provider notification without losing its turn identity", () => { expect( unwrapRunnerdProviderNotification({ @@ -4817,528 +4701,6 @@ it.each([ 60_000, ); -it("rejects active work and buffered tools from a resumed stopped checkpoint", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-stopped-resume-active-"), - ); - const identity = { - runnerInstanceId: "runner-stopped-active", - environmentLeaseId: "lease-stopped-active", - runId: "run-stopped-first", - normalizedSessionId: "session-stopped-active", - turnId: "turn-stopped-first", - itemId: "item-stopped-first", - }; - const options = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--linger-after-turn-start", - "--resume-unowned-turn-when-marked", - "--emit-tool-call-on-resume", - ), - stateDirectory, - lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, - }; - const first = createCapabilityRunnerdCodexTransport({ - ...options, - prpIdentity: identity, - }); - let second: - ReturnType | undefined; - const semanticHandler = vi.fn(async () => ({ - success: true, - contentItems: [], - })); - first.transport.setServerRequestHandler(semanticHandler); - try { - const opened = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [ - { - name: "get_task_context", - description: "Read the current task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, - }, - }, - ], - }); - await first.transport.request("turn/start", { - input: [{ type: "text", text: "Wait for another instruction." }], - }); - await expectTurnStarted( - first.transport.notifications()[Symbol.asyncIterator](), - ); - await first.transport.close(); - const providerPath = join( - stateDirectory, - "runner", - "codex-provider-state.json", - ); - expect(JSON.parse(await readFile(providerPath, "utf8"))).toMatchObject({ - lifecycle: "prepared", - activeProviderTurnId: null, - }); - await writeFile(join(stateDirectory, "resume-unowned-turn"), "armed"); - const secondIdentity = { - ...identity, - runId: "run-stopped-second", - turnId: "turn-stopped-second", - itemId: "item-stopped-second", - }; - second = createCapabilityRunnerdCodexTransport({ - ...options, - prpIdentity: secondIdentity, - }); - second.transport.setServerRequestHandler(semanticHandler); - await expect(second.transport.request("thread/read", {})).rejects.toThrow( - "prepared provider checkpoint resumed unexpected active work", - ); - // Exercise the normal durable transfer before inspecting the rejection: - // provider pendingEvents is an acknowledged queue, not an event journal. - await vi.waitFor( - async () => { - const provider = JSON.parse(await readFile(providerPath, "utf8")); - expect(provider.pendingEvents).toEqual([]); - const control = JSON.parse( - await readFile( - join(stateDirectory, "control-plane", "control-plane-state.json"), - "utf8", - ), - ) as { - committedEvents: Array<{ - eventType: string; - envelope: { payload: { payload: { code?: string } } }; - }>; - }; - const rejections = control.committedEvents.filter( - (event) => - event.eventType === "harness.diagnostic" && - event.envelope.payload.payload.code === - "prepared_provider_checkpoint_has_active_work", - ); - expect(rejections).toEqual([ - expect.objectContaining({ - logicalEffectCount: 1, - envelope: expect.objectContaining({ - ...secondIdentity, - payload: expect.objectContaining({ - payload: expect.objectContaining({ - code: "prepared_provider_checkpoint_has_active_work", - paperclipAccepted: false, - providerReportedActive: true, - }), - }), - }), - }), - ]); - }, - { timeout: 5_000 }, - ); - const closed = JSON.parse(await readFile(providerPath, "utf8")); - expect(closed).toMatchObject({ - lifecycle: "closed", - threadId: (opened.thread as Record).id, - activeProviderTurnId: null, - }); - expect(semanticHandler).not.toHaveBeenCalled(); - expect(closed.pendingEvents).toEqual([]); - } finally { - await Promise.allSettled([ - first.transport.close(), - second?.transport.close(), - ]); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); -it("does not retry a real memoized transport close whose suspension proof is unavailable", async () => { - const identity = { - runId: "run-recovery", - sessionId: "session-recovery", - companyId: "company-recovery", - issueId: "issue-recovery", - agentId: "agent-recovery", - }; - - const result: PrpStructuredRunResult = { - schema: "paperclip.run_result.v1", - reportedWorkDisposition: "done", - summary: "Recovered native work completed.", - completionClaim: { - contractRevision: "1", - objectiveSatisfied: true, - criteria: [ - { criterionId: "objective", status: "satisfied", evidenceRefs: [] }, - ], - remainingWork: [], - }, - evidence: [], - verification: [{ commandOrCheck: "recovery", status: "passed" }], - attentionRequests: [], - artifacts: [], - }; - - const terminal: PrpTerminalState = { - schema: "paperclip.prp.terminal.v1", - turnTerminalState: "completed", - runTerminalState: "succeeded", - reportedWorkDisposition: "done", - }; - - const input: NativeExecutionInputV1 = { - schema: "paperclip.native-execution-input.v1", - binding: { - companyId: identity.companyId, - runId: identity.runId, - issueId: identity.issueId, - agentId: identity.agentId, - executionWorkspaceId: "workspace-recovery", - }, - task: { - identifier: "PAP-RECOVERY", - title: "Recover native work", - description: null, - prompt: "# PAP-RECOVERY: Recover native work", - workMode: "standard", - }, - workspace: { - cwd: "/workspace", - repoUrl: null, - repoRef: null, - branchName: null, - }, - session: { - normalizedSessionId: identity.sessionId, - driverKind: "codex_app_server", - protocolVersion: 1, - }, - provider: { kind: "codex", model: null }, - completionContract: { - id: "contract-recovery", - sha256: "contract-recovery-sha", - schemaVersion: "paperclip.completion-contract.v1", - contract: { - revision: "1", - objective: "Recover native work", - criteria: [{ id: "objective", requirement: "Complete after recovery" }], - }, - }, - interactionResponses: [], - credentialBindings: [], - }; - - function runnerEvent( - sourceSeq: number, - eventType: PrpEvent["eventType"], - payload: Record = {}, - ): PrpEvent { - return { - schema: "paperclip.prp.event.v1", - sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, - sourceSeq, - sourceInstanceId: "runner-recovery", - sourceKind: "runner", - runId: identity.runId, - normalizedSessionId: identity.sessionId, - turnId: "turn-recovery", - eventType, - schemaVersion: 1, - priority: 0, - emittedAt: "2026-08-09T00:00:00.000Z", - payload, - }; - } - const stateDirectory = await mkdtemp( - join(tmpdir(), "native-close-quarantine-"), - ); - const readRunnerState = vi.fn(async () => ({ - schema: "paperclip.runner.durable.state.v1", - runnerInstanceId: "runner-close-quarantine", - environmentLeaseId: "lease-close-quarantine", - runId: identity.runId, - normalizedSessionId: identity.sessionId, - turnId: "turn-close-quarantine", - itemId: "item-close-quarantine", - lifecycle: "ready", - })); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: resolve( - import.meta.dirname, - "../../runner/target/debug/fake-codex-app-server", - ), - codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")], - stateDirectory, - closeGraceMs: 400, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: await readRunnerState(), - readRunnerState, - // A checkpoint owner requires durable suspension proof. A local transport - // without a checkpoint can simply terminate its process on close. - controlPlaneRegistration: async (authority) => { - await authority.start(); - return { - connectUrl: authority.connectUrl, - checkpoint: async () => {}, - release: async () => {}, - }; - }, - }); - try { - await bundle.transport.request("thread/start", { - cwd: stateDirectory, - dynamicTools: [], - }); - const failedClose = bundle.transport.close(); - const failure = await failedClose.catch((error: unknown) => error); - expect(failure).toMatchObject({ - code: "native_session_close_unrecoverable", - }); - expect(bundle.transport.close()).toBe(failedClose); - - vi.useFakeTimers(); - const close = vi.fn(({ reason }: { reason: string }) => - bundle.transport.close(reason), - ); - const capabilities = { - resume: true, - typedEvents: true, - steering: false, - interruption: true, - structuredResult: true, - }; - const session: NativeSession = { - identity: () => identity, - async capabilities() { - return capabilities; - }, - async *events() { - yield runnerEvent(1, "turn.completed"); - }, - async startTurn() { - return { turnId: "turn-recovery" }; - }, - async result() { - return { result, terminal, turnId: "turn-recovery" }; - }, - async snapshot() { - return { - backendKind: "mock", - sessionId: identity.sessionId, - identity, - providerSessionId: "provider-recovery", - cursor: null, - activeTurnId: null, - pendingRuntimeRequests: [], - lineage: [], - }; - }, - close, - }; - const openSession = vi.fn(async () => session); - const backend: NativeSessionBackend = { - async descriptor() { - return { - kind: "mock", - name: "real-memoized-close-quarantine", - version: "1", - capabilities, - }; - }, - openSession, - }; - const port: ControlPlanePort = { - async openRun() {}, - async checkpointSession() {}, - async appendEvent() { - return { - cursor: 1, - highestContiguousSourceSeq: 1, - disposition: "committed", - }; - }, - async replayEvents() { - return { events: [], highestContiguousSourceSeq: 0 }; - }, - async completeRun() {}, - }; - const execute = () => - executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - requireSessionCloseBeforeReturn: true, - }); - await expect(execute()).rejects.toBe(failure); - const readsAfterClose = readRunnerState.mock.calls.length; - await expect(execute()).rejects.toMatchObject({ - code: "native_session_cleanup_quarantined", - recovery: "operator_required", - }); - await vi.advanceTimersByTimeAsync(600_000); - await expect(execute()).rejects.toMatchObject({ - code: "native_session_cleanup_quarantined", - }); - expect(openSession).toHaveBeenCalledOnce(); - expect(close).toHaveBeenCalledOnce(); - expect(readRunnerState).toHaveBeenCalledTimes(readsAfterClose); - } finally { - vi.useRealTimers(); - await bundle.transport.close().catch(() => undefined); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 10_000); - -it.each(["not_suspended", "wrong_identity"] as const)( - "does not report local runner close healthy with %s durable evidence", - async (mode) => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-local-close-unproven-"), - ); - const identity = { - runnerInstanceId: "runner-close-unproven", - environmentLeaseId: "lease-close-unproven", - runId: "run-close-unproven", - normalizedSessionId: "session-close-unproven", - turnId: "turn-close-unproven", - itemId: "item-close-unproven", - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - closeGraceMs: 400, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: identity, - readRunnerState: async () => ({ - schema: "paperclip.runner.durable.state.v1", - ...identity, - ...(mode === "wrong_identity" ? { runId: "some-other-run" } : {}), - lifecycle: mode === "not_suspended" ? "ready" : "suspended", - }), - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - await expect(bundle.transport.close()).rejects.toThrow( - "runner did not durably suspend before checkpoint", - ); - const control = JSON.parse( - await readFile( - join(stateDirectory, "control-plane", "control-plane-state.json"), - "utf8", - ), - ); - expect(control.identity).toEqual(identity); - expect(await readdir(stateDirectory)).toContain("runner"); - } finally { - await bundle.transport.close().catch(() => undefined); - await rm(stateDirectory, { recursive: true, force: true }); - } - }, - 10_000, -); - -it.each([ - { - label: "interrupted", - args: ["--hold-turn"], - terminal: "turn.interrupted", - disposition: "needs_review", - }, - { - label: "failed", - args: ["--fail-turn-immediately"], - terminal: "turn.failed", - disposition: "needs_review", - }, - { - label: "completed", - args: [], - terminal: "turn.completed", - disposition: "done", - }, -])("retains a $label runner's result before closing its provider turn", async ({ - label, args, terminal, disposition, -}) => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-stop-result-")); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, ...args), - stateDirectory, - }); - const driver = new CodexAppServerDriver({ - taskEnvelope: createCodexTaskEnvelope({ - objective: "Stop and retain unfinished work.", - }), - environment: { - PATH: process.env.PATH, - HOME: join(tmpdir(), "runnerd-stop-host-home"), - PAPERCLIP_WORKSPACE_CWD: stateDirectory, - }, - approvalPolicy: "never", - transportFactory: () => bundle.transport, - }); - const session = await driver.openSession({ - runId: "run-stop-result", - normalizedSessionId: "session-stop-result", - workingDirectory: stateDirectory, - }); - try { - const turn = await session.startTurn({ - message: { role: "user", text: "Keep working until interrupted." }, - }); - if (label === "interrupted") { - await session.interrupt({ - turnId: turn.turnId, - reason: "Stopped by the user", - }); - } - const events: PrpEvent[] = []; - for await (const event of session.events()) { - events.push(event); - if (event.eventType === "session.failed" || event.eventType === terminal) { - break; - } - } - // Let already committed durable suffix events reach the facade as well. - await bundle.transport.request("thread/read", {}); - const snapshot = await session.snapshot(); - expect(events.some((event) => event.eventType === "session.failed")).toBe(false); - expect(events.map((event) => event.eventType)).toContain("run.result.proposed"); - expect(events.at(-1)?.eventType).toBe(terminal); - expect(snapshot).toMatchObject({ - activeTurnId: null, - semanticResult: { - turnId: turn.turnId, - result: { reportedWorkDisposition: disposition }, - }, - }); - } finally { - await session.close().catch(() => undefined); - await rm(stateDirectory, { - recursive: true, - force: true, - maxRetries: 5, - retryDelay: 25, - }); - } -}, 30_000); - it("binds an immediately failed durable turn before exposing its terminal", async () => { const stateDirectory = await mkdtemp( join(tmpdir(), "runnerd-fast-terminal-"), @@ -5908,3406 +5270,6 @@ it("steers the active provider turn through the durable PRP command path", async } }, 30_000); -it.each([ - ["held-ack", "normal"], - ["lost-ack", "normal"], - ["rejected-attach", "normal"], - ["lost-ack", "after-activation"], -] as const)( - "preserves old warm-attach authority and event ownership across %s (%s observer)", - async (mode, observer) => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-ack-")); - const callsPath = join(stateDirectory, "calls.log"); - const cores: DurablePrpControlPlane[] = []; - const effects = new Map(); - const handles: ReturnType[] = []; - let armed = false; - let heldEvent: PrpEvent | null = null; - let releaseCommit!: () => void; - let enteredCommit!: () => void; - const commitGate = new Promise((resolveCommit) => { - releaseCommit = resolveCommit; - }); - const commitEntered = new Promise((resolveEntered) => { - enteredCommit = resolveEntered; - }); - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], - ) { - const core = new OriginalCore({ - ...options, - onCommittedEvent: async (event) => { - await options.onCommittedEvent?.(event); - const prior = effects.get(event.sourceEventId); - if (prior) { - expect(event).toEqual(prior.event); - prior.deliveries += 1; - } else { - effects.set(event.sourceEventId, { - event: structuredClone(event), - deliveries: 1, - }); - } - if ( - armed && - mode !== "rejected-attach" && - heldEvent === null && - event.eventType === "run.attached" - ) { - heldEvent = structuredClone(event); - enteredCommit(); - await commitGate; - if (mode === "lost-ack") { - // The external durable effect exists, but this connection - // disappears before its local cursor/ACK can be published. - throw new Error( - "fixture lost the old authority ACK after commit", - ); - } - } - }, - }); - cores.push(core); - return core; - } as unknown as typeof OriginalCore); - const launch = durableControlPlane.spawnRunner; - const launchSpy = vi - .spyOn(durableControlPlane, "spawnRunner") - .mockImplementation((options) => { - const handle = launch(options); - handles.push(handle); - return handle; - }); - const within = async ( - label: string, - promise: Promise, - timeout = 5_000, - ) => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolveWait, rejectWait) => { - timer = setTimeout( - () => rejectWait(new Error(`${label} timeout`)), - timeout, - ); - }), - ]); - } finally { - if (timer !== undefined) clearTimeout(timer); - } - }; - const dead = (pid: number) => { - try { - process.kill(pid, 0); - return false; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH"; - } - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--call-log", - callsPath, - "--record-process-start", - ), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 5_000, - }); - const readRunner = async () => - JSON.parse( - await readFile( - join(stateDirectory, "runner/runner-state.json"), - "utf8", - ), - ) as { - runId: string; - ackedSourceSeq: number; - processedCommands: Record; - outbox: { envelope: { payload: PrpEvent } }[]; - }; - let providerPid: number | null = null; - let primaryError: unknown; - let cleanupProven = false; - try { - const opened = await within( - "initial thread", - bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }), - ) as { thread: { id: string } }; - const core = cores[0]!; - expect(cores).toHaveLength(1); - const oldIdentity = structuredClone(core.store.state.identity); - const runnerPid = bundle.evidence().runnerPid; - providerPid = bundle.evidence().codexPid; - const rotations: (typeof core.store.state)[] = []; - const preparedStates: (typeof core.store.state)[] = []; - const commit = core.store.commit.bind(core.store); - vi.spyOn(core.store, "commit").mockImplementation((candidate) => { - commit(candidate); - // Capture the durable old epoch at publication. The authenticated - // successor may activate before attachRun observes the completed command. - if ( - candidate.identity.runId === oldIdentity.runId && - candidate.warmTransition?.phase === "prepared" - ) { - preparedStates.push(structuredClone(core.store.state)); - } - }); - if (observer === "after-activation") { - const getCommand = core.getCommand.bind(core); - vi.spyOn(core, "getCommand").mockImplementation((commandId) => { - const command = getCommand(commandId); - if ( - command?.type === "run.attach" && - command.status === "completed" && - core.store.state.completedWarmTransition === undefined - ) { - // Deterministically exercise an observer that misses the old epoch. - return { ...command, status: "pending", result: null }; - } - return command; - }); - } - const rotate = core.rotateRunIdentity.bind(core); - vi.spyOn(core, "rotateRunIdentity").mockImplementation( - (identity, template) => { - rotations.push(structuredClone(core.store.state)); - return rotate(identity, template); - }, - ); - if (mode === "rejected-attach") { - const queue = core.queueCommand.bind(core); - vi.spyOn(core, "queueCommand").mockImplementation( - (type, payload = {}, id, immediate) => - queue( - type, - type === "run.attach" - ? { - ...payload, - provider: { - ...(payload.provider as Record), - model: "foreign-profile", - }, - } - : payload, - id, - immediate, - ), - ); - } - armed = true; - const attachment = bundle.transport.attachRun!({ - runId: "run-warm-ack-next", - turnId: "turn-warm-ack-next", - itemId: "item-warm-ack-next", - }); - void attachment.catch(() => undefined); - if (mode === "rejected-attach") { - await expect(within("rejected attach", attachment)).rejects.toThrow( - "run.attach cannot change the durable Codex provider profile", - ); - expect(rotations).toHaveLength(0); - expect(core.store.state.identity).toEqual(oldIdentity); - expect((await readRunner()).runId).toBe(oldIdentity.runId); - const read = await within( - "read under unchanged authority", - bundle.transport.request("thread/read", {}), - ); - expect((read.thread as { id: string }).id).toBe(opened.thread.id); - expect( - core.store.state.commands.find((entry) => entry.type === "run.attach") - ?.status, - ).toBe("failed"); - } else { - await within("old authority commit barrier", commitEntered); - const retained = await readRunner(); - expect(retained.runId).toBe(oldIdentity.runId); - expect( - Object.values(retained.processedCommands).find((entry) => entry.commandType === "run.attach") - ?.status, - ).toBe("completed"); - expect( - retained.outbox.some( - (entry) => - entry.envelope.payload.sourceEventId === heldEvent!.sourceEventId, - ), - ).toBe(true); - expect( - core.store.state.commands.find((entry) => entry.type === "run.attach") - ?.status, - ).toBe("pending"); - expect(rotations).toHaveLength(0); - if (mode === "lost-ack") core.disconnectActiveRunner(); - releaseCommit(); - await within("warm attach after old ACK", attachment, 10_000); - expect(rotations).toHaveLength(1); - expect(preparedStates.length).toBeGreaterThan(0); - const retired = preparedStates[0]!; - if (observer === "after-activation") { - expect(rotations[0]!.identity.runId).toBe("run-warm-ack-next"); - expect( - rotations[0]!.committedEvents.some( - (entry) => entry.sourceEventId === heldEvent!.sourceEventId, - ), - ).toBe(false); - } - const attachedEvent = retired.committedEvents.find( - (entry) => entry.sourceEventId === heldEvent!.sourceEventId, - )!; - expect(attachedEvent.logicalEffectCount).toBe(1); - expect(retired.ackedSourceSeq).toBeGreaterThanOrEqual( - attachedEvent.sourceSeq, - ); - expect( - retired.committedEvents.slice(-4).map((entry) => entry.eventType), - ).toEqual([ - "session.resumed", - "session.capabilities.updated", - "session.goal.snapshot", - "run.attached", - ]); - expect( - retired.committedEvents.every( - (entry) => entry.envelope.runId === oldIdentity.runId, - ), - ).toBe(true); - if (mode === "lost-ack") { - expect(retired.connectionCount).toBeGreaterThanOrEqual(2); - expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(2); - } else { - expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(1); - } - await vi.waitFor(async () => - expect((await readRunner()).runId).toBe("run-warm-ack-next"), - ); - const read = await within( - "read under new authority", - bundle.transport.request("thread/read", {}), - ); - expect((read.thread as { id: string }).id).toBe(opened.thread.id); - } - expect(bundle.evidence()).toMatchObject({ - runnerPid, - codexPid: providerPid, - runnerExited: false, - }); - const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); - expect(calls.filter((call) => call === "process-start")).toHaveLength(1); - expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); - expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); - } catch (error) { - primaryError = error; - throw error; - } finally { - releaseCommit(); - try { - await within( - "warm fixture close", - bundle.transport.close(), - 10_000, - ).catch(() => undefined); - for (const handle of handles) { - await durableControlPlane - .waitForProcess(handle, 250) - .catch(() => undefined); - await within("exact warm fixture runner exit", handle.completion); - if (handle.processGroupId && !dead(-handle.processGroupId)) - process.kill(-handle.processGroupId, "SIGKILL"); - await vi.waitFor(() => { - expect(handle.child.pid && dead(handle.child.pid)).toBe(true); - expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( - true, - ); - }); - } - if (providerPid && !dead(-providerPid)) - process.kill(-providerPid, "SIGKILL"); - if (providerPid) - await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); - cleanupProven = true; - } catch (error) { - if (primaryError === undefined) throw error; - console.error( - "Warm ACK fixture cleanup unproven; retaining its private state directory.", - ); - } finally { - try { - for (const core of cores) await core.stop().catch(() => undefined); - } finally { - launchSpy.mockRestore(); - coreSpy.mockRestore(); - if (cleanupProven) - await rm(stateDirectory, { recursive: true, force: true }); - } - } - } - }, - 30_000, -); - -it.each([false, true])( - "retains warm attach authority when its result is lost before controller persistence (held observer=%s)", - async (holdObserver) => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-attach-result-loss-"), - ); - const callsPath = join(stateDirectory, "calls.log"); - const cores: DurablePrpControlPlane[] = []; - const handles: ReturnType[] = []; - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], - ) { - const core = new OriginalCore(options); - cores.push(core); - return core; - } as unknown as typeof OriginalCore); - const launch = durableControlPlane.spawnRunner; - const launchSpy = vi - .spyOn(durableControlPlane, "spawnRunner") - .mockImplementation((options) => { - const handle = launch(options); - handles.push(handle); - return handle; - }); - const within = async ( - label: string, - promise: Promise, - timeout = 5_000, - ) => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolve, reject) => { - timer = setTimeout( - () => reject(new Error(`${label} timeout`)), - timeout, - ); - }), - ]); - } finally { - if (timer !== undefined) clearTimeout(timer); - } - }; - const dead = (pid: number) => { - try { - process.kill(pid, 0); - return false; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH"; - } - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: - process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--call-log", - callsPath, - "--record-process-start", - ), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 2_000, - }); - let providerPid: number | null = null; - let cleanupProven = false; - let saveSpy: ReturnType | undefined; - let observerSpy: { mockRestore(): void } | undefined; - let attachment: Promise | undefined; - try { - await within( - "initial thread", - bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }), - ); - expect(cores).toHaveLength(1); - const core = cores[0]!; - const oldIdentity = structuredClone(core.store.state.identity); - if (holdObserver) { - const getCommand = core.getCommand.bind(core); - observerSpy = vi - .spyOn(core, "getCommand") - .mockImplementation((commandId) => { - const command = getCommand(commandId); - if ( - command?.type === "run.attach" && - core.store.state.completedWarmTransition?.command.commandId !== - commandId - ) { - return { ...command, status: "pending", result: null }; - } - return command; - }); - } - providerPid = bundle.evidence().codexPid; - const store = core.store as typeof core.store & { - commit(candidate: typeof core.store.state): void; - }; - const commit = store.commit.bind(store); - let lostResult = false; - let observeLoss!: () => void; - const loss = new Promise((resolveLoss) => { - observeLoss = resolveLoss; - }); - saveSpy = vi.spyOn(store, "commit").mockImplementation((candidate) => { - const attach = candidate.commands.find( - (entry) => entry.type === "run.attach", - ); - if (!lostResult && attach?.status === "completed") { - lostResult = true; - // The authenticated result reached the receiver, but the durable write - // did not. The clone has not been exposed in memory or on disk. - const persisted = JSON.parse(readFileSync(core.store.path, "utf8")); - expect(persisted.identity).toEqual(oldIdentity); - expect( - persisted.commands.find( - (entry: { type: string }) => entry.type === "run.attach", - ).status, - ).toBe("pending"); - core.disconnectActiveRunner(); - observeLoss(); - throw new Error( - "fixture lost attach result before durable controller commit", - ); - } - commit(candidate); - }); - attachment = bundle.transport.attachRun!({ - runId: "run-result-loss-next", - turnId: "turn-result-loss-next", - itemId: "item-result-loss-next", - }); - void attachment.catch(() => undefined); - await within("lost attach result", loss); - expect(lostResult).toBe(true); - expect(core.store.state.identity).toEqual(oldIdentity); - expect( - core.store.state.commands.find((entry) => entry.type === "run.attach") - ?.status, - ).toBe("pending"); - const outcome = await within( - "exact result replay after reconnect", - attachment.then( - () => ({ status: "completed" as const }), - (error: unknown) => ({ - status: "failed" as const, - message: String(error), - }), - ), - 10_000, - ); - const runner = JSON.parse( - await readFile( - join(stateDirectory, "runner/runner-state.json"), - "utf8", - ), - ); - const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); - expect(calls.filter((call) => call === "process-start")).toHaveLength(1); - expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); - expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); - expect( - { - outcome, - runnerRunId: runner.runId, - controllerRunId: core.store.state.identity.runId, - }, - "lost attach result must replay without leaving the two durable authorities split", - ).toEqual({ - outcome: { status: "completed" }, - runnerRunId: "run-result-loss-next", - controllerRunId: "run-result-loss-next", - }); - if (holdObserver) - expect( - core.store.state.completedWarmTransition?.receipt.newIdentity.runId, - ).toBe("run-result-loss-next"); - } finally { - saveSpy?.mockRestore(); - observerSpy?.mockRestore(); - try { - await within( - "result-loss fixture close", - bundle.transport.close(), - 5_000, - ).catch(() => undefined); - for (const handle of handles) { - await durableControlPlane - .waitForProcess(handle, 250) - .catch(() => undefined); - await within("exact result-loss runner exit", handle.completion); - if (handle.processGroupId && !dead(-handle.processGroupId)) - process.kill(-handle.processGroupId, "SIGKILL"); - await vi.waitFor(() => { - expect(handle.child.pid && dead(handle.child.pid)).toBe(true); - expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( - true, - ); - }); - } - if (providerPid && !dead(-providerPid)) - process.kill(-providerPid, "SIGKILL"); - if (providerPid) - await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); - cleanupProven = true; - } finally { - for (const core of cores) await core.stop().catch(() => undefined); - launchSpy.mockRestore(); - coreSpy.mockRestore(); - if (cleanupProven) - await rm(stateDirectory, { recursive: true, force: true }); - } - } - }, - 30_000, -); - -it.each([ - ...[ - "before-result", - "after-result", - "after-activation", - "before-confirmation", - "after-confirmation", - ].flatMap((lossPoint) => - [false, true].map((routed) => ({ - lossPoint, - routed, - recoveryFault: "none", - })), - ), - ...["endpoint", "missing-capability", "listen", "malformed-core"].map( - (recoveryFault) => ({ - lossPoint: "after-result", - routed: true, - recoveryFault, - }), - ), - ...["before_bootstrap", "before_spawn", "before_authentication"].map( - (stage) => ({ - lossPoint: "after-result", - routed: true, - recoveryFault: `authorize_${stage}`, - }), - ), - ...[ - "attach-wait", - ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER - ? ["attach-capability"] - : []), - ].map((recoveryFault) => ({ - lossPoint: "before-result", - routed: true, - recoveryFault, - })), - ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER - ? [ - { - lossPoint: "after-result", - routed: false, - recoveryFault: "legacy-parser", - }, - ] - : []), - ...["before-result", "after-result", "after-activation", "after-confirmation"].map(lossPoint => ({ - lossPoint, routed: true, recoveryFault: "none", executionPrepared: true, - })), - { - lossPoint: "after-result", - routed: true, - recoveryFault: "none", - externalArtifact: true, - }, - { - lossPoint: "after-confirmation", - routed: true, - recoveryFault: "none", - ordinaryFollowup: true, - }, - ...[ - "missing_snapshot", - "rejected_snapshot", - "wrong_thread", - "callback_failure", - "callback_async_failure", - ].map((ordinaryFollowup) => ({ - lossPoint: "after-confirmation", - routed: true, - recoveryFault: "none", - ordinaryFollowup, - })), -])( - "recovers a warm attachment with a fresh controller and runner ($lossPoint, routed=$routed, fault=$recoveryFault, followup=$ordinaryFollowup, external=$externalArtifact, prepared=$executionPrepared)", - async (testCase) => { - const { lossPoint, routed, recoveryFault } = testCase; - const externalArtifact = "externalArtifact" in testCase && testCase.externalArtifact; - const executionPrepared = "executionPrepared" in testCase && testCase.executionPrepared; - const configurations = new Map(); - const ordinaryFollowup = - "ordinaryFollowup" in testCase && testCase.ordinaryFollowup; - const snapshotFault = - typeof ordinaryFollowup === "string" && - !ordinaryFollowup.startsWith("callback_"); - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-attach-restart-"), - ); - const callsPath = join(stateDirectory, "calls.log"); - const cores: DurablePrpControlPlane[] = []; - const handles: ReturnType[] = []; - const routes = new Map< - string, - { core: DurablePrpControlPlane; generation: symbol } - >(); - const routeCalls: string[] = []; - let recovering = false; - let recoveryClaimCurrent = true; - let recoveryFenceActive = true; - const completionSnapshotIds: string[] = []; - let rejectHeldCompletion: ((error: Error) => void) | undefined; - let markHeldCompletionEntered!: () => void; - const heldCompletionEntered = new Promise((resolve) => { - markHeldCompletionEntered = resolve; - }); - let snapshotObserverSpy: ReturnType | undefined; - const routeServer = createServer((_request, response) => - response.writeHead(404).end(), - ); - routeServer.on("upgrade", (request, socket, head) => { - const entry = routes.get(request.url ?? ""); - if (!entry) { - socket.destroy(); - return; - } - entry.core.handleUpgrade(request, socket, request.url!, head); - }); - if (routed) - await new Promise((resolveListen) => - routeServer.listen(0, "127.0.0.1", resolveListen), - ); - const routeAddress = routeServer.address(); - const routePort = - routeAddress && typeof routeAddress === "object" ? routeAddress.port : 0; - const registration = async ( - core: DurablePrpControlPlane, - identity = core.store.state.identity, - ) => { - if ( - recovering && - ordinaryFollowup && - recoveryFenceActive && - !recoveryClaimCurrent - ) { - throw new Error("fixture old recovery claim is no longer current"); - } - const path = `/api/runner/v1/connect/${identity.runId}`; - const generation = Symbol(); - routes.set(path, { core, generation }); - routeCalls.push(identity.runId); - return { - connection: - recovering && recoveryFault === "listen" - ? { - mode: "listen" as const, - listenAddress: "0.0.0.0" as const, - listenPort: routePort, - listenPath: path, - } - : { - mode: "connect" as const, - connectUrl: `ws://127.0.0.1:${routePort}${path}${recovering && recoveryFault === "endpoint" ? "/changed" : ""}`, - }, - release: () => { - if (routes.get(path)?.generation === generation) routes.delete(path); - }, - }; - }; - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], - ) { - const core = new OriginalCore(options); - cores.push(core); - if (recovering && snapshotFault) { - const getCommand = core.getCommand.bind(core); - snapshotObserverSpy = vi - .spyOn(core, "getCommand") - .mockImplementation((id) => { - const command = getCommand(id); - if ( - command?.type !== "session.snapshot" || - command.status !== "completed" - ) - return command; - const observed = structuredClone(command); - if (ordinaryFollowup === "rejected_snapshot") { - observed.status = "failed"; - observed.result = { - result: { message: "fixture snapshot observation rejected" }, - }; - } else if (ordinaryFollowup === "wrong_thread") { - observed.result = { - ...observed.result, - result: { - ...(observed.result?.result as Record), - driverSessionId: "foreign-thread", - providerSessionId: "foreign-thread", - }, - }; - } else observed.result = { result: { status: "session_open" } }; - return observed; - }); - } - return core; - } as unknown as typeof OriginalCore); - const launch = durableControlPlane.spawnRunner; - const launchSpy = vi - .spyOn(durableControlPlane, "spawnRunner") - .mockImplementation((options) => { - const handle = launch({ ...options, - ...(executionPrepared ? { processLauncher: undefined } : {}), - ...(externalArtifact && recovering ? { runnerBinaryPath: defaultCapabilityRunnerdBinary() } : {}), - }); - handles.push(handle); - return handle; - }); - const dead = (pid: number) => { - try { - process.kill(pid, 0); - return false; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH"; - } - }; - const stopOwnedProvider = async (pid: number) => { - // Exact runner completion can race the OS reaping its already-signalled - // provider. Absence, not the outcome of a redundant signal, is required. - try { - await vi.waitFor(() => expect(dead(-pid)).toBe(true), { - timeout: 500, - interval: 10, - }); - } catch { - try { - process.kill(-pid, "SIGKILL"); - } catch (error) { - if ( - !["ESRCH", "EPERM"].includes( - String((error as NodeJS.ErrnoException).code), - ) - ) - throw error; - } - await vi.waitFor(() => expect(dead(-pid)).toBe(true), { - timeout: 2_000, - interval: 10, - }); - } - expect(dead(pid)).toBe(true); - }; - const fingerprintTree = async (root: string): Promise => { - const rows: unknown[] = []; - const visit = async (path: string, relative: string) => { - const metadata = await lstat(path); - rows.push({ - path: relative, - inode: metadata.ino, - mode: metadata.mode, - mtimeMs: metadata.mtimeMs, - digest: metadata.isFile() - ? createHash("sha256") - .update(await readFile(path)) - .digest("hex") - : metadata.isSymbolicLink() - ? createHash("sha256") - .update(await readlink(path)) - .digest("hex") - : null, - }); - if (metadata.isDirectory()) - for (const child of (await readdir(path)).sort()) - await visit(join(path, child), `${relative}/${child}`); - }; - await visit(root, "."); - return rows; - }; - const within = async (promise: Promise, timeout = 5_000) => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolve, reject) => { - timer = setTimeout( - () => reject(new Error("warm restart fixture timed out")), - timeout, - ); - }), - ]); - } finally { - if (timer) clearTimeout(timer); - } - }; - const options = { - runnerBinary: - recoveryFault === "attach-capability" - ? process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER! - : (process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - defaultCapabilityRunnerdBinary()), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--call-log", - callsPath, - "--record-process-start", - ), - stateDirectory, - lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 2_000, - prpIdentity: (() => { - const runId = randomUUID(); - return { - runnerInstanceId: randomUUID(), - environmentLeaseId: - process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_SCOPE === - "transient" - ? runId - : randomUUID(), - runId, - normalizedSessionId: randomUUID(), - turnId: `turn-${runId}`, - itemId: `item-${runId}`, - }; - })(), - ...(routed - ? { - controlPlaneRegistration: registration, - warmTransitionRegistrationMode: "routed_connect" as const, - } - : {}), - ...(executionPrepared ? { - runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), - readRunnerState: async () => JSON.parse(await readFile(join(stateDirectory, "runner/runner-state.json"), "utf8")), - // The fixture's spawn spy supplies the real local process owner. - runnerProcessLauncher: () => { throw new Error("fixture process owner was bypassed"); }, - prepareLaunch: async (requirements: RunnerProviderLaunchRequirements, recovery?: PreparedRunnerProviderLaunch) => { - const existing = configurations.get(requirements.identity.runId); - if (existing) { - expect(requirements).toEqual(existing.requirements); - if (recovering) expect(recovery).toEqual(existing.prepared); - return existing.prepared; - } - const prepared = { configurationId: `prepared-${requirements.identity.runId}`, - requirementsDigest: runnerLaunchRequirementsDigest(requirements), - provider: createRunnerProviderLaunch(requirements, { - options: { codexArgs: fakeCodexArgs(stateDirectory, "--call-log", callsPath, "--record-process-start") }, - root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), - codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "", - }), - }; - configurations.set(requirements.identity.runId, { requirements: structuredClone(requirements), prepared }); - return prepared; - }, - } : {}), - }; - const first = createCapabilityRunnerdCodexTransport(options); - let resumed: - ReturnType | undefined; - const providerPids = new Set(); - let commitSpy: ReturnType | undefined; - let commandObserverSpy: ReturnType | undefined; - let detached: Promise | undefined; - let cleanupProven = false; - const legacyProbeDirectories: string[] = []; - try { - const opened = await within( - first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }), - ); - const openedThread = opened.thread as { id: string; sessionId: string }; - const thread = { id: openedThread.id, sessionId: openedThread.sessionId }; - const firstThreadSnapshot = await within( - first.transport.request("thread/read", {}), - ); - const firstEvidence = first.evidence(); - const cleanRunnerStateBytes = await readFile( - join(stateDirectory, "runner/runner-state.json"), - ); - const runnerProcessStartedAt = - process.platform === "darwin" - ? new Date( - execFileSync( - "ps", - ["-o", "lstart=", "-p", String(handles[0]!.child.pid)], - { encoding: "utf8", timeout: 1_500 }, - ).trim(), - ).toISOString() - : null; - if ((first.evidence().codexPid ?? 0) > 0) - providerPids.add(first.evidence().codexPid!); - const core = cores[0]!; - const oldIdentity = structuredClone(core.store.state.identity); - const nextRunId = randomUUID(); - const desired = { - ...oldIdentity, - runId: nextRunId, - turnId: `turn-${nextRunId}`, - itemId: `item-${nextRunId}`, - }; - if ( - recoveryFault === "attach-capability" || - recoveryFault === "attach-wait" - ) { - if (recoveryFault === "attach-wait") { - const getCommand = core.getCommand.bind(core); - commandObserverSpy = vi - .spyOn(core, "getCommand") - .mockImplementation((id) => { - if ( - core.store.state.commands.find( - (entry) => entry.commandId === id, - )?.type === "run.attach" - ) { - throw new Error("fixture result observer unavailable"); - } - return getCommand(id); - }); - } - const bootstrapCount = core.store.state.freshBootstraps; - await expect( - first.transport.attachRun!({ - runId: desired.runId, - turnId: desired.turnId, - itemId: desired.itemId, - }), - ).rejects.toThrow( - recoveryFault === "attach-capability" - ? "capability is required" - : "result observer unavailable", - ); - expect(routeCalls).toEqual([oldIdentity.runId, desired.runId]); - expect([...routes.keys()]).toEqual([ - `/api/runner/v1/connect/${oldIdentity.runId}`, - ]); - expect(core.store.state.freshBootstraps).toBe(bootstrapCount); - expect(handles).toHaveLength(1); - if (recoveryFault === "attach-capability") - expect( - core.store.state.commands.some( - (entry) => entry.type === "run.attach", - ), - ).toBe(false); - return; - } - const commit = core.store.commit.bind(core.store); - let lossObserved = false; - let signalLoss!: () => void; - const loss = new Promise((resolveLoss) => { - signalLoss = resolveLoss; - }); - commitSpy = vi - .spyOn(core.store, "commit") - .mockImplementation((candidate) => { - // The selected process-crash boundary stays unavailable until the - // exact owned runner is joined. A later replay must not silently - // settle this fixture before its retained pair is exported. - if (lossObserved) - throw new Error( - "fixture controller persistence unavailable after loss", - ); - const phase = candidate.warmTransition?.phase; - const target = - lossPoint === "after-activation" ? "activated" : "prepared"; - const atBoundary = lossPoint.includes("confirmation") - ? candidate.completedWarmTransition !== undefined && - candidate.warmTransition === undefined - : phase === target; - if (!lossObserved && atBoundary) { - lossObserved = true; - if ( - lossPoint !== "before-result" && - lossPoint !== "before-confirmation" - ) - commit(candidate); - // A dead controller cannot accept a reconnect through the second - // route installed for the in-flight authority rotation. Detaching - // only the current route leaves that overlapping fixture listener - // alive; completed-receipt replay needs no further store commit. - for (const [path, entry] of routes) { - if (entry.core === core) routes.delete(path); - } - core.disconnectActiveRunner(); - detached = first.transport.detachControllerForRestart!(); - signalLoss(); - // A committed confirmation is still before its ACK. Returning from - // this hook lets the current frame handler send that ACK even after - // close begins, collapsing the intended crash window on fast peers. - throw new Error("fixture interrupted exact result commit"); - } - commit(candidate); - }); - const attachment = first.transport.attachRun!({ - runId: desired.runId, - turnId: desired.turnId, - itemId: desired.itemId, - }); - void attachment.catch(() => undefined); - await within(loss); - await within(detached!); - expect(lossObserved).toBe(true); - const runnerPath = join(stateDirectory, "runner/runner-state.json"); - const runner = JSON.parse(await readFile(runnerPath, "utf8")); - expect(runner.warmTransition.phase).toBe( - lossPoint === "after-activation" || lossPoint.includes("confirmation") - ? "activating" - : "prepared", - ); - const persisted = JSON.parse(await readFile(core.store.path, "utf8")); - expect( - persisted.commands.find( - (entry: { type: string }) => entry.type === "run.attach", - )?.status, - ).toBe( - lossPoint === "before-result" - ? "pending" - : lossPoint === "after-result" - ? "completed" - : undefined, - ); - // Stop only this fixture's exact owned process handles. No stored receipt - // or PID absence is used as authority to terminate an unknown owner. - await durableControlPlane - .waitForProcess(handles[0]!, 100) - .catch(() => undefined); - await within(handles[0]!.completion); - for (const pid of providerPids) { - await stopOwnedProvider(pid); - } - await within(attachment.catch(() => undefined)); - const joinedRunner = JSON.parse(await readFile(runnerPath, "utf8")); - const joinedCore = JSON.parse(await readFile(core.store.path, "utf8")); - expect(joinedRunner.warmTransition).toEqual(runner.warmTransition); - expect(joinedCore.schema).toBe(persisted.schema); - expect(joinedCore.warmTransition).toEqual(persisted.warmTransition); - expect(joinedCore.completedWarmTransition).toEqual( - persisted.completedWarmTransition, - ); - commitSpy.mockRestore(); - const fixtureOutput = - process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_DIRECTORY; - if (fixtureOutput && recoveryFault === "none") { - const retainedArtifact = join(fixtureOutput, "paperclip-runnerd"); - await cp(options.runnerBinary, retainedArtifact, { force: false }); - expect( - createHash("sha256") - .update(await readFile(retainedArtifact)) - .digest("hex"), - ).toBe( - createHash("sha256") - .update(await readFile(options.runnerBinary)) - .digest("hex"), - ); - const retained = join( - fixtureOutput, - `${lossPoint}-${routed ? "routed" : "local"}`, - ); - await cp(stateDirectory, retained, { - recursive: true, - errorOnExist: true, - force: false, - }); - await writeFile( - join(retained, "transition-fixture-metadata.json"), - JSON.stringify( - { - schema: "paperclip.test.warm-transition-fixture.v1", - oldIdentity, - newIdentity: desired, - lossPoint, - routed, - runner: { - pid: handles[0]!.child.pid, - processGroupId: handles[0]!.processGroupId, - startedAt: runnerProcessStartedAt, - spawnObservedAt: handles[0]!.startedAt, - completion: await handles[0]!.completion, - processAbsent: dead(handles[0]!.child.pid!), - groupAbsent: dead(-handles[0]!.processGroupId!), - }, - providers: [...providerPids].map((pid) => ({ - pid, - processGroupId: pid, - startedAt: firstEvidence.providerProcessStartedAt, - processAbsent: dead(pid), - groupAbsent: dead(-pid), - })), - artifact: { - path: retainedArtifact, - version: runner.warmTransition.receipt.runnerVersion, - digest: runner.warmTransition.receipt.runnerDigest, - }, - thread, - firstThreadSnapshot, - firstEvidence, - }, - null, - 2, - ), - { mode: 0o600 }, - ); - } - routes.clear(); - const routeCallCount = routeCalls.length; - const beforeCalls = (await readFile(callsPath, "utf8")) - .trim() - .split(/\r?\n/); - expect( - beforeCalls.filter((call) => call === "process-start"), - ).toHaveLength(1); - if (recoveryFault === "legacy-parser") { - const legacyBinary = - process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER!; - const legacyDigest = `sha256:${createHash("sha256") - .update(await readFile(legacyBinary)) - .digest("hex")}`; - for (const [mode, bytes] of [ - ["pending", await readFile(runnerPath)], - ["clean", cleanRunnerStateBytes], - ] as const) { - const probe = await mkdtemp( - join(tmpdir(), "runnerd-legacy-schema-probe-"), - ); - legacyProbeDirectories.push(probe); - await writeFile(join(probe, "runner-state.json"), bytes, { - mode: 0o600, - }); - const handle = durableControlPlane.spawnRunner({ - connection: runner.warmTransition.receipt.connection, - stateDirectory: probe, - identity: oldIdentity, - runnerBinaryPath: legacyBinary, - runnerVersion: "0.3.0", - runnerDigest: legacyDigest, - ticket: "bootstrap_legacy_parser_probe", - maxOutboxBytes: runner.maxOutboxBytes, - p0ReserveBytes: runner.p0ReserveBytes, - maxRuntimeMs: 200, - reconnectGraceMs: 200, - }); - const result = await within(handle.completion, 5_000); - expect(result.code).not.toBe(0); - if (mode === "pending") { - expect(result.stderr).toContain( - "durable state binding does not match", - ); - expect(await readFile(join(probe, "runner-state.json"))).toEqual( - bytes, - ); - } else { - expect(result.stderr).not.toContain( - "durable state binding does not match", - ); - expect( - JSON.parse( - await readFile(join(probe, "runner-state.json"), "utf8"), - ).nextSourceSeq, - ).toBeGreaterThan(JSON.parse(bytes.toString("utf8")).nextSourceSeq); - } - } - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual(beforeCalls); - return; - } - recovering = true; - if (recoveryFault === "malformed-core") - await writeFile(core.store.path, "{", { mode: 0o600 }); - const beforeRecoveryBytes = await Promise.all( - [core.store.path, runnerPath].map((path) => readFile(path)), - ); - const beforeRecoveryTree = - recoveryFault === "none" ? null : await fingerprintTree(stateDirectory); - const authorizationStages: string[] = []; - let releaseHeldAuthorization: (() => void) | undefined; - let markHeldAuthorizationEntered!: () => void; - const heldAuthorizationEntered = new Promise((resolve) => { - markHeldAuthorizationEntered = resolve; - }); - const authorizationFault = recoveryFault.startsWith("authorize_"); - const failedAuthorizationStage = recoveryFault.slice("authorize_".length); - const beforeProviderBytes = await readFile( - join(stateDirectory, "runner", "codex-provider-state.json"), - ); - resumed = createCapabilityRunnerdCodexTransport({ - ...options, - ...(externalArtifact ? { - runnerBinary: "/controller-missing-runner", - runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), - } : {}), - ...(ordinaryFollowup - ? { - authorizeWarmTransitionRecovery: async () => { - if (!recoveryClaimCurrent) - throw new Error( - "fixture old recovery claim is no longer current", - ); - }, - onWarmTransitionRecoveryCompleted: (completion: { - transitionId: string; - }) => { - expect(completion.transitionId).toBe( - runner.warmTransition.receipt.transitionId, - ); - const completedSnapshot = cores[1]!.store.state.commands - .filter((command) => command.type === "session.snapshot") - .at(-1)!; - expect(completedSnapshot.status).toBe("completed"); - expect(cores[1]!.store.state.identity).toEqual(desired); - expect(cores[1]!.store.state.warmTransition).toBeUndefined(); - completionSnapshotIds.push(completedSnapshot.commandId); - if ( - ordinaryFollowup === "callback_failure" && - completionSnapshotIds.length === 1 - ) { - throw new Error( - "fixture recovery completion callback failed", - ); - } - if ( - ordinaryFollowup === "callback_async_failure" && - completionSnapshotIds.length === 1 - ) { - return new Promise((_resolve, reject) => { - rejectHeldCompletion = reject; - markHeldCompletionEntered(); - }); - } - recoveryFenceActive = false; - }, - } - : {}), - ...(authorizationFault - ? { - authorizeWarmTransitionRecovery: async (stage: string) => { - authorizationStages.push(stage); - if ( - stage === "before_bootstrap" && - failedAuthorizationStage === "before_bootstrap" - ) { - await new Promise((resolveGate) => { - releaseHeldAuthorization = resolveGate; - markHeldAuthorizationEntered(); - }); - } - if (stage === failedAuthorizationStage) - throw new Error("fixture recovery authority revoked"); - }, - } - : {}), - ...(recoveryFault === "missing-capability" - ? { warmTransitionRegistrationMode: undefined } - : {}), - ...(recoveryFault !== "none" - ? { - environment: { - CODEX_API_KEY: "synthetic-refused-route-credential", - }, - } - : {}), - prpIdentity: desired, - resumeProviderSession: { - driverSessionId: thread.id, - providerSessionId: thread.sessionId, - }, - }); - if (recoveryFault !== "none") { - const recoveryRequest = within( - resumed.transport.request("thread/read", {}), - ); - void recoveryRequest.catch(() => undefined); - if ( - authorizationFault && - failedAuthorizationStage === "before_bootstrap" - ) { - await within(Promise.race([ - heldAuthorizationEntered, - recoveryRequest.then(() => { - throw new Error("warm restart authorization callback was not entered"); - }), - ])); - expect(releaseHeldAuthorization).toBeTypeOf("function"); - const queuedBefore = cores[1]!.store.state.commands.map( - (command) => command.commandId, - ); - try { - await expect( - resumed.transport.request("turn/start", { - input: [{ text: "must not queue before bootstrap" }], - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.attachRun!({ - runId: "must-not-attach", - turnId: "must-not-attach", - itemId: "must-not-attach", - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.resolveRuntimeRequest!({ - requestId: "must-not-resolve", - turnId: desired.turnId, - resolution: { action: "cancel" }, - }), - ).rejects.toThrow("warm_transition_completion_pending"); - expect( - cores[1]!.store.state.commands.map( - (command) => command.commandId, - ), - ).toEqual(queuedBefore); - expect(handles).toHaveLength(1); - } finally { - releaseHeldAuthorization!(); - } - } - const failedRecovery = expect(recoveryRequest).rejects; - if (authorizationFault) { - await failedRecovery.toThrow( - failedAuthorizationStage === "before_authentication" - ? "native_runner_warm_transition_recovery_pending" - : "fixture recovery authority revoked", - ); - const expectedStages = [ - "before_bootstrap", - "before_spawn", - "before_authentication", - ]; - expect([...new Set(authorizationStages)]).toEqual( - expectedStages.slice( - 0, - expectedStages.indexOf(failedAuthorizationStage) + 1, - ), - ); - expect(handles).toHaveLength( - failedAuthorizationStage === "before_authentication" ? 2 : 1, - ); - expect( - await readFile( - join(stateDirectory, "runner", "codex-provider-state.json"), - ), - ).toEqual(beforeProviderBytes); - const refusedRunner = JSON.parse(await readFile(runnerPath, "utf8")); - expect(refusedRunner.warmTransition).toEqual(runner.warmTransition); - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual(beforeCalls); - if (failedAuthorizationStage !== "before_authentication") - expect(await readFile(runnerPath)).toEqual(beforeRecoveryBytes[1]); - if (failedAuthorizationStage === "before_bootstrap") - expect(await readFile(core.store.path)).toEqual( - beforeRecoveryBytes[0], - ); - await within(resumed.transport.close()).catch(() => undefined); - expect([...routes.keys()]).toEqual([]); - return; - } - if (recoveryFault === "malformed-core") await failedRecovery.toThrow(); - else - await failedRecovery.toThrow( - recoveryFault === "missing-capability" - ? "requires_exact_owned_endpoint" - : "registered_endpoint_mismatch", - ); - expect(handles).toHaveLength(1); - expect( - await Promise.all( - [core.store.path, runnerPath].map((path) => readFile(path)), - ), - ).toEqual(beforeRecoveryBytes); - expect(await fingerprintTree(stateDirectory)).toEqual( - beforeRecoveryTree, - ); - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual(beforeCalls); - expect([...routes.keys()]).toEqual([]); - return; - } - if ( - snapshotFault || - ordinaryFollowup === "callback_failure" || - ordinaryFollowup === "callback_async_failure" - ) { - const firstRead = within(resumed.transport.request("thread/read", {})); - void firstRead.catch(() => undefined); - if (ordinaryFollowup === "callback_async_failure") { - await within(Promise.race([ - heldCompletionEntered, - firstRead.then(() => { - throw new Error("warm restart completion callback was not entered"); - }), - ])); - expect(rejectHeldCompletion).toBeTypeOf("function"); - try { - expect(recoveryFenceActive).toBe(true); - const queuedBefore = cores[1]!.store.state.commands.map( - (command) => command.commandId, - ); - await expect( - resumed.transport.request("turn/start", { - input: [{ text: "must not pass held completion" }], - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.request("thread/resume", {}), - ).rejects.toThrow("warm_transition_completion_pending"); - expect( - cores[1]!.store.state.commands.map( - (command) => command.commandId, - ), - ).toEqual(queuedBefore); - } finally { - rejectHeldCompletion!( - new Error("fixture recovery completion callback failed"), - ); - } - } - await expect(firstRead).rejects.toThrow( - snapshotFault - ? ordinaryFollowup === "rejected_snapshot" - ? "snapshot observation rejected" - : "completion_unproven" - : "recovery completion callback failed", - ); - expect(recoveryFenceActive).toBe(true); - expect(completionSnapshotIds).toHaveLength(snapshotFault ? 0 : 1); - const beforeDeniedWork = cores[1]!.store.state.commands.map( - (command) => command.commandId, - ); - const beforeDeniedProviderCalls = await readFile(callsPath, "utf8"); - await expect( - resumed.transport.request("turn/start", { - input: [{ text: "must remain fenced" }], - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.request("thread/resume", {}), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.attachRun!({ - runId: "must-not-attach", - turnId: "must-not-attach", - itemId: "must-not-attach", - }), - ).rejects.toThrow("warm_transition_completion_pending"); - expect( - cores[1]!.store.state.commands.map((command) => command.commandId), - ).toEqual(beforeDeniedWork); - expect(await readFile(callsPath, "utf8")).toBe( - beforeDeniedProviderCalls, - ); - snapshotObserverSpy?.mockRestore(); - const providerCallsBeforeRetry = await readFile(callsPath, "utf8"); - expect( - (await within(resumed.transport.request("thread/read", {}))).thread, - ).toMatchObject(thread); - expect(recoveryFenceActive).toBe(false); - expect(completionSnapshotIds).toHaveLength(snapshotFault ? 1 : 2); - expect(new Set(completionSnapshotIds).size).toBe( - completionSnapshotIds.length, - ); - expect(await readFile(callsPath, "utf8")).toBe( - providerCallsBeforeRetry, - ); - return; - } - const read = await within( - resumed.transport.request("thread/read", {}), - 10_000, - ); - if ((resumed.evidence().codexPid ?? 0) > 0) - providerPids.add(resumed.evidence().codexPid!); - expect(read.thread).toMatchObject(thread); - expect(cores).toHaveLength(2); - expect(cores[1]!.store.state.identity).toEqual(desired); - expect(cores[1]!.store.state.warmTransition).toBeUndefined(); - expect( - cores[1]!.store.state.completedWarmTransition?.receipt.transitionId, - ).toBe(runner.warmTransition.receipt.transitionId); - if (routed) { - expect(routeCalls.slice(routeCallCount)).toEqual([ - oldIdentity.runId, - desired.runId, - ]); - expect([...routes.keys()]).toEqual([ - `/api/runner/v1/connect/${desired.runId}`, - ]); - } - const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); - expect(calls.filter((call) => call === "process-start")).toHaveLength(2); - expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); - expect(calls.filter((call) => call === "thread/resume")).toHaveLength(1); - expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); - if (ordinaryFollowup) { - recoveryClaimCurrent = false; - const resumedCore = cores[1]!; - resumedCore.disconnectActiveRunner(); - await vi.waitFor( - () => expect(resumedCore.activeRunnerConnectionCount()).toBe(1), - { timeout: 2_000 }, - ); - expect(recoveryFenceActive).toBe(false); - expect( - (await within(resumed.transport.request("thread/read", {}))).thread, - ).toMatchObject(thread); - const thirdRunId = randomUUID(); - await within( - resumed.transport.attachRun!({ - runId: thirdRunId, - turnId: `turn-${thirdRunId}`, - itemId: `item-${thirdRunId}`, - }), - ); - expect(resumedCore.store.state.identity.runId).toBe(thirdRunId); - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual([...calls, "thread/goal/get"]); - } - } finally { - snapshotObserverSpy?.mockRestore(); - commitSpy?.mockRestore(); - commandObserverSpy?.mockRestore(); - try { - await within(resumed?.transport.close() ?? Promise.resolve()).catch( - () => undefined, - ); - await within(first.transport.detachControllerForRestart!()).catch( - () => undefined, - ); - for (const handle of handles) { - await durableControlPlane - .waitForProcess(handle, 250) - .catch(() => undefined); - await within(handle.completion); - if (handle.processGroupId && !dead(-handle.processGroupId)) - process.kill(-handle.processGroupId, "SIGKILL"); - await vi.waitFor(() => { - expect(handle.child.pid && dead(handle.child.pid)).toBe(true); - expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( - true, - ); - }); - } - for (const pid of providerPids) { - await stopOwnedProvider(pid); - } - cleanupProven = true; - } finally { - for (const core of cores) await core.stop().catch(() => undefined); - if (routed) - await new Promise((resolveClose) => - routeServer.close(() => resolveClose()), - ); - coreSpy.mockRestore(); - launchSpy.mockRestore(); - if (cleanupProven) - await rm(stateDirectory, { recursive: true, force: true }); - if (cleanupProven) - for (const directory of legacyProbeDirectories) - await rm(directory, { recursive: true, force: true }); - } - } - }, - 30_000, -); - -it("rotates PRP authority in place for a warm cross-run attachment", async () => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-")); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - const within = async (label: string, promise: Promise) => - await Promise.race([ - promise, - new Promise((_, reject) => - setTimeout(() => reject(new Error(`${label} timeout`)), 5_000), - ), - ]); - try { - await within("initialize", bundle.transport.request("initialize", {})); - await within( - "thread start", - bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [ - { - name: "get_task_context", - description: "Read the active task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, - }, - }, - ], - completionContract: { - revision: "sha256:warm-three-turn-contract", - criterionIds: ["objective"], - }, - }), - ); - const runnerPid = bundle.evidence().runnerPid; - const providerPid = bundle.evidence().codexPid; - const notifications = bundle.transport - .notifications() - [Symbol.asyncIterator](); - const waitForCompletion = async (label: string) => { - const deadline = Date.now() + 5_000; - while (Date.now() < deadline) { - const next = await Promise.race([ - notifications.next(), - new Promise((_, reject) => - setTimeout( - () => reject(new Error(`${label} notification timeout`)), - 1_000, - ), - ), - ]); - if (next.value?.method === "turn/completed") return; - } - throw new Error(`${label} completion timeout`); - }; - await within( - "first turn start", - bundle.transport.request("turn/start", { - input: [{ type: "text", text: "first run" }], - }), - ); - await waitForCompletion("first run"); - - await within( - "warm attach", - bundle.transport.attachRun!({ - runId: "run-warm-second", - turnId: "turn-warm-second", - itemId: "item-warm-second", - }), - ); - await within( - "second turn start", - bundle.transport.request("turn/start", { - input: [{ type: "text", text: "second run" }], - }), - ); - await waitForCompletion("second run"); - - await within( - "second warm attach", - bundle.transport.attachRun!({ - runId: "run-warm-third", - turnId: "turn-warm-third", - itemId: "item-warm-third", - }), - ); - await within( - "third turn start", - bundle.transport.request("turn/start", { - input: [{ type: "text", text: "third run" }], - }), - ); - await waitForCompletion("third run"); - - expect(bundle.evidence()).toMatchObject({ - runnerPid, - codexPid: providerPid, - runnerExited: false, - }); - } finally { - await bundle.transport.close(); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it("recovers an active second warm business run after deleting its controller context assets", async () => { - const root = await mkdtemp(join(tmpdir(), "runnerd-warm-second-active-")); - const firstRoot = join(root, "first-controller"), nextRoot = join(root, "next-controller"); - const assetRoot = join(root, "retired-controller-assets"); - const skillRoot = join(assetRoot, "skill"), instructionRoot = join(assetRoot, "instructions"); - await mkdir(skillRoot, { recursive: true }); await mkdir(instructionRoot, { recursive: true }); - await writeFile(join(skillRoot, "SKILL.md"), "# Pinned skill\n"); - await writeFile(join(instructionRoot, "AGENTS.md"), "Pinned instructions\n"); - const server = createServer(); - let authority: DurablePrpControlPlane | null = null, generation = 0; - let checkpoint: durableControlPlane.DurablePrpCheckpoint | undefined; - const registered = new WeakSet(); - server.on("upgrade", (request, socket, head) => authority - ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy()); - await new Promise(done => server.listen(0, "127.0.0.1", done)); - const address = server.address(); - if (!address || typeof address === "string") throw new Error("Missing owned Runner route"); - const registration = async (core: DurablePrpControlPlane) => { - if (!registered.has(core)) { - core.setCheckpointPersistence(async value => { checkpoint = value; }); - registered.add(core); - } - authority = core; const current = ++generation; - return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: async () => { if (generation === current) authority = null; } }; - }; - const initialIdentity = { runnerInstanceId: "runner-warm-second-active", environmentLeaseId: "lease-warm-second-active", - runId: "run-warm-first-healthy", normalizedSessionId: "session-warm-second-active", turnId: "turn-first", itemId: "item-first" }; - const options = { runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(root, "--emit-tool-call", "--durable-tool-ids"), stateDirectory: firstRoot, - runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), - prpIdentity: initialIdentity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, controlPlaneRegistration: registration }; - const first = createCapabilityRunnerdCodexTransport(options); - const within = async (label: string, promise: Promise) => { - let timer: ReturnType | undefined; - try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`warm active recovery stage timed out: ${label}`)), 5_000); })]); } - finally { clearTimeout(timer); } - }; - const waitComplete = async (transport: typeof first.transport) => { - for await (const notification of transport.notifications()) if (notification.method === "turn/completed") return; - }; - let calls = 0, runnerPid: number | null = null, providerPid: number | null = null; - let recovered: ReturnType | undefined; - const result = { ok: true, task: { id: "task-1" } }; - first.transport.setServerRequestHandler(async () => ++calls === 1 - ? { success: true, contentItems: [{ type: "inputText", text: JSON.stringify(result) }] } - : new Promise>(() => undefined)); - try { - const opened = await within("first thread/start", first.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [{ name: "get_task_context", description: "Read current task", - inputSchema: { type: "object", properties: {}, additionalProperties: false } }] })); - runnerPid = first.evidence().runnerPid; - providerPid = first.evidence().codexPid; - expect(providerPid).toEqual(expect.any(Number)); - const providerStart = execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim(); - const startedAt = first.transport.processInfo().startedAt; - const firstCompleted = waitComplete(first.transport); - await within("first turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "first healthy business run" }] })); - await within("healthy first turn completion", firstCompleted); - const attachment = { runId: "run-warm-second-active", turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; - await within("warm second business attachment", first.transport.attachRun!(attachment)); - const active = await within("second active turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "second business run held at tool" }] })); - await vi.waitFor(() => expect(calls).toBe(2), { timeout: 5_000 }); - const activeCore = authority!; - const savedIdentity = { ...activeCore.store.state.identity }; - expect(savedIdentity).toEqual({ ...initialIdentity, ...attachment }); - await activeCore.flushCheckpoint(); - await within("controller detach", first.detachControllerForRestart()); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - await rm(assetRoot, { recursive: true, force: true }); - expect(existsSync(assetRoot)).toBe(false); - checkpoint!.restoreSessionDirectory(nextRoot); - const noSpawn = vi.fn(() => { throw new Error("Duplicate Runner launch"); }); - const openedThread = opened.thread as Record; - const activeTurn = active.turn as Record; - expect(openedThread.id).toEqual(expect.any(String)); - expect(activeTurn.id).toEqual(expect.any(String)); - const shared = { ...options, stateDirectory: nextRoot, - resumeProviderSession: { driverSessionId: String(openedThread.id), providerSessionId: String(openedThread.sessionId) }, - resumeActiveTurnId: String(activeTurn.id), runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), - runnerStateDirectory: join(firstRoot, "runner"), readRunnerState: async () => JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8")), - prepareExternalRunnerState: async () => {}, archiveExternalRunnerState: async () => { throw new Error("Active authority must not rotate"); }, - runnerProcessLauncher: noSpawn, adoptExistingRunner: { pid: runnerPid!, processGroupId: runnerPid, startedAt, - isAlive: () => { try { process.kill(runnerPid!, 0); return true; } catch { return false; } }, - signal: (signal: NodeJS.Signals) => { try { process.kill(runnerPid!, signal); return true; } catch { return false; } } } }; - // The old server's synthetic pair must continue failing the strict Runner guard. - const wrong = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: { ...savedIdentity, - turnId: `turn-${attachment.runId}`, itemId: `item-${attachment.runId}` } }); - await expect(wrong.transport.request("thread/read", {})).rejects.toThrow("PRP recovery identity does not match the durable session binding"); - await wrong.detachControllerForRestart(); - // Leave the pending tool outcome unknown. Reattachment must authenticate - // and preserve it without fabricating a receipt or redispatching its handler. - recovered = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: savedIdentity }); - await within("replacement thread/read", recovered.transport.request("thread/read", {})); - expect(recovered.evidence()).toMatchObject({ runnerPid, runnerExited: false }); - expect(authority!.activeRunnerConnectionCount()).toBe(1); - expect(() => process.kill(providerPid!, 0)).not.toThrow(); - expect(execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim()).toBe(providerStart); - expect(calls).toBe(2); - expect(authority!.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(0); - expect(recovered.transport.processInfo().startedAt).toBe(startedAt); - expect(authority!.store.state.identity).toEqual(savedIdentity); - expect(authority!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); - expect(noSpawn).not.toHaveBeenCalled(); - } catch (error) { - const saved = JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8").catch(() => "{}")); - const stderr = await readFile(join(firstRoot, "diagnostics/runnerd.stderr.log"), "utf8").catch(() => ""); - throw new Error(`${String(error)}; owned protocol evidence ${JSON.stringify({ - transport: recovered?.evidence(), connections: authority?.activeRunnerConnectionCount(), - identity: authority?.store.state.identity, commands: authority?.store.state.commands.map(c => ({type:c.type,status:c.status})), - events: authority?.store.state.committedEvents.map(e => e.eventType).slice(-8), - runner: { lifecycle:saved.lifecycle, runId:saved.runId, turnId:saved.turnId, itemId:saved.itemId, - outbox:saved.outbox?.length }, stderr:stderr.slice(-1500), - })}`); - } finally { - await (recovered ? within("recovered cleanup", recovered.transport.close()).catch(() => undefined) : undefined); - await within("first cleanup", first.transport.close()).catch(() => undefined); - if (providerPid) { try { process.kill(providerPid, "SIGKILL"); } catch {} } - await (authority as DurablePrpControlPlane | null)?.stop(); - if (runnerPid) { try { process.kill(-runnerPid, "SIGKILL"); } catch {} } - server.closeAllConnections(); - if (server.listening) await new Promise(done => server.close(() => done())); - for (const controller of [firstRoot, nextRoot]) await releaseMaterializedNativeRuntimeSkills(join(controller, "codex-home", "skills")); - await rm(root, { recursive: true, force: true }); - } -}, 30_000); - -it("waits for a warm runner to re-authenticate before probing attachment readiness", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-warm-reattach-before-probe-"), - ); - const server = createServer(); - const authorities = new Map(); - let blockFirstAuthorityReconnect = false; - let resolveRejectedReconnect!: () => void; - const rejectedReconnect = new Promise((resolvePromise) => { - resolveRejectedReconnect = resolvePromise; - }); - server.on("upgrade", (request, socket, head) => { - const route = request.url ?? ""; - if (route === "/runner-1" && blockFirstAuthorityReconnect) { - resolveRejectedReconnect(); - socket.destroy(); - return; - } - const authority = authorities.get(route); - if (!authority) { - socket.destroy(); - return; - } - authority.handleUpgrade(request, socket, route, head); - }); - await new Promise((resolveListen) => - server.listen(0, "127.0.0.1", resolveListen), - ); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("Expected warm reconnect test listener"); - } - let registrationCount = 0; - const diagnostics: string[] = []; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 5_000, - onDiagnostic: (message) => diagnostics.push(message), - controlPlaneRegistration: async (authority) => { - registrationCount += 1; - const route = `/runner-${registrationCount}`; - authorities.set(route, authority); - return { - connectUrl: `ws://127.0.0.1:${address.port}${route}`, - release: () => { - if (authorities.get(route) === authority) authorities.delete(route); - }, - }; - }, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let runnerPid: number | null = null; - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: codexSemanticToolSpecs(), - }); - runnerPid = bundle.evidence().runnerPid; - const firstAuthority = authorities.get("/runner-1"); - if (!firstAuthority) throw new Error("Missing first warm authority"); - const priorSnapshotCount = firstAuthority.store.state.commands.filter( - (command) => command.type === "session.snapshot", - ).length; - - blockFirstAuthorityReconnect = true; - firstAuthority.disconnectActiveRunner(); - const attachment = bundle.transport.attachRun!({ - runId: "run-warm-after-reconnect", - turnId: "turn-warm-after-reconnect", - itemId: "item-warm-after-reconnect", - }); - await Promise.race([ - rejectedReconnect, - new Promise((_, reject) => - setTimeout( - () => reject(new Error("runner did not attempt to reconnect")), - 5_000, - ), - ), - ]); - - // No command may be queued while its sole authenticated consumer is - // absent. The generic 30-second command timeout used to turn this state - // into same-run recovery and replace the healthy warm runner process. - expect( - firstAuthority.store.state.commands.filter( - (command) => command.type === "session.snapshot", - ), - ).toHaveLength(priorSnapshotCount); - - blockFirstAuthorityReconnect = false; - await Promise.race([ - attachment, - new Promise((_, reject) => - setTimeout(() => reject(new Error("warm attachment timeout")), 10_000), - ), - ]); - expect(bundle.evidence()).toMatchObject({ - runnerPid, - runnerExited: false, - }); - expect(diagnostics).toContain( - "warm runner connection interrupted; waiting for re-authentication before authority rotation", - ); - expect(diagnostics).toContain( - "warm runner re-authenticated before authority rotation", - ); - } finally { - await bundle.transport.close().catch(() => undefined); - if (runnerPid) { - try { - process.kill(-runnerPid, "SIGKILL"); - } catch { - // A successful durable close already stopped the runner process group. - } - } - server.closeAllConnections(); - if (server.listening) { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); - } - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it("releases both PRP authorities when warm rotation activation fails", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-warm-attach-activation-failure-"), - ); - const server = createServer(); - const authorities = new Map(); - const released: string[] = []; - server.on("upgrade", (request, socket, head) => { - const route = request.url ?? ""; - const authority = authorities.get(route); - if (!authority) { - socket.destroy(); - return; - } - authority.handleUpgrade(request, socket, route, head); - }); - await new Promise((resolveListen) => - server.listen(0, "127.0.0.1", resolveListen), - ); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("Expected warm activation failure test listener"); - } - let registrationCount = 0; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - controlPlaneRegistration: async (authority) => { - registrationCount += 1; - const route = `/runner-${registrationCount}`; - authorities.set(route, authority); - return { - connectUrl: `ws://127.0.0.1:${address.port}${route}`, - ...(registrationCount === 1 - ? {} - : { - activate: () => { - throw new Error("rotation activation failed"); - }, - }), - release: () => { - released.push(route); - if (authorities.get(route) === authority) authorities.delete(route); - }, - }; - }, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let runnerPid: number | null = null; - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: codexSemanticToolSpecs(), - }); - runnerPid = bundle.evidence().runnerPid; - - await expect( - bundle.transport.attachRun!({ - runId: "run-warm-activation-failure", - turnId: "turn-warm-activation-failure", - itemId: "item-warm-activation-failure", - }), - ).rejects.toThrow("rotation activation failed"); - expect(new Set(released)).toEqual(new Set(["/runner-1", "/runner-2"])); - expect(authorities.size).toBe(0); - await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( - "rotation activation failed", - ); - } finally { - await bundle.transport.close().catch(() => undefined); - if (runnerPid) { - try { - process.kill(-runnerPid, "SIGKILL"); - } catch { - // A successful durable close already stopped the runner process group. - } - } - server.closeAllConnections(); - if (server.listening) { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); - } - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it.each([ - { - binding: "runner instance", - priorRunnerInstanceId: "runner-other", - priorEnvironmentLeaseId: "lease-current", - }, - { - binding: "environment lease", - priorRunnerInstanceId: "runner-current", - priorEnvironmentLeaseId: "lease-other", - }, -])( - "quarantines a mismatched $binding instead of reusing its provider session", - async ({ priorRunnerInstanceId, priorEnvironmentLeaseId }) => { - const container = await mkdtemp( - join(tmpdir(), "runnerd-mismatched-authority-"), - ); - const stateDirectory = join(container, "state"); - await mkdir(join(stateDirectory, "control-plane"), { recursive: true }); - await writeFile( - join(stateDirectory, "control-plane", "control-plane-state.json"), - JSON.stringify({ - identity: { - runnerInstanceId: priorRunnerInstanceId, - environmentLeaseId: priorEnvironmentLeaseId, - runId: "run-old", - normalizedSessionId: "session-current", - turnId: "turn-old", - itemId: "item-old", - }, - }), - { mode: 0o600 }, - ); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - stateDirectory, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: { - runnerInstanceId: "runner-current", - environmentLeaseId: "lease-current", - runId: "run-new", - normalizedSessionId: "session-current", - turnId: "turn-new", - itemId: "item-new", - }, - }); - try { - await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( - "native_runner_state_quarantined", - ); - expect(await readdir(stateDirectory)).toEqual([]); - expect( - (await readdir(container)).some((entry) => - entry.startsWith("state.quarantine-"), - ), - ).toBe(true); - } finally { - await bundle.transport.close(); - await rm(container, { recursive: true, force: true }); - } - }, -); - -it.each([0, 193 * 1024 * 1024])( - "probes an exact-authority resume with %i extra journal bytes and confirms its live provider identity", - async (extraJournalBytes) => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-exact-authority-resume-"), - ); - const identity = { - runnerInstanceId: "runner-exact-resume", - environmentLeaseId: "lease-exact-resume", - runId: "run-exact-resume", - normalizedSessionId: "session-exact-resume", - turnId: "turn-exact-resume", - itemId: "item-exact-resume", - }; - const options = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, "--durable-turn-ids"), - stateDirectory, - lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, - prpIdentity: identity, - }; - const first = createCapabilityRunnerdCodexTransport(options); - first.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let providerThread: { id: string; sessionId: string } | null = null; - try { - const opened = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - const thread = opened.thread as Record; - providerThread = { - id: String(thread.id), - sessionId: String(thread.sessionId), - }; - } finally { - await first.transport.close(); - } - if (providerThread === null) { - throw new Error( - "exact-authority fixture did not return a provider thread", - ); - } - - const statePath = join( - stateDirectory, - "control-plane", - "control-plane-state.json", - ); - if (extraJournalBytes > 0) { - const padding = Buffer.alloc(1024 * 1024, 0x20); - const stateHandle = await open(statePath, "a"); - try { - for (let remaining = extraJournalBytes; remaining > 0;) { - const bytesToWrite = Math.min(remaining, padding.length); - await stateHandle.write(padding, 0, bytesToWrite); - remaining -= bytesToWrite; - } - } finally { - await stateHandle.close(); - } - expect((await stat(statePath)).size).toBeGreaterThan( - 64 * 1024 * 1024, - ); - } - const beforeResume = JSON.parse(await readFile(statePath, "utf8")) as { - commands: Array<{ type: string }>; - committedEvents: Array<{ eventType: string }>; - }; - expect( - beforeResume.commands.some((command) => command.type === "run.attach"), - ).toBe(false); - const priorResumeEvents = beforeResume.committedEvents.filter( - (event) => event.eventType === "session.resumed", - ).length; - const priorSnapshots = beforeResume.commands.filter( - (command) => command.type === "session.snapshot", - ).length; - - const resumed = createCapabilityRunnerdCodexTransport({ - ...options, - resumeProviderSession: { - driverSessionId: providerThread.id, - providerSessionId: providerThread.sessionId, - }, - }); - resumed.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - const read = await resumed.transport.request("thread/read", {}); - expect(read.thread).toMatchObject(providerThread); - const afterResume = JSON.parse(await readFile(statePath, "utf8")) as { - commands: Array<{ commandId: string; type: string; status: string }>; - committedEvents: Array<{ eventType: string }>; - }; - expect(afterResume.commands).toContainEqual( - expect.objectContaining({ - commandId: expect.stringMatching(/^command_resume_probe_/), - type: "runner.drain", - status: "completed", - }), - ); - expect(afterResume.commands).toContainEqual( - expect.objectContaining({ - type: "session.snapshot", - status: "completed", - }), - ); - expect( - afterResume.commands.filter( - (command) => command.type === "session.snapshot", - ), - ).toHaveLength(priorSnapshots + 2); - // The authenticated snapshot above proves the live provider identity. - // Control-first dispatch may deliver that command before the independent - // session event is ingested. Still require exactly one durable event; - // don't mistake an immediate file read for an event-delivery barrier. - await vi.waitFor( - async () => { - const delivered = JSON.parse(await readFile(statePath, "utf8")) as { - committedEvents: Array<{ eventType: string }>; - }; - expect( - delivered.committedEvents.filter( - (event) => event.eventType === "session.resumed", - ), - ).toHaveLength(priorResumeEvents + 1); - }, - { timeout: 3_000, interval: 25 }, - ); - } finally { - await resumed.transport.close(); - await rm(stateDirectory, { recursive: true, force: true }); - } - }, - 30_000, -); - -it("still fails closed when a real close grace period cannot fit a durable suspension round trip", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-close-grace-too-small-"), - ); - const identity = { - runnerInstanceId: "runner-close-grace-too-small", - environmentLeaseId: "lease-close-grace-too-small", - runId: "run-close-grace-too-small", - normalizedSessionId: "session-close-grace-too-small", - turnId: "turn-close-grace-too-small", - itemId: "item-close-grace-too-small", - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - // No real durable command round trip can complete this fast. A wider - // budget for the provider-drain proof must not turn this barrier into - // one that always passes; it still needs the actual proof to arrive. - closeGraceMs: 1, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: identity, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - const closed = bundle.transport.close(); - await expect(closed).rejects.toMatchObject({ - code: "native_session_close_unrecoverable", - semanticToolsSettled: true, - }); - // Retirement preserves the original rejected promise and its proof. - expect(bundle.transport.close()).toBe(closed); - } finally { - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accounting-rejected", "credentials-cleaned"] as const)("cold-restores a suspended provider session under its durable run binding (pending tail: %s)", async (pendingTail) => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-cold-attach-")); - const tracePath = join(stateDirectory, "provider-trace.ndjson"); - const skillRoot = join(stateDirectory, "runtime-skill"); - const instructionRoot = join(stateDirectory, "runtime-instructions"); - await Promise.all([mkdir(skillRoot), mkdir(instructionRoot)]); - await writeFile(join(skillRoot, "SKILL.md"), "# Assigned runtime skill\n"); - await writeFile(join(instructionRoot, "AGENTS.md"), "Runtime instructions\n"); - const credentialsCleaned = pendingTail === "credentials-cleaned"; - const sourceCodexHome = join(stateDirectory, "selected-account-first"); - if (credentialsCleaned) { - await mkdir(sourceCodexHome); - await writeFile(join(sourceCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); - } - const baseIdentity = { - runnerInstanceId: "runner-cold-attach", - environmentLeaseId: "lease-cold-attach", - runId: "run-cold-first", - normalizedSessionId: "session-cold-attach", - turnId: "turn-cold-first", - itemId: "item-cold-first", - }; - const options = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--include-skill-instructions", - "--durable-turn-ids", - ...(credentialsCleaned ? ["--require-codex-home-auth"] : []), - "-c", - 'shell_environment_policy.set={PATH="/run/A"}', - ), - stateDirectory, - ...(credentialsCleaned ? { sourceCodexHome } : {}), - environment: { - PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-A-capability", - PAPERCLIP_PROVIDER_TRACE_PATH: tracePath, - PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024), - ...(credentialsCleaned ? { - PAPERCLIP_NATIVE_MCP_NAME: "assigned", - PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/first", - PAPERCLIP_NATIVE_MCP_TOKEN: "first-assigned-gateway-credential", - } : {}), - }, - lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, - runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), - }; - const dynamicTools = [ - { - name: "get_task_context", - description: "Read the active task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, - }, - }, - ]; - const first = createCapabilityRunnerdCodexTransport({ - ...options, - prpIdentity: baseIdentity, - }); - first.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let firstProviderThread: { id: string; sessionId: string } | null = null; - try { - const started = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [...dynamicTools, ...codexSemanticToolSpecs()], - completionContract: { - revision: "contract-first", - criterionIds: ["criterion-first"], - }, - }); - const startedThread = started.thread as Record; - firstProviderThread = { - id: String(startedThread.id), - sessionId: String(startedThread.sessionId), - }; - expect( - (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) - .mode & 0o222, - ).toBe(0); - expect( - ( - await stat( - join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), - ) - ).mode & 0o222, - ).toBe(0); - await first.transport.request("turn/start", { - input: [{ type: "text", text: "$assigned first process" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], - }); - for await (const event of first.transport.notifications()) { - if (event.method === "turn/completed") break; - } - } finally { - await first.transport.close(); - } - if (!firstProviderThread) { - throw new Error("cold attach fixture did not return a provider thread"); - } - const successorCodexHome = join(stateDirectory, "selected-account-next"); - if (credentialsCleaned) { - await rm(join(stateDirectory, "codex-home", "auth.json")); - await rm(sourceCodexHome, { recursive: true }); - await mkdir(successorCodexHome); - await writeFile(join(successorCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); - } - - // Model a retained provider-owned suffix: rotation must deliver and durably - // accept it before attempting run.attach, rather than dropping or ACKing it. - const retainedDelta = "cold-recovery-retained-tail"; - if (pendingTail) { - const providerFile = join(stateDirectory, "runner", "codex-provider-state.json"); - const provider = JSON.parse(await readFile(providerFile, "utf8")); - expect(provider.pendingEvents).toEqual([]); - provider.pendingEvents.push({ - executorEventId: `codex_provider_${String(provider.nextProviderEventSeq++).padStart(16, "0")}`, - eventType: pendingTail === "unknown-semantic" ? "semantic_tool.input" : "usage.reported", - priority: "p0", - payload: pendingTail === "unknown-semantic" ? { semantic_tool: { - schema: "paperclip.prp.semantic_tool.v1", schemaVersion: 1, phase: "input", - callId: "unknown-source-call", operationId: "get_task_context", input: {}, idempotencyKey: null, - correlation: { runId: baseIdentity.runId, normalizedSessionId: baseIdentity.normalizedSessionId, - turnId: baseIdentity.turnId, itemId: baseIdentity.itemId }, - content: { digest: `sha256:${createHash("sha256").update("{}").digest("hex")}`, redactionDisposition: "digest_only", references: [] }, - } } : { provider: "codex", kind: "usage", delta: retainedDelta, - providerTurnId: provider.completed_provider_turn_id, runDelta: { inputTokens: 20, outputTokens: 2 }, - runDeltaAvailable: true, runDeltaBasis: "run" }, - }); - await writeFile(providerFile, JSON.stringify(provider)); - } - const retainedEvents: PrpEvent[] = []; - let releaseRetained!: () => void, retainedEntered!: () => void; - const retainedAccepted = new Promise(done => { releaseRetained = done; }); - const retainedAdmission = new Promise(done => { retainedEntered = done; }); - const secondIdentity = { - ...baseIdentity, - runId: "run-cold-second", - turnId: "turn-cold-second", - itemId: "item-cold-second", - }; - let stagedCheckpoint: durableControlPlane.DurablePrpCheckpoint | undefined; - let rejectStagedCommit = pendingTail === "restart-before-bootstrap"; - const rotatedOptions = { - ...options, - ...(credentialsCleaned ? { sourceCodexHome: successorCodexHome } : {}), - onStagedRunAttachment: pendingTail === "restart-before-bootstrap" ? async (authority: DurablePrpControlPlane) => { - authority.setCheckpointPersistence(async checkpoint => { - stagedCheckpoint = checkpoint; - if (rejectStagedCommit) throw new Error("fixture lost controller after durable intent"); - }); - } : undefined, - onRecoveryAccountingEvent: pendingTail ? async (event: PrpEvent) => { - if (event.payload.delta === retainedDelta) { - retainedEvents.push(structuredClone(event)); - retainedEntered(); - if (pendingTail === "accounting-rejected") throw new Error("source accounting receipt unavailable"); - await retainedAccepted; - } - } : undefined, - environment: { ...options.environment, PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-B-capability", - ...(credentialsCleaned ? { - PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/current", - PAPERCLIP_NATIVE_MCP_TOKEN: "current-assigned-gateway-credential", - } : {}), - }, - codexArgs: options.codexArgs.map((arg) => arg.replace('/run/A', '/run/B')), - resumeDynamicTools: dynamicTools, - resumeCompletionContract: { - revision: "contract-second", - criterionIds: ["criterion-second"], - }, - prpIdentity: secondIdentity, - }; - let rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); - if (pendingTail === "restart-before-bootstrap") { - await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("fixture lost controller after durable intent"); - expect(rotated.evidence().runnerPid).toBeNull(); - expect(stagedCheckpoint?.runId).toBe(secondIdentity.runId); - expect([...stagedCheckpoint!.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value).type)).not.toContain("run.attach"); - await rotated.detachControllerForRestart().catch(() => undefined); - rejectStagedCommit = false; - await rm(join(stateDirectory, "control-plane"), { recursive: true, force: true }); - stagedCheckpoint!.restoreSessionDirectory(stateDirectory); - rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); - } - const currentHandler = vi.fn(async () => ({ success: true, contentItems: [] })); - rotated.transport.setServerRequestHandler(currentHandler); - if (pendingTail === "accounting-rejected") { - const reading = rotated.transport.request("thread/read", {}); - void reading.catch(() => undefined); - try { - await Promise.race([retainedAdmission, reading]); - const source = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(source.identity).toEqual(baseIdentity); - expect(source.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); - expect(source.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); - const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); - expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => - entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); - expect(currentHandler).not.toHaveBeenCalled(); - } finally { - await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); - await expect(reading).rejects.toThrow(); - await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); - await rm(stateDirectory, { recursive: true, force: true }); - } - return; - } - if (pendingTail === "unknown-semantic") { - try { - await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("native_runner_source_semantic_reconciliation_required"); - expect(currentHandler).not.toHaveBeenCalled(); - const retained = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(retained.identity).toEqual(baseIdentity); - expect(retained.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); - expect(retained.commands.some((command: { type: string }) => ["run.attach", "semantic_tool.result"].includes(command.type))).toBe(false); - expect(retained.committedEvents.some((event: PrpEvent) => event.eventType === "semantic_tool.input")).toBe(true); - } finally { - await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); - await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); - await rm(stateDirectory, { recursive: true, force: true }); - } - return; - } - let recoveryError: unknown; - try { - const reading = rotated.transport.request("thread/read", {}); - void reading.catch(() => undefined); - if (pendingTail) { - await Promise.race([retainedAdmission, reading]); - const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); - expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => - entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); - const retainedCore = stagedCheckpoint ? { - identity: JSON.parse(stagedCheckpoint.records.get("header")!).state.identity, - commands: [...stagedCheckpoint.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value)), - } : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(retainedCore.identity).toEqual(baseIdentity); - expect(retainedCore.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); - if (credentialsCleaned) { - const config = await readFile(join(stateDirectory, "codex-home", "config.toml"), "utf8"); - expect(config).toContain("https://paperclip.example/mcp/current"); - expect(config).toContain("current-assigned-gateway-credential"); - expect(config).not.toContain("first-assigned-gateway-credential"); - expect(currentHandler).not.toHaveBeenCalled(); - } - } - releaseRetained(); - const read = await reading; - expect(retainedEvents).toHaveLength(pendingTail ? 1 : 0); - if (pendingTail) expect(retainedEvents[0]).toMatchObject({ - runId: baseIdentity.runId, turnId: baseIdentity.turnId, itemId: baseIdentity.itemId, - }); - const persistedProvider = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); - expect(persistedProvider.pendingEvents).toEqual([]); - expect(persistedProvider.config.args.join("\n")).toContain('/run/B'); - expect(JSON.stringify(persistedProvider)).not.toContain("test-run-B-capability"); - expect(read.thread).toMatchObject({ - id: firstProviderThread.id, - sessionId: firstProviderThread.sessionId, - cwd: tmpdir(), - }); - await rotated.transport.request("turn/start", { - input: [{ type: "text", text: "$assigned second authority epoch" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], - }); - for await (const event of rotated.transport.notifications()) { - if (event.method === "paperclip/runResult") break; - } - expect(rotated.evidence().diagnostics).toContain( - "runnerd attached the durable provider session to a fresh PRP run authority", - ); - if (pendingTail) { - const target = stagedCheckpoint ? JSON.parse(stagedCheckpoint.records.get("header")!).state - : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(target.stagedRunAttachment).toBeUndefined(); - expect(target.completedWarmTransition.receipt).toMatchObject({ oldIdentity: baseIdentity, newIdentity: secondIdentity }); - } else expect(await stat(join(stateDirectory, "authority-epochs"))).toBeDefined(); - } catch (error) { - recoveryError = error; - throw error; - } finally { - releaseRetained(); - await rotated.transport.close().catch(error => { - if (recoveryError) throw new AggregateError([recoveryError, error], "Provider recovery and cleanup failed"); - throw error; - }); - } - const requestFrames = (await readFile(tracePath, "utf8")) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record) - .filter( - (entry) => - entry.kind === "frame" && entry.direction === "client_to_provider", - ) - .map( - (entry) => - JSON.parse( - Buffer.from(String(entry.rawBase64), "base64").toString("utf8"), - ) as Record, - ) - ; - const resumeFrames = requestFrames.filter((frame) => frame.method === "thread/resume"); - for (const frame of requestFrames.filter((f) => ["thread/start", "thread/resume"].includes(String(f.method)))) { - expect(frame.params).toMatchObject({ config: { "skills.include_instructions": true } }); - } - const turnFrames = requestFrames.filter((f) => f.method === "turn/start"); - expect(turnFrames).toHaveLength(2); - for (const frame of turnFrames) { - expect(frame.params).toMatchObject({ input: expect.arrayContaining([{ type: "skill", name: "assigned", path: join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md") }]) }); - } - expect(resumeFrames.length).toBeGreaterThanOrEqual(1); - for (const frame of resumeFrames) { - expect(frame).toEqual( - expect.objectContaining({ - method: "thread/resume", - params: expect.objectContaining({ threadId: firstProviderThread.id }), - }), - ); - } - - if (pendingTail) { - await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); - await rm(stateDirectory, { recursive: true, force: true }); - return; - } - - // A remote process owner keeps runner-state outside the controller's local - // session root. Resume must defer to its explicit state reader instead of - // rejecting recovery before the remote checkpoint can be made available. - const externallyOwnedRunnerStateDirectory = join( - stateDirectory, - "externally-owned-runner", - ); - await rename( - join(stateDirectory, "runner"), - externallyOwnedRunnerStateDirectory, - ); - const readRunnerState = async () => - JSON.parse( - await readFile( - join(externallyOwnedRunnerStateDirectory, "runner-state.json"), - "utf8", - ), - ) as Record; - const mismatched = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - resumeDynamicTools: dynamicTools, - prpIdentity: { - ...secondIdentity, - runId: "run-cold-other", - turnId: "turn-cold-other", - itemId: "item-cold-other", - }, - }); - await expect(mismatched.transport.request("thread/read", {})).rejects.toThrow( - "native_runner_prp_run_rotation_unavailable", - ); - await mismatched.transport.close(); - - const externalIdentity = { - ...secondIdentity, - runId: "run-cold-external", - turnId: "turn-cold-external", - itemId: "item-cold-external", - }; - const rejectedExternalRotationSteps: string[] = []; - let externalArchiveDirectory: string | null = null; - const rejectedExternalRotation = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - prepareExternalRunnerState: async () => { - rejectedExternalRotationSteps.push("prepared"); - }, - archiveExternalRunnerState: async ({ archiveKey }) => { - await expect( - stat(join(stateDirectory, "control-plane")), - ).rejects.toMatchObject({ code: "ENOENT" }); - expect( - await stat( - join( - stateDirectory, - "authority-epochs", - `epoch-${archiveKey}`, - "control-plane", - ), - ), - ).toBeDefined(); - externalArchiveDirectory = join( - stateDirectory, - "external-authority-epochs", - archiveKey, - ); - await mkdir(externalArchiveDirectory, { recursive: true }); - await rename( - join(externallyOwnedRunnerStateDirectory, "runner-state.json"), - join(externalArchiveDirectory, "runner-state.json"), - ); - rejectedExternalRotationSteps.push("remote-archived"); - throw new Error("controller crashed after external archive"); - }, - resumeDynamicTools: dynamicTools, - prpIdentity: externalIdentity, - }); - await expect( - rejectedExternalRotation.transport.request("thread/read", {}), - ).rejects.toThrow("controller crashed after external archive"); - await rejectedExternalRotation.transport.close(); - expect(rejectedExternalRotationSteps).toEqual([ - "prepared", - "remote-archived", - ]); - await expect(stat(join(stateDirectory, "control-plane"))).rejects.toThrow(); - await expect( - stat(join(externallyOwnedRunnerStateDirectory, "runner-state.json")), - ).rejects.toThrow(); - - const externalRotationSteps: string[] = []; - const externallyRotated = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - prepareExternalRunnerState: async () => { - throw new Error("retry must not prepare a new external runner"); - }, - archiveExternalRunnerState: async ({ archiveKey }) => { - expect(externalArchiveDirectory).toBe( - join(stateDirectory, "external-authority-epochs", archiveKey), - ); - externalRotationSteps.push("archived"); - return JSON.parse( - await readFile( - join(externalArchiveDirectory!, "runner-state.json"), - "utf8", - ), - ) as Record; - }, - resumeDynamicTools: dynamicTools, - resumeCompletionContract: { - revision: "contract-external", - criterionIds: ["criterion-external"], - }, - prpIdentity: externalIdentity, - }); - externallyRotated.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - const read = await externallyRotated.transport.request("thread/read", {}); - expect(read.thread).toMatchObject({ - id: firstProviderThread.id, - sessionId: firstProviderThread.sessionId, - cwd: tmpdir(), - }); - expect(externalRotationSteps).toEqual(["archived"]); - } finally { - await externallyRotated.transport.close(); - } - - const restored = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - resumeDynamicTools: dynamicTools, - prpIdentity: externalIdentity, - }); - restored.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - const read = await restored.transport.request("thread/read", {}); - expect(read.thread).toMatchObject({ - id: "codex-thread-1", - cwd: tmpdir(), - }); - expect( - (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) - .mode & 0o222, - ).toBe(0); - expect( - ( - await stat( - join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), - ) - ).mode & 0o222, - ).toBe(0); - const providerState = JSON.parse( - await readFile( - join(externallyOwnedRunnerStateDirectory, "codex-provider-state.json"), - "utf8", - ), - ) as { toolBridge?: { authorized?: Record } }; - expect(Object.keys(providerState.toolBridge?.authorized ?? {})).toEqual([ - "get_task_context", - "paperclip_block", - "paperclip_finish", - ]); - await restored.transport.request("turn/start", { - input: [{ type: "text", text: "restored process" }], - }); - for await (const event of restored.transport.notifications()) { - if (event.method === "turn/completed") break; - } - expect(restored.evidence()).toMatchObject({ - runnerExited: false, - codexPid: expect.any(Number), - }); - } finally { - await restored.transport.close(); - await releaseMaterializedNativeRuntimeSkills( - join(stateDirectory, "codex-home", "skills"), - ); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -async function verifyLiveRunnerAdoption( - mismatchedCheckpoint: boolean, - mismatchedArtifact = false, - goalMidTurn = false, - detachBeforeCleanup = false, - startWithoutCheckpoint = false, - preparedTask = false, - adoptionAuthorization: "allow" | "deny" | "legacy" | "legacy-missing" | "missing-binding" = "allow", - toolHandlerOrder?: "before" | "after" | "completion-before" | "completion-after", - uncertainAtClose = false, - semanticRecoveryMode?: "denied-before-cancel" | "cleanup-only", - stopExitProofFault?: "missing" | "false", -) { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-live-adopt-")); - const pendingCompletion = toolHandlerOrder?.startsWith("completion-") === true; - const completionPath = join(stateDirectory, "pending-completion.json"); - if (pendingCompletion) await writeFile(completionPath, JSON.stringify({ - schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed the synthetic work", - completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] }, - evidence: [], verification: [], attentionRequests: [], artifacts: [], - })); - const server = createServer(); - let authority: DurablePrpControlPlane | null = null; - const checkpoint = vi.fn(async () => undefined); - server.on("upgrade", (request, socket, head) => { - if (!authority) { - socket.destroy(); - return; - } - authority.handleUpgrade(request, socket, "/runner", head); - }); - await new Promise((resolveListen) => - server.listen(0, "127.0.0.1", resolveListen), - ); - const address = server.address(); - if (!address || typeof address === "string") - throw new Error("Expected adoption test listener"); - const registration = vi.fn(async (next: DurablePrpControlPlane) => { - authority = next; - return { - connectUrl: `ws://127.0.0.1:${address.port}/runner`, - ...(mismatchedArtifact || uncertainAtClose || semanticRecoveryMode === "cleanup-only" ? { checkpoint } : {}), - release: async () => { - if (authority === next) authority = null; - }, - }; - }); - const identity = { - runnerInstanceId: "runner-live-adopt", - environmentLeaseId: "lease-live-adopt", - runId: "run-live-adopt", - normalizedSessionId: "session-live-adopt", - turnId: "turn-live-adopt", - itemId: "item-live-adopt", - }; - const sharedOptions = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, ...(pendingCompletion ? ["--terminal-result-file", completionPath] : toolHandlerOrder ? ["--emit-tool-call"] : []), ...(goalMidTurn ? ["--goal-autostart", "--goal-item-trigger", join(stateDirectory, "emit-goal-item")] : [])), - stateDirectory, - prpIdentity: identity, - lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, - controlPlaneRegistration: registration, - }; - const prepareLaunch = vi.fn(async (requirements: RunnerProviderLaunchRequirements) => ({ - configurationId: "adopted-preparation", requirementsDigest: runnerLaunchRequirementsDigest(requirements), - provider: createRunnerProviderLaunch(requirements, { options: { codexArgs: sharedOptions.codexArgs }, - root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), - codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "" }), - })); - const options = { ...sharedOptions, ...(preparedTask ? { - prepareLaunch, runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), - runnerProcessLauncher: (spec: durableControlPlane.RunnerProcessLaunchSpec) => { - const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: { ...process.env, ...spec.environment }, - detached: process.platform !== "win32", stdio: "ignore" }); - return { child, processGroupId: child.pid ?? null, completion: new Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }>((resolveExit, reject) => { - child.once("error", reject); - child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" })); - }) }; - }, - } : {}) }; - const first = createCapabilityRunnerdCodexTransport(options); - const pendingTool = vi.fn(() => new Promise>(() => undefined)); - if (toolHandlerOrder) first.transport.setServerRequestHandler(pendingTool); - let releaseRecovery!: () => void; - const recoveryGate = new Promise(resolve => { releaseRecovery = resolve; }); - const recoveryDenied = new Error("execution authority revoked after Stop"); - const recoveredTool = vi.fn(async () => { - if (semanticRecoveryMode) { - // Model server authority denying replay after Stop cleared the task lock. - // The gate holds that denial until reattachment has returned its session, - // before the caller invokes the eventual onSession cancellation callback. - await recoveryGate; - throw recoveryDenied; - } - return { result: { receipt: "known-outcome" }, isError: false }; - }); - const redispatchedTool = vi.fn(async () => ({ success: true, contentItems: [] })); - let runnerPid: number | null = null; - let runnerStartedAt: string | null = null; - let adopted: ReturnType | null = - null; - let restoreIntervalSpies: () => void = () => undefined; - try { - let opened: Record; - try { - opened = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: toolHandlerOrder && !pendingCompletion ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : codexSemanticToolSpecs(), - }); - } catch (error) { - const stderr = await readFile( - join(stateDirectory, "diagnostics", "runnerd.stderr.log"), - "utf8", - ).catch(() => ""); - throw new Error( - `${String(error)}\n${JSON.stringify(first.evidence())}${stderr ? `\n${stderr}` : ""}`, - ); - } - runnerPid = first.evidence().runnerPid; - runnerStartedAt = first.transport.processInfo().startedAt; - expect(runnerPid).toEqual(expect.any(Number)); - - if (goalMidTurn) { - await first.transport.request("thread/goal/set", { objective: "Recover a live goal", status: "active" }); - for await (const event of first.transport.notifications()) { - if (event.method === "turn/started") break; - } - } - - if (toolHandlerOrder) { - await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); - await vi.waitFor(() => expect(pendingTool).toHaveBeenCalledOnce(), { timeout: 5_000 }); - } - await first.detachControllerForRestart(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - if (goalMidTurn) { - await writeFile(join(stateDirectory, "emit-goal-item"), "emit"); - // runnerd need not poll the provider into its PRP outbox while disconnected. - // Wait for flushed provider output, not a platform-dependent final poll - // racing the disconnect. Adoption must still bind that buffered item. - await vi.waitFor(async () => { - expect(await readFile(join(stateDirectory, "emit-goal-item.sent"), "utf8")).toBe("sent"); - }, { timeout: 5_000 }); - } - - const controlPlaneStatePath = join( - stateDirectory, - "control-plane", - "control-plane-state.json", - ); - const compactProviderIdentityEvents = async () => { - const controlPlaneState = JSON.parse( - await readFile(controlPlaneStatePath, "utf8"), - ) as { committedEvents: Array<{ eventType: string }> }; - controlPlaneState.committedEvents = - controlPlaneState.committedEvents.filter( - (event) => - event.eventType !== "harness.ready" && - event.eventType !== "session.started" && - event.eventType !== "session.resumed", - ); - await writeFile( - controlPlaneStatePath, - `${JSON.stringify(controlPlaneState, null, 2)}\n`, - { mode: 0o600 }, - ); - }; - if (!startWithoutCheckpoint) await compactProviderIdentityEvents(); - - if (preparedTask) { - if (adoptionAuthorization !== "legacy") await rm(join(stateDirectory, "launch-preparation.json")); - prepareLaunch.mockClear(); - registration.mockClear(); - if (adoptionAuthorization === "missing-binding") { - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - delete retained.preparedLaunchConfigurations; - await writeFile(controlPlaneStatePath, JSON.stringify(retained), { mode: 0o600 }); - } - } - const authorizeRunnerAdoption = vi.fn(async () => { - expect(registration).not.toHaveBeenCalled(); - if (adoptionAuthorization === "deny") throw new Error("project grant revoked"); - }); - const duplicateLauncher = vi.fn(() => { - throw new Error("duplicate runner spawn attempted"); - }); - const openedThread = opened.thread as Record; - const signal = vi.fn(() => true); - const probeError = new Error("fixture remote process identity is unavailable"); - let rejectLiveness = false; - const intervals = uncertainAtClose ? vi.spyOn(globalThis, "setInterval") : null; - const clearedIntervals = uncertainAtClose ? vi.spyOn(globalThis, "clearInterval") : null; - restoreIntervalSpies = () => { intervals?.mockRestore(); clearedIntervals?.mockRestore(); }; - adopted = createCapabilityRunnerdCodexTransport({ - ...options, - ...(preparedTask && !adoptionAuthorization.startsWith("legacy") ? { authorizeRunnerAdoption } : {}), - ...(uncertainAtClose ? { closeGraceMs: 2_000 } : {}), - ...(semanticRecoveryMode === "cleanup-only" ? { recoveryPurpose: "stop_cleanup" as const } : {}), - // Hash different stable bytes without replacing the real runner artifact - // used by concurrent tests. Adoption must never execute this path. - ...(mismatchedArtifact - ? { - runnerBinary: resolve(import.meta.dirname, "../../package.json"), - runnerReconnectGraceMs: 150, - } - : {}), - ...(toolHandlerOrder && !pendingCompletion ? { recoverSemanticToolInput: recoveredTool } : {}), - resumeDynamicTools: pendingCompletion ? codexSemanticToolSpecs() : toolHandlerOrder ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : [], - resumeProviderSession: startWithoutCheckpoint ? undefined : { - driverSessionId: String(openedThread.id), - providerSessionId: mismatchedCheckpoint - ? "wrong-provider-session" - : String(openedThread.sessionId), - }, - runnerProcessLauncher: duplicateLauncher, - adoptExistingRunner: { - pid: runnerPid!, - processGroupId: runnerPid, - startedAt: runnerStartedAt, - signal, - isAlive: () => { - if (rejectLiveness) return Promise.reject(probeError); - try { - process.kill(runnerPid!, 0); - return true; - } catch { - return false; - } - }, - }, - }); - if (preparedTask && ["deny", "legacy-missing", "missing-binding"].includes(adoptionAuthorization)) { - await expect(adopted.transport.request("thread/read", {})).rejects.toThrow( - adoptionAuthorization === "deny" ? "project grant revoked" : adoptionAuthorization === "legacy-missing" - ? "runner_launch_recovery_requirements_unavailable" : "runner_launch_recovery_configuration_unavailable"); - expect(registration).not.toHaveBeenCalled(); - expect(prepareLaunch).not.toHaveBeenCalled(); - expect(duplicateLauncher).not.toHaveBeenCalled(); - expect(signal).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - expect(authorizeRunnerAdoption).toHaveBeenCalledTimes(adoptionAuthorization === "deny" ? 1 : 0); - return; - } - if (mismatchedArtifact) { - await expect( - adopted.transport.request("thread/read", {}), - ).rejects.toThrow("native_adopted_runner_authentication_timeout"); - expect(authority?.activeRunnerConnectionCount()).toBe(0); - await expect( - adopted.transport.request("turn/start", { - input: [{ type: "text", text: "must not be dispatched" }], - }), - ).rejects.toThrow("native_adopted_runner_authentication_timeout"); - await adopted.transport.close(); - expect(signal).not.toHaveBeenCalled(); - expect(checkpoint).not.toHaveBeenCalled(); - expect(duplicateLauncher).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - const retained = JSON.parse( - await readFile(controlPlaneStatePath, "utf8"), - ) as { - identity: unknown; - commands: Array<{ type: string; status: string }>; - }; - expect(retained.identity).toEqual(identity); - expect( - retained.commands.some( - (command) => - command.type === "runner.drain" && command.status === "pending", - ), - ).toBe(true); - expect(retained.commands.map((command) => command.type)).not.toEqual( - expect.arrayContaining(["runner.suspend"]), - ); - expect(retained.commands.map((command) => command.type)).not.toEqual( - expect.arrayContaining(["turn.stop"]), - ); - return; - } - if (mismatchedCheckpoint) { - await expect( - adopted.transport.request("thread/read", {}), - ).rejects.toThrow("native_adopted_provider_identity_mismatch"); - expect(duplicateLauncher).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - return; - } - if (toolHandlerOrder?.endsWith("before")) adopted.transport.setServerRequestHandler(redispatchedTool); - await expect(adopted.transport.request(startWithoutCheckpoint ? "thread/start" : "thread/read", {})).resolves.toEqual( - expect.objectContaining({ - thread: expect.objectContaining({ id: "codex-thread-1" }), - }), - ); - if (toolHandlerOrder?.endsWith("after")) adopted.transport.setServerRequestHandler(redispatchedTool); - if (semanticRecoveryMode) { - const retained = authority as DurablePrpControlPlane | null; - expect(retained).not.toBeNull(); - expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); - expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); - expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); - if (semanticRecoveryMode === "denied-before-cancel") { - await vi.waitFor(() => expect(recoveredTool).toHaveBeenCalledOnce()); - // This is a real Core replay callback, not an artificial transport error. - // Deliver its denial before giving the caller its cancellation turn. - releaseRecovery(); - await vi.waitFor(async () => { - await expect(adopted!.transport.request("thread/read", {})).rejects.toBe(recoveryDenied); - }); - await expect(adopted.transport.request("turn/interrupt", {})).rejects.toBe(recoveryDenied); - expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); - } else { - // Let all startup/handler microtasks and a real reconciliation read run. - // Cleanup must not enter replay even while its authority would deny it. - await adopted.transport.request("thread/read", {}); - expect(recoveredTool).not.toHaveBeenCalled(); - expect(redispatchedTool).not.toHaveBeenCalled(); - releaseRecovery(); - await expect(adopted.transport.request("turn/interrupt", {})).resolves.toEqual({}); - expect(retained!.store.state.commands.filter(command => command.type === "turn.interrupt")).toMatchObject([{ status: "completed" }]); - } - expect(redispatchedTool).not.toHaveBeenCalled(); - expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); - expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); - expect(duplicateLauncher).not.toHaveBeenCalled(); - if (semanticRecoveryMode === "cleanup-only") { - const faultedStopProof = vi.fn(); - if (stopExitProofFault) { - const queue = retained!.queueCommand.bind(retained!); - vi.spyOn(retained!, "queueCommand").mockImplementation((type, ...args) => { - const command = queue(type, ...args); - if (type === "turn.stop") { - let result = command.result; - Object.defineProperty(command, "result", { - enumerable: true, - configurable: true, - get: () => result, - set: value => { - const receipt = value as { result?: Record } | null; - if (receipt?.result && typeof receipt.result === "object") { - faultedStopProof(receipt.result.providerExitConfirmed); - const stopped = { ...receipt.result }; - if (stopExitProofFault === "missing") delete stopped.providerExitConfirmed; - else stopped.providerExitConfirmed = false; - result = { ...receipt, result: stopped }; - } else result = value; - }, - }); - } - return command; - }); - await expect(adopted.transport.close()).rejects.toBeInstanceOf(NativeSessionCloseUnrecoverableError); - expect(faultedStopProof).toHaveBeenCalledWith(true); - expect(checkpoint).toHaveBeenCalledExactlyOnceWith("unsettled"); - } else { - await expect(adopted.transport.close()).resolves.toBeUndefined(); - expect(checkpoint).toHaveBeenCalledExactlyOnceWith("settled"); - expect(retained!.store.state.commands.filter(command => command.type === "turn.stop")).toMatchObject([{ - status: "completed", result: { result: { providerExitConfirmed: true } }, - }]); - } - expect(retained!.semanticToolResultsSettled()).toBe(false); - expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); - expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); - if (!stopExitProofFault) { - expect(retained!.store.state.commands.filter(command => command.type === "runner.suspend")).toMatchObject([{ status: "completed" }]); - const runnerState = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); - expect(runnerState).toMatchObject({ ...identity, lifecycle: "suspended" }); - } - const providerState = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); - expect(providerState).toMatchObject({ activeProviderTurnId: null, pendingEvents: [], queuedEvents: [] }); - expect(recoveredTool).not.toHaveBeenCalled(); - expect(redispatchedTool).not.toHaveBeenCalled(); - } - return; - } - if (toolHandlerOrder) { - await vi.waitFor(() => { - expect(pendingCompletion ? redispatchedTool : recoveredTool).toHaveBeenCalledOnce(); - const result = authority!.store.state.commands.find(command => command.type === "semantic_tool.result"); - expect(result?.status).toBe("completed"); - }, { timeout: 5_000 }); - if (pendingCompletion) { - expect(recoveredTool).not.toHaveBeenCalled(); - expect(redispatchedTool).toHaveBeenCalledWith(expect.objectContaining({ params: expect.objectContaining({ tool: "paperclip_finish" }) })); - } else expect(redispatchedTool).not.toHaveBeenCalled(); - } - expect(adopted.evidence().runnerPid).toBe(runnerPid); - expect(adopted.transport.processInfo().startedAt).toBe(runnerStartedAt); - if (startWithoutCheckpoint) { - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - for (const type of ["run.prepare", "session.open"]) { - expect(retained.commands.filter((command: { type: string }) => command.type === type)).toHaveLength(1); - } - } - if (goalMidTurn) { - const observed = await Promise.race([ - (async () => { - for await (const notification of adopted!.transport.notifications()) { - if (notification.method === "item/started") return notification; - } - throw new Error("recovered goal item was lost"); - })(), - new Promise((_, reject) => setTimeout(() => reject(new Error("recovered item timed out")), 5_000)), - ]); - expect(observed.params).toMatchObject({ threadId: "codex-thread-1", turnId: "provider-goal-turn-1" }); - } - expect(duplicateLauncher).not.toHaveBeenCalled(); - if (preparedTask) { - if (adoptionAuthorization === "legacy") { - expect(prepareLaunch).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ identity }), expect.objectContaining({ configurationId: "adopted-preparation" })); - expect(authorizeRunnerAdoption).not.toHaveBeenCalled(); - } else { - expect(prepareLaunch).not.toHaveBeenCalled(); - expect(authorizeRunnerAdoption).toHaveBeenCalledExactlyOnceWith("adopted-preparation", identity); - } - expect((await adopted.transport.request("thread/read", {})).thread).toMatchObject({ cwd: tmpdir() }); - } - if (uncertainAtClose) { - const before = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - const retainedAuthority = authority as DurablePrpControlPlane | null; - rejectLiveness = true; - await expect(adopted.transport.close()).rejects.toBe(probeError); - expect(retainedAuthority?.activeRunnerConnectionCount()).toBe(0); - expect(intervals!.mock.results.length).toBeGreaterThan(0); - for (const interval of intervals!.mock.results) if (interval.type === "return") expect(clearedIntervals).toHaveBeenCalledWith(interval.value); - const notificationsClosed = (async () => { for await (const _event of adopted!.transport.notifications()) {} return true; })(); - let queueDeadline: ReturnType | undefined; - try { - expect(await Promise.race([notificationsClosed, new Promise(resolve => { queueDeadline = setTimeout(() => resolve(false), 250); })])).toBe(true); - } finally { clearTimeout(queueDeadline); } - await expect(adopted.transport.close()).rejects.toBe(probeError); - expect(adopted.transport.processInfo()).toMatchObject({ exited: false }); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - expect(signal).not.toHaveBeenCalled(); - expect(checkpoint).not.toHaveBeenCalled(); - expect(duplicateLauncher).not.toHaveBeenCalled(); - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - expect(retained.identity).toEqual(before.identity); - expect(retained.commands).toEqual(before.commands); - return; - } - expect(adopted.evidence().diagnostics).toContain( - `adopted runner ${runnerPid} authenticated to its durable PRP authority`, - ); - if (!startWithoutCheckpoint) { - expect(adopted.evidence().diagnostics).toContain( - "restored adopted provider identity from the exact durable checkpoint after PRP event compaction; awaiting live confirmation", - ); - expect(adopted.evidence().diagnostics).toContain( - "confirmed adopted provider identity against authenticated recovery session.snapshot", - ); - } - if (detachBeforeCleanup) { - await adopted.detachControllerForRestart(); - await adopted.transport.close("old controller finalizer"); - expect(signal).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - const commandTypes = retained.commands.map((command: { type: string }) => command.type); - expect(commandTypes).not.toContain("turn.stop"); - expect(commandTypes).not.toContain("runner.suspend"); - } - } finally { - releaseRecovery(); - await adopted?.transport.close().catch(() => undefined); - if (uncertainAtClose) { - const providerPid = adopted?.evidence().codexPid; - if (providerPid) { - try { process.kill(providerPid, "SIGKILL"); } - catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } - } - await (authority as DurablePrpControlPlane | null)?.stop(); - } - restoreIntervalSpies(); - if (runnerPid) { - try { - process.kill(-runnerPid, "SIGKILL"); - } catch { - // The adopted runner normally exits after its durable suspend command. - } - } - server.closeAllConnections(); - if (server.listening) { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); - } - await rm(stateDirectory, { recursive: true, force: true }); - } -} - -it( - "adopts a live runner on the same durable authority without spawning a duplicate", - () => verifyLiveRunnerAdoption(false), - 30_000, -); - -it("adopts a prepared runner after losing its controller launch cache without preparing resources again", - () => verifyLiveRunnerAdoption(false, false, false, false, false, true), 30_000); - -it.each(["deny", "legacy", "legacy-missing", "missing-binding"] as const)("preserves adoption authorization with %s", - mode => verifyLiveRunnerAdoption(false, false, false, false, false, true, mode), 30_000); -it("retains authenticated adopted-runner ownership when a close probe rejects", () => verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", undefined, true), 30_000); - -it( - "blocks adopted runner artifact drift without duplicate launch, checkpoint replacement, or process signals", - () => verifyLiveRunnerAdoption(false, true), - 15_000, -); - -it( - "rejects a live runner whose provider identity mismatches the compacted checkpoint", - () => verifyLiveRunnerAdoption(true), - 30_000, -); - -it("binds buffered mid-goal items only after the authenticated recovery snapshot", () => verifyLiveRunnerAdoption(false, false, true), 30_000); - -it("keeps an adopted runner alive when the detached controller finalizer closes", () => verifyLiveRunnerAdoption(false, false, true, true), 30_000); - -it("adopts an opening session without a checkpoint instead of bootstrapping a duplicate provider", () => verifyLiveRunnerAdoption(false, false, false, false, true), 30_000); - it("cancels startup readiness before releasing the route that settles its ready callback", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-ready-release-cycle-")); let releaseReady!: () => void; @@ -9470,52 +5432,6 @@ it("rejects the notification stream promptly when runnerd exits after accepting } }, 30_000); -it("persists an active provider as settled before bounded suspension", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-active-suspension-"), - ); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, "--linger-after-turn-start"), - stateDirectory, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - await bundle.transport.request("initialize", {}); - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - await bundle.transport.request("turn/start", { - input: [{ type: "text", text: "Wait for another instruction." }], - }); - const notifications = bundle.transport - .notifications() - [Symbol.asyncIterator](); - await expectTurnStarted(notifications); - await bundle.transport.close(); - - const providerState = JSON.parse( - await readFile( - join(stateDirectory, "runner", "codex-provider-state.json"), - "utf8", - ), - ) as Record; - expect(providerState).toMatchObject({ - lifecycle: "prepared", - activeProviderTurnId: null, - }); - } finally { - await bundle.transport.close(); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - - it.each(["claude", "codex"] as const)("keeps the explicitly assigned gateway in the %s runner environment", (agent) => { const gateway = { PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned", @@ -9550,7 +5466,6 @@ it("resolves explicit skills to the remote provider home and rejects unassigned expect(() => resolveRunnerdCodexSkillInputs([skill], null, "/runner/codex-home")).toThrow("assigned runtime skill"); }); - it("admits only exact live Pi turn controls", () => { expect(parseAcpxTurnControlCapabilities(undefined, "pi")).toEqual({ steering: false, queuedFollowUp: false }); expect(parseAcpxTurnControlCapabilities({ steering: true, queuedFollowUp: true }, "pi")).toEqual({ steering: true, queuedFollowUp: true }); @@ -9843,7 +5758,6 @@ it.each([ }); }, 30_000); - it.each([ { provider: "codex", acpxAgent: "cursor", acpxMode: "plan" }, { provider: "acpx", acpxAgent: "copilot", acpxMode: "" }, @@ -9856,11 +5770,11 @@ it.each([ it.each([undefined, "medium", "minimal", "xhigh", null])("rejects non-exact Pi transport thinking level %s before spawn", piThinkingLevel => { expect(() => createCapabilityRunnerdCodexTransport({ provider: "acpx", acpxAgent: "pi", piThinkingLevel: piThinkingLevel as never })).toThrow(/thinking/); }); + it("rejects Pi thinking settings on a different transport provider", () => { expect(() => createCapabilityRunnerdCodexTransport({ provider: "codex", piThinkingLevel: "low" })).toThrow(/only supported/); }); - describe("cold process admission budget", () => { it("bounds Pi cold startup and recovery at 60 seconds", () => { expect(coldAdmissionTimeoutMs("acpx", "pi", true)).toBe(60_000); @@ -9929,7 +5843,6 @@ it("reports a generic failure when an ACPX rejection does not match Runner's clo expect((failure as Error).message).not.toContain("Private provider explanation."); }); - it("shares Pi's absolute cold deadline across recovery barriers and rejects a late ACK", async () => { vi.useFakeTimers(); try { @@ -9978,6 +5891,7 @@ it("aborts startup promptly on close while allowing the owned cleanup command to await drain; } finally { vi.useRealTimers(); } }); + it("launches and closes native Codex without materializing or editing the preconfigured home", async () => { const root = await mkdtemp(join(tmpdir(), "runner-preconfigured-launch-")); const home = join(root, "user-home"); @@ -10005,19 +5919,3 @@ it("launches and closes native Codex without materializing or editing the precon await rm(root, { force: true, recursive: true }); } }, 30_000); - - -it.each(["before", "after"] as const)("settles a saved tool outcome with the handler installed %s recovery startup", order => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); - -it.each(["completion-before", "completion-after"] as const)("recovers a pending finish without an ordinary-tool lookup (%s)", order => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); - -it("exposes semantic recovery before cancellation when the authority has been revoked", () => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "denied-before-cancel"), 30_000); - -it.each(["before", "after", "completion-before", "completion-after"] as const)("allows cleanup-only adoption to cancel without replaying a pending semantic input (%s)", order => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order, false, "cleanup-only"), 30_000); - -it.each(["missing", "false"] as const)("refuses cleanup settlement when the exact stop receipt has %s provider exit proof", fault => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "cleanup-only", fault), 30_000); diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 11f665119dd..784f95ed6f4 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -15,10 +15,12 @@ import type { PersistedNativeSession, } from "./contracts/native-session-backend.js"; import { + NativeProviderStateIncompatibleError, NativeSessionCloseUnrecoverableError, NativeSessionCleanupQuarantinedError, NativeSessionProtocolIntegrityError, } from "./contracts/native-session-backend.js"; +import { PROVIDER_STATE_INCOMPATIBLE } from "./contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, @@ -5641,6 +5643,40 @@ describe("executeNativeSession recovery", () => { expect(openSession).not.toHaveBeenCalled(); }); + it("types a runner-proven incompatible session when same-session recovery is required", async () => { + for (const providerRecoveryPolicy of ["same_session_only", "allow_replacement_after_governed_wait"] as const) { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", sessionId: "driver-incompatible", identity, + providerSessionId: "provider-incompatible", providerRecoveryPolicy, + cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], + }; + const openRun = vi.fn(async () => undefined); + const openSession = vi.fn(); + const openReplacementSession = vi.fn(); + const backend: NativeSessionBackend = { + descriptor: async () => ({ kind: "mock", name: "incompatible", version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true } }), + openSession, openReplacementSession, + recoverSession: async () => ({ recovered: false, reason: "saved launch profile changed", + failureClass: PROVIDER_STATE_INCOMPATIBLE }), + }; + const port: ControlPlanePort = { + openRun, appendEvent: vi.fn(), completeRun: vi.fn(), + loadSessionCheckpoint: async () => structuredClone(checkpoint), + replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), + }; + const failure = await executeNativeSession({ + input, backend, controlPlane: port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", + }).catch((error: unknown) => error); + expect(failure, providerRecoveryPolicy).toBeInstanceOf(NativeProviderStateIncompatibleError); + expect((failure as Error).message).toBe("native_session_recovery_failed: saved launch profile changed"); + expect(openRun).not.toHaveBeenCalled(); + expect(openSession).not.toHaveBeenCalled(); + expect(openReplacementSession).not.toHaveBeenCalled(); + } + }); + it("bounds paginated recovery replay with one signal and observes a late rejection", async () => { vi.useFakeTimers(); let rejectStalledReplay = (_error: Error) => {}; diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index fbbb6a98133..a5f55fdf47a 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -19,8 +19,9 @@ import type { NativeSessionBackend, } from "./contracts/native-session-backend.js"; import type { PersistedNativeSession } from "./contracts/native-session-backend.js"; -import type { HarnessThreadGoal } from "./contracts/harness-driver.js"; +import { PROVIDER_STATE_INCOMPATIBLE, type HarnessThreadGoal } from "./contracts/harness-driver.js"; import { + NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionCloseUnrecoverableError, NativeSessionCleanupQuarantinedError, @@ -2164,6 +2165,9 @@ export async function executeNativeSession( }; if (!recovery.recovered || !recovery.session) { if (!replacementAllowed) { + if (recovery.failureClass === PROVIDER_STATE_INCOMPATIBLE) { + throw new NativeProviderStateIncompatibleError(recovery.reason ?? "unknown"); + } throw new Error( `native_session_recovery_failed: ${recovery.reason ?? "unknown"}`, ); diff --git a/packages/paperclip-runner/test/support/runnerd-transport.ts b/packages/paperclip-runner/test/support/runnerd-transport.ts new file mode 100644 index 00000000000..171414f2415 --- /dev/null +++ b/packages/paperclip-runner/test/support/runnerd-transport.ts @@ -0,0 +1,88 @@ +import { join, resolve } from "node:path"; +import { expect } from "vitest"; +import { + NATIVE_RUNTIME_ASSET_SCHEMA, + PAPERCLIP_EXECUTION_PROMPT, + PAPERCLIP_EXECUTION_PROMPT_REVISION, + canonicalNativeRuntimeContextDigest, + nativeRuntimePromptDigest, + type NativeRuntimeContextSnapshot, +} from "../../src/contracts/runtime-context.js"; +import { defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "../../src/live/runnerd-codex-transport.js"; + +export const defaultCapabilityRunnerdBinary = () => + process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? + qualifiedCapabilityRunnerdBinary(); + +export async function expectTurnStarted( + notifications: AsyncIterator<{ method: string }>, +) { + for (let index = 0; index < 32; index += 1) { + const next = await notifications.next(); + expect(next.done).not.toBe(true); + if (next.value.method === "turn/started") return; + // Startup ownership and capability facts can precede the active turn. + expect(next.value.method).toBe("paperclip/canonicalProviderEvent"); + } + throw new Error( + "provider turn did not start within the bounded notification prefix", + ); +} + +export const fakeCodex = resolve( + import.meta.dirname, + "../../runner/target/debug/fake-codex-app-server", +); + +export function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { + return [ + "--state-file", + join(stateDirectory, "fake-codex-state.json"), + ...args, + ]; +} + +export function assignedRuntimeContext( + skillRoot: string, + instructionRoot: string, +): NativeRuntimeContextSnapshot { + const digest = "0".repeat(64); + const value = { + prompt: { + revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, + text: PAPERCLIP_EXECUTION_PROMPT, + digest: nativeRuntimePromptDigest(), + }, + instructions: { + entryPath: "AGENTS.md", + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest, + manifestDigest: digest, + rootPath: instructionRoot, + fileCount: 1, + totalBytes: 1, + }, + }, + skills: [ + { + key: "company/assigned", + runtimeName: "assigned", + versionId: "version-1", + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest, + manifestDigest: digest, + rootPath: skillRoot, + fileCount: 1, + totalBytes: 1, + }, + }, + ], + mcp: { assignmentSetId: "assigned", digest, bindingId: "binding" }, + } satisfies Omit; + return { + ...value, + aggregateDigest: canonicalNativeRuntimeContextDigest(value), + }; +} diff --git a/packages/paperclip-runner/vitest.config.ts b/packages/paperclip-runner/vitest.config.ts index 3514fa1ea01..4c5817aec31 100644 --- a/packages/paperclip-runner/vitest.config.ts +++ b/packages/paperclip-runner/vitest.config.ts @@ -1,7 +1,9 @@ +import { availableParallelism } from "node:os"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { configDefaults, defineConfig } from "vitest/config"; +import { RunnerSequencer } from "./scripts/vitest-sequencer.mjs"; const packageRoot = fileURLToPath(new URL(".", import.meta.url)); @@ -17,6 +19,8 @@ export default defineConfig({ ? { reporters: [...configDefaults.reporters, resolve(packageRoot, "../../scripts/ci-profile/vitest-reporter.mjs")] } : {}), environment: "node", + ...(process.env.GITHUB_ACTIONS === "true" ? { maxWorkers: Math.min(4, availableParallelism()) } : {}), + sequence: { sequencer: RunnerSequencer }, include: [ "src/**/*.test.ts", "src/**/*.test.tsx", @@ -27,6 +31,7 @@ export default defineConfig({ "examples/scenario-explorer/src/**/*.test.tsx", "scripts/local-runner-browser-server.test.mjs", "scripts/live-console-browser-server.test.mjs", + "scripts/vitest-sequencer.test.mjs", ], }, }); diff --git a/packages/shared/src/telemetry/README.md b/packages/shared/src/telemetry/README.md index da0403c260b..e792827b9f1 100644 --- a/packages/shared/src/telemetry/README.md +++ b/packages/shared/src/telemetry/README.md @@ -65,6 +65,24 @@ provider-supplied error name that may contain a credential. This is a privacy boundary, not enum canonicalization. Test the value reaching the telemetry helper as well as the separate crash-reporting and local logging sinks. +## Server Startup Privacy + +The startup helper emits the raw Node.js operating system family once when an +enabled server telemetry client initializes. It omits OS versions, CPU +architecture, hostnames, and other machine details. The usual telemetry opt-out +and CI suppression apply before client initialization. If state storage fails +at startup, the server disables telemetry for that process and continues to run. +The warning contains no storage path or underlying error. Container and WSL servers +report their runtime platform; this does not identify the underlying physical +host's OS. Count distinct installation identifiers by platform to compare +installations, since restart event counts also reflect restart frequency. + +Contract syncs may also register existing proposals. Remove each adopted +proposal marker and update its helper and transport tests in the same change. +The connector helpers preserve raw enum values and send only catalog connector +keys and the approved lifecycle and invocation dimensions. Connection names, +credentials, tool arguments, and tool results stay out of those events. + ## Interaction Resolver Events `interaction.created` records the interaction kind and whether the create diff --git a/packages/shared/src/telemetry/proposed-connector-events.test.ts b/packages/shared/src/telemetry/connector-events.test.ts similarity index 75% rename from packages/shared/src/telemetry/proposed-connector-events.test.ts rename to packages/shared/src/telemetry/connector-events.test.ts index a8a6826e9de..c060017c7a6 100644 --- a/packages/shared/src/telemetry/proposed-connector-events.test.ts +++ b/packages/shared/src/telemetry/connector-events.test.ts @@ -28,7 +28,7 @@ function makeClient(config?: { enabled?: boolean }) { }; } -function trackAllProposedConnectorEvents(client: TelemetryClient) { +function trackAllConnectorEvents(client: TelemetryClient) { trackConnectionCreated(client, { connector_key: "github", transport: "mcp_remote", @@ -56,28 +56,33 @@ function trackAllProposedConnectorEvents(client: TelemetryClient) { }); } -describe("proposed connector events against the real TelemetryClient", () => { +describe("registered connector events against the real TelemetryClient", () => { afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); + vi.unstubAllGlobals(); }); - it("cannot queue or send: state stays untouched and nothing hits the wire", async () => { + it("queues and sends the three registered connector events", async () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); const { client, stateFactory } = makeClient(); - trackAllProposedConnectorEvents(client); + trackAllConnectorEvents(client); await client.flush(); - expect(stateFactory).not.toHaveBeenCalled(); - expect(fetch).not.toHaveBeenCalled(); + expect(stateFactory).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledTimes(1); + const body = JSON.parse(String(vi.mocked(fetch).mock.calls[0]?.[1]?.body)); + expect(body.events.map((event: { name: string }) => event.name)).toEqual([ + "connection.created", "connection.updated", "connection.invoked", + ]); }); - it("differential control: a registered event through the same client does send", async () => { + it("batches connector events with other registered events", async () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); const { client } = makeClient(); - trackAllProposedConnectorEvents(client); + trackAllConnectorEvents(client); client.track("project.created", {}); await client.flush(); @@ -86,15 +91,18 @@ describe("proposed connector events against the real TelemetryClient", () => { String((vi.mocked(fetch).mock.calls[0]?.[1] as RequestInit).body), ); expect(body.events).toEqual([ + expect.objectContaining({ name: "connection.created" }), + expect.objectContaining({ name: "connection.updated" }), + expect.objectContaining({ name: "connection.invoked" }), expect.objectContaining({ name: "project.created" }), ]); }); - it("reports the three connector event names as unregistered", () => { + it("reports the three connector event names as registered", () => { const { client } = makeClient(); - expect(client.isRegisteredEventName("connection.created")).toBe(false); - expect(client.isRegisteredEventName("connection.updated")).toBe(false); - expect(client.isRegisteredEventName("connection.invoked")).toBe(false); + expect(client.isRegisteredEventName("connection.created")).toBe(true); + expect(client.isRegisteredEventName("connection.updated")).toBe(true); + expect(client.isRegisteredEventName("connection.invoked")).toBe(true); expect(client.isRegisteredEventName("project.created")).toBe(true); }); @@ -102,6 +110,7 @@ describe("proposed connector events against the real TelemetryClient", () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); const { client, stateFactory } = makeClient({ enabled: false }); + trackAllConnectorEvents(client); client.track("project.created", {}); await client.flush(); diff --git a/packages/shared/src/telemetry/events.ts b/packages/shared/src/telemetry/events.ts index 71f78bb2f3f..78f2031313f 100644 --- a/packages/shared/src/telemetry/events.ts +++ b/packages/shared/src/telemetry/events.ts @@ -156,70 +156,69 @@ export function trackAgentTaskRun( export function trackConnectionCreated( client: TelemetryClient, dims: { - connector_key: RawDimension<"custom">; - transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">; - auth_kind: RawDimension<"oauth" | "api_key" | "none">; - setup_flow: RawDimension<"gallery" | "api" | "example">; - status: RawDimension<"draft" | "active" | "disabled" | "archived">; + connector_key: string; + transport: RawDimension; + auth_kind: RawDimension; + setup_flow: RawDimension; + status: RawDimension; enabled: boolean; }, ): void { - client.track( - // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure which catalog connectors installations create connections for - "connection.created", - dims, - ); + client.track("connection.created", { + ...dims, + transport: asEventDimension(dims.transport), + auth_kind: asEventDimension(dims.auth_kind), + setup_flow: asEventDimension(dims.setup_flow), + status: asEventDimension(dims.status), + }); } export function trackConnectionUpdated( client: TelemetryClient, dims: { - connector_key: RawDimension<"custom">; - transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">; - auth_kind: RawDimension<"oauth" | "api_key" | "none">; - change_source: RawDimension< - | "api" - | "gallery" - | "oauth_callback" - | "credential_refresh" - | "archive" - | "example" - >; - previous_status: RawDimension<"draft" | "active" | "disabled" | "archived">; - status: RawDimension<"draft" | "active" | "disabled" | "archived">; + connector_key: string; + transport: RawDimension; + auth_kind: RawDimension; + change_source: RawDimension; + previous_status: RawDimension; + status: RawDimension; previous_enabled: boolean; enabled: boolean; }, ): void { - client.track( - // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure connector lifecycle transitions (configured, paused, archived) after creation - "connection.updated", - dims, - ); + client.track("connection.updated", { + ...dims, + transport: asEventDimension(dims.transport), + auth_kind: asEventDimension(dims.auth_kind), + change_source: asEventDimension(dims.change_source), + previous_status: asEventDimension(dims.previous_status), + status: asEventDimension(dims.status), + }); } export function trackConnectionInvoked( client: TelemetryClient, dims: { - connector_key: RawDimension<"custom">; - transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">; - status: RawDimension< - | "succeeded" - | "failed" - | "denied" - | "cancelled" - | "timed_out" - | "rate_limited" - >; - origin: RawDimension<"setup_test" | "agent" | "user" | "system" | "plugin">; + connector_key: string; + transport: RawDimension; + status: RawDimension; + origin: RawDimension; duration_seconds?: number; }, ): void { - client.track( - // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure whether connected connectors are successfully used and where invocations fail; records completed invocation attempts and their terminal status, never invocation starts - "connection.invoked", - dims, - ); + client.track("connection.invoked", { + ...dims, + transport: asEventDimension(dims.transport), + status: asEventDimension(dims.status), + origin: asEventDimension(dims.origin), + }); +} + +export function trackServerStarted( + client: TelemetryClient, + dims: { os_platform: RawDimension }, +): void { + client.track("server.started", { os_platform: asEventDimension(dims.os_platform) }); } export function trackErrorHandlerCrash( diff --git a/packages/shared/src/telemetry/generated/paperclip-telemetry.ts b/packages/shared/src/telemetry/generated/paperclip-telemetry.ts index 0502b6e2a1c..b44d37d3428 100644 --- a/packages/shared/src/telemetry/generated/paperclip-telemetry.ts +++ b/packages/shared/src/telemetry/generated/paperclip-telemetry.ts @@ -37,6 +37,34 @@ source_ref?: string source_ref_hashed?: boolean } +export interface PaperclipConnectionCreatedDimensions { +auth_kind: ("oauth" | "api_key" | "none") +connector_key: string +enabled: boolean +setup_flow: ("gallery" | "api" | "example") +status: ("draft" | "active" | "disabled" | "archived") +transport: ("mcp_remote" | "rest_api" | "local_stdio") +} + +export interface PaperclipConnectionInvokedDimensions { +connector_key: string +duration_seconds?: number +origin: ("setup_test" | "agent" | "user" | "system" | "plugin") +status: ("succeeded" | "failed" | "denied" | "cancelled" | "timed_out" | "rate_limited") +transport: ("mcp_remote" | "rest_api" | "local_stdio") +} + +export interface PaperclipConnectionUpdatedDimensions { +auth_kind: ("oauth" | "api_key" | "none") +change_source: ("api" | "gallery" | "oauth_callback" | "credential_refresh" | "archive" | "example") +connector_key: string +enabled: boolean +previous_enabled: boolean +previous_status: ("draft" | "active" | "disabled" | "archived") +status: ("draft" | "active" | "disabled" | "archived") +transport: ("mcp_remote" | "rest_api" | "local_stdio") +} + export interface PaperclipErrorHandlerCrashDimensions { error_code: string } @@ -58,7 +86,6 @@ interaction_kind: ("suggest_tasks" | "ask_user_questions" | "request_confirmatio used_deprecated_resolver_policy_alias: boolean } -/** Emit only after the complete resolution transaction commits, including response provenance. */ export interface PaperclipInteractionResolvedDimensions { interaction_kind: ("suggest_tasks" | "ask_user_questions" | "request_confirmation" | "request_checkbox_confirmation" | "request_item_verdicts" | "other") status: ("accepted" | "rejected" | "answered" | "cancelled" | "expired" | "failed" | "other") @@ -95,9 +122,11 @@ source: ("schedule" | "manual" | "api" | "webhook" | "other") status: ("received" | "coalesced" | "skipped" | "issue_created" | "completed" | "failed" | "other") } +export interface PaperclipServerStartedDimensions { +os_platform: ("aix" | "android" | "darwin" | "freebsd" | "haiku" | "linux" | "openbsd" | "sunos" | "win32" | "cygwin" | "netbsd" | "other") +} + export interface PaperclipSkillImportedDimensions { -/** skill_ref may identify a known public catalog source only. Omit it for - * GitHub-synced sources, which may be private; never emit repository paths or names. */ source_type: ("local_path" | "github" | "url" | "catalog" | "skills_sh" | "unknown") skill_ref?: string } @@ -108,6 +137,9 @@ export type PaperclipEventName = | "agent.task_completed" | "agent.task_run" | "company.imported" + | "connection.created" + | "connection.invoked" + | "connection.updated" | "error.handler_crash" | "goal.created" | "install.completed" @@ -117,6 +149,7 @@ export type PaperclipEventName = | "project.created" | "routine.created" | "routine.run" + | "server.started" | "skill.imported"; export interface EventDimensionsMap { @@ -125,6 +158,9 @@ export interface EventDimensionsMap { "agent.task_completed": PaperclipAgentTaskCompletedDimensions; "agent.task_run": PaperclipAgentTaskRunDimensions; "company.imported": PaperclipCompanyImportedDimensions; + "connection.created": PaperclipConnectionCreatedDimensions; + "connection.invoked": PaperclipConnectionInvokedDimensions; + "connection.updated": PaperclipConnectionUpdatedDimensions; "error.handler_crash": PaperclipErrorHandlerCrashDimensions; "goal.created": PaperclipGoalCreatedDimensions; "install.completed": PaperclipInstallCompletedDimensions; @@ -134,6 +170,7 @@ export interface EventDimensionsMap { "project.created": PaperclipProjectCreatedDimensions; "routine.created": PaperclipRoutineCreatedDimensions; "routine.run": PaperclipRoutineRunDimensions; + "server.started": PaperclipServerStartedDimensions; "skill.imported": PaperclipSkillImportedDimensions; } @@ -143,6 +180,9 @@ export const PAPERCLIP_EVENTS = { "agent.task_completed": "agent.task_completed", "agent.task_run": "agent.task_run", "company.imported": "company.imported", + "connection.created": "connection.created", + "connection.invoked": "connection.invoked", + "connection.updated": "connection.updated", "error.handler_crash": "error.handler_crash", "goal.created": "goal.created", "install.completed": "install.completed", @@ -152,6 +192,7 @@ export const PAPERCLIP_EVENTS = { "project.created": "project.created", "routine.created": "routine.created", "routine.run": "routine.run", + "server.started": "server.started", "skill.imported": "skill.imported", } as const; @@ -277,6 +318,83 @@ export const PAPERCLIP_ENUM_DESCRIPTIONS = { "unknown": "Source type could not be classified." } }, + "connection.created": { + "auth_kind": { + "oauth": "Connection authenticates with an OAuth flow.", + "api_key": "Connection authenticates with a stored API key.", + "none": "Connection requires no authentication." + }, + "setup_flow": { + "gallery": "Connection was created through the app-gallery setup flow.", + "api": "Connection was created through the connections API.", + "example": "Connection was created or upserted as a bundled example connection." + }, + "status": { + "draft": "Connection row exists but setup has not produced a usable configuration yet.", + "active": "Connection is fully configured and available for use.", + "disabled": "Connection is disabled and excluded from tool use.", + "archived": "Connection is archived and excluded from tool use." + }, + "transport": { + "mcp_remote": "Connection talks to a remote MCP server over HTTP.", + "rest_api": "Connection calls the provider's REST API directly.", + "local_stdio": "Connection runs a local MCP server over stdio." + } + }, + "connection.invoked": { + "origin": { + "setup_test": "A human ran the connection's setup test, identified by durable invocation columns: user actor with no run, issue, or gateway reference and a connection id.", + "agent": "An agent run invoked the tool.", + "user": "A human invoked the tool outside the setup-test shape.", + "system": "A system-initiated process invoked the tool.", + "plugin": "A plugin invoked the tool." + }, + "status": { + "succeeded": "The tool invocation completed successfully.", + "failed": "The tool invocation reached a terminal failure.", + "denied": "The tool invocation was denied by policy or by a human approval decision.", + "cancelled": "The tool invocation was cancelled before completion. Declared in the application status enum; no current application code path writes this value.", + "timed_out": "The tool invocation exceeded its time limit.", + "rate_limited": "The tool invocation was rejected by rate limiting." + }, + "transport": { + "mcp_remote": "Connection talks to a remote MCP server over HTTP.", + "rest_api": "Connection calls the provider's REST API directly.", + "local_stdio": "Connection runs a local MCP server over stdio." + } + }, + "connection.updated": { + "auth_kind": { + "oauth": "Connection authenticates with an OAuth flow.", + "api_key": "Connection authenticates with a stored API key.", + "none": "Connection requires no authentication." + }, + "change_source": { + "api": "A connections API update changed the persisted lifecycle state.", + "gallery": "An app-gallery setup step (finish, OAuth access finalization, or reconnect; Apps gallery and inline task cards) changed the persisted lifecycle state.", + "oauth_callback": "An OAuth callback completion (provider OAuth, Paperclip Cloud connector, or Vercel Connect) changed the persisted lifecycle state.", + "credential_refresh": "A credential refresh changed the persisted lifecycle state.", + "archive": "An archive operation changed the persisted lifecycle state.", + "example": "An example-connection upsert changed the persisted lifecycle state." + }, + "previous_status": { + "draft": "Connection row exists but setup has not produced a usable configuration yet.", + "active": "Connection is fully configured and available for use.", + "disabled": "Connection is disabled and excluded from tool use.", + "archived": "Connection is archived and excluded from tool use." + }, + "status": { + "draft": "Connection row exists but setup has not produced a usable configuration yet.", + "active": "Connection is fully configured and available for use.", + "disabled": "Connection is disabled and excluded from tool use.", + "archived": "Connection is archived and excluded from tool use." + }, + "transport": { + "mcp_remote": "Connection talks to a remote MCP server over HTTP.", + "rest_api": "Connection calls the provider's REST API directly.", + "local_stdio": "Connection runs a local MCP server over stdio." + } + }, "goal.created": { "goal_level": { "company": "Goal applies at company scope.", @@ -402,6 +520,22 @@ export const PAPERCLIP_ENUM_DESCRIPTIONS = { "other": "Fallback when the status is unknown or not represented by the tracked enum." } }, + "server.started": { + "os_platform": { + "aix": "Server host runs IBM AIX.", + "android": "Server host runs Android.", + "darwin": "Server host runs macOS.", + "freebsd": "Server host runs FreeBSD.", + "haiku": "Server host runs Haiku.", + "linux": "Server host runs Linux, including WSL and container hosts.", + "openbsd": "Server host runs OpenBSD.", + "sunos": "Server host runs SunOS, Solaris, or illumos.", + "win32": "Server host runs Windows, both 32-bit and 64-bit.", + "cygwin": "Server host runs under Cygwin on Windows.", + "netbsd": "Server host runs NetBSD.", + "other": "Fallback when the platform is unknown or not represented by the tracked enum." + } + }, "skill.imported": { "source_type": { "local_path": "Import source came from a filesystem path on the operator's machine.", diff --git a/packages/shared/src/telemetry/index.ts b/packages/shared/src/telemetry/index.ts index 61f945ad834..d6436ffd5ba 100644 --- a/packages/shared/src/telemetry/index.ts +++ b/packages/shared/src/telemetry/index.ts @@ -21,6 +21,7 @@ export { trackConnectionCreated, trackConnectionUpdated, trackConnectionInvoked, + trackServerStarted, } from "./events.js"; export type { TelemetryConfig, diff --git a/packages/shared/src/telemetry/retention.ts b/packages/shared/src/telemetry/retention.ts index 543e1085298..023004d4e49 100644 --- a/packages/shared/src/telemetry/retention.ts +++ b/packages/shared/src/telemetry/retention.ts @@ -47,4 +47,5 @@ export const EVENT_RETENTION_CLASS: Partial> = { "codex.credential_health": "operational_enum_count", "interaction.created": "operational_enum_count", "interaction.resolved": "operational_enum_count", + "server.started": "operational_enum_count", }; diff --git a/packages/shared/src/telemetry/server-started-event.test.ts b/packages/shared/src/telemetry/server-started-event.test.ts new file mode 100644 index 00000000000..3d0e0e2289b --- /dev/null +++ b/packages/shared/src/telemetry/server-started-event.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { TelemetryClient } from "./client.js"; +import { trackServerStarted } from "./events.js"; +import type { TelemetryState } from "./types.js"; + +const TEST_STATE: TelemetryState = { + installId: "test-install", + salt: "test-salt", + createdAt: "2026-01-01T00:00:00Z", + firstSeenVersion: "0.0.0", +}; + +function makeClient(enabled = true) { + const stateFactory = vi.fn(() => TEST_STATE); + return { + client: new TelemetryClient( + { enabled, endpoint: "http://localhost:9999/ingest" }, + stateFactory, + "0.0.0-test", + () => 0.5, + ), + stateFactory, + }; +} + +describe("registered server.started event", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + it("passes the raw process.platform value through without normalization", () => { + const { client } = makeClient(); + const track = vi.spyOn(client, "track"); + + trackServerStarted(client, { os_platform: "win32" }); + trackServerStarted(client, { os_platform: "some-future-platform" }); + + expect(track.mock.calls).toEqual([ + ["server.started", { os_platform: "win32" }], + ["server.started", { os_platform: "some-future-platform" }], + ]); + }); + + it("sends the registered event with only the raw OS family", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); + const { client, stateFactory } = makeClient(); + + expect(client.isRegisteredEventName("server.started")).toBe(true); + trackServerStarted(client, { os_platform: process.platform }); + await client.flush(); + + expect(stateFactory).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledTimes(1); + const body = JSON.parse(String(vi.mocked(fetch).mock.calls[0]?.[1]?.body)); + expect(body.events).toEqual([ + { name: "server.started", occurredAt: expect.any(String), dimensions: { os_platform: process.platform } }, + ]); + }); + + it("leaves state and network untouched when disabled", async () => { + vi.stubGlobal("fetch", vi.fn()); + const { client, stateFactory } = makeClient(false); + trackServerStarted(client, { os_platform: process.platform }); + await client.flush(); + expect(stateFactory).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 5167a32193f..a507f3034f8 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -61,6 +61,39 @@ test("Docker Hub mirrors apply to both image builders without widening publicati "the registry-only manifest merge does not need a Docker Hub mirror"); }); +test("source proof gates publication without occupying a publication runner", () => { + const workflow = readWorkflow("docker.yml"); + const observer = workflow.split(" source_verification:\n")[1].split(" reuse_canary_image:")[0]; + const gate = workflow.split(" publication_verification:\n")[1].split(" merge-and-push:")[0]; + const publisher = workflow.split(" merge-and-push:\n")[1].split(" promote_canary_channel:")[0]; + assert.match(observer, /verified: \$\{\{ steps.watch.outputs.verified \}\}/); + assert.match(gate, /run: node scripts\/watch-release-source.mjs --require-proof/); + assert.match(gate, /actions: write # restart only exact-source producers deferred before admission/); + assert.match(gate, /needs: \[build-and-push, source_verification\]/); + assert.match(publisher, /needs: \[build-and-push, source_verification, publication_verification\]/); + assert.doesNotMatch(publisher, /--require-proof|actions: write/); + assert.ok(publisher.indexOf("run: node scripts/watch-release-source.mjs --check-proof") < publisher.indexOf("- name: Create manifest list and push")); + assert.match(publisher, /actions: read/); + assert.match(publisher, /timeout-minutes: 20/); + const expression = publisher.match(/^ if: \$\{\{ (.+) \}\}$/m)[1].replaceAll("needs.build-and-push", "needs.build"); + for (const [ref, observed, fallback, expected, repository = "paperclipai/paperclip", repository_id = "1170821064"] of [ + ["refs/heads/master", "true", "skipped", true], + ["refs/heads/master", "false", "skipped", false], + ["refs/heads/master", undefined, "skipped", false], + ["refs/heads/master", "false", "success", true], + ["refs/heads/master", "false", "failure", false], + ["refs/tags/v1", undefined, "skipped", true], + ["refs/heads/master", undefined, "skipped", true, "fork/paperclip", "123"], + ["refs/heads/master", undefined, "skipped", true, "paperclipai/paperclip", "123"], + ]) { + const canonical = repository === "paperclipai/paperclip" && repository_id === "1170821064" && ref === "refs/heads/master"; + assert.equal(runInNewContext(expression, { cancelled: () => false, github: { ref, repository, repository_id }, needs: { + build: { result: "success" }, source_verification: { result: canonical ? "success" : "skipped", outputs: { verified: observed } }, + publication_verification: { result: fallback }, + } }), expected); + } +}); + test("chaos verification isolates callers that verify the same source commit", () => { const chaosWorkflow = readWorkflow("runner-chaos-evals.yml"); const group = chaosWorkflow.match(/^ group: (.+)$/m)?.[1]; @@ -82,11 +115,11 @@ test("chaos verification isolates callers that verify the same source commit", ( assert.match(chaosWorkflow, /cancel-in-progress: true/); }); -test("canary reuses exact-source proof while stable keeps full verification", () => { +test("canary can admit deferred exact-source checks while stable keeps full verification", () => { const releaseWorkflow = readWorkflow("release.yml"); const canary = releaseWorkflow.match(/ verify_canary:\n[\s\S]*?(?=\n [a-z_]+:)/)[0]; - assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/); - assert.match(canary, /actions: read/); + assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.repository_id == '1170821064' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/); + assert.match(canary, /actions: write # restart only an exact-source push deferred before admission/); assert.match(canary, /ref: \$\{\{ github\.sha \}\}/); assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/); assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/); @@ -933,7 +966,9 @@ test('native dependency caches are target-bound and unmerged master callers cann const cache = producer.split(' - name: Cache native Rust dependencies\n')[1].split('\n - ')[0]; assert.match(cache, /Swatinem\/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6/); assert.match(cache, /workspaces: \$\{\{ steps.native_rust_workspace.outputs.path \}\} -> target/); - assert.match(cache, /shared-key: release-native-v2-\$\{\{ matrix.host \}\}-\$\{\{ matrix.target \}\}-\$\{\{ hashFiles\('packages\/paperclip-runner\/protocol\/\*\*', 'packages\/paperclip-runner\/src\/\*\*', 'packages\/paperclip-runner\/\*.json'\) \}\}/); + const key = cache.match(/^\s*shared-key: (.+)$/m)?.[1]; + assert.ok(key?.startsWith("release-native-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', ")); + assert.ok(key?.endsWith("-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }}")); for (const option of ['cache-workspace-crates', 'cache-bin', 'cache-on-failure']) { assert.match(cache, new RegExp(option + ': false')); } diff --git a/scripts/ci-dependency-cache.mjs b/scripts/ci-dependency-cache.mjs index b447c01d362..cca636cfb45 100644 --- a/scripts/ci-dependency-cache.mjs +++ b/scripts/ci-dependency-cache.mjs @@ -7,11 +7,19 @@ import { readPluginsUnder } from "./link-plugin-dev-sdk.mjs"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const postinstallHooks = new Map([ + [".", "node scripts/link-plugin-dev-sdk.mjs"], + ["server", "node scripts/install-copilot-assets.mjs"], + ["packages/paperclip-runner", "node scripts/install-copilot-assets.mjs"], +]); + export function cachePlan({ files, read, scope, packageDirs }) { const dependencyInputs = files.filter((file) => /(^|\/)package\.json$/.test(file) || ["pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc", ".pnpmfile.cjs", "pnpmfile.cjs"].includes(file) || file.startsWith("scripts/") + || file.startsWith("server/scripts/") + || file.startsWith("packages/paperclip-runner/scripts/") || file.startsWith("patches/")); const digest = (inputs, salt) => { const hash = createHash("sha256").update(JSON.stringify(salt)); @@ -28,8 +36,8 @@ export function cachePlan({ files, read, scope, packageDirs }) { if (manifest === null) return false; const scripts = JSON.parse(manifest).scripts ?? {}; return ['preinstall', 'install', 'postinstall', 'prepare'].every((hook) => - scripts[hook] === undefined || (directory === '.' && hook === 'postinstall' - && scripts[hook] === 'node scripts/link-plugin-dev-sdk.mjs')); + scripts[hook] === undefined || (hook === 'postinstall' + && scripts[hook] === postinstallHooks.get(directory))); }); const buildInputs = files.filter((file) => file.startsWith("packages/shared/") || file.startsWith("packages/plugins/sdk/") diff --git a/scripts/cloud-release-admission.mjs b/scripts/cloud-release-admission.mjs new file mode 100644 index 00000000000..31ca67fa9db --- /dev/null +++ b/scripts/cloud-release-admission.mjs @@ -0,0 +1,123 @@ +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { createActionsReader } from "./github-actions-reader.mjs"; + +const repository = "paperclipai/paperclip"; +const repositoryId = 1170821064; +const workflows = new Set(["docker.yml", "release.yml", "cloud-readiness.yml", "cloud-migrator-artifacts.yml"]); +export const sourceAdmissionJob = "Cloud candidate admission v1"; +export const migratorAdmissionJob = "Cloud image candidate admission v1"; + +function assertSha(sha) { + if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("Admission requires a full source SHA."); +} + +function trusted(run, sha, workflow) { + return run.workflow_id === workflow.id && run.path === workflow.path && run.head_sha === sha && + run.repository?.id === repositoryId && run.head_repository?.id === repositoryId && + run.repository?.full_name === repository && run.head_repository?.full_name === repository && + run.head_branch === "master" && run.event === "push" && + Number.isSafeInteger(run.id) && run.id > 0 && Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0; +} + +async function producer(file, sha, api, status) { + assertSha(sha); + if (!workflows.has(file)) throw new Error("Unknown release workflow."); + const workflow = await api(`/repos/${repository}/actions/workflows/${file}`); + if (workflow.path !== `.github/workflows/${file}` || !Number.isSafeInteger(workflow.id) || workflow.id < 1) throw new Error("Release workflow identity mismatch."); + const list = await api(`/repos/${repository}/actions/workflows/${workflow.id}/runs?branch=master&event=push&head_sha=${sha}&per_page=100${status ? `&status=${status}` : ""}`); + if (!Array.isArray(list.workflow_runs) || !Number.isSafeInteger(list.total_count) || list.total_count < 0 || + list.total_count > 100 || list.workflow_runs.length !== list.total_count) throw new Error("Incomplete release workflow listing."); + const run = list.workflow_runs.filter(run => trusted(run, sha, workflow) && (!status || run.status === status)).sort((a, b) => b.id - a.id)[0]; + return { workflow, run }; +} + +// A retry is an explicit request for this exact immutable source. Normal push +// work starts only for a candidate admitted by an image build or npm canary. +export async function releaseAdmitted(sha, { api, event = "push", attempt = 1, imageOnly = false }) { + assertSha(sha); + if (!Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release attempt."); + if (!["push", "workflow_dispatch"].includes(event)) return false; + if (event === "workflow_dispatch" || (Number.isSafeInteger(attempt) && attempt > 1)) return true; + for (const file of imageOnly ? ["docker.yml"] : ["docker.yml", "release.yml"]) { + const { workflow, run } = await producer(file, sha, api, "in_progress"); + if (!run) continue; + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (current.id === run.id && trusted(current, sha, workflow) && current.run_attempt === run.run_attempt && current.status === "in_progress") return true; + } + return false; +} + +export function deferredProducer(jobs, run, admissionJob) { + const admission = jobs.filter(job => job.name === admissionJob); + const remaining = jobs.filter(job => job.name !== admissionJob); + return run.status === "completed" && run.conclusion === "success" && admission.length === 1 && admission[0].status === "completed" && admission[0].conclusion === "success" && remaining.length > 0 && + remaining.every(job => job.status === "completed" && job.conclusion === "skipped") && + jobs.every(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === run.head_sha); +} + +// Only skipped admission is restartable here. A failed check or build never +// becomes an automatic retry, and every write revalidates canonical identity. +export async function requestDeferredProducer({ file, sha, admissionJob, api, rerun, requested = new Set() }) { + if (({ "cloud-readiness.yml": sourceAdmissionJob, "cloud-migrator-artifacts.yml": migratorAdmissionJob })[file] !== admissionJob) throw new Error("Unknown deferred producer contract."); + const { workflow, run } = await producer(file, sha, api); + if (!run) return "pending"; + const key = `${run.id}/${run.run_attempt}`; + if (requested.has(key)) return "pending"; + const jobs = []; + let total; + for (let page = 1; page <= 10; page++) { + const batch = await api(`/repos/${repository}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`); + if (!Array.isArray(batch.jobs) || !Number.isSafeInteger(batch.total_count) || batch.total_count < 0) throw new Error("Invalid release producer job list."); + if ((total !== undefined && total !== batch.total_count) || + batch.jobs.length !== Math.min(100, Math.max(0, batch.total_count - (page - 1) * 100))) throw new Error("Incomplete release producer job list."); + total = batch.total_count; + jobs.push(...batch.jobs); + if (page * 100 >= total) break; + if (page === 10) throw new Error("Incomplete release producer job list."); + } + if (run.status !== "completed") { + const current = jobs.filter(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === sha); + // Admission itself succeeds for both outcomes. Only an actual build is + // evidence that its output admitted work; queued jobs may still be skipped. + return current.some(job => job.name !== admissionJob && job.labels?.length > 0 && ["in_progress", "completed"].includes(job.status) && job.conclusion !== "skipped") + ? "admitted" : "pending"; + } + if (!deferredProducer(jobs, run, admissionJob)) return "finished"; + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (current.id !== run.id || !trusted(current, sha, workflow) || current.run_attempt !== run.run_attempt || current.status !== "completed" || current.conclusion !== "success") return "pending"; + try { + await rerun(`/repos/${repository}/actions/runs/${run.id}/rerun`); + } catch (error) { + // Docker and npm may select the same source at the same time. A rejected + // duplicate write is harmless only when a fresh read proves a retry began. + const latest = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (latest.id !== run.id || !trusted(latest, sha, workflow)) throw error; + if (!(latest.run_attempt > run.run_attempt || (latest.run_attempt === run.run_attempt && + ["queued", "in_progress", "requested", "pending", "waiting"].includes(latest.status)))) { + // GitHub may reject a duplicate before its read API shows the retry. + // Leave it pending for the next poll; this is not proof of admission. + if ([409, 422].includes(error.status) && latest.run_attempt === run.run_attempt && latest.status === "completed" && latest.conclusion === "success") return "pending"; + throw error; + } + } + requested.add(key); + return "pending"; +} + +export function createReleaseRerunner(token, fetchImpl = fetch) { + return async path => { + if (!/^\/repos\/paperclipai\/paperclip\/actions\/runs\/[1-9]\d*\/rerun$/.test(path)) throw new Error("Invalid release retry path."); + const response = await fetchImpl(`https://api.github.com${path}`, { method: "POST", redirect: "error", + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw Object.assign(new Error(`Could not admit deferred release work: HTTP ${response.status}.`), { status: response.status }); + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + if (process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master" || !process.env.GITHUB_TOKEN) throw new Error("Canonical master admission context required."); + const admitted = await releaseAdmitted(process.env.GITHUB_SHA, { api: createActionsReader({ token: process.env.GITHUB_TOKEN }), + event: process.env.GITHUB_EVENT_NAME, attempt: Number(process.env.GITHUB_RUN_ATTEMPT), imageOnly: process.argv[2] === "image-only" }); + console.log(admitted ? "Release candidate admitted." : "Full build deferred until a release consumer admits this source."); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `admitted=${admitted}\n`); +} diff --git a/scripts/cloud-source-verification.mjs b/scripts/cloud-source-verification.mjs index bd40e33cd3c..09c4687fed1 100644 --- a/scripts/cloud-source-verification.mjs +++ b/scripts/cloud-source-verification.mjs @@ -1,9 +1,19 @@ import { appendFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; +import { createActionsReader } from "./github-actions-reader.mjs"; +import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; const repository = "paperclipai/paperclip"; const workflowPath = ".github/workflows/cloud-readiness.yml"; export const sourceVerificationJob = "Cloud source verified v1"; +export class SourceVerificationFailed extends Error { + constructor(message, run) { + super(message); + this.runId = run.id; + this.attempt = run.run_attempt; + } +} +export class SourceVerificationDeferred extends Error {} function assertSha(sha) { if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("A full lowercase source SHA is required."); @@ -19,7 +29,7 @@ function trustedRun(run, sha, workflowId) { // Consume one versioned job, independent of image/migrator availability. A // failed image build must not invalidate source checks that already passed. -export async function readSourceVerification(sha, api) { +export async function readSourceVerification(sha, api, { failFast = false, onJobs } = {}) { assertSha(sha); const workflow = await api(`/repos/${repository}/actions/workflows/cloud-readiness.yml`); if (workflow.path !== workflowPath || !Number.isSafeInteger(workflow.id) || workflow.id < 1) { @@ -46,6 +56,17 @@ export async function readSourceVerification(sha, api) { } const matches = jobs.filter((job) => job.name === sourceVerificationJob); if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous."); + await onJobs?.(jobs.filter(job => job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt)); + if (failFast) { + const failure = jobs.find((job) => job.status === "completed" && + ["failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"].includes(job.conclusion) && + job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt); + if (failure) { + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined; + throw new SourceVerificationFailed(`Cloud source verification failed: ${failure.name} (${failure.conclusion}).`, run); + } + } const job = matches[0]; if (job?.status === "completed" && job.conclusion === "success" && job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt) { @@ -56,95 +77,33 @@ export async function readSourceVerification(sha, api) { return { sha, runId: run.id, attempt: run.run_attempt, jobId: job.id }; } if (job?.status === "completed" || run.status === "completed") { - throw new Error(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`); + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined; + if (job?.conclusion === "skipped" && current.status !== "completed") return undefined; + if (deferredProducer(jobs, current, sourceAdmissionJob)) throw new SourceVerificationDeferred("Source checks are deferred until release admission."); + throw new SourceVerificationFailed(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`, run); } return undefined; } -// Statuses that say "ask again", not "the answer is no". A release must not be -// blocked because GitHub returned a gateway error during a 45-minute poll. -const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); - -// A rate-limited read also says "ask again", and GitHub reports both primary -// and secondary rate limits as 403. Only the headers separate that from a -// token that may not read Actions, which must still fail at once. -function rateLimited(response) { - if (response.status !== 403) return false; - const header = (name) => response.headers?.get?.(name) ?? null; - return header("retry-after") !== null || header("x-ratelimit-remaining") === "0"; -} - -/** Milliseconds from a Retry-After header, when it carries a sane delay. */ -function retryAfterMs(response, capMs) { - const value = Number(response?.headers?.get?.("retry-after")); - if (!Number.isFinite(value) || value <= 0) return null; - return Math.min(value * 1_000, capMs); -} - -/** - * The GitHub Actions read used by the polling below, with transient transport - * failures retried: network errors, the statuses above, rate-limited 403s, and - * a body that fails while it is being read. Any other non-OK status throws on - * the first response, because waiting out a wrong token only delays the news. - * - * `deadlineAt` bounds retries by the caller's own polling deadline, so a read - * cannot extend the wait past the timeout it belongs to. - */ -export function createActionsReader({ - token, fetchImpl = fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), - attempts = 4, backoffMs = 1_000, log = console.log, - now = Date.now, deadlineAt = () => Infinity, maxRetryAfterMs = 60_000, -} = {}) { - return async (path) => { - for (let attempt = 1; ; attempt += 1) { - // Retry only while both the attempt budget and the caller's deadline - // leave room for the wait this attempt would cost. - const waitMs = backoffMs * attempt; - const retryable = attempt < attempts && now() + waitMs < deadlineAt(); - const pause = (response) => sleep(retryAfterMs(response, maxRetryAfterMs) ?? waitMs); - let response; - try { - response = await fetchImpl(`https://api.github.com${path}`, { - headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, - signal: AbortSignal.timeout(30_000), redirect: "error", - }); - } catch (cause) { - if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); - log(`GitHub Actions read failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); - await pause(); - continue; - } - if (response.ok) { - try { - // A 200 whose body is truncated or undecodable is a transport - // failure like any other, so it belongs inside the retry. - return await response.json(); - } catch (cause) { - if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); - log(`GitHub Actions read body failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); - await pause(); - continue; - } - } - if (!retryable || !(TRANSIENT_READ_STATUSES.has(response.status) || rateLimited(response))) { - throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`); - } - log(`GitHub Actions read failed (HTTP ${response.status}); retrying (${attempt}/${attempts - 1}).`); - await pause(response); - } - }; -} +export { createActionsReader }; export async function waitForSourceVerification(sha, { api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), - timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log, + timeoutMs = 60 * 60_000, intervalMs = 30_000, log = console.log, rerun, } = {}) { assertSha(sha); const deadline = now() + timeoutMs; + const requested = new Set(); log(`Waiting for ${sourceVerificationJob} for ${sha}.`); while (now() < deadline) { - const result = await readSourceVerification(sha, api); - if (result) return result; + try { + const result = await readSourceVerification(sha, api); + if (result) return result; + } catch (error) { + if (!(error instanceof SourceVerificationDeferred) || !rerun) throw error; + await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested }); + } const remaining = deadline - now(); if (remaining > 0) await sleep(Math.min(intervalMs, remaining)); } @@ -156,10 +115,11 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required."); // One deadline for both layers: the reader stops retrying when the poll it // serves is out of time, instead of extending the wait past its timeout. - const timeoutMs = 45 * 60_000; + const timeoutMs = 60 * 60_000; const deadline = Date.now() + timeoutMs; const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline }); - const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs }); + const rerun = process.env.GITHUB_REPOSITORY === repository && process.env.GITHUB_REF === "refs/heads/master" ? createReleaseRerunner(process.env.GITHUB_TOKEN) : undefined; + const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs, rerun }); const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`; console.log(message); if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`); diff --git a/scripts/cloud-source-verification.test.mjs b/scripts/cloud-source-verification.test.mjs index 4a47abfd164..6e055607267 100644 --- a/scripts/cloud-source-verification.test.mjs +++ b/scripts/cloud-source-verification.test.mjs @@ -1,8 +1,68 @@ import assert from "node:assert/strict"; -import test from "node:test"; -import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs"; +import test, { describe } from "node:test"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; + +import { releaseAdmitted, deferredProducer, requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; + +import { cancelImageRun, runnerWaits, watchReleaseSource } from "./watch-release-source.mjs"; const sha = "a".repeat(40); + +function runAdmissionCli({ active = false, imageOnly = false, repositoryName = "paperclipai/paperclip" } = {}) { + const root = mkdtempSync(join(tmpdir(), "release-admission-cli-")); + try { + const preload = join(root, "github-fixture.mjs"); + const installFixture = (active) => { + const repository = { id: 1170821064, full_name: "paperclipai/paperclip" }; + const run = { id: 77, workflow_id: 21, path: ".github/workflows/docker.yml", head_sha: "a".repeat(40), + repository, head_repository: repository, head_branch: "master", event: "push", run_attempt: 1, status: "in_progress" }; + globalThis.fetch = async url => { + const parsed = new URL(url); + if (parsed.origin !== "https://api.github.com") throw new Error("Unexpected admission request origin"); + const path = parsed.pathname; + if (path.endsWith("/docker.yml")) return Response.json({ id: 21, path: ".github/workflows/docker.yml" }); + if (path.endsWith("/release.yml")) return Response.json({ id: 22, path: ".github/workflows/release.yml" }); + if (path.endsWith("/workflows/21/runs")) return Response.json({ total_count: active ? 1 : 0, workflow_runs: active ? [run] : [] }); + if (path.endsWith("/workflows/22/runs")) return Response.json({ total_count: 0, workflow_runs: [] }); + if (path.endsWith("/runs/77")) return Response.json(run); + throw new Error("Unexpected admission request path"); + }; + }; + writeFileSync(preload, "(" + installFixture.toString() + ")(" + JSON.stringify(active) + ");\n"); + const output = join(root, "output"); + const result = spawnSync(process.execPath, ["--import", preload, fileURLToPath(new URL("./cloud-release-admission.mjs", import.meta.url)), ...(imageOnly ? ["image-only"] : [])], { + encoding: "utf8", timeout: 5_000, + env: { ...process.env, GITHUB_REPOSITORY: repositoryName, GITHUB_REF: "refs/heads/master", GITHUB_SHA: "a".repeat(40), + GITHUB_EVENT_NAME: "push", GITHUB_RUN_ATTEMPT: "1", GITHUB_TOKEN: "fixture", GITHUB_OUTPUT: output }, + }); + return { ...result, output: result.status === 0 ? readFileSync(output, "utf8") : "" }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +describe("admission CLI startup", () => { + for (const imageOnly of [false, true]) { + for (const active of [false, true]) { + test("executes " + (imageOnly ? "image-only" : "source") + " admission with active consumer=" + active, () => { + const result = runAdmissionCli({ active, imageOnly }); + assert.equal(result.status, 0, result.stderr || result.error?.message); + assert.equal(result.output, "admitted=" + active + "\n"); + }); + } + } + test("rejects a noncanonical CLI context before discovering consumers", () => { + const result = runAdmissionCli({ repositoryName: "fork/paperclip" }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Canonical master admission context required/); + assert.equal(result.output, ""); + }); +}); const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" }; const baseRun = { id: 456, workflow_id: workflow.id, path: workflow.path, run_attempt: 2, @@ -62,7 +122,8 @@ test("job identities and terminal failures fail closed", async () => { { head_sha: "b".repeat(40) }, { run_id: 999 }, { run_attempt: 1 }, { conclusion: "failure" }, { conclusion: "cancelled" }, { conclusion: "skipped" }, ]) { - const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }] }); + const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }], + current: overrides.conclusion === "skipped" ? { ...baseRun, status: "completed", conclusion: "success" } : baseRun }); await assert.rejects(readSourceVerification(sha, api), /did not pass/); } }); @@ -231,3 +292,357 @@ test("retries stop at the caller's deadline rather than outliving the poll", asy assert.equal(seen.length, 1); assert.deepEqual(waits, []); }); + + +describe("image build source monitor", () => { + const sha = "a".repeat(40); + const runId = 50; + const attempt = 1; + const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" }; + const sourceRun = { id: 99, workflow_id: 123, path: workflow.path, run_attempt: 1, + repository: { full_name: "paperclipai/paperclip" }, head_repository: { full_name: "paperclipai/paperclip" }, + head_sha: sha, head_branch: "master", event: "push", status: "in_progress" }; + const imageRun = { id: runId, run_attempt: attempt, head_sha: sha, head_branch: "master", event: "push", + repository: { id: 1170821064 }, head_repository: { id: 1170821064 }, + path: ".github/workflows/docker.yml", status: "in_progress" }; + const proof = { id: 111, name: "Cloud source verified v1", run_id: 99, run_attempt: 1, + head_sha: sha, status: "completed", conclusion: "success" }; + + function fixture({ jobs = [proof], run = sourceRun, current = run, image = imageRun, imageJobs = [], missing = false } = {}) { + const cancellations = []; + const messages = []; + const api = async path => { + if (path.endsWith("/workflows/cloud-readiness.yml")) return workflow; + if (path.includes("/workflows/123/runs?")) return { workflow_runs: missing ? [] : [run], total_count: missing ? 0 : 1 }; + if (path.includes("/runs/99/attempts/1/jobs?")) return { jobs }; + if (path.endsWith("/runs/99")) return current; + if (path.includes(`/runs/${runId}/attempts/${attempt}/jobs?`)) return { jobs: imageJobs, total_count: imageJobs.length }; + if (path.endsWith(`/runs/${runId}`)) return image; + throw new Error(`Unexpected request: ${path}`); + }; + return { sha, runId, attempt, api, cancel: async path => cancellations.push(path), + cancellations, messages, log: message => messages.push(message), summary: async message => messages.push(message) }; + } + + test("successful exact-source proof permits publication without canceling image work", async () => { + const f = fixture(); + assert.equal((await watchReleaseSource(f)).jobId, proof.id); + assert.deepEqual(f.cancellations, []); + }); + + test("a required check failing cancels the same image attempt before siblings finish", async () => { + const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }] }); + await assert.rejects(watchReleaseSource(f), /General tests/); + assert.deepEqual(f.cancellations, ["/repos/paperclipai/paperclip/actions/runs/99/cancel", `/repos/paperclipai/paperclip/actions/runs/${runId}/cancel`]); + }); + + test("a superseded readiness attempt cancels image work with no deployable future", async () => { + const f = fixture({ jobs: [], run: { ...sourceRun, status: "completed", conclusion: "cancelled" } }); + await assert.rejects(watchReleaseSource(f), /did not pass/); + assert.equal(f.cancellations.length, 1); + }); + + test("a rerun racing a failed job is retried rather than canceling the image", async () => { + const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }], + current: { ...sourceRun, run_attempt: 2 } }); + let time = 0; + const api = f.api; + f.api = async path => time > 0 ? fixture().api(path) : api(path); + const result = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; } }); + assert.equal(result.jobId, proof.id); + assert.deepEqual(f.cancellations, []); + }); + + test("a transport failure does not issue a destructive cancellation", async () => { + const f = fixture(); + await assert.rejects(watchReleaseSource({ ...f, api: async () => { throw new Error("GitHub unavailable"); } }), /unavailable/); + assert.deepEqual(f.cancellations, []); + }); + + test("the observation window ends without cancelling healthy pending verification", async () => { + const f = fixture({ missing: true }); + let time = 0; + const proof = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; }, timeoutMs: 20 }); + assert.equal(proof, undefined); + assert.equal(time, 20); + assert.deepEqual(f.cancellations, []); + assert.ok(f.messages.some(message => message.includes("final publication still requires exact-source verification"))); + }); + + test("final publication requires proof even after the early observation window ends", async () => { + const f = fixture({ missing: true }); + let time = 0; + await assert.rejects(watchReleaseSource({ ...f, requireProof: true, now: () => time, + sleep: async ms => { time += ms; }, timeoutMs: 20 }), /before the publication deadline/); + assert.deepEqual(f.cancellations, []); + assert.equal((await watchReleaseSource({ ...fixture(), requireProof: true })).jobId, proof.id); + }); + + test("cancellation rejects another SHA, repository, ref, workflow, or attempt", async () => { + for (const change of [{ head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "pull_request" }, + { path: ".github/workflows/release.yml" }, { run_attempt: 2 }, { id: 51 }, + { repository: { id: 42 } }, { head_repository: { id: 42 } }]) { + const f = fixture({ image: { ...imageRun, ...change } }); + await assert.rejects(cancelImageRun(f), /different image workflow identity/); + assert.deepEqual(f.cancellations, []); + } + }); + + test("queue measurement does not count running time or skipped dependency jobs", () => { + const created_at = "2020-01-01T00:00:00Z"; + const started_at = "2020-01-01T00:00:20Z"; + const job = { id: 1, name: "amd64", labels: ["release"], created_at, started_at, runner_id: 10, status: "in_progress" }; + const now = Date.parse(created_at) + 120_000; + assert.deepEqual(runnerWaits([job, { ...job, status: "completed", conclusion: "skipped" }], now), [{ id: 1, job: "amd64", seconds: 20 }]); + assert.equal(runnerWaits([{ ...job, status: "queued", runner_id: 0 }], now)[0].seconds, 120); + }); + + test("a runner wait over 60 seconds produces an Actions warning and summary", async () => { + const f = fixture({ imageJobs: [{ id: 1, name: "amd64", labels: ["release"], status: "queued", runner_id: 0, created_at: "2020-01-01T00:00:00Z" }] }); + await watchReleaseSource({ ...f, now: () => Date.parse("2020-01-01T00:02:00Z") }); + assert.ok(f.messages.some(message => message.startsWith("::warning") && message.includes("120s"))); + assert.ok(f.messages.some(message => message.includes("Release runner wait: amd64"))); + }); +}); + +describe("release candidate admission", () => { + const repository = { id: 1170821064, full_name: "paperclipai/paperclip" }; + const ids = { "docker.yml": 101, "release.yml": 102, "cloud-readiness.yml": 123, "cloud-migrator-artifacts.yml": 124 }; + function run(file, change = {}) { + return { id: ids[file] + 1000, workflow_id: ids[file], path: `.github/workflows/${file}`, + repository, head_repository: repository, head_sha: sha, head_branch: "master", event: "push", + run_attempt: 1, status: "in_progress", conclusion: null, ...change }; + } + function job(run, name, conclusion = "skipped") { + return { id: 789, run_id: run.id, run_attempt: run.run_attempt, head_sha: run.head_sha, + name, status: "completed", conclusion }; + } + function fixture(file = "cloud-readiness.yml", change = {}) { + const producer = run(file, { status: "completed", conclusion: "success", ...change }); + const admission = file === "cloud-migrator-artifacts.yml" ? migratorAdmissionJob : sourceAdmissionJob; + const f = { runs: { [file]: [producer] }, jobs: { [producer.id]: [job(producer, admission, "success"), job(producer, sourceVerificationJob)] }, + current: {}, writes: [], calls: [], requested: new Set(), file, sha, admissionJob: admission }; + f.api = async path => { + f.calls.push(path); + const url = new URL(`https://api.github.com${path}`); + const fileMatch = path.match(/\/workflows\/([^/?]+\.yml)$/); + if (fileMatch) return { id: ids[fileMatch[1]], path: `.github/workflows/${fileMatch[1]}` }; + const listMatch = path.match(/\/workflows\/(\d+)\/runs\?/); + if (listMatch) { + const name = Object.keys(ids).find(file => ids[file] === Number(listMatch[1])); + const runs = f.runs[name] ?? []; + return { workflow_runs: runs, total_count: f.total ?? runs.length }; + } + const jobMatch = path.match(/\/runs\/(\d+)\/attempts\/(\d+)\/jobs\?/); + if (jobMatch) { + const page = Number(url.searchParams.get("page") ?? "1"); + const jobs = f.jobs[Number(jobMatch[1])] ?? []; + return { total_count: f.jobTotal ?? jobs.length, jobs: jobs.slice((page - 1) * 100, page * 100) }; + } + const currentMatch = path.match(/\/runs\/(\d+)$/); + if (currentMatch) return f.current[Number(currentMatch[1])] ?? Object.values(f.runs).flat().find(run => run.id === Number(currentMatch[1])); + throw new Error(`Unexpected admission API path: ${path}`); + }; + f.rerun = async path => f.writes.push(path); + return f; + } + + test("only a running image or npm release admits a normal push", async () => { + for (const file of ["docker.yml", "release.yml"]) { + const f = fixture(); + assert.equal(await releaseAdmitted(sha, f), false); + f.runs[file] = [run(file)]; + assert.equal(await releaseAdmitted(sha, f), true); + assert.equal(await releaseAdmitted(sha, { ...f, imageOnly: true }), file === "docker.yml"); + for (const status of ["queued", "pending", "completed"]) { + f.runs[file] = [run(file, { status })]; + assert.equal(await releaseAdmitted(sha, f), false, status); + } + } + }); + + test("manual and retry requests admit exact source without consumer discovery", async () => { + const api = () => assert.fail("explicit request needs no discovery"); + assert.equal(await releaseAdmitted(sha, { api, event: "workflow_dispatch" }), true); + assert.equal(await releaseAdmitted(sha, { api, attempt: 2 }), true); + assert.equal(await releaseAdmitted(sha, { api, event: "pull_request", attempt: 2 }), false); + for (const attempt of [0, "2"]) { + await assert.rejects(releaseAdmitted(sha, { api, attempt }), /Invalid release attempt/); + } + }); + + test("forks, stale identities, tags, and non-push consumers do not admit work", async () => { + for (const change of [{ repository: { ...repository, id: 2 } }, { head_repository: { ...repository, id: 2 } }, + { repository: { ...repository, full_name: "fork/paperclip" } }, { path: ".github/workflows/pr.yml" }, + { workflow_id: 999 }, { head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "workflow_dispatch" }, + { id: 0 }, { run_attempt: 0 }]) { + const f = fixture(); + f.runs["docker.yml"] = [run("docker.yml", change)]; + assert.equal(await releaseAdmitted(sha, f), false, JSON.stringify(change)); + } + const f = fixture(); + const active = run("docker.yml"); + f.runs["docker.yml"] = [active]; + f.current[active.id] = { ...active, run_attempt: 2 }; + assert.equal(await releaseAdmitted(sha, f), false); + }); + + test("incomplete discovery and transport failure fail closed", async () => { + for (const total of [1, 101, -1]) { + const f = fixture(); + f.total = total; + await assert.rejects(releaseAdmitted(sha, f), /Incomplete/); + assert.deepEqual(f.writes, []); + } + await assert.rejects(releaseAdmitted(sha, { api: async () => { throw new Error("API unavailable"); } }), /unavailable/); + await assert.rejects(releaseAdmitted("master", { api: () => assert.fail() }), /full source SHA/); + }); + + test("a selected consumer restarts only a strictly deferred producer, once", async () => { + for (const file of ["cloud-readiness.yml", "cloud-migrator-artifacts.yml"]) { + const f = fixture(file); + assert.equal(await requestDeferredProducer(f), "pending"); + assert.deepEqual(f.writes, [`/repos/paperclipai/paperclip/actions/runs/${ids[file] + 1000}/rerun`]); + await requestDeferredProducer(f); + assert.equal(f.writes.length, 1); + } + }); + + test("failed, incomplete, ambiguous, or mismatched work is never automatically retried", async () => { + for (const change of [jobs => { jobs[1].conclusion = "failure"; }, jobs => { jobs[0].conclusion = "failure"; }, + jobs => { jobs[1].run_attempt = 2; }, jobs => { jobs[1].head_sha = "b".repeat(40); }, + jobs => { jobs[1].run_id = 2; }, jobs => { jobs[1].status = "queued"; }, + jobs => { jobs.push({ ...jobs[0] }); }, jobs => { jobs.splice(1); }]) { + const f = fixture(); + change(f.jobs[1123]); + assert.equal(await requestDeferredProducer(f), "finished"); + assert.deepEqual(f.writes, []); + } + const f = fixture(); + for (const change of [{ run_attempt: 2 }, { status: "in_progress" }, { head_sha: "b".repeat(40) }, { id: 2 }]) { + f.current[1123] = { ...f.runs[f.file][0], ...change }; + assert.equal(await requestDeferredProducer(f), "pending"); + assert.deepEqual(f.writes, []); + } + }); + + test("an explicitly cancelled or failed producer cannot be restarted by admission", async () => { + for (const conclusion of ["cancelled", "failure", "timed_out"]) { + const f = fixture("cloud-readiness.yml", { conclusion }); + assert.equal(await requestDeferredProducer(f), "finished"); + assert.deepEqual(f.writes, []); + } + }); + + test("a truncated job page cannot hide a failed check and trigger a retry", async () => { + const f = fixture(); + f.jobTotal = 3; + await assert.rejects(requestDeferredProducer(f), /Incomplete release producer job list/); + assert.deepEqual(f.writes, []); + f.jobTotal = "2"; + await assert.rejects(requestDeferredProducer(f), /Invalid release producer job list/); + assert.deepEqual(f.writes, []); + }); + + test("producer retries reject arbitrary workflow and admission contracts", async () => { + const f = fixture(); + for (const change of [{ file: "docker.yml" }, { admissionJob: migratorAdmissionJob }]) { + await assert.rejects(requestDeferredProducer({ ...f, ...change }), /Unknown deferred/); + } + assert.deepEqual(f.calls, []); + }); + + test("a deferred completion racing the run listing does not become a source failure", async () => { + const f = fixture(); + const completed = f.runs[f.file][0]; + f.runs[f.file] = [{ ...completed, status: "in_progress", conclusion: null }]; + f.current[completed.id] = completed; + await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred); + delete f.current[completed.id]; + assert.equal(await readSourceVerification(sha, f.api), undefined); + }); + + test("a competing consumer's accepted retry does not fail the losing consumer", async () => { + const f = fixture(); + f.rerun = async () => { + f.current[1123] = run(f.file, { run_attempt: 2 }); + throw new Error("duplicate restart rejected"); + }; + assert.equal(await requestDeferredProducer(f), "pending"); + assert.equal(f.requested.size, 1); + const rejected = fixture(); + rejected.rerun = async () => { throw new Error("token denied"); }; + await assert.rejects(requestDeferredProducer(rejected), /token denied/); + assert.equal(rejected.requested.size, 0); + }); + + test("a queued migrator is pending until its build actually starts", async () => { + const f = fixture("cloud-migrator-artifacts.yml", { status: "in_progress", conclusion: null }); + const producer = f.runs[f.file][0]; + const build = { ...job(producer, "Build migrator", null), status: "queued", labels: ["cloud-release-32"] }; + f.jobs[producer.id] = [job(producer, migratorAdmissionJob, "success"), build]; + assert.equal(await requestDeferredProducer(f), "pending"); + build.status = "in_progress"; + assert.equal(await requestDeferredProducer(f), "admitted"); + build.run_attempt = 2; + assert.equal(await requestDeferredProducer(f), "pending", "a stale build cannot admit work"); + assert.deepEqual(f.writes, []); + }); + + test("a rejected duplicate with stale read visibility remains pending without claiming admission", async () => { + for (const status of [409, 422, 403]) { + const f = fixture(); + f.rerun = async () => { throw Object.assign(new Error("retry rejected"), { status }); }; + if (status === 403) await assert.rejects(requestDeferredProducer(f), /retry rejected/); + else assert.equal(await requestDeferredProducer(f), "pending"); + assert.equal(f.requested.size, 0); + } + }); + + test("source proof waits for admission, restarts it, and accepts only the new attempt", async () => { + const f = fixture(); + await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred); + let time = 0; + const proof = await waitForSourceVerification(sha, { ...f, now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, + sleep: async ms => { + time += ms; + const selected = run(f.file, { run_attempt: 2 }); + f.runs[f.file] = [selected]; + f.jobs[selected.id] = [job(selected, sourceAdmissionJob, "success"), job(selected, sourceVerificationJob, "success")]; + } }); + assert.equal(proof.attempt, 2); + assert.equal(time, 10); + assert.equal(f.writes.length, 1); + }); + + test("the image monitor admits a late migrator even when source proof is ready", async () => { + const f = fixture("cloud-migrator-artifacts.yml"); + const source = run("cloud-readiness.yml"); + f.runs["cloud-readiness.yml"] = [source]; + f.jobs[source.id] = [job(source, sourceVerificationJob, "success")]; + const image = run("docker.yml"); + f.runs["docker.yml"] = [image]; + f.jobs[image.id] = []; + let time = 0; + const proof = await watchReleaseSource({ ...f, runId: image.id, attempt: 1, cancel: () => assert.fail("must not cancel"), + now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, summary: async () => {}, sleep: async ms => { + time += ms; + const selected = run(f.file, { run_attempt: 2 }); + f.runs[f.file] = [selected]; + f.jobs[selected.id] = [{ ...job(selected, "Build migrator", null), status: "in_progress", labels: ["cloud-release-32"] }]; + } }); + assert.equal(proof.sha, sha); + assert.equal(time, 10); + assert.equal(f.writes.length, 1); + }); + + test("the retry transport is a bounded POST to a canonical run", async () => { + const calls = []; + const rerun = createReleaseRerunner("fixture", async (url, options) => { calls.push({ url, options }); return { ok: true }; }); + await assert.rejects(rerun("/repos/fork/paperclip/actions/runs/1/rerun"), /Invalid release retry/); + await rerun("/repos/paperclipai/paperclip/actions/runs/1/rerun"); + assert.equal(calls.length, 1); + assert.equal(calls[0].options.method, "POST"); + assert.equal(calls[0].options.redirect, "error"); + }); +}); diff --git a/scripts/github-actions-reader.mjs b/scripts/github-actions-reader.mjs new file mode 100644 index 00000000000..1898164792a --- /dev/null +++ b/scripts/github-actions-reader.mjs @@ -0,0 +1,73 @@ +// Statuses that say "ask again", not "the answer is no". A release must not be +// blocked because GitHub returned a gateway error during a 60-minute poll. +const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); + +// A rate-limited read also says "ask again", and GitHub reports both primary +// and secondary rate limits as 403. Only the headers separate that from a +// token that may not read Actions, which must still fail at once. +function rateLimited(response) { + if (response.status !== 403) return false; + const header = (name) => response.headers?.get?.(name) ?? null; + return header("retry-after") !== null || header("x-ratelimit-remaining") === "0"; +} + +/** Milliseconds from a Retry-After header, when it carries a sane delay. */ +function retryAfterMs(response, capMs) { + const value = Number(response?.headers?.get?.("retry-after")); + if (!Number.isFinite(value) || value <= 0) return null; + return Math.min(value * 1_000, capMs); +} + +/** + * The GitHub Actions read used by the polling below, with transient transport + * failures retried: network errors, the statuses above, rate-limited 403s, and + * a body that fails while it is being read. Any other non-OK status throws on + * the first response, because waiting out a wrong token only delays the news. + * + * `deadlineAt` bounds retries by the caller's own polling deadline, so a read + * cannot extend the wait past the timeout it belongs to. + */ +export function createActionsReader({ + token, fetchImpl = fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + attempts = 4, backoffMs = 1_000, log = console.log, + now = Date.now, deadlineAt = () => Infinity, maxRetryAfterMs = 60_000, +} = {}) { + return async (path) => { + for (let attempt = 1; ; attempt += 1) { + // Retry only while both the attempt budget and the caller's deadline + // leave room for the wait this attempt would cost. + const waitMs = backoffMs * attempt; + const retryable = attempt < attempts && now() + waitMs < deadlineAt(); + const pause = (response) => sleep(retryAfterMs(response, maxRetryAfterMs) ?? waitMs); + let response; + try { + response = await fetchImpl(`https://api.github.com${path}`, { + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, + signal: AbortSignal.timeout(30_000), redirect: "error", + }); + } catch (cause) { + if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); + log(`GitHub Actions read failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); + await pause(); + continue; + } + if (response.ok) { + try { + // A 200 whose body is truncated or undecodable is a transport + // failure like any other, so it belongs inside the retry. + return await response.json(); + } catch (cause) { + if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); + log(`GitHub Actions read body failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); + await pause(); + continue; + } + } + if (!retryable || !(TRANSIENT_READ_STATUSES.has(response.status) || rateLimited(response))) { + throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`); + } + log(`GitHub Actions read failed (HTTP ${response.status}); retrying (${attempt}/${attempts - 1}).`); + await pause(response); + } + }; +} diff --git a/scripts/sleep-qualification/evidence.d.mts b/scripts/sleep-qualification/evidence.d.mts new file mode 100644 index 00000000000..25a091fe9d0 --- /dev/null +++ b/scripts/sleep-qualification/evidence.d.mts @@ -0,0 +1,54 @@ +export interface SleepQualificationIdentity { + appSha: string; + cloudSha: string; + target: string; + configDigest: string; + collectorOrigins?: { telemetry: string; otel: string }; +} +export interface SleepQualificationReport { + version: number; + target: string; + appSha: string; + cloudSha: string; + configDigest: string; + layers: Record; +} +export function logicalSleepConfiguration(configuration: unknown): { + automaticBackups: { intervalMinutes: number; idleCheckpoint: boolean }; + announcements: boolean; + telemetry: boolean; + otel: { enabled: boolean; protocol: 'http/json' }; + serveUi: boolean; + managedInstance: boolean; + deploymentMode: 'authenticated' | 'local_trusted'; + storageProvider: 's3' | 'local_disk'; +}; +export function sleepConfigurationDigest(configuration: unknown): string; +export interface SleepSignalObservation { + telemetryRequests: number; traceRequests: number; exportedSpans: number; qualificationSpans: number; + metricRequests: number; logRequests: number; malformedTraceRequests: number; unexpectedRequests: number; +} +export interface IncomingHttpSnapshot { + source: 'node-http-server-events'; + captureId: string; + installed: boolean; + valid: boolean; + requests: number; + observedAt: string; +} +export interface IncomingHttpWindow { + before: IncomingHttpSnapshot; + after: IncomingHttpSnapshot; + requests: number; +} +export function observeIncomingHttpWindow(before: IncomingHttpSnapshot, after: IncomingHttpSnapshot): IncomingHttpWindow; +export function observedNoIncomingHttp(observation: unknown, quiet: { startedAt: string; finishedAt: string }): boolean; +export function observeConfiguredSignals(requests: ReadonlyArray<{ path: string; body: unknown }>): SleepSignalObservation; +export function configuredSleepFeatures(observations: Record): { + telemetry: boolean; announcements: boolean; automaticBackups: boolean; otel: boolean; exportedSpans: number; +}; +export function evaluateSleepQualification(receipt: unknown, expected: SleepQualificationIdentity): SleepQualificationReport; +export function finalizeLocalSleepReceipt>(observations: T, expected: SleepQualificationIdentity, now?: () => string): { + receipt: T & { finishedAt: string }; + report: SleepQualificationReport; +}; diff --git a/scripts/sleep-qualification/evidence.mjs b/scripts/sleep-qualification/evidence.mjs new file mode 100644 index 00000000000..d19b54fb034 --- /dev/null +++ b/scripts/sleep-qualification/evidence.mjs @@ -0,0 +1,231 @@ +import { createHash } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; + +const sha = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value); +const finite = value => typeof value === 'number' && Number.isFinite(value) && value >= 0; +const stamp = value => typeof value === 'string' && Number.isFinite(Date.parse(value)); +const layers = ['workerQuiet', 'eligibility', 'databaseSuspension', 'hostSuspension', 'backupPersistence', 'inboundWake', 'timedWake', 'returnToIdle']; +const result = (verdict, reason) => ({ verdict, reason }); +const announcementFeed = 'https://pages.paperclip.ing/announcements/v1/current.json'; +const observedAnnouncement = announcement => announcement?.intercepted === true + && Array.isArray(announcement.requests) && announcement.requests.length > 0 + && announcement.requests.every(url => url === announcementFeed) + && Array.isArray(announcement.unmatched) && announcement.unmatched.length === 0; +const signalKeys = ['telemetryRequests', 'traceRequests', 'exportedSpans', 'qualificationSpans', 'metricRequests', 'logRequests', 'malformedTraceRequests', 'unexpectedRequests']; +const knownSignalCounts = signal => signal && signalKeys.every(key => Number.isSafeInteger(signal[key]) && signal[key] >= 0); + +/** Preserve actual cumulative app-server snapshots; IPC reads create no HTTP traffic. */ +export function observeIncomingHttpWindow(before, after) { + return { before, after, requests: after?.requests - before?.requests }; +} +const knownIncomingHttpSnapshot = snapshot => snapshot?.source === 'node-http-server-events' + && typeof snapshot.captureId === 'string' && /^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/.test(snapshot.captureId) + && snapshot.installed === true && snapshot.valid === true + && Number.isSafeInteger(snapshot.requests) && snapshot.requests >= 0 && stamp(snapshot.observedAt); +export const observedNoIncomingHttp = (observation, quiet) => knownIncomingHttpSnapshot(observation?.before) + && knownIncomingHttpSnapshot(observation?.after) && observation.before.captureId === observation.after.captureId + && stamp(quiet?.startedAt) && stamp(quiet?.finishedAt) && Date.parse(quiet.finishedAt) >= Date.parse(quiet.startedAt) + && Date.parse(observation.before.observedAt) <= Date.parse(quiet.startedAt) + && Date.parse(observation.after.observedAt) >= Date.parse(quiet.finishedAt) + && Number.isSafeInteger(observation.requests) && observation.requests === 0 + && observation.after.requests - observation.before.requests === observation.requests; + +/** Count actual collector requests and OTLP span records, never request-as-span estimates. */ +export function observeConfiguredSignals(requests) { + const observed = Object.fromEntries(signalKeys.map(key => [key, 0])); + for (const request of requests) { + if (typeof request?.path !== 'string' || !request.path.startsWith('/') || request.path.startsWith('//')) { observed.unexpectedRequests++; continue; } + const pathname = new URL(request.path, 'http://owned-collector').pathname; + if (pathname === '/telemetry') { observed.telemetryRequests++; continue; } + if (pathname === '/v1/metrics') { observed.metricRequests++; continue; } + if (pathname === '/v1/logs') { observed.logRequests++; continue; } + if (pathname !== '/v1/traces') { observed.unexpectedRequests++; continue; } + observed.traceRequests++; + try { + if (!Array.isArray(request.body?.resourceSpans)) throw new Error('Missing resourceSpans'); + let spans = 0, qualificationSpans = 0; + for (const resource of request.body.resourceSpans) { + if (!Array.isArray(resource?.scopeSpans)) throw new Error('Missing scopeSpans'); + for (const scope of resource.scopeSpans) { + if (!Array.isArray(scope?.spans)) throw new Error('Missing spans'); + for (const span of scope.spans) { + if (typeof span?.name !== 'string' || !span.name) throw new Error('Invalid span'); + spans++; if (span.name === 'sleep-qualification-stimulus') qualificationSpans++; + } + } + } + observed.exportedSpans += spans; observed.qualificationSpans += qualificationSpans; + } catch { observed.malformedTraceRequests++; } + } + return observed; +} + +/** Coverage booleans come from recorded activity; the evaluator also checks their provenance. */ +export function configuredSleepFeatures({ announcement, signalObservation, backupCheckpoint }) { + const known = knownSignalCounts(signalObservation); + return { telemetry: Boolean(known && signalObservation.telemetryRequests > 0), announcements: Boolean(observedAnnouncement(announcement)), + automaticBackups: backupCheckpoint?.owner === 'normal-idle-checkpoint' && backupCheckpoint.wakeMarkerObserved === true + && backupCheckpoint.failureObserved === false && Array.isArray(backupCheckpoint.archives) && backupCheckpoint.archives.length > 0, + otel: Boolean(known && signalObservation.traceRequests > 0 && signalObservation.exportedSpans > 0 + && signalObservation.qualificationSpans > 0 && signalObservation.qualificationSpans <= signalObservation.exportedSpans + && signalObservation.metricRequests === 0 && signalObservation.logRequests === 0 + && signalObservation.malformedTraceRequests === 0 && signalObservation.unexpectedRequests === 0), + exportedSpans: known ? signalObservation.exportedSpans : 0 }; +} +const validBackupCheckpoint = (backup, inRun, quietFinishedAt) => backup?.owner === 'normal-idle-checkpoint' && backup.wakeMarkerObserved === true + && backup.failureObserved === false && inRun(backup.startedAt) && inRun(backup.observedAt) + && Date.parse(backup.startedAt) >= Date.parse(quietFinishedAt) && Date.parse(backup.observedAt) >= Date.parse(backup.startedAt) && Array.isArray(backup.archives) && backup.archives.length > 0 + && backup.archives.every(archive => typeof archive.name === 'string' && /^[^/\\]+\.sql\.gz$/.test(archive.name) + && finite(archive.bytes) && archive.bytes > 0 && finite(archive.sqlBytes) && archive.sqlBytes > 0 + && /^[a-f0-9]{64}$/.test(archive.sha256 ?? '') && archive.gzipVerified === true && archive.fixtureIdentifierFound === true + && ['birthtime', 'mtime'].includes(archive.createdAtSource) + && inRun(archive.createdAt) && Date.parse(archive.createdAt) >= Date.parse(backup.startedAt) && Date.parse(archive.createdAt) <= Date.parse(backup.observedAt)); + +/** Canonical logical configuration: ephemeral collector addresses are bound separately. */ +export function logicalSleepConfiguration(configuration) { + if (!configuration || typeof configuration.announcements !== 'boolean' || typeof configuration.serveUi !== 'boolean' + || typeof configuration.telemetryOrigin !== 'string' || typeof configuration.otelOrigin !== 'string' + || (configuration.managedInstance !== undefined && typeof configuration.managedInstance !== 'boolean') + || (configuration.deploymentMode !== undefined && !['authenticated', 'local_trusted'].includes(configuration.deploymentMode)) + || (configuration.storageProvider !== undefined && !['s3', 'local_disk'].includes(configuration.storageProvider)) + || typeof configuration.automaticBackups?.idleCheckpoint !== 'boolean' + || !finite(configuration.automaticBackups?.intervalMinutes) || configuration.automaticBackups.intervalMinutes < 1) { + throw new Error('sleep_qualification_configuration_invalid'); + } + return { automaticBackups: { intervalMinutes: configuration.automaticBackups.intervalMinutes, + idleCheckpoint: configuration.automaticBackups.idleCheckpoint }, announcements: configuration.announcements, + telemetry: Boolean(configuration.telemetryOrigin), otel: { enabled: Boolean(configuration.otelOrigin), protocol: 'http/json' }, serveUi: configuration.serveUi, managedInstance: configuration.managedInstance ?? false, + deploymentMode: configuration.deploymentMode ?? 'local_trusted', storageProvider: configuration.storageProvider ?? 'local_disk' }; +} +export const sleepConfigurationDigest = configuration => createHash('sha256').update(JSON.stringify(logicalSleepConfiguration(configuration))).digest('hex'); + +/** Finish after observations, then validate the exact serialized local receipt. */ +export function finalizeLocalSleepReceipt(observations, expected, now = () => new Date().toISOString()) { + const receipt = { ...observations, finishedAt: now() }; + const report = evaluateSleepQualification(receipt, expected); + if (report.layers.workerQuiet.verdict !== 'VERIFIED' || report.layers.eligibility.verdict !== 'VERIFIED') { + throw new Error('sleep_qualification_local_evidence_invalid'); + } + return { receipt, report }; +} + +/** Evaluate this campaign's measured boundaries. A local run cannot certify a provider. */ +export function evaluateSleepQualification(receipt, expected) { + if (!receipt || receipt.version !== 1 || !expected || !sha(expected.appSha) + || !sha(expected.cloudSha) || typeof expected.target !== 'string' || !expected.target + || typeof expected.configDigest !== 'string' || !/^[a-f0-9]{64}$/.test(expected.configDigest) + || !stamp(receipt.startedAt) || !stamp(receipt.finishedAt) + || Date.parse(receipt.finishedAt) < Date.parse(receipt.startedAt) + || ['appSha', 'cloudSha', 'target', 'configDigest'].some(key => receipt[key] !== expected[key])) { + throw new Error('sleep_qualification_identity_or_window_invalid'); + } + if (sleepConfigurationDigest(receipt.configuration) !== expected.configDigest) throw new Error('sleep_qualification_configuration_digest_invalid'); + const origins = receipt.collectorOrigins; + if (!origins || origins.telemetry !== receipt.configuration.telemetryOrigin || origins.otel !== receipt.configuration.otelOrigin + || (expected.collectorOrigins && (origins.telemetry !== expected.collectorOrigins.telemetry || origins.otel !== expected.collectorOrigins.otel))) { + throw new Error('sleep_qualification_collector_origins_invalid'); + } + const output = Object.fromEntries(layers.map(layer => [layer, result('INCONCLUSIVE', 'No evidence for this layer')])); + const inRun = value => stamp(value) && Date.parse(value) >= Date.parse(receipt.startedAt) && Date.parse(value) <= Date.parse(receipt.finishedAt); + const quiet = receipt.quiet; + if (quiet) { + const observedFeatures = configuredSleepFeatures(receipt); + const valid = inRun(quiet.startedAt) && inRun(quiet.finishedAt) && finite(quiet.elapsedMs) + && Date.parse(quiet.finishedAt) - Date.parse(quiet.startedAt) >= 90_000 && quiet.elapsedMs >= 90_000 + && quiet.clock === 'real' && finite(quiet.sqlQueries) && finite(quiet.connectionsOpened) + && finite(quiet.connectionsRemaining) && finite(quiet.nonDatabaseRequests) && finite(quiet.nonDatabaseBytes) + && observedNoIncomingHttp(quiet.observerRequests, quiet) + && inRun(quiet.observerRequests.before.observedAt) && inRun(quiet.observerRequests.after.observedAt) + && observedFeatures.telemetry && observedFeatures.announcements && observedFeatures.otel + && Object.entries(observedFeatures).every(([key, value]) => receipt.features?.[key] === value) + && receipt.features?.telemetry === true && receipt.features?.announcements === true + && receipt.features?.automaticBackups === true && validBackupCheckpoint(receipt.backupCheckpoint, inRun, quiet.finishedAt) + && receipt.features?.otel === true && receipt.features?.exportedSpans > 0; + const idle = valid && quiet.sqlQueries === 0 && quiet.connectionsOpened === 0 && quiet.connectionsRemaining === 0 + && quiet.nonDatabaseRequests === 0 && quiet.nonDatabaseBytes === 0; + output.workerQuiet = result(idle ? 'VERIFIED' : 'NOT VERIFIED', idle ? 'Real configured quiet window, including closed DB connections' : 'Configured real quiet window missing, contaminated, or active'); + } + const eligibility = receipt.eligibility; + if (eligibility) { + const valid = inRun(eligibility.observedAt) && eligibility.backgroundWork === 'none' + && eligibility.ownedHold === true && eligibility.unchangedGeneration === true + && finite(eligibility.apiCooldownElapsedMs) && eligibility.apiCooldownElapsedMs >= 600_000; + output.eligibility = result(valid ? 'VERIFIED' : 'NOT VERIFIED', valid ? 'Owned stable safety proof after real API activity cooldown' : 'Safety proof or real API activity cooldown not established'); + } + const provider = receipt.provider; + if (provider) { + if (!provider.live || provider.target !== expected.target || provider.configDigest !== expected.configDigest + || provider.appSha !== expected.appSha || provider.cloudSha !== expected.cloudSha + || !inRun(provider.observedAt)) { + throw new Error('sleep_qualification_provider_identity_invalid'); + } + const untouched = Array.isArray(provider.observerRequests) + && provider.observerRequests.every(entry => entry.boundary === 'provider-control-plane' && (entry.method === 'GET' || (entry.provider === 'railway' && entry.method === 'POST' && entry.operation === 'query-only'))); + const endpoint = provider.neon; + if (untouched && endpoint?.identityVerified === true && endpoint.currentState === 'idle' && endpoint.pendingState === null) { + output.databaseSuspension = result('VERIFIED', 'Exact endpoint idle from live Neon control-plane metadata'); + } + if (untouched && endpoint?.identityVerified === true && (endpoint.currentState === 'active' + || (endpoint.currentState === 'idle' && endpoint.pendingState === 'active'))) { + output.databaseSuspension = result('NOT VERIFIED', 'Exact Neon endpoint is active or becoming active'); + } + const services = provider.railway?.services; + if (untouched && provider.railway?.identityVerified === true && provider.railway.state === 'OBSERVED' && Array.isArray(services) && services.length > 0 + && services.every(service => service.identityVerified === true && service.state === 'OBSERVED' && service.allDeploymentsStopped === true + && typeof service.backupsEnabled === 'boolean' && (service.backupsEnabled === false || service.backupStorageVerified === true))) { + output.hostSuspension = result('VERIFIED', 'Every exact owned service has stopped deployments'); + } + if (untouched && provider.railway?.identityVerified === true && Array.isArray(services) + && services.some(service => service.identityVerified === true && service.state === 'OBSERVED' && service.allDeploymentsStopped === false)) { + output.hostSuspension = result('NOT VERIFIED', 'An exact observed owned service still has running deployments'); + } + const storage = provider.storage; + if (output.hostSuspension.verdict === 'VERIFIED' && storage?.identityVerified === true + && storage.volumeId && storage.beforeVolumeId === storage.volumeId && storage.afterVolumeId === storage.volumeId + && storage.mountPath === '/paperclip' && storage.backupsEnabledBefore === true && storage.backupsEnabledAfter === true + && /^[a-f0-9]{64}$/.test(storage.sentinelBefore ?? '') && storage.sentinelBefore === storage.sentinelAfter + && inRun(storage.beforeAt) && inRun(storage.afterAt) && Date.parse(storage.beforeAt) < Date.parse(provider.observedAt) + && Date.parse(storage.afterAt) > Date.parse(provider.observedAt)) { + output.backupPersistence = result('VERIFIED', 'Attached volume and fsynced sentinel survive owned stop/wake with backups enabled'); + } + } + for (const [layer, kind] of [['inboundWake', 'inbound'], ['timedWake', 'deadline']]) { + const wake = receipt[layer]; + if (!wake) continue; + const sleeping = output.databaseSuspension.verdict === 'VERIFIED' && output.hostSuspension.verdict === 'VERIFIED'; + const chronology = provider && inRun(wake.stimulusAt) && inRun(wake.finishedAt) + && Date.parse(wake.stimulusAt) > Date.parse(provider.observedAt) && Date.parse(wake.finishedAt) >= Date.parse(wake.stimulusAt); + const alarm = kind !== 'deadline' || (inRun(wake.alarmPersistedAt) + && Date.parse(wake.alarmPersistedAt) < Date.parse(provider.observedAt) && wake.sameFinalGeneration === true); + if (!sleeping) { output[layer] = result('INCONCLUSIVE', 'Exact provider sleep and known backup policy have not been established'); continue; } + const verified = sleeping && chronology && alarm && wake.ordinaryProducer === true + && wake.authorizationVerified === true && wake.persistedOutcomeVerified === true && wake.duplicateEffects === 0; + output[layer] = result(verified ? 'VERIFIED' : 'NOT VERIFIED', verified ? `Useful authorized ${kind} wake after provider sleep` : `Missing provider sleep, chronology, authorization or useful ${kind} outcome`); + } + if (receipt.returnToIdle) { + const presentWakes = ['inboundWake', 'timedWake'].filter(layer => Object.hasOwn(receipt, layer)); + const everyWakeInRun = presentWakes.every(layer => inRun(receipt[layer]?.stimulusAt) && inRun(receipt[layer]?.finishedAt) + && Date.parse(receipt[layer].finishedAt) >= Date.parse(receipt[layer].stimulusAt)); + const lastWake = Math.max(...presentWakes.map(layer => Date.parse(receipt[layer]?.finishedAt))); + const idle = receipt.returnToIdle; + const usefulWake = ['inboundWake', 'timedWake'].some(layer => output[layer].verdict === 'VERIFIED'); + const valid = usefulWake && everyWakeInRun && Number.isFinite(lastWake) && inRun(idle.observedAt) && Date.parse(idle.observedAt) > lastWake + && inRun(idle.quietStartedAt) && inRun(idle.quietFinishedAt) + && Date.parse(idle.quietStartedAt) >= lastWake && Date.parse(idle.quietFinishedAt) <= Date.parse(idle.observedAt) + && Date.parse(idle.quietFinishedAt) - Date.parse(idle.quietStartedAt) >= 90_000 + && finite(idle.freshQuietWindowMs) && idle.freshQuietWindowMs >= 90_000 + && idle.freshQuietWindowMs <= Date.parse(idle.quietFinishedAt) - Date.parse(idle.quietStartedAt) && idle.backgroundWork === 'none' && idle.ownedHold === true; + output.returnToIdle = result(valid ? 'VERIFIED' : 'NOT VERIFIED', valid ? 'Fresh quiet owned proof after useful wake' : 'No fresh quiet proof after useful wake'); + } + return { version: 1, target: expected.target, appSha: expected.appSha, cloudSha: expected.cloudSha, configDigest: expected.configDigest, layers: output }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [receiptPath, expectedPath, outputPath] = process.argv.slice(2); + if (!receiptPath || !expectedPath || !outputPath) throw new Error('Usage: node scripts/sleep-qualification/evidence.mjs receipt.json expected.json output.json'); + const report = evaluateSleepQualification(JSON.parse(await readFile(receiptPath, 'utf8')), JSON.parse(await readFile(expectedPath, 'utf8'))); + await writeFile(outputPath, JSON.stringify(report, null, 2) + '\n', { mode: 0o600 }); + process.stdout.write(JSON.stringify(report) + '\n'); + if (Object.values(report.layers).some(layer => layer.verdict === 'NOT VERIFIED')) process.exitCode = 1; +} diff --git a/scripts/sleep-qualification/setup-otel.mjs b/scripts/sleep-qualification/setup-otel.mjs new file mode 100644 index 00000000000..684d3b119fb --- /dev/null +++ b/scripts/sleep-qualification/setup-otel.mjs @@ -0,0 +1,125 @@ +import { lstat, mkdtemp, mkdir, readFile, readlink, rename, rm, symlink, unlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { randomUUID } from 'node:crypto'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +// Explicit opt-in test setup. No manifests, lockfiles or other worktrees change. +const server = fileURLToPath(new URL('../../server/', import.meta.url)); +export const otelPeerNames = ['@opentelemetry/sdk-node', '@opentelemetry/auto-instrumentations-node', '@opentelemetry/exporter-trace-otlp-http', '@opentelemetry/resources', '@opentelemetry/semantic-conventions']; +const ledgerPath = root => path.join(root, 'node_modules', '.paperclip-sleep-otel-links.json'); +async function entryAt(destination) { + try { return await lstat(destination); } catch (error) { if (error.code !== 'ENOENT') throw error; } +} +async function versionAt(root, name) { + try { return JSON.parse(await readFile(path.join(root, 'node_modules', name, 'package.json'), 'utf8')).version; } + catch (error) { if (error.code !== 'ENOENT') throw error; return undefined; } +} +async function readOwnership(root) { + const file = ledgerPath(root), entry = await entryAt(file); + if (!entry) return []; + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error('OTel ownership ledger must be a regular file'); + const record = JSON.parse(await readFile(file, 'utf8')); + if (record.version !== 1 || record.serverDirectory !== root || !Array.isArray(record.links) + || record.links.length > otelPeerNames.length || new Set(record.links.map(link => link.name)).size !== record.links.length + || record.links.some(link => !otelPeerNames.includes(link.name) || typeof link.target !== 'string' + || !path.isAbsolute(link.target) || path.resolve(link.target) !== link.target + || !link.target.endsWith(path.join('node_modules', link.name)))) throw new Error('Invalid OTel ownership ledger'); + return record.links; +} +async function writeOwnership(root, links) { + const file = ledgerPath(root); + if (!links.length) { await rm(file, { force: true }); return; } + await mkdir(path.dirname(file), { recursive: true }); + const temporary = `${file}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, JSON.stringify({ version: 1, serverDirectory: root, links }, null, 2), { flag: 'wx' }); + await rename(temporary, file); + } finally { await rm(temporary, { force: true }); } +} +async function ownsLink(destination, record) { + const entry = await entryAt(destination); + return Boolean(record && entry?.isSymbolicLink() + && path.resolve(path.dirname(destination), await readlink(destination)) === record.target); +} +async function installWithNpm(directory) { + await new Promise((resolve, reject) => { + const child = spawn('npm', ['install', '--prefix', directory, '--ignore-scripts', '--no-audit', '--no-fund', '--prefer-offline'], { stdio: 'inherit', shell: false }); + child.once('error', reject); child.once('exit', code => code === 0 ? resolve() : reject(new Error(`OTel fixture installation failed (${code})`))); + }); +} + +/** Only the exact recorded symlink may be removed; install prefixes retain evidence. */ +export async function cleanupSleepOtelLinks({ serverDirectory = server } = {}) { + const root = path.resolve(serverDirectory), records = await readOwnership(root), remaining = [], removed = [], errors = []; + for (const record of records) { + const destination = path.join(root, 'node_modules', record.name); + if (!await entryAt(destination)) continue; + if (!await ownsLink(destination, record)) { + remaining.push(record); errors.push(new Error(`Refusing to clean unrelated ${record.name}`)); continue; + } + await unlink(destination); removed.push(record.name); + } + await writeOwnership(root, remaining); + if (errors.length) throw new AggregateError(errors, 'Some OTel destinations no longer match owned links'); + return { removed, installPrefixes: [...new Set(records.map(record => path.dirname(path.dirname(path.dirname(record.target)))))], boundary: 'Only recorded matching symlinks removed; temporary install prefixes are not deleted' }; +} + +export async function setupSleepOtelPeers({ serverDirectory = server, temporaryRoot = tmpdir(), install = installWithNpm } = {}) { + const root = path.resolve(serverDirectory), manifest = JSON.parse(await readFile(path.join(root, 'package.json'), 'utf8')); + const owned = await readOwnership(root), missing = []; + for (const name of otelPeerNames) { + const exact = manifest.peerDependencies[name]; + if (!/^\d+\.\d+\.\d+$/.test(exact ?? '')) throw new Error(`Exact existing peer version required for ${name}`); + const installed = await versionAt(root, name); + if (installed && installed !== exact) throw new Error(`Installed ${name} does not match ${exact}; refusing to replace it`); + if (installed) continue; + const destination = path.join(root, 'node_modules', name); + if (await entryAt(destination) && !await ownsLink(destination, owned.find(link => link.name === name))) { + throw new Error(`Refusing to replace unrelated ${name}; restore its target or verify ownership before removing the unrecorded entry`); + } + missing.push(name); + } + if (!missing.length) return { linked: [], alreadyInstalled: true }; + const directory = await mkdtemp(path.join(temporaryRoot, 'paperclip-sleep-otel-')); + let linked = 0; + try { + await writeFile(path.join(directory, 'package.json'), JSON.stringify({ private: true, dependencies: Object.fromEntries(otelPeerNames.map(name => [name, manifest.peerDependencies[name]])) }, null, 2)); + await install(directory); + const resolved = Object.fromEntries(await Promise.all(otelPeerNames.map(async name => [name, await versionAt(directory, name)]))); + for (const name of otelPeerNames) if (resolved[name] !== manifest.peerDependencies[name]) throw new Error(`Resolved ${name} differs from pinned peer`); + const records = [...owned]; + for (const name of missing) { + const destination = path.join(root, 'node_modules', name), previous = owned.find(record => record.name === name); + await mkdir(path.dirname(destination), { recursive: true }); + if (await entryAt(destination)) { + if (!await ownsLink(destination, previous)) throw new Error(`OTel destination changed; refusing to replace ${name}`); + await unlink(destination); + } + const target = path.join(directory, 'node_modules', name); + await symlink(target, destination, 'dir'); linked++; + const updated = [...records.filter(record => record.name !== name), { name, target }]; + try { await writeOwnership(root, updated); } + catch (error) { + if (await ownsLink(destination, { name, target })) { await unlink(destination); linked--; } + throw error; + } + records.splice(0, records.length, ...updated); + } + const receipt = { version: 1, directory, packages: resolved, linked: missing, ownershipLedger: ledgerPath(root), cleanup: 'node scripts/sleep-qualification/setup-otel.mjs --cleanup' }; + await writeFile(path.join(directory, 'qualification-setup.json'), JSON.stringify(receipt, null, 2)); + return receipt; + } catch (error) { + // A prefix with published links stays available for explicit owned cleanup. + if (!linked) await rm(directory, { recursive: true, force: true }); + throw error; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const args = process.argv.slice(2); + if (args.length > 1 || (args.length && args[0] !== '--cleanup')) throw new Error('Usage: node scripts/sleep-qualification/setup-otel.mjs [--cleanup]'); + const receipt = args[0] === '--cleanup' ? await cleanupSleepOtelLinks() : await setupSleepOtelPeers(); + process.stdout.write(JSON.stringify(receipt) + '\n'); +} diff --git a/scripts/watch-release-source.mjs b/scripts/watch-release-source.mjs new file mode 100644 index 00000000000..bf4fe032774 --- /dev/null +++ b/scripts/watch-release-source.mjs @@ -0,0 +1,108 @@ +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { createActionsReader, readSourceVerification, SourceVerificationFailed, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; +import { requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; + +const repository = "paperclipai/paperclip"; + +export function runnerWaits(jobs, now = Date.now()) { + return jobs.filter(job => ["queued", "in_progress", "completed"].includes(job.status) && + job.conclusion !== "skipped" && job.labels?.length > 0).map(job => ({ + job: job.name, id: job.id, + seconds: Math.max(0, Math.floor(((job.runner_id > 0 && job.started_at ? Date.parse(job.started_at) : now) - Date.parse(job.created_at)) / 1000)), + })).filter(job => Number.isFinite(job.seconds)); +} + +// The write is limited to this canonical master Docker run. Never cancel +// another workflow, a tag release, or a run selected from untrusted input. +export async function cancelImageRun({ api, cancel, runId, attempt, sha }) { + const run = await api(`/repos/${repository}/actions/runs/${runId}`); + if (run.id !== runId || run.run_attempt !== attempt || run.head_sha !== sha || run.head_branch !== "master" || + run.repository?.id !== 1170821064 || run.head_repository?.id !== 1170821064 || + run.path !== ".github/workflows/docker.yml" || !["push", "workflow_dispatch"].includes(run.event)) { + throw new Error("Refusing to cancel a different image workflow identity."); + } + if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${runId}/cancel`); +} + +async function cancelFailedVerification({ api, cancel, sha }, failure) { + const run = await api(`/repos/${repository}/actions/runs/${failure.runId}`); + // A concurrent retry owns the new attempt; a late observer leaves it alone. + if (run.run_attempt !== failure.attempt) return; + if (run.id !== failure.runId || run.head_sha !== sha || run.head_branch !== "master" || run.event !== "push" || + run.repository?.full_name !== repository || run.head_repository?.full_name !== repository || + run.path !== ".github/workflows/cloud-readiness.yml") throw new Error("Failed verification identity changed."); + if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${failure.runId}/cancel`); +} + +export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rerun, now = Date.now, + sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), log = console.log, + timeoutMs = 40 * 60_000, intervalMs = 30_000, requireProof = false, summary = async () => {} }) { + if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !Number.isSafeInteger(runId) || runId < 1 || + !Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release identity."); + const deadline = now() + timeoutMs; + const warned = new Set(); + const requested = new Set(); + let migratorAdmitted = !rerun; + const reportWaits = async jobs => { + for (const wait of runnerWaits(jobs, now())) { + if (wait.seconds > 60 && !warned.has(wait.id)) { + warned.add(wait.id); + log(`::warning title=Release runner waited over 60 seconds::Job ${wait.id} waited ${wait.seconds}s for a runner.`); + await summary(`Release runner wait: ${wait.job} — ${wait.seconds}s (job ${wait.id}).\n`); + } + } + }; + while (now() < deadline) { + if (!migratorAdmitted) { + migratorAdmitted = await requestDeferredProducer({ file: "cloud-migrator-artifacts.yml", sha, admissionJob: migratorAdmissionJob, api, rerun, requested }) !== "pending"; + } + const batch = await api(`/repos/${repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100`); + if (!Array.isArray(batch.jobs) || batch.total_count > 100) throw new Error("Incomplete image job listing."); + await reportWaits(batch.jobs); + try { + const proof = await readSourceVerification(sha, api, { failFast: true, onJobs: reportWaits }); + if (proof && migratorAdmitted) { + await summary(`Exact source verified: ${sha}; readiness run ${proof.runId}, attempt ${proof.attempt}.\n`); + return proof; + } + } catch (error) { + if (error instanceof SourceVerificationDeferred && rerun) { + await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested }); + } else { + if (error instanceof SourceVerificationFailed) { + await cancelFailedVerification({ api, cancel, sha }, error); + await cancelImageRun({ api, cancel, runId, attempt, sha }); + } + throw error; + } + } + await sleep(Math.min(intervalMs, Math.max(0, deadline - now()))); + } + // This early failure observer is not the publication deadline. Healthy + // checks can outlive a Spot worker; a separate exact-source gate still + // waits for proof before publishing any image. + if (requireProof) throw new Error("Exact-source verification did not become ready before the publication deadline."); + await summary("Source monitoring window ended; final publication still requires exact-source verification.\n"); + return undefined; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const token = process.env.GITHUB_TOKEN; + if (!token || process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Canonical master token required."); + const requireProof = process.argv.includes("--require-proof"); + const checkProof = process.argv.includes("--check-proof"); + const api = createActionsReader({ token }); + const proof = checkProof ? await readSourceVerification(process.env.GITHUB_SHA, api, { failFast: true }) : await watchReleaseSource({ requireProof, sha: process.env.GITHUB_SHA, runId: Number(process.env.GITHUB_RUN_ID), attempt: Number(process.env.GITHUB_RUN_ATTEMPT), + api, + rerun: createReleaseRerunner(token), + cancel: async path => { + const response = await fetch(`https://api.github.com${path}`, { method: "POST", redirect: "error", + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw new Error(`Could not cancel failed-source image work: HTTP ${response.status}.`); + }, + summary: text => process.env.GITHUB_STEP_SUMMARY ? appendFile(process.env.GITHUB_STEP_SUMMARY, text) : Promise.resolve(), + }); + if (checkProof && !proof) throw new Error("Exact-source proof changed before publication."); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `verified=${Boolean(proof)}\n`); +} diff --git a/server/scripts/sleep-otel-setup.test.mjs b/server/scripts/sleep-otel-setup.test.mjs new file mode 100644 index 00000000000..ffbedb038f1 --- /dev/null +++ b/server/scripts/sleep-otel-setup.test.mjs @@ -0,0 +1,67 @@ +import { it } from 'vitest'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, readlink, readdir, lstat, rm, symlink, unlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { cleanupSleepOtelLinks, otelPeerNames, setupSleepOtelPeers } from '../../scripts/sleep-qualification/setup-otel.mjs'; + +async function fixture(run) { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-otel-setup-test-')); + const serverDirectory = path.join(directory, 'server'), temporaryRoot = path.join(directory, 'installs'); + await mkdir(serverDirectory); await mkdir(temporaryRoot); + await writeFile(path.join(serverDirectory, 'package.json'), JSON.stringify({ peerDependencies: Object.fromEntries(otelPeerNames.map(name => [name, '1.2.3'])) })); + let installs = 0; + const install = async prefix => { + installs++; + const manifest = JSON.parse(await readFile(path.join(prefix, 'package.json'), 'utf8')); + for (const [name, version] of Object.entries(manifest.dependencies)) { + const packageDirectory = path.join(prefix, 'node_modules', name); + await mkdir(packageDirectory, { recursive: true }); + await writeFile(path.join(packageDirectory, 'package.json'), JSON.stringify({ version })); + } + }; + const options = { serverDirectory, temporaryRoot, install }; + try { await run({ directory, options, install, installs: () => installs, destination: name => path.join(serverDirectory, 'node_modules', name) }); } + finally { await rm(directory, { recursive: true, force: true }); } +} +it('reruns after losing its temporary peers and cleans only its recorded matching links', async () => fixture(async f => { + const first = await setupSleepOtelPeers(f.options); + assert.equal(first.linked.length, 5); assert.equal(f.installs(), 1); + assert.equal((await setupSleepOtelPeers(f.options)).alreadyInstalled, true); assert.equal(f.installs(), 1); + await rm(first.directory, { recursive: true }); + assert.equal((await lstat(f.destination(otelPeerNames[0]))).isSymbolicLink(), true); + const repaired = await setupSleepOtelPeers(f.options); + assert.equal(f.installs(), 2); assert.notEqual(repaired.directory, first.directory); + for (const name of otelPeerNames) assert.equal(await readlink(f.destination(name)), path.join(repaired.directory, 'node_modules', name)); + const cleaned = await cleanupSleepOtelLinks(f.options); assert.equal(cleaned.removed.length, 5); + for (const name of otelPeerNames) await assert.rejects(lstat(f.destination(name)), { code: 'ENOENT' }); + assert.ok(await readFile(path.join(repaired.directory, 'qualification-setup.json'), 'utf8')); // Evidence retained. +})); +it('preserves unowned dangling links, incomplete directories and mismatched installed packages', async () => { + for (const kind of ['dangling', 'directory', 'mismatch']) await fixture(async f => { + const destination = f.destination(otelPeerNames[0]); await mkdir(path.dirname(destination), { recursive: true }); + if (kind === 'dangling') await symlink(path.join(f.directory, 'foreign-missing'), destination); + else { await mkdir(destination); await writeFile(path.join(destination, kind === 'mismatch' ? 'package.json' : 'keep.txt'), kind === 'mismatch' ? JSON.stringify({ version: '9.9.9' }) : 'foreign'); } + await assert.rejects(setupSleepOtelPeers(f.options), /refusing|Refusing/); assert.equal(f.installs(), 0); + if (kind === 'dangling') assert.equal(await readlink(destination), path.join(f.directory, 'foreign-missing')); + else assert.ok(await readFile(path.join(destination, kind === 'mismatch' ? 'package.json' : 'keep.txt'), 'utf8')); + }); +}); +it('keeps healthy unrelated packages and refuses cleanup after an owned link is replaced', async () => fixture(async f => { + const destination = f.destination(otelPeerNames[0]); await mkdir(destination, { recursive: true }); + await writeFile(path.join(destination, 'package.json'), JSON.stringify({ version: '1.2.3' })); + const receipt = await setupSleepOtelPeers(f.options); assert.equal(receipt.linked.length, 4); + const replaced = f.destination(otelPeerNames[1]); await unlink(replaced); + const foreign = path.join(f.directory, 'foreign'); await mkdir(foreign); await writeFile(path.join(foreign, 'keep.txt'), 'foreign'); + await symlink(foreign, replaced); + await assert.rejects(cleanupSleepOtelLinks(f.options), /no longer match/); + assert.equal(await readlink(replaced), foreign); assert.equal(await readFile(path.join(foreign, 'keep.txt'), 'utf8'), 'foreign'); + assert.equal(JSON.parse(await readFile(path.join(destination, 'package.json'), 'utf8')).version, '1.2.3'); +})); +it('preserves an entry introduced during installation and removes an unpublished failed install prefix', async () => fixture(async f => { + await assert.rejects(setupSleepOtelPeers({ ...f.options, install: async prefix => { + await f.install(prefix); const destination = f.destination(otelPeerNames[0]); await mkdir(destination, { recursive: true }); await writeFile(path.join(destination, 'keep.txt'), 'new foreign entry'); + } }), /destination changed/); + assert.equal(await readFile(path.join(f.destination(otelPeerNames[0]), 'keep.txt'), 'utf8'), 'new foreign entry'); + assert.equal((await readdir(f.options.temporaryRoot)).length, 0); +})); diff --git a/server/scripts/sleep-qualification.test.mjs b/server/scripts/sleep-qualification.test.mjs new file mode 100644 index 00000000000..c0a275152a5 --- /dev/null +++ b/server/scripts/sleep-qualification.test.mjs @@ -0,0 +1,233 @@ +import { it as test } from 'vitest'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { configuredSleepFeatures, evaluateSleepQualification, observeConfiguredSignals, observeIncomingHttpWindow, sleepConfigurationDigest, finalizeLocalSleepReceipt } from '../../scripts/sleep-qualification/evidence.mjs'; +const configuration = { automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, telemetryOrigin: 'http://127.0.0.1:4318/telemetry', otelOrigin: 'http://127.0.0.1:4318', serveUi: true }; +// Independent preflight contract, not copied from the receipt/digest implementation. +const configDigest = createHash('sha256').update(JSON.stringify({ automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, telemetry: true, otel: { enabled: true, protocol: 'http/json' }, serveUi: true, managedInstance: false, deploymentMode: 'local_trusted', storageProvider: 'local_disk' })).digest('hex'); +const expected = { appSha: 'a'.repeat(40), cloudSha: 'b'.repeat(40), target: 'owned-fixture', configDigest }; +function fixture() { + return { version: 1, ...expected, configuration: structuredClone(configuration), collectorOrigins: { telemetry: configuration.telemetryOrigin, otel: configuration.otelOrigin }, + announcement: { intercepted: true, requests: ['https://pages.paperclip.ing/announcements/v1/current.json'], unmatched: [] }, + signalObservation: { telemetryRequests: 1, traceRequests: 1, exportedSpans: 1, qualificationSpans: 1, metricRequests: 0, logRequests: 0, malformedTraceRequests: 0, unexpectedRequests: 0 }, + backupCheckpoint: { owner: 'normal-idle-checkpoint', startedAt: '2099-01-01T00:12:31Z', observedAt: '2099-01-01T00:13:00Z', wakeMarkerObserved: true, failureObserved: false, archives: [{ name: 'owned.sql.gz', bytes: 128, sqlBytes: 256, sha256: 'd'.repeat(64), gzipVerified: true, fixtureIdentifierFound: true, createdAt: '2099-01-01T00:12:45Z', createdAtSource: 'birthtime' }] }, startedAt: '2099-01-01T00:00:00Z', finishedAt: '2099-01-01T01:00:00Z', + features: { telemetry: true, announcements: true, automaticBackups: true, otel: true, exportedSpans: 1 }, + quiet: { startedAt: '2099-01-01T00:11:00Z', finishedAt: '2099-01-01T00:12:30Z', elapsedMs: 90_000, clock: 'real', sqlQueries: 0, connectionsOpened: 0, connectionsRemaining: 0, nonDatabaseRequests: 0, nonDatabaseBytes: 0, + observerRequests: observeIncomingHttpWindow( + { source: 'node-http-server-events', captureId: '11111111-1111-4111-8111-111111111111', installed: true, valid: true, requests: 12, observedAt: '2099-01-01T00:11:00Z' }, + { source: 'node-http-server-events', captureId: '11111111-1111-4111-8111-111111111111', installed: true, valid: true, requests: 12, observedAt: '2099-01-01T00:12:30Z' }) }, + eligibility: { observedAt: '2099-01-01T00:13:00Z', backgroundWork: 'none', ownedHold: true, unchangedGeneration: true, apiCooldownElapsedMs: 600_000 } }; +} +function deployed() { + const f = fixture(); + f.provider = { ...expected, live: true, observedAt: '2099-01-01T00:14:00Z', observerRequests: [{ method: 'GET', boundary: 'provider-control-plane' }], + neon: { identityVerified: true, currentState: 'idle', pendingState: null }, + railway: { identityVerified: true, state: 'OBSERVED', services: [{ identityVerified: true, state: 'OBSERVED', backupsEnabled: false, backupStorageVerified: false, allDeploymentsStopped: true }] } }; + return f; +} +test('local configured quiet leaves every provider and wake claim inconclusive', () => { + const r = evaluateSleepQualification(fixture(), expected).layers; + assert.equal(r.workerQuiet.verdict, 'VERIFIED'); assert.equal(r.eligibility.verdict, 'VERIFIED'); + for (const key of ['hostSuspension', 'databaseSuspension', 'backupPersistence', 'inboundWake', 'timedWake']) assert.equal(r[key].verdict, 'INCONCLUSIVE'); +}); +test('wrong source, target and configuration evidence cannot be combined', () => { + for (const key of Object.keys(expected)) { const f = fixture(); f[key] = 'wrong'; assert.throws(() => evaluateSleepQualification(f, expected), /identity/); } + const f = deployed(); f.provider.target = 'foreign'; assert.throws(() => evaluateSleepQualification(f, expected), /provider_identity/); +}); +test('fake time, a no-op exporter, open connections, observer health requests and a shortened cooldown fail', () => { + for (const change of [f => f.quiet.clock = 'qualified', f => f.features.exportedSpans = 0, f => f.quiet.connectionsRemaining = 1, + f => { f.quiet.observerRequests.after.requests++; f.quiet.observerRequests.requests++; }, f => f.quiet.nonDatabaseBytes = 1, f => f.quiet.elapsedMs = 89_999]) { + const f = fixture(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } + const f = fixture(); f.eligibility.apiCooldownElapsedMs = 599_999; assert.equal(evaluateSleepQualification(f, expected).layers.eligibility.verdict, 'NOT VERIFIED'); +}); +test('incoming HTTP observation must be installed, intact and cover the whole window; legacy empty lists are not evidence', () => { + const changes = [f => f.quiet.observerRequests = [], f => delete f.quiet.observerRequests, + f => f.quiet.observerRequests.before.installed = false, f => f.quiet.observerRequests.after.installed = false, + f => f.quiet.observerRequests.before.valid = false, f => f.quiet.observerRequests.after.valid = false, + f => delete f.quiet.observerRequests.before, f => delete f.quiet.observerRequests.after, + f => f.quiet.observerRequests.before.source = 'harness-api-only', f => f.quiet.observerRequests.after.source = 'unknown', + f => f.quiet.observerRequests.after.captureId = '22222222-2222-4222-8222-222222222222', + f => f.quiet.observerRequests.before.captureId = '', + f => f.quiet.observerRequests.before.observedAt = '2099-01-01T00:11:00.001Z', + f => f.quiet.observerRequests.after.observedAt = '2099-01-01T00:12:29.999Z', + f => f.quiet.observerRequests.before.observedAt = '2098-12-31T23:59:59Z', + f => f.quiet.observerRequests.after.observedAt = '2099-01-01T01:00:01Z', + f => f.quiet.observerRequests.after.observedAt = 'invalid', + f => f.quiet.observerRequests.after.requests--, + // A forged zero cannot hide an actual request while SQL and outbound counters remain zero. + f => f.quiet.observerRequests.after.requests++, + f => f.quiet.observerRequests.requests = 1]; + for (const point of ['before', 'after']) { + for (const value of [-1, 0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, '12']) { + changes.push(f => f.quiet.observerRequests[point].requests = value); + } + } + for (const value of [-1, 0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, '0']) changes.push(f => f.quiet.observerRequests.requests = value); + for (const change of changes) { + const f = fixture(); change(f); + assert.equal(f.quiet.sqlQueries, 0); assert.equal(f.quiet.nonDatabaseRequests, 0); + assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } +}); +test('configured feature claims require complete announcement and integer collector observations', () => { + const changes = [f => delete f.announcement, f => f.announcement.intercepted = false, + f => f.announcement.requests = [], f => f.announcement.requests = ['https://foreign.example/feed'], + f => f.announcement.unmatched.push('https://foreign.example/feed'), f => delete f.announcement.unmatched, + f => delete f.signalObservation, f => f.signalObservation.telemetryRequests = 0, + f => f.signalObservation.traceRequests = 0, f => f.signalObservation.exportedSpans = 0, + f => f.signalObservation.qualificationSpans = 0, f => f.signalObservation.qualificationSpans = 2, + f => f.signalObservation.metricRequests = 1, f => f.signalObservation.logRequests = 1, + f => f.signalObservation.malformedTraceRequests = 1, f => f.signalObservation.unexpectedRequests = 1, + f => f.features.telemetry = false, f => f.features.announcements = false, f => f.features.otel = false]; + for (const key of Object.keys(fixture().signalObservation)) { + changes.push(f => delete f.signalObservation[key]); + for (const value of [-1, 0.5, NaN, Infinity, '1']) changes.push(f => f.signalObservation[key] = value); + } + for (const change of changes) { const f = fixture(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); } +}); +test('assembled receipts derive coverage from actual named spans and requests, including unwanted exporters', () => { + const requests = [{ path: '/telemetry', body: {} }, { path: '/v1/traces', body: { resourceSpans: [{ scopeSpans: [{ spans: [ + { name: 'ordinary-http' }, { name: 'sleep-qualification-stimulus' }, + ] }] }] } }]; + const f = fixture(); f.signalObservation = observeConfiguredSignals(requests); f.features = configuredSleepFeatures(f); + assert.deepEqual(f.signalObservation, { telemetryRequests: 1, traceRequests: 1, exportedSpans: 2, qualificationSpans: 1, + metricRequests: 0, logRequests: 0, malformedTraceRequests: 0, unexpectedRequests: 0 }); + assert.equal(f.features.exportedSpans, 2); // One trace request contained two real span records. + assert.equal(finalizeLocalSleepReceipt(f, expected, () => f.finishedAt).report.layers.workerQuiet.verdict, 'VERIFIED'); + for (const extra of [{ path: '/v1/metrics?batch=1', body: {} }, { path: '/v1/logs', body: {} }, + { path: '/unmatched', body: {} }, { path: '/v1/traces', body: { resourceSpans: [{}] } }]) { + const changed = fixture(); changed.signalObservation = observeConfiguredSignals([...requests, extra]); changed.features = configuredSleepFeatures(changed); + assert.equal(evaluateSleepQualification(changed, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } + const noop = fixture(); noop.signalObservation = observeConfiguredSignals([{ path: '/telemetry', body: {} }, + { path: '/v1/traces', body: { resourceSpans: [{ scopeSpans: [{ spans: [{ name: 'other', attributes: [{ value: 'sleep-qualification-stimulus' }] }] }] }] } }]); + noop.features = configuredSleepFeatures(noop); assert.equal(noop.signalObservation.qualificationSpans, 0); + assert.equal(evaluateSleepQualification(noop, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); +}); +test('registry sleep, incomplete deployment inventory and a failed metadata read cannot prove providers stopped', () => { + const f = deployed(); f.provider.neon = { currentState: 'sleeping', identityVerified: true }; f.provider.railway.services = []; + let r = evaluateSleepQualification(f, expected).layers; assert.equal(r.databaseSuspension.verdict, 'INCONCLUSIVE'); assert.equal(r.hostSuspension.verdict, 'INCONCLUSIVE'); + f.provider.neon = { currentState: 'idle', pendingState: 'active', identityVerified: true }; r = evaluateSleepQualification(f, expected).layers; assert.equal(r.databaseSuspension.verdict, 'NOT VERIFIED'); +}); +test('useful authorized deadline outcome requires a durable alarm before the exact stop proof', () => { + const f = deployed(); f.timedWake = { stimulusAt: '2099-01-01T00:20:00Z', finishedAt: '2099-01-01T00:21:00Z', ordinaryProducer: true, authorizationVerified: true, persistedOutcomeVerified: true, duplicateEffects: 0, sameFinalGeneration: true, alarmPersistedAt: '2099-01-01T00:13:30Z' }; + assert.equal(evaluateSleepQualification(f, expected).layers.timedWake.verdict, 'VERIFIED'); + for (const change of [x => x.timedWake.alarmPersistedAt = '2099-01-01T00:15:00Z', x => x.timedWake.sameFinalGeneration = false, x => x.timedWake.persistedOutcomeVerified = false, x => x.timedWake.duplicateEffects = 1]) { + const x = structuredClone(f); change(x); assert.equal(evaluateSleepQualification(x, expected).layers.timedWake.verdict, 'NOT VERIFIED'); + } +}); +test('writable path alone cannot establish backup persistence across stop and wake', () => { + const f = deployed(); f.provider.storage = { identityVerified: true, volumeId: 'volume', beforeVolumeId: 'volume', afterVolumeId: 'volume', mountPath: '/paperclip', backupsEnabledBefore: true, backupsEnabledAfter: true, sentinelBefore: 'd'.repeat(64), sentinelAfter: 'd'.repeat(64), beforeAt: '2099-01-01T00:13:00Z', afterAt: '2099-01-01T00:25:00Z' }; + assert.equal(evaluateSleepQualification(f, expected).layers.backupPersistence.verdict, 'VERIFIED'); + for (const change of [x => delete x.provider.storage.volumeId, x => x.provider.storage.afterVolumeId = 'replacement', x => x.provider.storage.backupsEnabledAfter = false, x => x.provider.storage.sentinelAfter = 'e'.repeat(64)]) { + const x = structuredClone(f); change(x); assert.equal(evaluateSleepQualification(x, expected).layers.backupPersistence.verdict, 'INCONCLUSIVE'); + } +}); + +test('unknown backup policy and unverified enabled storage cannot certify a stopped host or wake', () => { + for (const change of [x => delete x.provider.railway.state, x => x.provider.railway.state = 'INCONCLUSIVE', + x => delete x.provider.railway.services[0].backupsEnabled, x => x.provider.railway.services[0].backupsEnabled = null, + x => x.provider.railway.services[0].state = 'INCONCLUSIVE', x => x.provider.railway.services[0].backupsEnabled = true]) { + const f = deployed(); change(f); + f.inboundWake = { stimulusAt: '2099-01-01T00:20:00Z', finishedAt: '2099-01-01T00:21:00Z', ordinaryProducer: true, authorizationVerified: true, persistedOutcomeVerified: true, duplicateEffects: 0 }; + f.timedWake = { ...f.inboundWake, alarmPersistedAt: '2099-01-01T00:13:30Z', sameFinalGeneration: true }; + const r = evaluateSleepQualification(f, expected).layers; + assert.equal(r.hostSuspension.verdict, 'INCONCLUSIVE'); assert.equal(r.inboundWake.verdict, 'INCONCLUSIVE'); assert.equal(r.timedWake.verdict, 'INCONCLUSIVE'); + } + const f = deployed(); assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'VERIFIED'); + f.provider.railway.services[0].backupsEnabled = true; f.provider.railway.services[0].backupStorageVerified = true; + assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'VERIFIED'); +}); + +function woken() { + const f = deployed(); + f.inboundWake = { stimulusAt: '2099-01-01T00:20:00Z', finishedAt: '2099-01-01T00:21:00Z', ordinaryProducer: true, authorizationVerified: true, persistedOutcomeVerified: true, duplicateEffects: 0 }; + f.returnToIdle = { quietStartedAt: '2099-01-01T00:31:00Z', quietFinishedAt: '2099-01-01T00:32:30Z', observedAt: '2099-01-01T00:33:00Z', freshQuietWindowMs: 90_000, backgroundWork: 'none', ownedHold: true }; + return f; +} +test('return to idle requires a real 90-second window after the latest wake', () => { + assert.equal(evaluateSleepQualification(woken(), expected).layers.returnToIdle.verdict, 'VERIFIED'); + for (const finishedAt of ['invalid', '2098-12-31T23:59:59Z', '2099-01-01T01:00:01Z']) { + const f = woken(); f.timedWake = { ...f.inboundWake, finishedAt, alarmPersistedAt: '2099-01-01T00:13:30Z', sameFinalGeneration: true }; + assert.equal(evaluateSleepQualification(f, expected).layers.inboundWake.verdict, 'VERIFIED'); + assert.equal(evaluateSleepQualification(f, expected).layers.returnToIdle.verdict, 'NOT VERIFIED'); + } + + for (const change of [ + f => f.returnToIdle.quietFinishedAt = '2099-01-01T00:31:00.001Z', + f => { f.returnToIdle.quietStartedAt = '2099-01-01T00:19:00Z'; f.returnToIdle.quietFinishedAt = '2099-01-01T00:20:30Z'; }, + f => f.returnToIdle.quietStartedAt = '2098-12-31T23:00:00Z', + f => f.returnToIdle.quietFinishedAt = '2099-01-01T01:00:01Z', + f => f.returnToIdle.quietFinishedAt = '2099-01-01T00:30:00Z', + f => f.returnToIdle.observedAt = '2099-01-01T00:32:00Z', + f => f.returnToIdle.freshQuietWindowMs = Infinity, + f => f.returnToIdle.freshQuietWindowMs = 90_001, + f => delete f.returnToIdle.quietStartedAt, + ]) { + const f = woken(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.returnToIdle.verdict, 'NOT VERIFIED'); + } +}); +test('exact observed active providers are negative evidence while unknown identity stays inconclusive', () => { + for (const state of [{ currentState: 'active', pendingState: null }, { currentState: 'idle', pendingState: 'active' }]) { + const f = deployed(); Object.assign(f.provider.neon, state); + assert.equal(evaluateSleepQualification(f, expected).layers.databaseSuspension.verdict, 'NOT VERIFIED'); + f.provider.neon.identityVerified = false; + assert.equal(evaluateSleepQualification(f, expected).layers.databaseSuspension.verdict, 'INCONCLUSIVE'); + } + const f = deployed(); f.provider.railway.services[0].allDeploymentsStopped = false; + assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'NOT VERIFIED'); + f.provider.railway.services[0].identityVerified = false; + assert.equal(evaluateSleepQualification(f, expected).layers.hostSuspension.verdict, 'INCONCLUSIVE'); +}); + +test('logical digest is stable across origins and detects changed enabled configuration', () => { + assert.equal(sleepConfigurationDigest(configuration), configDigest); + const f = fixture(); f.configuration.telemetryOrigin = 'http://127.0.0.1:5555/telemetry'; f.configuration.otelOrigin = 'http://127.0.0.1:5555'; + f.collectorOrigins = { telemetry: f.configuration.telemetryOrigin, otel: f.configuration.otelOrigin }; + assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'VERIFIED'); + assert.throws(() => evaluateSleepQualification(f, { ...expected, collectorOrigins: fixture().collectorOrigins }), /collector_origins/); + for (const change of [x => x.configuration.announcements = false, x => x.configuration.serveUi = false, + x => x.configuration.automaticBackups.intervalMinutes = 5, x => x.configuration.automaticBackups.idleCheckpoint = false, + x => x.configuration.managedInstance = true, x => x.configuration.deploymentMode = 'authenticated', x => x.configuration.storageProvider = 's3', + x => x.configuration.otelOrigin = '', x => x.configuration.telemetryOrigin = '']) { + const changed = fixture(); change(changed); assert.throws(() => evaluateSleepQualification(changed, expected), /configuration_digest/); + } +}); +test('configured backup feature requires observed verified owned archives and a wake marker', () => { + for (const change of [f => delete f.backupCheckpoint, f => f.backupCheckpoint.archives = [], + f => f.backupCheckpoint.wakeMarkerObserved = false, f => f.backupCheckpoint.failureObserved = true, + f => f.backupCheckpoint.archives[0].gzipVerified = false, f => f.backupCheckpoint.archives[0].fixtureIdentifierFound = false, + f => f.backupCheckpoint.archives[0].bytes = 0, f => f.backupCheckpoint.archives[0].sha256 = 'invalid', + f => f.backupCheckpoint.archives[0].createdAt = '2098-01-01T00:00:00Z', + f => f.backupCheckpoint.archives[0].createdAtSource = 'unknown', + f => f.backupCheckpoint.startedAt = '2099-01-01T00:11:00Z']) { + const f = fixture(); change(f); assert.equal(evaluateSleepQualification(f, expected).layers.workerQuiet.verdict, 'NOT VERIFIED'); + } +}); +test('receipt finalization follows observations and validates the actual resulting receipt', () => { + const f = fixture(); delete f.finishedAt; + let tick = Date.parse(f.eligibility.observedAt); + const { receipt, report } = finalizeLocalSleepReceipt(f, expected, () => new Date(++tick).toISOString()); + assert.ok(Date.parse(receipt.finishedAt) > Date.parse(receipt.eligibility.observedAt)); + assert.equal(report.layers.eligibility.verdict, 'VERIFIED'); + assert.throws(() => finalizeLocalSleepReceipt(f, expected, () => f.startedAt), /local_evidence/); +}); +test('CLI exits one for known active providers and zero for inconclusive identity', async () => { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-evidence-cli-')); + try { + const input = path.join(directory, 'receipt.json'), expectedPath = path.join(directory, 'expected.json'), output = path.join(directory, 'report.json'); + await writeFile(expectedPath, JSON.stringify(expected)); + const f = deployed(); f.provider.neon.currentState = 'active'; + for (const [identityVerified, status] of [[true, 1], [false, 0]]) { + f.provider.neon.identityVerified = identityVerified; await writeFile(input, JSON.stringify(f)); + const child = spawnSync(process.execPath, [fileURLToPath(new URL('../../scripts/sleep-qualification/evidence.mjs', import.meta.url)), input, expectedPath, output], { encoding: 'utf8' }); + assert.equal(child.status, status, child.stderr); + } + } finally { await rm(directory, { recursive: true, force: true }); } +}); diff --git a/server/src/__tests__/chat-completion-delivery.test.ts b/server/src/__tests__/chat-completion-delivery.test.ts index 2dfd4290972..df769c99372 100644 --- a/server/src/__tests__/chat-completion-delivery.test.ts +++ b/server/src/__tests__/chat-completion-delivery.test.ts @@ -290,6 +290,9 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue. async function interruptedBootstrap() { const f = await seed(); await f.finish(); const run = await f.run(); const interrupted = await terminalizeLegacyExecution({ db, run, status: "interrupted", patch: { + // The real controller claim records preparation before terminal status + // records completion. The delivery fixture otherwise leaves both unset. + executionStage: "preparing", finishedAt: new Date(), errorCode: "server_shutdown_interrupted", runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, } }); expect(await getExecutionBlocker(db, f.companyId, f.sourceId)).toMatchObject({ runId: run.id }); @@ -338,8 +341,10 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue. expect(await settleInterruptedNativeBootstrap(db, { run, providerDispatchStarted: false })).not.toBeNull(); expect(await issueService(db).getById(f.sourceId)).toMatchObject({ status: "blocked" }); }); - it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted"])("retains an interrupted bootstrap hold for %s", async kind => { + it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted", "missing-stage", "unfinished"])("retains an interrupted bootstrap hold for %s", async kind => { const { f, run } = await interruptedBootstrap(); + if (kind === "missing-stage") await db.update(heartbeatRuns).set({ executionStage: null }).where(eq(heartbeatRuns.id, run.id)); + if (kind === "unfinished") await db.update(heartbeatRuns).set({ finishedAt: null }).where(eq(heartbeatRuns.id, run.id)); if (kind === "runtime-selected") await db.update(heartbeatRuns).set({ runtimeModeResolvedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); if (kind === "other-adapter") await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "process" } } }).where(eq(heartbeatRuns.id, run.id)); if (kind === "restore-unsafe") await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreFailure: "restore_unsafe_archive" } }).where(eq(heartbeatRuns.id, run.id)); diff --git a/server/src/__tests__/chat-run-milestone-work.test.ts b/server/src/__tests__/chat-run-milestone-work.test.ts index 9f532ec8fc3..710ba505c68 100644 --- a/server/src/__tests__/chat-run-milestone-work.test.ts +++ b/server/src/__tests__/chat-run-milestone-work.test.ts @@ -330,7 +330,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.transaction(async tx => { await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), { - companyId: f.companyId, activeExecutionRunId: seeded.id, mergedContextSnapshot: chatContextSnapshot(), + companyId: f.companyId, activeExecutionRunId: seeded.id, activeExecutionRunStatus: seeded.status, mergedContextSnapshot: chatContextSnapshot(), agentId: f.agentId, source: "on_demand", triggerDetail: null, payload: { issueId: f.issueId }, requestedByActorType: "user", requestedByActorId: f.userId, idempotencyKey: null, }); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 21738faa5c0..9231fb7a089 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -3808,17 +3808,23 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); - await expect(runtimeWithPlugin.acquireRunLease({ - companyId, - environment, - issueId: null, - agentId, - heartbeatRunId: runId, - persistedExecutionWorkspace: { - id: executionWorkspaceId, - mode: "shared_workspace", - }, - })).rejects.toThrow("the lease was preserved and no replacement was created"); + // Keep the exact configured budget stable across deadline creation and RPC dispatch. + const now = vi.spyOn(Date, "now").mockReturnValue(Date.now()); + try { + await expect(runtimeWithPlugin.acquireRunLease({ + companyId, + environment, + issueId: null, + agentId, + heartbeatRunId: runId, + persistedExecutionWorkspace: { + id: executionWorkspaceId, + mode: "shared_workspace", + }, + })).rejects.toThrow("the lease was preserved and no replacement was created"); + } finally { + now.mockRestore(); + } expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentResumeLease", expect.objectContaining({ driverKey: "fake-plugin", diff --git a/server/src/__tests__/ephemeral-migration.test.ts b/server/src/__tests__/ephemeral-migration.test.ts index 16ac2b79372..a2687c0e4ec 100644 --- a/server/src/__tests__/ephemeral-migration.test.ts +++ b/server/src/__tests__/ephemeral-migration.test.ts @@ -10,7 +10,7 @@ import { randomBytes, randomUUID } from "node:crypto"; import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; +import { activityLog, agentWakeupRequests, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "../services/remote-workspace-snapshot-storage.js"; @@ -53,6 +53,7 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll await db.delete(issues).where(eq(issues.companyId, companyId)); await db.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, companyId)); await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); await db.delete(agents).where(eq(agents.companyId, companyId)); await db.delete(companies).where(eq(companies.id, companyId)); let token: string | undefined; @@ -104,7 +105,8 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll const parent = path.join(home, "project"), child = path.join(parent, ".paperclip/worktrees/task"); await fs.mkdir(child, { recursive: true }); const fixtures = [{ cwd: parent, file: "project.txt", bytes: "project work" }, { cwd: child, file: "task.txt", bytes: "nested task work" }]; - const ids: string[] = []; + const ids: string[] = [], queued = []; + const queuedAgent = await agent(); for (const fixture of fixtures) { await fs.writeFile(path.join(fixture.cwd, fixture.file), fixture.bytes); const [issue] = await db.insert(issues).values({ companyId, title: fixture.file }).returning(); @@ -112,10 +114,20 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll name: fixture.file, mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: fixture.cwd }).returning(); ids.push(workspace!.id); await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issue!.id)); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId: queuedAgent.id, + source: "on_demand", payload: { issueId: issue!.id, text: "Continue my saved work" } }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: queuedAgent.id, + issueId: issue!.id, wakeupRequestId: wake!.id, status: "queued", contextSnapshot: { issueId: issue!.id, + executionWorkspaceId: workspace!.id } }).returning(); + queued.push({ run: run!, wake: wake! }); } expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 2 } }); await fs.rm(home, { recursive: true }); home = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-successor-")); vi.stubEnv("PAPERCLIP_HOME", home); + for (const { run, wake } of queued) { + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]).toEqual(wake); + } for (const [index, workspaceId] of ids.entries()) { const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)); const authority = readRemoteWorkspaceAuthority(workspace!.metadata)!; @@ -137,6 +149,8 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId: row.id, status: "interrupted", runtimeMode: "native", logStore: "local_file", logRef }); vi.stubEnv("PAPERCLIP_DECISION_SIGNING_SECRET", ""); + const [queued] = await db.insert(heartbeatRuns).values({ companyId, agentId: row.id, status: "queued", + contextSnapshot: { input: "Resume this turn after maintenance" } }).returning(); const original = resolveDecisionSigningSecret(); const listener = createServer(); await new Promise(resolve => listener.listen(0, "127.0.0.1", resolve)); @@ -167,6 +181,7 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll } expect(status).toMatchObject({ id, commit: stamp.commit, state: "ready", eligibility: { eligible: true } }); expect(JSON.stringify(status)).not.toContain(original); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued!.id)))[0]).toEqual(queued); const exported = await fetch(`${origin}/export`, { method: "POST", headers }).then(r => r.json()); expect(exported).toMatchObject({ id, commit: stamp.commit, decisionSigningSecret: original }); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); diff --git a/server/src/__tests__/generic-mcp-connection.test.ts b/server/src/__tests__/generic-mcp-connection.test.ts index 2978db9bbe1..78bf314dc4f 100644 --- a/server/src/__tests__/generic-mcp-connection.test.ts +++ b/server/src/__tests__/generic-mcp-connection.test.ts @@ -1445,7 +1445,7 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { ]); }); - it("does not let another user take over an archived personal URL connection", async () => { + it("creates a separate fresh personal URL connection and keeps explicit reconnect owner-bound", async () => { const fixture = installMcpOAuthFixture({ auth: "oauth" }); const company = await createCompany(db); const service = toolAccessService(db); @@ -1456,18 +1456,36 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { await db.insert(companyMemberships).values({ companyId: company.id, principalType: "user", principalId: "other-user", status: "active", membershipRole: "admin", }); + const archivedConnection = await db.select().from(toolConnections).where(eq(toolConnections.id, first.connectionId)); + const archivedApplication = await db.select().from(toolApplications).where(eq(toolApplications.id, first.application.id)); fixture.fetchMock.mockClear(); - await expect(service.connectGalleryApp(company.id, input, { - actorType: "user", actorId: "other-user", - })).rejects.toMatchObject({ status: 403, message: "Only the existing personal identity can reconnect this connection" }); + for (const target of [{ reconnectConnectionId: first.connectionId }, { applicationId: first.application.id }]) { + await expect(service.connectGalleryApp(company.id, { ...input, ...target }, { + actorType: "user", actorId: "other-user", + })).rejects.toMatchObject({ status: 403, message: "Only the existing personal identity can reconnect this connection" }); + } expect(fixture.fetchMock).not.toHaveBeenCalled(); + const fresh = await service.connectGalleryApp(company.id, input, { actorType: "user", actorId: "other-user" }); + expect(fresh.connectionId).not.toBe(first.connectionId); + expect(fresh.application.id).not.toBe(first.application.id); + expect(fresh.auth).toMatchObject({ kind: "oauth" }); + await expect(service.getConnection(fresh.connectionId)).resolves.toMatchObject({ + createdByUserId: "other-user", credentialPolicy: "per_user", credentialSecretRefs: [], + }); + await expect(db.select().from(toolConnections).where(eq(toolConnections.id, first.connectionId))) + .resolves.toEqual(archivedConnection); + await expect(db.select().from(toolApplications).where(eq(toolApplications.id, first.application.id))) + .resolves.toEqual(archivedApplication); + // OAuth grants are created only after authorization; discovery is not a grant. await expect(db.select().from(connectionGrants)).resolves.toHaveLength(0); await expect(service.getConnection(first.connectionId)).resolves.toMatchObject({ status: "archived", createdByUserId: "board-user", }); - const resumed = await service.connectGalleryApp(company.id, input, { actorType: "user", actorId: "board-user" }); + const resumed = await service.connectGalleryApp(company.id, { + ...input, reconnectConnectionId: first.connectionId, + }, { actorType: "user", actorId: "board-user" }); expect(resumed.connectionId).toBe(first.connectionId); expect(resumed.auth).toMatchObject({ kind: "oauth" }); }); @@ -1544,7 +1562,7 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { return jsonResponse({ jsonrpc: "2.0", id: "paperclip-catalog-refresh", result: { tools: FIXTURE_TOOLS } }); }); const failure = service.connectGalleryApp(company.id, { - link: MCP_URL, name: "Personal retry rollback", grantKind: "user", + link: MCP_URL, name: "Personal retry rollback", grantKind: "user", reconnectConnectionId: first.connectionId, }, actor).then(() => null, (error: unknown) => error); await Promise.race([catalogPending, failure.then((error) => { throw error ?? new Error("Retry finished before the catalog probe"); })]); try { @@ -1587,7 +1605,9 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { FOR EACH ROW EXECUTE FUNCTION test_personal_profile_failure() `); try { - await expect(service.connectGalleryApp(company.id, input, actor)).rejects.toThrow(); + await expect(service.connectGalleryApp(company.id, { + ...input, reconnectConnectionId: first.connectionId, + }, actor)).rejects.toThrow(); await expect(db.select().from(toolCatalogEntries).where(eq(toolCatalogEntries.connectionId, first.connectionId))) .resolves.toEqual(catalogBefore); await expect(db.select().from(toolProfiles).where(eq(toolProfiles.companyId, company.id))).resolves.toHaveLength(0); diff --git a/server/src/__tests__/heartbeat-ephemeral-queued-migration.test.ts b/server/src/__tests__/heartbeat-ephemeral-queued-migration.test.ts new file mode 100644 index 00000000000..a1876242ca7 --- /dev/null +++ b/server/src/__tests__/heartbeat-ephemeral-queued-migration.test.ts @@ -0,0 +1,214 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command } from "@aws-sdk/client-s3"; +import { agents, authUsers, companies, companyMemberships, createDb, closeRegisteredClients, environmentLeases, + environments, executionWorkspaces, heartbeatRuns, agentWakeupRequests, issues, issueWorkProducts, nativeRunResults } from "@paperclipai/db"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import type { NativeExecutionInput, NativeSession, NativeSessionBackend, PrpEvent } from "@paperclipai/paperclip-runner"; +import type { EnvironmentLease } from "@paperclipai/shared"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createIsolatedTestHome } from "./helpers/isolated-test-home.js"; +import { heartbeatService, startTaskDrain, stopTaskDrain } from "../services/heartbeat.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; +import * as environmentTarget from "../services/environment-execution-target.js"; +import { migrateEphemeralServerData } from "../maintenance/ephemeral-migration-data.js"; +import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js"; +import { materializeInstructionBytes } from "../services/agent-instruction-files.js"; +import { flushInFlightRunLogMirrors } from "../services/run-log-store.js"; +import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../vendor/paperclip-runner/testing.js"; + +vi.mock("@paperclipai/adapter-utils/execution-target", async importOriginal => ({ + ...await importOriginal(), + // The scripted session makes no sandbox-to-host API calls. + startAdapterExecutionTargetPaperclipBridge: async () => null, +})); + +// Real database, migration, object storage, queue claim, workspace restoration, +// instruction materialization and result settlement. Only the sandbox transport +// and provider session are scripted; neither can read the deleted predecessor. +describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("queued turn after preserving offline enrollment", () => { + let database: Awaited>; + let db: ReturnType, root: string, bucket: string, s3: S3Client; + let home: ReturnType; + let replacement: ReturnType | undefined; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("queued-enrollment-"); + db = createDb(database.connectionString); + home = createIsolatedTestHome(); root = await fs.realpath(home.home); + bucket = `queued-enrollment-${randomUUID()}`; + s3 = new S3Client({ endpoint: process.env.PAPERCLIP_TEST_S3_ENDPOINT, region: "us-east-1", forcePathStyle: true }); + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + vi.stubEnv("DATABASE_URL", database.connectionString); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "predecessor")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "74".repeat(32)); + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); + vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", process.env.PAPERCLIP_TEST_S3_ENDPOINT!); + vi.stubEnv("PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "true"); + }, 90_000); + afterAll(async () => { + stopTaskDrain(); + await replacement?.drainActiveRunExecutions(); + await flushInFlightRunLogMirrors(); + vi.restoreAllMocks(); + if (database) await closeRegisteredClients(database.connectionString, { scope: "database" }); + await database?.cleanup(); + if (s3) { + for (;;) { + const page = await s3.send(new ListObjectsV2Command({ Bucket: bucket })); + if (!page.Contents?.length) break; + await s3.send(new DeleteObjectsCommand({ Bucket: bucket, Delete: { Objects: page.Contents.map(row => ({ Key: row.Key! })) } })); + } + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); s3.destroy(); + } + home?.cleanup(); + vi.unstubAllEnvs(); + }); + + it("admits the original queued run once and reads its saved workspace and instructions without the old disk", async () => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), userId = randomUUID(); + const workProductId = randomUUID(), evidenceRef = `work_product:${workProductId}`; + await instanceSettingsService(db).updateExperimental({ enableNativeRunner: true, enableIsolatedWorkspaces: true }); + await db.insert(authUsers).values({ id: userId, name: "Owner", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Queued enrollment", issuePrefix: "QEM", defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" }); + const [environment] = await db.insert(environments).values({ name: `Scripted sandbox ${companyId}`, driver: "sandbox", + status: "active", config: { provider: "daytona" } }).returning(); + const instructions = "Preserved instructions: read saved.txt before completing this turn."; + const instructionsRoot = resolveManagedInstructionsRoot({ id: agentId, companyId, name: "Queued worker", adapterConfig: {} }); + await materializeInstructionBytes(instructionsRoot, "AGENTS.md", Buffer.from(instructions)); + await db.insert(agents).values({ id: agentId, companyId, name: "Queued worker", adapterType: "paperclip_runner", status: "idle", + defaultEnvironmentId: environment.id, adapterConfig: { provider: "codex", instructionsBundleMode: "managed", instructionsRootPath: instructionsRoot }, + runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }); + await db.insert(issues).values({ id: issueId, companyId, title: "Continue saved work", issueNumber: 1, identifier: "QEM-1", + status: "todo", assigneeAgentId: agentId, responsibleUserId: userId, executionWorkspacePreference: "reuse_existing", + executionWorkspaceSettings: { mode: "isolated_workspace" } }); + const oldHome = process.env.PAPERCLIP_HOME!, cwd = path.join(oldHome, "task"); + await fs.mkdir(cwd, { recursive: true }); await fs.writeFile(path.join(cwd, "saved.txt"), "unfinished work before migration"); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issueId, name: "Saved task", + mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd }).returning(); + await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issueId)); + await db.insert(issueWorkProducts).values({ id: workProductId, companyId, issueId, type: "artifact", provider: "paperclip", + title: "Migration fixture evidence", status: "ready_for_review", reviewState: "approved" }); + const predecessor = heartbeatService(db); + startTaskDrain({}); + await predecessor.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", reason: "manual", + requestedByActorType: "user", requestedByActorId: userId, payload: { issueId }, contextSnapshot: { issueId, skipIssueComment: true } }); + await predecessor.drainActiveRunExecutions(); + const [queued] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + expect(queued).toMatchObject({ status: "queued", startedAt: null, wakeupRequestId: wake.id }); + expect(wake.status).toBe("queued"); + expect(await migrateEphemeralServerData(db, { signal: AbortSignal.timeout(60_000), phase: () => {} })) + .toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 1 } }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)))[0]).toEqual(queued); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]).toEqual(wake); + await fs.rm(oldHome, { recursive: true }); + await expect(fs.stat(oldHome)).rejects.toMatchObject({ code: "ENOENT" }); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "replacement")); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "false"); + + const remoteCwd = path.join(root, "independent-sandbox"); await fs.mkdir(remoteCwd); + const exec = promisify(execFile); + const transfer: NonNullable = async operations => { + const results = []; + for (const operation of operations) { + for (const file of operation.files) { + await fs.mkdir(path.dirname(file.targetPath), { recursive: true }); + await fs.cp(file.sourcePath, file.targetPath, { recursive: file.kind === "directory" }); + } + results.push({ operationId: operation.operationId, filesTransferred: operation.files.length, bytesTransferred: 0 }); + } + return { operations: results }; + }; + const runner: CommandManagedRuntimeRunner = { syncIn: transfer, syncOut: transfer, execute: async input => { + const command = exec(input.command, input.args ?? [], { cwd: input.cwd ?? remoteCwd, env: { ...process.env, ...input.env }, + timeout: input.timeoutMs, maxBuffer: 4 * 1024 * 1024 }); + command.child.stdin?.end(input.stdin ?? ""); + const result = await command; + return { exitCode: 0, stdout: result.stdout, stderr: result.stderr, signal: null, timedOut: false, pid: process.pid, startedAt: new Date().toISOString() }; + } }; + vi.spyOn(environmentTarget, "resolveEnvironmentExecutionTarget").mockImplementation(async input => ({ + kind: "remote", transport: "sandbox", remoteCwd: typeof input.leaseMetadata?.remoteCwd === "string" ? input.leaseMetadata.remoteCwd : remoteCwd, + runner, provider: "daytona", leaseId: input.leaseId, sandboxId: `fixture-${queued.id}`, timeoutMs: 60_000, + }) as AdapterExecutionTarget); + const runtime = environmentRuntimeService(db); + vi.spyOn(runtime, "acquireRunLease").mockImplementation(async input => { + const [lease] = await db.insert(environmentLeases).values({ companyId, environmentId: environment.id, heartbeatRunId: input.heartbeatRunId, + issueId, executionWorkspaceId: workspace.id, provider: "daytona", providerLeaseId: `fixture-${queued.id}`, status: "active", metadata: { remoteCwd } }).returning(); + return { environment: input.environment, lease: lease as EnvironmentLease, + leaseContext: { executionWorkspaceId: workspace.id, executionWorkspaceMode: "isolated_workspace" } }; + }); + // The scripted provider owns no external allocation. Retain its directory + // until after the normal native workspace finalizer has completed. + vi.spyOn(runtime, "releaseRunLeases").mockResolvedValue([]); + const startTurn = vi.fn(); + const backendFactory = (execution: NativeExecutionInput): NativeSessionBackend => { + const turnId = `turn-${queued.id}`; + const identity = { companyId, agentId, issueId, runId: queued.id, sessionId: execution.session.normalizedSessionId! }; + const result = structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT); + result.completionClaim.contractRevision = execution.completionContract.contract.revision; + result.completionClaim.criteria = execution.completionContract.contract.criteria.map(criterion => ({ + ...result.completionClaim.criteria[0]!, criterionId: criterion.id, evidenceRefs: [evidenceRef], + })); + result.evidence = [{ kind: "work_product", ref: evidenceRef }]; result.verification[0]!.artifactRef = evidenceRef; + let started!: () => void; + const turnStarted = new Promise(resolve => { started = resolve; }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, + async *events() { + await turnStarted; + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)); + yield { schema: "paperclip.prp.event.v1", sourceEventId: `${queued.id}:terminal`, sourceSeq: 1, sourceInstanceId: run.runnerInstanceId!, + sourceKind: "runner", runId: queued.id, normalizedSessionId: identity.sessionId, turnId, eventType: "turn.completed", + schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(), payload: {} } satisfies PrpEvent; + }, + async startTurn(input) { + expect(execution.binding.runId).toBe(queued.id); + expect(execution.workspace.cwd).toBeTruthy(); + expect(await fs.readFile(path.join(execution.workspace.cwd!, "saved.txt"), "utf8")).toBe("unfinished work before migration"); + if (!("runtimeContext" in execution)) throw new Error("Expected native runtime context"); + const bundle = execution.runtimeContext.instructions; + const instructionPath = path.join(bundle.bundle.rootPath, bundle.entryPath); + expect(instructionPath.startsWith(`${oldHome}/`)).toBe(false); + expect(await fs.readFile(instructionPath, "utf8")).toBe(instructions); + await expect(fs.stat(path.join(cwd, "saved.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + startTurn(input); started(); return { turnId }; + }, + async result() { await turnStarted; return { result, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId }; }, + async snapshot() { return { backendKind: "mock", sessionId: "queued-migration", identity, providerSessionId: "queued-migration", + activeTurnId: null, pendingRuntimeRequests: [], lineage: [], cursor: "1" }; }, + async close() {}, + }; + return { async descriptor() { return { kind: "mock", name: "queued-migration", version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" } }; }, + async openSession() { return session; } }; + }; + replacement = heartbeatService(db, { environmentRuntime: runtime, nativeSessionBackendFactory: backendFactory }); + stopTaskDrain(); + await replacement.resumeQueuedRunsAfterDrain(); await replacement.drainActiveRunExecutions(); + const finished = await replacement.getRun(queued.id); + expect(finished?.status, JSON.stringify({ errorCode: finished?.errorCode, error: finished?.error, result: finished?.resultJson })).toBe("succeeded"); + expect(finished?.wakeupRequestId).toBe(wake.id); + await replacement.resumeQueuedRunsAfterDrain(); await replacement.drainActiveRunExecutions(); + expect(startTurn).toHaveBeenCalledOnce(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).toHaveLength(1); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, queued.id))).toHaveLength(1); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))) + .toEqual([expect.objectContaining({ id: wake.id, status: "completed", runId: queued.id, + claimedAt: expect.any(Date), finishedAt: expect.any(Date), payload: wake.payload })]); + }, 120_000); +}); diff --git a/server/src/__tests__/helpers/maintenance/announcement-boundary.ts b/server/src/__tests__/helpers/maintenance/announcement-boundary.ts new file mode 100644 index 00000000000..44d642fe9c0 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/announcement-boundary.ts @@ -0,0 +1,36 @@ +import { mock } from 'node:test'; +import { DEFAULT_ANNOUNCEMENT_FEED_URL, announcementManifestSchema } from '@paperclipai/shared'; + +/** Test-child transport seam: actual route/feed/cache, no DNS or HTTPS claim. */ +export async function installMaintenanceAnnouncementBoundary() { + const actual = await import('../../../app.js'); + const requests: string[] = [], unmatched: string[] = []; + let intercepted = false; + const manifest = announcementManifestSchema.parse({ schemaVersion: 1, announcement: { + id: 'new-projects', eyebrow: 'New', title: 'Projects', description: 'Organize your work.', + primaryAction: { kind: 'route', label: 'Open', path: '/projects' }, + } }); + const wrapped: typeof actual.createApp = (db, options) => { + const announcements = options.announcements; + if (!announcements?.enabled || announcements.feedUrl !== DEFAULT_ANNOUNCEMENT_FEED_URL) { + throw new Error('Configured announcement boundary requires the exact enabled default feed'); + } + intercepted = true; + const fetch = async (url: URL, init: RequestInit) => { + if (url.href !== DEFAULT_ANNOUNCEMENT_FEED_URL || init.method !== 'GET') { + unmatched.push(url.href); + throw new Error('Unmatched announcement transport is forbidden'); + } + requests.push(url.href); + return Response.json(manifest, { headers: { ETag: '"qualification-v1"' } }); + }; + const withTransport = { ...options, announcements: { ...announcements, fetch } }; + return actual.createApp(db, withTransport); + }; + mock.module(new URL('../../../app.ts', import.meta.url).href, { namedExports: { ...actual, createApp: wrapped } }); + // Fail before server startup if the TS loader/module cache bypasses the mock. + if ((await import('../../../app.js')).createApp !== wrapped) throw new Error('Announcement module boundary did not intercept the startup import'); + return { assertIntercepted() { if (!intercepted) throw new Error('Actual server bypassed the announcement boundary'); }, + snapshot: () => ({ intercepted, requests: [...requests], unmatched: [...unmatched], + boundary: 'Feed transport injected through existing fetch seam; actual route/feed validation/cache/UI. DNS and HTTPS guard are not exercised in this combined case.' }) }; +} diff --git a/server/src/__tests__/helpers/maintenance/backup-observation.ts b/server/src/__tests__/helpers/maintenance/backup-observation.ts new file mode 100644 index 00000000000..c3a273c1346 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/backup-observation.ts @@ -0,0 +1,54 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { lstat, open, readdir } from 'node:fs/promises'; +import path from 'node:path'; +import { Transform, Writable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { createGunzip } from 'node:zlib'; + +/** Inspect only archives created by the normal idle owner after the quiet counters freeze. + * Stream through EOF to validate the gzip trailer; retain no SQL/credential contents. */ +export async function observeIdleBackupCheckpoint(directory: string, beforeNames: string[], startedAt: string, fixtureId: string) { + if (!/^[a-f0-9-]{36}$/.test(fixtureId)) throw new Error('Invalid owned backup fixture identifier'); + const checkpointMs = Date.parse(startedAt); + if (!Number.isFinite(checkpointMs) || checkpointMs <= 0) throw new Error('Invalid owned checkpoint timestamp'); + const backupDir = path.join(directory, 'backups'); + const marker = await lstat(path.join(backupDir, '.idle-backup-wake-required')); + if (!marker.isFile() || marker.isSymbolicLink() || marker.size === 0) throw new Error('Missing normal idle backup wake marker'); + for (const failure of [path.join(directory, 'health/db-backup-to-s3.failure'), path.join(directory, 'db-backup-to-s3.failure'), path.join(backupDir, 'db-backup-to-s3.failure')]) { + try { await lstat(failure); throw new Error('Observed database backup failure marker'); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + } + const names = (await readdir(backupDir)).filter(name => name.endsWith('.sql.gz') && !beforeNames.includes(name)).sort(); + if (!names.length) throw new Error('Normal idle owner produced no new archive'); + const archives = []; + for (const name of names) { + const handle = await open(path.join(backupDir, name), constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stat = await handle.stat(); + // Some filesystems report birthtime=0. The new-name filter and opened + // non-symlink file still apply; use its content mtime only in that case. + // An old positive birthtime must not be rescued by a recent modification. + const createdAtSource = stat.birthtimeMs === 0 ? 'mtime' : 'birthtime'; + const createdMs = stat.birthtimeMs === 0 ? stat.mtimeMs : stat.birthtimeMs; + if (!stat.isFile() || stat.size <= 0 || !Number.isFinite(createdMs) || createdMs <= 0 + || createdMs < checkpointMs || Math.trunc(createdMs) > Date.now()) throw new Error('Archive predates owned checkpoint or is invalid'); + const hash = createHash('sha256'); let sqlBytes = 0, tail = '', fixtureIdentifierFound = false; + await pipeline(handle.createReadStream({ autoClose: false }), + new Transform({ transform(chunk, _encoding, done) { hash.update(chunk); done(null, chunk); } }), createGunzip(), + new Writable({ write(chunk, _encoding, done) { + sqlBytes += chunk.length; + if (sqlBytes > 512 * 1024 * 1024) { done(new Error('Owned fixture archive exceeded bounded observation size')); return; } + const text = tail + chunk.toString('utf8'); fixtureIdentifierFound ||= text.includes(fixtureId); + tail = text.slice(-fixtureId.length); done(); + } })); + if (!sqlBytes || !fixtureIdentifierFound) throw new Error('Archive does not contain the owned fixture'); + archives.push({ name, bytes: stat.size, sqlBytes, sha256: hash.digest('hex'), createdAt: new Date(createdMs).toISOString(), createdAtSource, gzipVerified: true, fixtureIdentifierFound }); + } finally { await handle.close(); } + } + return { owner: 'normal-idle-checkpoint', startedAt, observedAt: new Date().toISOString(), archives, wakeMarkerObserved: true, failureObserved: false }; +} +export async function idleBackupArchiveNames(directory: string) { + try { return (await readdir(path.join(directory, 'backups'))).filter(name => name.endsWith('.sql.gz')); } + catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; throw error; } +} diff --git a/server/src/__tests__/helpers/maintenance/browser-observation.ts b/server/src/__tests__/helpers/maintenance/browser-observation.ts new file mode 100644 index 00000000000..76b4488d434 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/browser-observation.ts @@ -0,0 +1,82 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { withNativeMaintenanceBrowser } from './native-browser.js'; + +/** Actual Chromium visibility; never substitutes a fake document or frozen lifecycle. */ +export async function observeMaintenanceBrowser(origin: string, announcementDescription?: string, tenantHeaders?: Record) { + return withNativeMaintenanceBrowser(async (browser, context) => { + const unmatchedOrigins: string[] = []; + if (tenantHeaders) { + // Both HTTP and native WS need the simulated Cloud front door identity. + // Gate every possible foreign transport BEFORE networking; no fixture + // header can reach another origin. about:blank does not use networking. + await context.route('**/*', async route => { + if (new URL(route.request().url()).origin !== origin) { unmatchedOrigins.push(route.request().url()); await route.abort('blockedbyclient'); return; } + await route.continue(); + }); + await context.routeWebSocket(url => url.origin.replace(/^ws/, 'http') !== origin, route => { + unmatchedOrigins.push(route.url()); route.close({ code: 1008, reason: 'Outside owned app origin' }); + }); + await context.setExtraHTTPHeaders(tenantHeaders); + } + const page = await context.newPage(); + const requests: Array<{ at: number; path: string }> = []; + page.on('request', request => { const url = new URL(request.url()); if (url.origin === origin) requests.push({ at: performance.now(), path: url.pathname }); }); + await page.goto(origin, { waitUntil: 'networkidle' }); + if (announcementDescription) await page.getByText(announcementDescription, { exact: true }).waitFor({ state: 'visible' }); + await page.bringToFront(); + if (await page.evaluate(() => document.visibilityState) !== 'visible') throw new Error('Browser active-use observation requires a visible page'); + const beforeActive = requests.length; + await page.reload({ waitUntil: 'networkidle' }); + if (!requests.slice(beforeActive).some(request => request.path.startsWith('/api/'))) throw new Error('Active browser did not make ordinary app requests'); + const visibilityEvents: Array<{ state: string; rendererAt: number }> = []; + const prefix = 'paperclip-sleep-visibility:'; + page.on('console', message => { if (message.text().startsWith(prefix)) visibilityEvents.push(JSON.parse(message.text().slice(prefix.length))); }); + // Passive native events only; never replace document visibility properties. + await page.evaluate(prefix => document.addEventListener('visibilitychange', () => console.log(prefix + JSON.stringify({ state: document.visibilityState, rendererAt: performance.now() }))), prefix); + const visibility = () => page.evaluate(() => ({ state: document.visibilityState, focused: document.hasFocus(), rendererAt: performance.now(), + ownVisibilityOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'visibilityState')), ownHiddenOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'hidden')), + ownFocusOverride: Boolean(Object.getOwnPropertyDescriptor(document, 'hasFocus')) })); + const initialVisibility = await visibility(); + if (initialVisibility.state !== 'visible') throw new Error('Native browser is not visible before the actual tab switch'); + const foreground = await context.newPage(); + await foreground.goto('about:blank'); await foreground.bringToFront(); + const native = await browser.newBrowserCDPSession(); + const windowIdentity = async (target: typeof page) => { + const session = await context.newCDPSession(target); + try { + const { targetInfo } = await session.send('Target.getTargetInfo'); + const { windowId } = await native.send('Browser.getWindowForTarget', { targetId: targetInfo.targetId }); + return { targetId: targetInfo.targetId, windowId }; + } finally { await session.detach(); } + }; + const appWindow = await windowIdentity(page), foregroundWindow = await windowIdentity(foreground); + if (appWindow.windowId !== foregroundWindow.windowId) throw new Error('Background browser qualification requires an actual same-window tab switch'); + const hiddenDeadline = performance.now() + 5_000; + while (await page.evaluate(() => document.visibilityState) !== 'hidden') { + if (performance.now() > hiddenDeadline) throw new Error('Browser could not establish a real background tab; visibility simulation is not qualification'); + await delay(50); + } + // Let requests already admitted by the transition settle before the untouched window. + await delay(2_000); + const initialHidden = await visibility(); + if (initialHidden.state !== 'hidden') throw new Error('Browser lost native hidden state during settling'); + if (!visibilityEvents.some(event => event.state === 'hidden' && event.rendererAt <= initialHidden.rendererAt)) throw new Error('Native hidden event was not observed'); + const hiddenRequests = requests.length, started = performance.now(); + await delay(90_000); + const finalHidden = await visibility(); + const hiddenRealElapsedMs = performance.now() - started; + if (hiddenRealElapsedMs < 90_000) throw new Error('Native hidden observation was shorter than 90 real seconds'); + const intervalVisibilityEvents = visibilityEvents.filter(event => event.rendererAt >= initialHidden.rendererAt && event.rendererAt <= finalHidden.rendererAt); + if (finalHidden.state !== 'hidden' || intervalVisibilityEvents.some(event => event.state !== 'hidden')) throw new Error(`Browser left native hidden state: ${JSON.stringify({ initialHidden, finalHidden, intervalVisibilityEvents })}`); + if ([initialVisibility, initialHidden, finalHidden].some(state => state.ownVisibilityOverride || state.ownHiddenOverride || state.ownFocusOverride)) throw new Error('Browser visibility or focus properties were overridden'); + const hidden = requests.slice(hiddenRequests); + if (hidden.some(request => request.path.startsWith('/api/'))) throw new Error(`Background tab produced tenant traffic: ${JSON.stringify(hidden)}`); + if (unmatchedOrigins.length) throw new Error(`Browser attempted non-owned transport: ${JSON.stringify(unmatchedOrigins)}`); + await page.bringToFront(); const restoredVisibility = await visibility(); + if (restoredVisibility.state !== 'visible') throw new Error('Native app tab did not become visible after activation'); + if (!visibilityEvents.some(event => event.state === 'visible' && event.rendererAt >= finalHidden.rendererAt)) throw new Error('Native visible restoration event was not observed'); + return { activeRequests: requests.slice(beforeActive, hiddenRequests).length, hiddenRealElapsedMs, + hiddenRequests: hidden, unmatchedOrigins, visibility: 'Native Chromium default context with no Playwright focus override', initialVisibility, initialHidden, finalHidden, + intervalVisibilityEvents, visibilityEvents, restoredVisibility, appWindow, foregroundWindow, announcementRendered: Boolean(announcementDescription), authBoundary: tenantHeaders ? 'Owned Cloud front-door headers; actual tenant resolver/membership' : 'Legacy local board', closed: true }; + }); +} diff --git a/server/src/__tests__/helpers/maintenance/chat-history-producer.ts b/server/src/__tests__/helpers/maintenance/chat-history-producer.ts new file mode 100644 index 00000000000..d2050cc5ba9 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/chat-history-producer.ts @@ -0,0 +1,21 @@ +import assert from 'node:assert/strict'; +import type { Db } from '@paperclipai/db'; +import { githubIngressFixture } from '../github-webhook-ingress.js'; +import { chatChannelService } from '../../../services/chat-channels.js'; + +/** The fixture supplies the named external signed-webhook boundary; domain owners write receipts. */ +export async function produceChatHistory(db: Db) { + const fixture = await githubIngressFixture(db); + const action = await fixture.insertAction(); + await fixture.service.processGitHubWebhookIngress(action.id); + const finished = await fixture.readAction(action.id); + assert.equal(fixture.calls.length, 1); + assert.equal(finished.status, 'processed'); + const service = chatChannelService(db, { heartbeat: { wakeup: async () => { throw new Error('Unexpected chat turn'); } }, + fetch: async () => { throw new Error('Unexpected provider HTTP outside signed-webhook fixture'); } }); + try { + await service.configure(fixture.endpoint.id, { action: 'remove' }, 'local-board'); + } finally { await service.shutdown(); } + return { companyId: fixture.companyId, actionId: action.id, endpointId: fixture.endpoint.id, + boundary: 'Existing signed webhook fixture input; actual chat ingress completion and ordinary endpoint removal preserve history' }; +} diff --git a/server/src/__tests__/helpers/maintenance/configuration-producers.ts b/server/src/__tests__/helpers/maintenance/configuration-producers.ts new file mode 100644 index 00000000000..a67d82912ec --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/configuration-producers.ts @@ -0,0 +1,48 @@ +import type { createDb } from '@paperclipai/db'; +import { expect } from 'vitest'; +import { randomUUID } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { startMaintenanceServer } from './server-harness.js'; + +/** Real routes own request creation, human rejection, run cancellation and outbox retirement. */ +export async function produceCompletedToolReview(server: Awaited>, db: ReturnType, companyId: string) { + await server.api('POST', `/companies/${companyId}/tools/policies`, { name: 'Review harmless qualification writes', policyType: 'require_approval', selectors: { toolName: 'mcp-remote-fixture:update_note' } }); + const readyPath = path.join(server.directory, `tool-review-ready-${randomUUID()}`); + const readyProgram = `const fs=require('node:fs');const marker=${JSON.stringify(readyPath)};fs.writeFileSync(marker+'.pending',process.env.PAPERCLIP_RUN_ID,{flag:'wx'});fs.renameSync(marker+'.pending',marker);setTimeout(()=>process.exit(0),60000)`; + const agent = await server.api<{ id: string }>('POST', `/companies/${companyId}/agents`, { name: 'Owned tool review process', role: 'general', adapterType: 'process', + adapterConfig: { command: process.execPath, args: ['-e', readyProgram], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + const issue = await server.api<{ id: string }>('POST', `/companies/${companyId}/issues`, { title: 'Harmless tool review history', status: 'todo', assigneeAgentId: agent.id }); + let runId: string | undefined; + await expect.poll(async () => { + const [run] = await db.$client`select id,status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; + runId = run?.id; + if (run?.status !== 'running') return false; + // The process runs only after its awaited adapter.invoke write can commit + // behind the final dispatch gate's run lock. Status alone precedes setup. + try { return await readFile(readyPath, 'utf8') === run.id; } + catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; throw error; } + }, { timeout: 30_000 }).toBe(true); + const readiness = { runId, marker: path.basename(readyPath), observedAt: new Date().toISOString(), source: 'actual-process-run-id-marker' }; + await writeFile(path.join(server.directory, 'sleep-tool-review-readiness.json'), JSON.stringify(readiness, null, 2)); + const session = await server.api<{ sessionId: string; token: string }>('POST', '/tool-gateway/sessions', { companyId, agentId: agent.id, runId, issueId: issue.id }); + const call = await fetch(`http://127.0.0.1:${server.port}/api/tool-gateway/tools/call`, { method: 'POST', signal: AbortSignal.timeout(15_000), + headers: { 'Content-Type': 'application/json', 'x-paperclip-tool-gateway-token': session.token }, + body: JSON.stringify({ tool: 'mcp-remote-fixture:update_note', parameters: { noteId: 'qualification', body: 'Harmless denied operation' } }) }); + expect(call.ok).toBe(false); + expect(JSON.stringify(await call.json())).toContain('approval_required'); + const [request] = await db.$client`select id,status from tool_action_requests where issue_id=${issue.id}`; + expect(request.status).toBe('pending'); + await server.api('POST', `/tool-gateway/action-requests/${request.id}/decline`, { companyId, reason: 'Qualification declines the harmless write' }); + await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' }); + await expect.poll(async () => { + const [run] = await db.$client`select status from heartbeat_runs where id=${runId!}`; + return ['succeeded', 'failed', 'cancelled', 'timed_out'].includes(run?.status); + }, { timeout: 75_000 }).toBe(true); + await expect.poll(async () => { + const [delivery] = await db.$client`select delivered_at is not null as delivered from tool_action_deliveries where action_request_id=${request.id}`; + return delivery?.delivered; + }, { timeout: 30_000 }).toBe(true); + await server.api('POST', `/tool-gateway/sessions/${session.sessionId}/revoke`, { companyId }); + return { requestId: request.id, issueId: issue.id, runId, readiness, boundary: 'Actual request/rejection/session revocation and closed-task retirement; no provider write executed' }; +} diff --git a/server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts b/server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts new file mode 100644 index 00000000000..c51c4e337c1 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/early-diagnostic-stabilization.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { assertEarlySleepDiagnostics } from './early-sleep-diagnostics.js'; +import type { startMaintenanceServer } from './server-harness.js'; + +type Report = { ownerId: string; draining: boolean; quiescent: boolean; expiresAt: string; + idleSleepSafety: { diagnostics?: { complete?: boolean; checks?: Array<{ reason: string; state: string }> } } }; +const transientStates = new Map([['inspection.generation', 'unknown'], ['local.tracked_work', 'present'], ['plugins.runtime_drain', 'not_checked']]); + +export function classifyEarlyDiagnostic(report: Report, elapsedSinceApiMs: number): 'strict' | 'transient' { + const checks = report.idleSleepSafety.diagnostics?.checks; + if (!Array.isArray(checks)) throw new Error('Missing early diagnostic checks'); + // A real durable blocker fails now, even if unrelated inspection work is in flight. + const expectedStates = new Map([['local.registered_probes', 'present'], ['local.inspection', 'present'], ['backup.checkpoint', 'not_checked']]); + for (const check of checks) { + if (check.state === (expectedStates.get(check.reason) ?? 'none')) continue; + if (transientStates.get(check.reason) !== check.state) throw new Error(`Unexpected early sleep blocker: ${check.reason}=${check.state}`); + } + if (checks.some(check => transientStates.get(check.reason) === check.state)) return 'transient'; + assertEarlySleepDiagnostics(report, elapsedSinceApiMs); + return 'strict'; +} + +/** Bounded setup-only observation, never a quiet-window poll or an uncertainty exemption. */ +export async function stabilizeEarlyDiagnostics(server: Awaited>, options: { + ownerId: string; apiSucceededAt: number; read: () => Promise; persist: (observations: unknown[]) => Promise; +}) { + const observations: unknown[] = []; + for (let round = 1; round <= 3; round++) { + const before = await server.settled(); + observations.push({ round, kind: 'settled-before', at: new Date().toISOString(), snapshot: before }); await options.persist(observations); + assert(before.managed?.apiActivityActive, 'Actual successful managed API key activity is not held'); + const reports: Report[] = []; let transient = false; + for (const position of ['initial', 'final']) { + const report = await options.read(); reports.push(report); + observations.push({ round, kind: position, at: new Date().toISOString(), report }); await options.persist(observations); + assert(report.ownerId === options.ownerId && report.draining && report.quiescent && Date.parse(report.expiresAt) > Date.now(), 'Early diagnostic lost exact live hold'); + const classification = classifyEarlyDiagnostic(report, performance.now() - options.apiSucceededAt); + transient = classification === 'transient' || transient; + } + const after = await server.snapshot(); + observations.push({ round, kind: 'snapshot-after', at: new Date().toISOString(), snapshot: after }); await options.persist(observations); + if (transient) continue; + assert(before.idle.active === 0 && after.idle.active === 0, 'Accepted early diagnostic has active owned work'); + // Each real HTTP authentication is tracked work. Its generation changes are + // observer stimuli; each report separately proves its own scan stayed stable. + assert.deepEqual(reports[0]!.idleSleepSafety, reports[1]!.idleSleepSafety, 'Accepted early diagnostic pair differs'); + assert(after.managed?.apiActivityActive, 'Real managed API activity expired during setup'); + return { rounds: round, initial: reports[0]!, final: reports[1]!, observations, + boundary: 'Strict complete stable setup inventory with real API cooldown active; not sleep eligibility' }; + } + throw new Error('Early inspection did not stabilize in three bounded owner-settled rounds'); +} diff --git a/server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts b/server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts new file mode 100644 index 00000000000..f7bda4c2369 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/early-sleep-diagnostics.ts @@ -0,0 +1,18 @@ +type Check = { reason: string; state: string }; +/** Early diagnostics are only a fail-fast blocker inventory, never sleep proof. + * The registered-probe aggregate includes the real 10-minute API activity hold; + * it cannot distinguish other probes, so the later full cooldown/proof is mandatory. */ +export function assertEarlySleepDiagnostics(report: { idleSleepSafety?: { diagnostics?: { complete?: boolean; checks?: Check[] } } }, elapsedSinceApiMs: number) { + const diagnostics = report.idleSleepSafety?.diagnostics; + if (!Number.isFinite(elapsedSinceApiMs) || elapsedSinceApiMs < 0 || elapsedSinceApiMs >= 600_000) throw new Error('Early diagnostic requires the actual API cooldown still active'); + if (diagnostics?.complete !== true || !Array.isArray(diagnostics.checks)) throw new Error('Early diagnostic coverage incomplete'); + const expected = new Map([['local.registered_probes', 'present'], ['local.inspection', 'present'], ['backup.checkpoint', 'not_checked']]); + const seen = new Set(); + for (const check of diagnostics.checks) { + if (seen.has(check.reason) || check.state !== (expected.get(check.reason) ?? 'none')) throw new Error(`Unexpected early sleep blocker: ${check.reason}=${check.state}`); + seen.add(check.reason); + } + for (const required of [...expected.keys(), 'inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog']) { + if (!seen.has(required)) throw new Error(`Missing early sleep diagnostic: ${required}`); + } +} diff --git a/server/src/__tests__/helpers/maintenance/github-configuration-producer.ts b/server/src/__tests__/helpers/maintenance/github-configuration-producer.ts new file mode 100644 index 00000000000..8431d6dbbc4 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/github-configuration-producer.ts @@ -0,0 +1,47 @@ +import { eq } from 'drizzle-orm'; +import { randomUUID } from 'node:crypto'; +import { toolApplications, toolConnections, connectionGrants, type Db } from '@paperclipai/db'; +import { writeConnectionCredential } from '../../../services/connection-credentials.js'; +import { secretService } from '../../../services/secrets.js'; +import { createOAuthCredentialStore } from '../../../services/tool-access/oauth/credential-store.js'; +import { createOAuthRefreshLeases } from '../../../services/tool-access/oauth/refresh-leases.js'; +import { createOAuthGrantRefresh } from '../../../services/tool-access/oauth/grant-refresh.js'; +import { createGitHubGrantContinuity } from '../../../services/tool-access/github-grant-continuity.js'; +import { createPaperclipCloudConnector, paperclipCloudConnectorConfigFromEnv } from '../../../services/paperclip-cloud-connector.js'; + +/** Bootstrap the named OAuth consent boundary, then use real vault, binding and continuity owners. */ +export async function prepareGitHubConfiguration(db: Db, companyId: string) { + const owner = 'local-board', now = Date.now(); + const [application] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: 'GitHub qualification', type: 'mcp_http', status: 'active' }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: application.id, uid: randomUUID(), name: 'Retained GitHub', status: 'active', enabled: false, + transport: 'mcp_remote', authKind: 'oauth', credentialPolicy: 'per_user', credentialSource: 'paperclip_vault', transportConfig: {}, + config: { url: 'https://api.github.com/mcp', sourceTemplateKey: 'github', oauth: { strategy: 'paperclip_cloud_connector', connectorProfile: 'github.code' } } }).returning(); + const refs = []; + for (const [configPath, value] of [['oauth.access_token', 'qualification-access'], ['oauth.refresh_token', 'qualification-refresh']] as const) { + const saved = await writeConnectionCredential(db, { companyId, connectionName: 'GitHub', configPath, label: configPath, value, ownerUserId: owner }); + refs.push({ configPath, secretId: saved.secret.id, versionSelector: 'latest' as const, required: true }); + } + const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection.id, kind: 'user', subjectUserId: owner, status: 'active', credentialSecretRefs: refs, + providerTenant: { oauth: { strategy: 'paperclip_cloud_connector', accessTokenExpiresAt: new Date(now + 8 * 3_600_000).toISOString(), refreshedAt: new Date(now).toISOString() } } }).returning(); + await createOAuthCredentialStore(db, secretService(db)).syncCredentialBindings(connection, refs); + return { connectionId: connection.id, grantId: grant.id }; +} + +export async function settleGitHubConfiguration(db: Db, input: { connectionId: string; grantId: string }) { + const getConnection = async () => (await db.select().from(toolConnections).where(eq(toolConnections.id, input.connectionId)))[0]; + const connection = await getConnection(); + const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, input.grantId)); + const config = paperclipCloudConnectorConfigFromEnv(); + if (!config) throw new Error('Owned qualification connector is not configured'); + const connector = createPaperclipCloudConnector({ config }); + const credentials = createOAuthCredentialStore(db, secretService(db)); + const leases = createOAuthRefreshLeases(db, () => new Date(), getConnection, credentials.getOAuthGrantRow); + const refresh = createOAuthGrantRefresh(db, { now: () => new Date(), store: credentials, leases, getConnectionRow: getConnection, + currentCloudConnector: () => connector, exchangeOAuthToken: async () => { throw new Error('Unexpected direct OAuth outside managed GitHub fixture'); } }); + const continuity = createGitHubGrantContinuity(db, { now: () => new Date(), currentCloudConnector: () => connector, + refreshOAuthGrantCredentials: refresh, resolveOAuthGrantSecret: credentials.resolveOAuthGrantSecret }); + const settled = await continuity.refreshManagedGitHubGrantAccess(connection, grant, { actorType: 'system', actorId: 'qualification' }); + if (settled.providerTenant?.github?.bindingSync || settled.providerTenant?.github?.continuityRetry) throw new Error('Real GitHub binding debt did not settle'); + return { connectionId: connection.id, grantId: grant.id, credentialRefs: settled.credentialSecretRefs.map(ref => ref.configPath), + boundary: 'Actual encrypted user credential/version/binding owners and GitHub authority refresh; real signed broker binding; named external GitHub HTTP only' }; +} diff --git a/server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts b/server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts new file mode 100644 index 00000000000..99218823a88 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/idle-work-capture-probe.ts @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict'; +import { installMaintenanceIdleWorkCapture } from './idle-work-capture.js'; + +// Isolated Node/TS-loader identity probe. Never imports app/index or opens a DB. +const original = await import('../../../services/task-admission.js'); +const before = process.argv[2] === 'unhandled' ? undefined : original.beginIdleTrackedWork(); +const capture = await installMaintenanceIdleWorkCapture(); +try { + const observed = await import('../../../services/task-admission.js'); + assert.equal(observed.idleWorkSnapshot, original.idleWorkSnapshot); + if (process.argv[2] === 'unhandled') { + const reason = new Error('original unhandled tracked rejection'); + let listener!: (reason: unknown, promise: Promise) => void; + let timer: ReturnType | undefined; + const event = new Promise<{ reason: unknown; promise: Promise }>((resolve, reject) => { + listener = (reason, promise) => resolve({ reason, promise }); + process.once('unhandledRejection', listener); + timer = setTimeout(() => reject(new Error('Original tracked rejection did not reach unhandledRejection')), 1500); + }); + try { + // Deliberately never await or attach a handler to this original promise. + const tracked = observed.trackIdleWork(Promise.reject(reason)); + const unhandled = await event; + assert.equal(unhandled.promise, tracked); + assert.equal(unhandled.reason, reason); + assert.equal(original.idleWorkSnapshot().active, 0); + assert.equal(capture.snapshot().observedActive, 0); + console.log(JSON.stringify({ unhandledOriginalPromise: true, unhandledOriginalReason: true, finalActive: 0 })); + } finally { + if (timer) clearTimeout(timer); + process.removeListener('unhandledRejection', listener); + } + } else { + assert.equal(observed.idleWorkSnapshot().active, 1); // Existing state was not reset. + assert.equal(capture.snapshot().observedActive, 0); // The pre-install path is explicitly missing. + before!(); + const finish = observed.beginIdleTrackedWork(); + assert.equal(original.idleWorkSnapshot().active, 1); + finish(); finish(); + assert.equal(original.idleWorkSnapshot().active, 0); + assert.equal(await observed.trackIdleWork(Promise.resolve('original-result')), 'original-result'); + assert.equal(original.idleWorkSnapshot().active, 0); + const result = capture.summary(); + assert(result.snapshot.missingPaths > 0); + console.log(JSON.stringify({ identity: true, originalStatePreserved: true, finalActive: original.idleWorkSnapshot().active, missingPathsExplicit: result.snapshot.missingPaths })); + } +} finally { + before?.(); + capture.dispose(); +} diff --git a/server/src/__tests__/helpers/maintenance/idle-work-capture.ts b/server/src/__tests__/helpers/maintenance/idle-work-capture.ts new file mode 100644 index 00000000000..732d1da4f27 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/idle-work-capture.ts @@ -0,0 +1,137 @@ +import { mock } from 'node:test'; +import { promiseHooks } from 'node:v8'; + +type Admission = typeof import('../../../services/task-admission.js'); +type Entry = { id: number; kind: 'begin' | 'track'; sourceId: number; startedAt: number; generation: number }; +type Completion = Entry & { finishedAt: number; finishedGeneration: number }; +export type IdleWorkCaptureSnapshot = { + actual: { active: number; generation: number }; observedActive: number; active: Entry[]; completionTail: Completion[]; + newSources: Array<{ id: number; callers: string[] }>; overflow: number; sourceOverflow: number; + missingCallsites: number; missingPaths: number; observerErrors: number; disposed: boolean; +}; + +/** Test-only metadata. No original work, promise, drain or counter is changed. */ +export function createMaintenanceIdleWorkCapture(actual: Pick, + controls: { now?: () => number; stack?: () => string | undefined } = {}) { + const active = new Map(), sources = new Map(); + const tail: Completion[] = []; + let trackedPromises = new WeakMap, number>(); + let disposed = false; + let serial = 0, emittedSourceId = 0, overflow = 0, sourceOverflow = 0, missingCallsites = 0, missingPaths = 0, observerErrors = 0; + const now = controls.now ?? Date.now; + function start(kind: Entry['kind']): number | undefined { + try { + if (disposed) return; + if (active.size >= 512) { overflow++; return; } + const callers: string[] = []; + for (const line of (controls.stack?.() ?? new Error().stack ?? '').split('\n')) { + const match = line.match(/[/\\]server[/\\](?:src|dist)[/\\]((?:(?:services|middleware|routes|modules)[/\\][A-Za-z0-9_./\\-]+|(?:app|index))\.[cm]?[jt]s):(\d+):(\d+)/); + if (!match || match[1]!.split(/[/\\]/).includes('..')) continue; + const caller = `${match[1]!.replaceAll('\\', '/')}:${match[2]}:${match[3]}`; + if (!callers.includes(caller)) callers.push(caller); + if (callers.length === 4) break; + } + if (!callers.length) missingCallsites++; + const key = JSON.stringify(callers); let source = sources.get(key); + if (!source) { + if (sources.size >= 512) { sourceOverflow++; return; } + source = { id: sources.size + 1, callers }; sources.set(key, source); + } + const id = ++serial; + active.set(id, { id, kind, sourceId: source.id, startedAt: now(), generation: actual.idleWorkSnapshot().generation }); + return id; + } catch { observerErrors++; return; } + } + // V8 observes settlement without attaching a continuation or marking a rejected + // original promise handled. Unrelated promises only perform a WeakMap lookup. + const stopSettlementHook = promiseHooks.onSettled(promise => { + try { + const id = trackedPromises.get(promise); + if (id === undefined) return; + trackedPromises.delete(promise); + finish(id); + } catch { observerErrors++; } + }); + function finish(id: number | undefined) { + try { + if (id === undefined) return; + const entry = active.get(id); if (!entry) return; + active.delete(id); + tail.push({ ...entry, finishedAt: now(), finishedGeneration: actual.idleWorkSnapshot().generation }); + if (tail.length > 64) tail.shift(); + } catch { observerErrors++; } + } + const beginIdleTrackedWork: Admission['beginIdleTrackedWork'] = () => { + const originalFinish = actual.beginIdleTrackedWork(); const id = start('begin'); + return () => { try { return originalFinish(); } finally { finish(id); } }; + }; + const trackIdleWork: Admission['trackIdleWork'] = (work: Promise) => { + const original = actual.trackIdleWork(work); const id = start('track'); + try { if (id !== undefined) trackedPromises.set(original, id); } catch { observerErrors++; } + return original; + }; + return { beginIdleTrackedWork, trackIdleWork, + dispose() { + if (disposed) return; + disposed = true; + try { stopSettlementHook(); } finally { trackedPromises = new WeakMap(); } + }, + snapshot(): IdleWorkCaptureSnapshot { + const state = actual.idleWorkSnapshot(); + if (state.active !== active.size) missingPaths++; + const newSources = [...sources.values()].filter(source => source.id > emittedSourceId).map(source => ({ ...source, callers: [...source.callers] })); + emittedSourceId = sources.size; + return { actual: state, observedActive: active.size, active: [...active.values()].map(entry => ({ ...entry })), completionTail: tail.map(entry => ({ ...entry })), + newSources, overflow, sourceOverflow, missingCallsites, missingPaths, observerErrors, disposed }; + }, + summary() { return { sourceCatalog: [...sources.values()], snapshot: this.snapshot() }; }, + }; +} + +export async function installMaintenanceIdleWorkCapture() { + const actual = await import('../../../services/task-admission.js'); + const capture = createMaintenanceIdleWorkCapture(actual); + let moduleCapture: ReturnType | undefined; + const disposeCapture = capture.dispose; + capture.dispose = () => { try { disposeCapture(); } finally { const installed = moduleCapture; moduleCapture = undefined; installed?.restore(); } }; + try { + moduleCapture = mock.module(new URL('../../../services/task-admission.ts', import.meta.url).href, { + namedExports: { ...actual, beginIdleTrackedWork: capture.beginIdleTrackedWork, trackIdleWork: capture.trackIdleWork }, + }); + const observed = await import('../../../services/task-admission.js'); + if (observed.beginIdleTrackedWork !== capture.beginIdleTrackedWork || observed.trackIdleWork !== capture.trackIdleWork + || Object.keys(observed).sort().join(',') !== Object.keys(actual).sort().join(',') + || Object.keys(actual).some(key => !['beginIdleTrackedWork', 'trackIdleWork'].includes(key) + && observed[key as keyof Admission] !== actual[key as keyof Admission])) throw new Error('Idle work module capture did not preserve original exports/identity'); + return capture; + } catch (error) { + try { capture.dispose(); } catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Idle work capture installation and cleanup failed'); } + throw error; + } +} + +/** Reuse existing settle-loop observations. No extra poll, timer, SQL or raw stack per sample. */ +export function createSettledWorkSamples(sourceCatalog: () => Array<{ id: number; callers: string[] }>) { + // Finite longest setup observation at the existing 25ms cadence, plus its + // terminal sample. Overflow still makes diagnostic completion fail closed. + const sampleLimit = Math.ceil(61_000 / 25) + 1; + const samples: Array<{ id: number; at: number; queryTotal: number; queryDelta: number; pending: number; + idle: { active: number; generation: number }; activeIds: number[] }> = []; + let previous: number | undefined, overflowSamples = 0, lastCapture: IdleWorkCaptureSnapshot | undefined; + const startedAt = Date.now(); + return { + sample(snapshot: { now: number; idle: { active: number; generation: number }; workCapture?: IdleWorkCaptureSnapshot }, queryTotal: number, pending: number) { + lastCapture = snapshot.workCapture; + if (samples.length < sampleLimit) samples.push({ id: samples.length + 1, at: snapshot.now, queryTotal, + queryDelta: previous === undefined ? 0 : queryTotal - previous, pending, idle: { ...snapshot.idle }, activeIds: lastCapture?.active.map(entry => entry.id) ?? [] }); + else overflowSamples++; + previous = queryTotal; + }, + report() { return { startedAt, finishedAt: Date.now(), settleDeadlineMs: 30_000, sampleLimit, overflowSamples, + samples, sourceCatalog: sourceCatalog(), lastCapture }; }, + assertComplete() { + if (!lastCapture || overflowSamples || lastCapture.overflow || lastCapture.sourceOverflow || lastCapture.missingCallsites + || lastCapture.missingPaths || lastCapture.observerErrors || lastCapture.disposed) throw new Error('Passive idle work diagnostic capture incomplete'); + }, + }; +} diff --git a/server/src/__tests__/helpers/maintenance/incoming-http-observation.ts b/server/src/__tests__/helpers/maintenance/incoming-http-observation.ts new file mode 100644 index 00000000000..e317165325f --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/incoming-http-observation.ts @@ -0,0 +1,40 @@ +import { randomUUID } from "node:crypto"; +import type { Server } from "node:http"; +import type { IncomingHttpSnapshot } from "../../../../../scripts/sleep-qualification/evidence.mjs"; + +/** Passive test-child observation of the actual app listener, including direct clients. */ +export function observeMaintenanceIncomingHttp(server: Server) { + // A sole upgrade listener suppresses Node's default unsupported-upgrade close. + // The real app installs its WebSocket handlers before startServer returns. + if (server.listenerCount("request") === 0 || server.listenerCount("upgrade") === 0) { + throw new Error("Maintenance HTTP observation requires existing app request and upgrade handlers"); + } + const captureId = randomUUID(); + let requests = 0, installed = true, valid = true; + const count = () => { + if (requests === Number.MAX_SAFE_INTEGER) valid = false; + else requests++; + }; + const detached = (event: string | symbol, listener: unknown) => { + if ((event === "request" || event === "upgrade") && listener === count) valid = false; + }; + // Count before application handlers without inspecting or retaining request data. + server.prependListener("request", count); + server.prependListener("upgrade", count); + server.on("removeListener", detached); + return { + snapshot(): IncomingHttpSnapshot { + return { source: "node-http-server-events", captureId, installed, + valid: valid && installed && server.listening + && server.listeners("request").includes(count) && server.listeners("upgrade").includes(count), + requests, observedAt: new Date().toISOString() }; + }, + dispose() { + if (!installed) return; + installed = false; valid = false; + server.off("removeListener", detached); + server.off("request", count); + server.off("upgrade", count); + }, + }; +} diff --git a/server/src/__tests__/helpers/maintenance/managed-profile.ts b/server/src/__tests__/helpers/maintenance/managed-profile.ts new file mode 100644 index 00000000000..ca8ff553f84 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/managed-profile.ts @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict'; +import { authUsers, companyMemberships, type Db } from '@paperclipai/db'; +import { and, eq } from 'drizzle-orm'; +import { cloudTenantPrimaryCompanyId } from '../../../services/cloud-instance.js'; +import type { MaintenanceConfiguration, MaintenanceServerOptions, startMaintenanceServer } from './server-harness.js'; + +export const managedTenantActor = { userId: 'local-board', email: 'sleep-owner@example.test', name: 'Owned sleep operator' }; +export function managedSleepConfiguration(collectorOrigin: string, serveUi: boolean): MaintenanceConfiguration { + return { automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, + telemetryOrigin: `${collectorOrigin}/telemetry`, otelOrigin: collectorOrigin, serveUi, + managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3' }; +} +export function managedSleepServerOptions(configuration: MaintenanceConfiguration, jwks: string, s3Origin: string, + broker: { origin: string; variables: Record }): MaintenanceServerOptions { + return { configuration, controlJwks: jwks, cloud: { jwks, s3Origin, standby: false }, tenantActor: managedTenantActor, + githubOrigin: broker.origin, connector: { origin: broker.origin, + instanceId: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID, + signPrivateKey: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY, + sealPrivateKey: broker.variables.PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY } }; +} +export async function provisionManagedSleepCompany(server: Awaited>, db: Db) { + // This real request invokes resolveCloudTenantActor, never a table seed or a bypassed company-create route. + const companies = await server.api>('GET', '/companies'); + const id = cloudTenantPrimaryCompanyId('maintenance-stack'); + assert(companies.some(company => company.id === id), 'Actual managed primary company was not provisioned'); + const [authority] = await db.select({ userId: authUsers.id, status: companyMemberships.status, membershipRole: companyMemberships.membershipRole }) + .from(authUsers).innerJoin(companyMemberships, and(eq(companyMemberships.principalId, authUsers.id), eq(companyMemberships.principalType, 'user'))) + .where(and(eq(authUsers.id, managedTenantActor.userId), eq(companyMemberships.companyId, id))); + assert.deepEqual(authority, { userId: managedTenantActor.userId, status: 'active', membershipRole: 'owner' }); + await server.api('PATCH', `/companies/${id}`, { defaultResponsibleUserId: managedTenantActor.userId }); + const observation = (await server.snapshot()).managed; + assert(observation?.isCloudManaged && observation.deploymentMode === 'authenticated' && observation.storageProvider === 's3', 'Actual supported managed profile not active'); + return { id, authority, observation, boundary: 'Fixture Cloud front-door headers; actual resolver creates canonical company/user/member/default grants' }; +} diff --git a/server/src/__tests__/helpers/maintenance/native-browser.ts b/server/src/__tests__/helpers/maintenance/native-browser.ts new file mode 100644 index 00000000000..189e7993509 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/native-browser.ts @@ -0,0 +1,65 @@ +import { spawn, type ChildProcess } from 'node:child_process'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { Browser, BrowserContext } from '@playwright/test'; + +/** A fresh owned browser, not a user's profile. Never terminate by process name. */ +export async function withNativeMaintenanceBrowser(run: (browser: Browser, context: BrowserContext) => Promise): Promise { + const { chromium } = await import('@playwright/test'); + const profile = await mkdtemp(path.join(tmpdir(), 'paperclip-sleep-browser-')); + let child: ChildProcess | undefined, browser: Browser | undefined, exited: Promise | undefined; + let exitObserved = false, launchError: Error | undefined, stderr = '', failure: { error: unknown } | undefined; + try { + child = spawn(chromium.executablePath(), [`--user-data-dir=${profile}`, '--remote-debugging-address=127.0.0.1', '--remote-debugging-port=0', + '--no-first-run', '--no-default-browser-check', '--disable-background-networking', '--disable-component-update', '--disable-sync', 'about:blank'], { stdio: ['ignore', 'ignore', 'pipe'] }); + const ownedChild = child; + exited = new Promise(resolve => { + ownedChild.once('exit', () => { exitObserved = true; resolve(); }); + ownedChild.once('error', error => { launchError = error; if (!ownedChild.pid) resolve(); }); + }); + child.stderr?.on('data', chunk => { stderr = (stderr + String(chunk)).slice(-16_384); }); + const deadline = performance.now() + 10_000; + let port: number | undefined; + while (!port) { + if (launchError) throw launchError; + if (child.exitCode !== null || child.signalCode !== null || performance.now() > deadline) throw new Error(`Owned Chromium startup failed: ${stderr}`); + try { + const candidate = Number((await readFile(path.join(profile, 'DevToolsActivePort'), 'utf8')).split('\n')[0]); + if (Number.isInteger(candidate) && candidate > 0 && candidate < 65536) port = candidate; + } catch { /* The owned startup file is not ready yet. */ } + if (!port) await delay(50); + } + // Playwright1.62 installs focus emulation on normal pages. Measured native + // tab switches stayed visible there. The supported noDefaults/default-context + // path avoids installing that override (types.d.ts ConnectOverCDPOptions). + browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`, { noDefaults: true, timeout: 10_000 }); + const context = browser.contexts()[0]; + if (!context) throw new Error('Owned Chromium has no native default context'); + return await run(browser, context); + } catch (error) { failure = { error }; throw error; } + finally { + const errors: unknown[] = []; + let disconnectTimer: ReturnType | undefined; + try { + await Promise.race([browser?.close(), new Promise((_, reject) => { disconnectTimer = setTimeout(() => reject(new Error('Owned browser disconnect timed out')), 5_000); })]); + } catch (error) { errors.push(error); } + finally { if (disconnectTimer) clearTimeout(disconnectTimer); } + if (child && exited) { + const waitExit = async () => { + let timer: ReturnType | undefined; + try { return await Promise.race([exited!.then(() => true), new Promise(resolve => { timer = setTimeout(() => resolve(false), 5_000); })]); } + finally { if (timer) clearTimeout(timer); } + }; + try { + if (child.exitCode === null && child.signalCode === null && !launchError) child.kill('SIGTERM'); + if (!(await waitExit())) { child.kill('SIGKILL'); if (!(await waitExit())) throw new Error('Owned Chromium did not exit after termination'); } + } catch (error) { errors.push(error); } + } + if (!child?.pid || exitObserved) { + try { await rm(profile, { recursive: true, force: true }); } catch (error) { errors.push(error); } + } else errors.push(new Error(`Owned Chromium exit was not confirmed; profile preserved at ${profile}`)); + if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Native browser work or cleanup failed'); + } +} diff --git a/server/src/__tests__/helpers/maintenance/otel-child.ts b/server/src/__tests__/helpers/maintenance/otel-child.ts new file mode 100644 index 00000000000..507fa5e529a --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/otel-child.ts @@ -0,0 +1,10 @@ +import { getStartupTracer, instrumentationReady, shutdownInstrumentation } from '../../../instrumentation.js'; +await instrumentationReady; +const span = getStartupTracer('paperclip.sleep-qualification').startSpan('sleep-qualification-stimulus'); +span.end(); +// Keep the real SDK alive beyond its 60s default metric-reader interval. This +// observes unwanted exporter traffic without disabling any exporter through env. +if (process.argv.includes('--observe-exporters')) { + await new Promise(resolve => setTimeout(resolve, 90_000)); +} +await shutdownInstrumentation(); diff --git a/server/src/__tests__/helpers/maintenance/s3-boundary.ts b/server/src/__tests__/helpers/maintenance/s3-boundary.ts new file mode 100644 index 00000000000..dc857f5d187 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/s3-boundary.ts @@ -0,0 +1,79 @@ +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; + +/** Adapted from the real-SDK loopback fixture in managed-cloud-prerequisites.test.ts. + * Small owned objects only; no provider credentials/IAM/durability qualification. */ +export function createMaintenanceS3Boundary() { + const objects = new Map(); + const requests: Array<{ method: string; path: string; status: number }> = []; + const errors: string[] = []; + const xml = (text: string) => text.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>'); + const server = createServer(async (req, res) => { + const url = new URL(req.url!, 'http://fixture.invalid'); + const operation = { method: req.method!, path: url.pathname + url.search, status: 0 }; requests.push(operation); + const reply = (status: number, body?: string | Buffer) => { operation.status = status; res.statusCode = status; res.end(body); }; + try { + if (!url.pathname.startsWith('/maintenance-test/') && url.pathname !== '/maintenance-test') throw new Error('Unknown S3 fixture bucket'); + const key = decodeURIComponent(url.pathname.slice('/maintenance-test/'.length)); + if (req.method === 'GET' && ['/maintenance-test', '/maintenance-test/'].includes(url.pathname) && url.searchParams.get('list-type') === '2' + && [...url.searchParams.keys()].every(key => ['list-type', 'prefix', 'max-keys', 'x-id'].includes(key))) { + const keys = [...objects.keys()].filter(key => key.startsWith(url.searchParams.get('prefix') ?? '')).sort(); + if (keys.length > Math.min(1000, Number(url.searchParams.get('max-keys') ?? 1000))) throw new Error('Listing exceeds bounded single-page fixture'); + res.setHeader('content-type', 'application/xml'); + reply(200, `false${keys.map(key => `${xml(key)}${objects.get(key)!.bytes.length}`).join('')}`); return; + } + if (!key || [...url.searchParams.keys()].some(key => key !== 'x-id')) throw new Error('Unknown S3 fixture operation/query'); + if (req.method === 'PUT') { + const chunks: Buffer[] = []; let length = 0; + for await (const chunk of req) { length += chunk.length; if (length > 17 * 1024 * 1024) throw new Error('S3 fixture object too large'); chunks.push(Buffer.from(chunk)); } + let bytes = Buffer.concat(chunks); + if (req.headers['content-encoding'] === 'aws-chunked') bytes = decodeMaintenanceAwsChunks(bytes); + else if (req.headers['content-encoding']) throw new Error('Unknown S3 fixture content framing'); + const decodedLength = req.headers['x-amz-decoded-content-length']; + if (decodedLength !== undefined && Number(decodedLength) !== bytes.length) throw new Error('S3 decoded length mismatch'); + const previous = objects.get(key); + if ((req.headers['if-none-match'] !== undefined && req.headers['if-none-match'] !== '*') + || (req.headers['if-match'] !== undefined && typeof req.headers['if-match'] !== 'string')) throw new Error('Unknown conditional S3 request'); + if ((req.headers['if-none-match'] === '*' && previous) || (req.headers['if-match'] && previous?.etag !== req.headers['if-match'])) { + res.setHeader('content-type', 'application/xml'); reply(412, 'PreconditionFailed'); return; + } + const total = [...objects.values()].reduce((sum, object) => sum + object.bytes.length, 0) - (previous?.bytes.length ?? 0) + bytes.length; + if (bytes.length > 16 * 1024 * 1024 || total > 256 * 1024 * 1024 || objects.size >= 2048) throw new Error('S3 fixture storage bound exceeded'); + const object = { bytes, etag: `"${createHash('md5').update(bytes).digest('hex')}"`, contentType: String(req.headers['content-type'] ?? 'application/octet-stream') }; + objects.set(key, object); res.setHeader('ETag', object.etag); reply(200); return; + } + if (req.method === 'DELETE') { objects.delete(key); reply(204); return; } + if (req.method !== 'GET' && req.method !== 'HEAD') throw new Error('Unknown S3 fixture method'); + const object = objects.get(key); + if (!object) { res.setHeader('content-type', 'application/xml'); reply(404, req.method === 'GET' ? 'NoSuchKey' : undefined); return; } + if (req.headers.range) throw new Error('Range is outside this small-object fixture contract'); + res.setHeader('ETag', object.etag); res.setHeader('content-type', object.contentType); res.setHeader('content-length', object.bytes.length); + reply(200, req.method === 'GET' ? object.bytes : undefined); + } catch (error) { + errors.push(`${req.method} ${url.pathname + url.search}: ${error instanceof Error ? error.message : String(error)}`); + if (!res.headersSent) { res.setHeader('content-type', 'application/xml'); reply(501, 'NotImplemented'); } else res.destroy(); + } + }); + return { server, snapshot: () => ({ boundary: 'Actual AWS SDK HTTP; bounded in-memory S3 protocol fixture, no IAM/SigV4/provider durability proof', requests: requests.map(request => ({ ...request })), errors: [...errors], + objects: [...objects].map(([key, object]) => ({ key, bytes: object.bytes.length, sha256: createHash('sha256').update(object.bytes).digest('hex') })) }) }; +} + +/** Decode SDK checksum framing without pretending to validate provider signatures. */ +export function decodeMaintenanceAwsChunks(wire: Buffer) { + const chunks: Buffer[] = []; let offset = 0; + while (offset < wire.length) { + const end = wire.indexOf('\r\n', offset); if (end < 0) throw new Error('Malformed AWS chunk header'); + const header = wire.subarray(offset, end).toString('ascii'); + if (!/^[0-9a-f]+$/i.test(header)) throw new Error('Unknown AWS chunk extension'); + const length = Number.parseInt(header, 16); offset = end + 2; + if (!Number.isSafeInteger(length) || length < 0 || offset + length + 2 > wire.length) throw new Error('Truncated AWS chunk'); + if (length === 0) { + const trailers = wire.subarray(offset).toString('ascii'); + if (!/^(?:x-amz-checksum-[a-z0-9]+:[A-Za-z0-9+/=]+\r\n)*\r\n$/.test(trailers)) throw new Error('Unknown AWS checksum trailer'); + return Buffer.concat(chunks); + } + chunks.push(wire.subarray(offset, offset + length)); offset += length; + if (wire.subarray(offset, offset + 2).toString('ascii') !== '\r\n') throw new Error('Malformed AWS chunk terminator'); offset += 2; + } + throw new Error('Missing AWS final chunk'); +} diff --git a/server/src/__tests__/helpers/maintenance/server-child.ts b/server/src/__tests__/helpers/maintenance/server-child.ts index 5569b29d2e9..833726d930e 100644 --- a/server/src/__tests__/helpers/maintenance/server-child.ts +++ b/server/src/__tests__/helpers/maintenance/server-child.ts @@ -2,6 +2,7 @@ import { channel } from "node:diagnostics_channel"; import { mock } from "node:test"; import type { Db } from "@paperclipai/db"; import type { Socket } from "node:net"; +import { observeMaintenanceIncomingHttp } from "./incoming-http-observation.js"; const realTimeout = globalThis.setTimeout, realInterval = globalThis.setInterval, clearRealInterval = globalThis.clearInterval, realNow = Date.now; const fakeClock = process.env.PAPERCLIP_MAINTENANCE_FAKE_CLOCK === "true"; @@ -40,14 +41,42 @@ const observeRequest = () => { outgoingHttpRequests++; }; channel("undici:request:create").subscribe(observeRequest); channel("http.client.request.start").subscribe(observeRequest); +let workCapture: Awaited> | undefined; +let databaseCapture: ReturnType; +let incomingHttp: ReturnType | undefined; try { + // Must precede the announcement wrapper's first import of the real app graph. + if (process.env.PAPERCLIP_MAINTENANCE_IDLE_WORK_CAPTURE === '1') { + workCapture = await (await import('./idle-work-capture.js')).installMaintenanceIdleWorkCapture(); + } + const announcement = process.env.PAPERCLIP_MAINTENANCE_ANNOUNCEMENT_FIXTURE === '1' + ? await (await import('./announcement-boundary.js')).installMaintenanceAnnouncementBoundary() : undefined; const { startServer } = await import("../../../index.js"); const { idleWorkSnapshot, startTaskDrain, stopTaskDrain } = await import("../../../services/task-admission.js"); const { getStartupRecoveryState } = await import("../../../startup-recovery-state.js"); const server = await startServer(); + if (workCapture) databaseCapture = (await import('@paperclipai/db/response-capture')).databaseResponseCaptureFromEnv(); + const managedObservation = process.env.PAPERCLIP_MAINTENANCE_MANAGED_TENANT === '1' ? await (async () => { + const { loadConfig } = await import('../../../config.js'); + const { isCloudManagedInstance } = await import('../../../services/cloud-instance.js'); + const { idleApiActivity } = await import('../../../services/idle-api-activity.js'); + const config = loadConfig(); + return () => ({ isCloudManaged: isCloudManagedInstance(), deploymentMode: config.deploymentMode, + storageProvider: config.storageProvider, apiActivityActive: idleApiActivity.active() }); + })() : undefined; const app = server.server.listeners("request")[0] as unknown as { locals: { paperclipDb: Db } }; + try { incomingHttp = observeMaintenanceIncomingHttp(server.server); } + catch (error) { + try { await server.shutdown(); } + catch (cleanupError) { throw new AggregateError([error, cleanupError], "Incoming HTTP observation and owned startup cleanup failed"); } + throw error; + } + async function shutdownServer() { + try { await server.shutdown(); } finally { incomingHttp?.dispose(); } + } + try { announcement?.assertIntercepted(); } catch (error) { await shutdownServer(); throw error; } process.send?.({ kind: "ready", port: server.listenPort }); - process.on("message", async (message: { id: number; command: string; ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } }) => { + process.on("message", async (message: { id: number; command: string; ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }; github?: { companyId?: string; connectionId?: string; grantId?: string } }) => { try { if (message.command === "snapshot" || message.command === "advance" || message.command === "drain" || message.command === "resume" || message.command === "freeze" || message.command === "realtime") { if (message.command === "freeze" && bootClock) { clearRealInterval(bootClock); bootClock = realInterval(() => mock.timers.tick(0), 10); } @@ -59,7 +88,23 @@ try { if (message.command === "drain") startTaskDrain({ purpose: "idle" }); if (message.command === "resume") stopTaskDrain(); if (fakeClock) mock.timers.tick(0); - process.send?.({ id: message.id, kind: "snapshot", idle: idleWorkSnapshot(), startup: getStartupRecoveryState(), outgoingHttpRequests, network: network(), now: Date.now() }); + process.send?.({ id: message.id, kind: "snapshot", idle: idleWorkSnapshot(), startup: getStartupRecoveryState(), outgoingHttpRequests, network: network(), incomingHttp: incomingHttp?.snapshot(), now: Date.now(), announcement: announcement?.snapshot(), managed: managedObservation?.(), workCapture: workCapture?.snapshot() }); + } else if (message.command === "prepareGitHub" || message.command === "settleGitHub") { + const { prepareGitHubConfiguration, settleGitHubConfiguration } = await import("./github-configuration-producer.js"); + const input = message.github; + if (!input) throw new Error("Missing GitHub fixture identity"); + const result = message.command === "prepareGitHub" + ? await prepareGitHubConfiguration(app.locals.paperclipDb, input.companyId!) + : await settleGitHubConfiguration(app.locals.paperclipDb, { connectionId: input.connectionId!, grantId: input.grantId! }); + process.send?.({ id: message.id, kind: "githubConfiguration", result }); + } else if (message.command === "produceChatHistory") { + const { produceChatHistory } = await import("./chat-history-producer.js"); + process.send?.({ id: message.id, kind: "chatHistory", result: await produceChatHistory(app.locals.paperclipDb) }); + } else if (message.command === "trace") { + const { getStartupTracer } = await import("../../../instrumentation.js"); + const span = getStartupTracer("paperclip.sleep-qualification").startSpan("sleep-qualification-stimulus"); + span.end(); + process.send?.({ id: message.id, kind: "traceEnded" }); } else if (message.command === "createOrphanLease") { if (!message.orphanLease) throw new Error("Missing orphan lease fixture"); const { environmentLeases } = await import("@paperclipai/db"); @@ -83,7 +128,20 @@ try { process.send?.({ id: message.id, kind: "published" }); } else if (message.command === "shutdown") { if (fakeClock) runClock(); - await server.shutdown(); + if (workCapture) { + const failures: unknown[] = []; + let before: ReturnType | undefined; + try { before = workCapture.summary(); } catch (error) { failures.push(error); } + try { await shutdownServer(); } catch (error) { failures.unshift(error); } + // A passive flush failure must not replace the original shutdown error. + try { databaseCapture?.flush(); } catch (error) { failures.push(error); } + try { + await (await import('node:fs/promises')).writeFile((await import('node:path')).join(process.env.PAPERCLIP_HOME!, 'idle-work-capture-cleanup.json'), JSON.stringify({ before, after: workCapture.summary() }, null, 2)); + } catch (error) { failures.push(error); } + finally { try { workCapture.dispose(); } catch (error) { failures.push(error); } } + if (failures.length > 1) throw new AggregateError(failures, 'Owned shutdown and passive evidence failed'); + if (failures.length) throw failures[0]; + } else await shutdownServer(); if (bootClock) { clearRealInterval(bootClock); bootClock = undefined; } process.send?.({ id: message.id, kind: "stopped", outgoingHttpRequests }); if (fakeClock) mock.timers.reset(); @@ -93,6 +151,16 @@ try { } catch (error) { process.send?.({ id: message.id, kind: "error", error: error instanceof Error ? error.stack : String(error) }); } }); } catch (error) { + incomingHttp?.dispose(); + if (workCapture) { + try { + // Existing bounded DB capture performs no SQL while flushing metadata. + databaseCapture ??= (await import('@paperclipai/db/response-capture')).databaseResponseCaptureFromEnv(); + databaseCapture?.flush(); + await (await import('node:fs/promises')).writeFile((await import('node:path')).join(process.env.PAPERCLIP_HOME!, 'idle-work-capture-startup-failure.json'), JSON.stringify(workCapture.summary(), null, 2)); + } catch { console.warn('Passive startup evidence could not be persisted; original error preserved'); } + finally { try { workCapture.dispose(); } catch { console.warn('Passive capture cleanup failed; original startup error preserved'); } } + } if (bootClock) clearRealInterval(bootClock); process.send?.({ kind: "error", error: error instanceof Error ? error.stack : String(error) }); process.exitCode = 1; diff --git a/server/src/__tests__/helpers/maintenance/server-harness.ts b/server/src/__tests__/helpers/maintenance/server-harness.ts index 922f90292a8..15e1c6ca04e 100644 --- a/server/src/__tests__/helpers/maintenance/server-harness.ts +++ b/server/src/__tests__/helpers/maintenance/server-harness.ts @@ -1,3 +1,5 @@ +import type { IdleWorkCaptureSnapshot } from './idle-work-capture.js'; +import type { IncomingHttpSnapshot } from '../../../../../scripts/sleep-qualification/evidence.mjs'; import { fork } from "node:child_process"; import type { Server as HttpServer } from "node:http"; import { randomBytes, randomUUID } from "node:crypto"; @@ -42,9 +44,54 @@ export async function withMaintenanceProvider(provider: HttpServer, run: (ori } } -type Snapshot = { idle: { active: number; generation: number }; startup: { phase: string }; outgoingHttpRequests: number; network: { nonDatabaseConnections: number; databaseConnections: number; nonDatabaseBytesWritten: number }; now: number }; +type Snapshot = { idle: { active: number; generation: number }; startup: { phase: string }; outgoingHttpRequests: number; network: { nonDatabaseConnections: number; databaseConnections: number; nonDatabaseBytesWritten: number }; now: number; + incomingHttp: IncomingHttpSnapshot; + managed?: { isCloudManaged: boolean; deploymentMode: string; storageProvider: string; apiActivityActive: boolean }; + workCapture?: IdleWorkCaptureSnapshot; + announcement?: { intercepted: boolean; requests: string[]; unmatched: string[]; boundary: string } }; -export async function startMaintenanceServer(databaseUrl: string, options: { artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string }; cloud?: { jwks: string; s3Origin: string; standby: boolean } } = {}) { +export interface MaintenanceConfiguration { + automaticBackups?: { intervalMinutes: number; idleCheckpoint: boolean }; + announcements?: boolean; + telemetryOrigin?: string; + otelOrigin?: string; + serveUi?: boolean; + managedInstance?: boolean; + deploymentMode?: "local_trusted" | "authenticated"; + storageProvider?: "local_disk" | "s3"; +} +export interface MaintenanceServerOptions { + artifactRoot?: string; enabled?: boolean; clock?: boolean; suppressed?: boolean; githubOrigin?: string; + connector?: { origin: string; signPrivateKey: string; sealPrivateKey: string; instanceId?: string }; + cloud?: { jwks: string; s3Origin: string; standby: boolean }; + configuration?: MaintenanceConfiguration; + controlJwks?: string; + captureIdleWork?: boolean; + tenantActor?: { userId: string; email: string; name: string }; +} +export async function startMaintenanceServer(databaseUrl: string, options: MaintenanceServerOptions = {}) { + const configuration = options.configuration; + if (options.captureIdleWork && options.clock) throw new Error("Passive setup diagnostics require the real clock"); + if (options.tenantActor && (!options.cloud || !configuration?.managedInstance || configuration.deploymentMode !== 'authenticated' || configuration.storageProvider !== 's3')) { + throw new Error('Managed tenant fixture requires the supported authenticated S3 profile'); + } + const tenantHeaders: Record = options.tenantActor ? { + 'x-paperclip-cloud-tenant-token': 'maintenance-tenant-token', 'x-paperclip-cloud-stack-id': 'maintenance-stack', + 'x-paperclip-cloud-stack-role': 'owner', 'x-paperclip-cloud-user-id': options.tenantActor.userId, + 'x-paperclip-cloud-user-email': options.tenantActor.email, 'x-paperclip-cloud-user-name': options.tenantActor.name, + 'x-paperclip-cloud-paperclip-company-name': 'Configured sleep acceptance', + } : {}; + for (const origin of [configuration?.telemetryOrigin, configuration?.otelOrigin]) { + if (!origin) continue; + const endpoint = new URL(origin); + if (endpoint.protocol !== "http:" || endpoint.hostname !== "127.0.0.1" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash) { + throw new Error("Maintenance collectors must be explicit loopback HTTP endpoints"); + } + } + if (configuration?.automaticBackups && (!Number.isInteger(configuration.automaticBackups.intervalMinutes) || configuration.automaticBackups.intervalMinutes < 1)) { + throw new Error("Maintenance backup interval must be a positive integer"); + } + if (configuration && options.clock) throw new Error("Configured acceptance requires the real clock"); const artifacts = await createMaintenanceArtifacts(options.artifactRoot); const { directory, retainedDirectory } = artifacts; const reservation = createServer(); @@ -72,43 +119,57 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art const configPath = path.join(directory, "config.json"); await writeFile(configPath, JSON.stringify({ $meta: { version: 1, updatedAt: new Date().toISOString(), source: "onboard" }, - database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: false, dir: path.join(directory, "backups") } }, + database: { mode: "postgres", connectionString: proxy.url, backup: { enabled: Boolean(configuration?.automaticBackups), intervalMinutes: configuration?.automaticBackups?.intervalMinutes ?? 60, dir: path.join(directory, "backups") } }, logging: { mode: "file", logDir: path.join(directory, "logs") }, - server: { deploymentMode: "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: false }, + server: { deploymentMode: options.tenantActor ? "authenticated" : "local_trusted", exposure: "private", bind: "loopback", host: "127.0.0.1", port: address.port, serveUi: configuration?.serveUi ?? false }, storage: options.cloud ? { provider: "s3", s3: { bucket: "maintenance-test", region: "us-east-1", endpoint: options.cloud.s3Origin, forcePathStyle: true } } : { provider: "local_disk", localDisk: { baseDir: path.join(directory, "storage") } }, secrets: { provider: "local_encrypted", strictMode: false, localEncrypted: { keyFilePath: path.join(directory, "master.key") } }, - telemetry: { enabled: false }, updates: { checkEnabled: false }, + ...(options.tenantActor ? { auth: { baseUrlMode: "explicit", publicBaseUrl: `http://127.0.0.1:${address.port}`, disableSignUp: true } } : {}), + telemetry: { enabled: Boolean(configuration?.telemetryOrigin) }, updates: { checkEnabled: false }, }), { mode: 0o600 }); await new Promise(resolve => reservation.close(() => resolve())); + const captureStartedAt = options.captureIdleWork ? Date.now() : undefined; + const captureSources = options.captureIdleWork ? new Map() : undefined; + let settledDiagnosticId = 0; const child = fork(fileURLToPath(new URL("./server-child.ts", import.meta.url)), [], { cwd: fileURLToPath(new URL("../../../../..", import.meta.url)), - execArgv: ["--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], + execArgv: [...((configuration?.announcements || options.captureIdleWork) ? ['--experimental-test-module-mocks'] : []), "--import", fileURLToPath(new URL("../../../../node_modules/tsx/dist/loader.mjs", import.meta.url))], env: { + ...(captureStartedAt === undefined ? {} : { + PAPERCLIP_MAINTENANCE_IDLE_WORK_CAPTURE: '1', + PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW: `${new Date(captureStartedAt).toISOString()}/${new Date(captureStartedAt + 60_000).toISOString()}`, + }), PATH: process.env.PATH, TMPDIR: process.env.TMPDIR, NODE_ENV: "production", PAPERCLIP_HOME: directory, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "maintenance-test", PAPERCLIP_RUNNER_STATE_DIR: path.join(directory, "durable-sessions"), - DATABASE_URL: proxy.url, DATABASE_POOL_MAX: "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test", + DATABASE_URL: proxy.url, DATABASE_POOL_MAX: configuration ? "4" : "6", DATABASE_APPLICATION_NAME: "maintenance-integration-test", PAPERCLIP_AGENT_JWT_SECRET: randomUUID(), PAPERCLIP_DECISION_SIGNING_SECRET: randomUUID(), PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: randomUUID(), BETTER_AUTH_SECRET: randomUUID(), - CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false", - PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_UI_DEV_MIDDLEWARE: "false", + CODEX_HOME: path.join(directory, "codex"), PAPERCLIP_ANNOUNCEMENTS_ENABLED: String(configuration?.announcements ?? false), + ...(configuration?.announcements ? { PAPERCLIP_MAINTENANCE_ANNOUNCEMENT_FIXTURE: '1' } : {}), + PAPERCLIP_DB_BACKUP_ENABLED: String(Boolean(configuration?.automaticBackups)), PAPERCLIP_UI_DEV_MIDDLEWARE: "false", + ...(configuration?.automaticBackups ? { PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED: configuration.automaticBackups.idleCheckpoint ? "1" : "0" } : {}), + ...(configuration?.telemetryOrigin ? { PAPERCLIP_TELEMETRY_ENDPOINT: configuration.telemetryOrigin } : {}), + ...(configuration?.otelOrigin ? { OTEL_EXPORTER_OTLP_ENDPOINT: configuration.otelOrigin, OTEL_EXPORTER_OTLP_PROTOCOL: "http/json" } : {}), HEARTBEAT_SCHEDULER_ENABLED: String(options.enabled ?? true), PAPERCLIP_MAINTENANCE_FAKE_CLOCK: String(options.clock ?? false), ...(options.suppressed ? { PAPERCLIP_IN_WORKTREE: "true" } : {}), ...(options.githubOrigin ? { PAPERCLIP_MAINTENANCE_GITHUB_ORIGIN: options.githubOrigin } : {}), ...(options.connector ? { - PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test", + PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID: options.connector.instanceId ?? "maintenance-connector", PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT: "test", PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY: options.connector.signPrivateKey, PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY: options.connector.sealPrivateKey, PAPERCLIP_CLOUD_CONNECTOR_BASE_URL: options.connector.origin, } : {}), + ...(options.controlJwks ? { PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.controlJwks } : {}), ...(options.cloud ? { PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] } }), PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString("hex"), PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "maintenance-tenant-token", PAPERCLIP_CLOUD_STACK_ID: "maintenance-stack", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: options.cloud.jwks, PAPERCLIP_CLOUD_WARM_STANDBY: options.cloud.standby ? "1" : "0", - PAPERCLIP_PUBLIC_URL: "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: "pool-maintenance.staging.paperclip.app", + PAPERCLIP_PUBLIC_URL: options.tenantActor ? `http://127.0.0.1:${address.port}` : "https://pool-maintenance.staging.paperclip.app", PAPERCLIP_PRIMARY_HOST: options.tenantActor ? "127.0.0.1" : "pool-maintenance.staging.paperclip.app", + ...(options.tenantActor ? { PAPERCLIP_MAINTENANCE_MANAGED_TENANT: "1" } : {}), AWS_ACCESS_KEY_ID: "maintenance-fixture", AWS_SECRET_ACCESS_KEY: "maintenance-fixture", AWS_EC2_METADATA_DISABLED: "true", } : {}), PAPERCLIP_MIGRATION_PROMPT: "never", PAPERCLIP_MIGRATION_AUTO_APPLY: "true", PAPERCLIP_MANAGED_RUNTIME_HTTPS: "off", @@ -139,12 +200,18 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art for (const request of requests.values()) request.reject(error); requests.clear(); }); - async function request(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string } } = {}): Promise { + async function request(command: string, payload: { ms?: number; issueUpdate?: { id: string; status: "done" }; orphanLease?: { companyId: string }; publication?: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }; github?: { companyId?: string; connectionId?: string; grantId?: string } } = {}): Promise { const id = ++requestId; const result = new Promise((resolve, reject) => requests.set(id, { resolve, reject })); child.send({ id, command, ...payload }); - try { return await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`); } - finally { requests.delete(id); } + try { + const value = await withDeadline(result, 30_000, () => `Maintenance child ${command} timed out: ${output.slice(-8_000)}`); + if (captureSources) { + const capture = (value as { workCapture?: IdleWorkCaptureSnapshot }).workCapture; + for (const source of capture?.newSources ?? []) captureSources.set(source.id, source.callers); + } + return value; + } finally { requests.delete(id); } } stopChild = async () => { try { @@ -165,7 +232,7 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art if (options.clock) await request("realtime"); try { const response = await fetch(`http://127.0.0.1:${port}/api${route}`, { method, signal: AbortSignal.timeout(30_000), - headers: { "Content-Type": "application/json", ...headers }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); + headers: { "Content-Type": "application/json", ...(Object.keys(headers ?? {}).some(key => ["authorization", "x-paperclip-cloud-control"].includes(key.toLowerCase())) ? {} : tenantHeaders), ...headers }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); const value = await response.json(); if (!response.ok) throw new Error(`Maintenance API ${method} ${route}: ${response.status} ${JSON.stringify(value)}`); return value as T; @@ -182,6 +249,16 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art finally { if (options.clock) await request("freeze"); } }, publishBackup: (publication: { companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string }) => request("publishBackup", { publication }), + async prepareGitHub(companyId: string) { + const result = await request<{ result: { connectionId: string; grantId: string } }>("prepareGitHub", { github: { companyId } }); + return result.result; + }, + async settleGitHub(input: { connectionId: string; grantId: string }) { + return request("settleGitHub", { github: input }); + }, + tenantHeaders, + produceChatHistory: () => request("produceChatHistory"), + trace: () => request("trace"), snapshot: () => request("snapshot"), drain: () => request("drain"), resume: () => request("resume"), realtime: () => request("realtime"), @@ -197,17 +274,38 @@ export async function startMaintenanceServer(databaseUrl: string, options: { art }, freeze: () => request("freeze"), get output() { return output; }, - async settled(quietMs = 500, deferredStartupHolds = 0) { + async settled(quietMs = 500, deferredStartupHolds = 0, settleOptions: { managedSetupRecovery?: true } = {}) { + // A successful managed health check self-notifies, then reconciles its + // future deadline on the next existing 30s pass. Observe both passes + + // 500ms quiet + 500ms margin without changing worker timing/admission. + const settleTimeoutMs = settleOptions.managedSetupRecovery ? 61_000 : 30_000; + const diagnostic = options.captureIdleWork ? (await import('./idle-work-capture.js')).createSettledWorkSamples( + () => [...captureSources!].map(([id, callers]) => ({ id, callers }))) : undefined; + const diagnosticId = diagnostic ? ++settledDiagnosticId : undefined; + let failure: { error: unknown } | undefined; let previous = -1, quietSince = Date.now(); - for (const deadline = Date.now() + 30_000; Date.now() < deadline;) { + try { + for (const deadline = Date.now() + settleTimeoutMs; Date.now() < deadline;) { + const snapshot = await request("snapshot"); + diagnostic?.sample(snapshot, proxy.queries.length, proxy.pending); + if (proxy.errors.length) throw new AggregateError(proxy.errors, "PostgreSQL observer failed"); + if (snapshot.idle.active !== deferredStartupHolds || proxy.pending !== 0 || proxy.queries.length !== previous) quietSince = Date.now(); + previous = proxy.queries.length; + if (snapshot.idle.active === deferredStartupHolds && proxy.pending === 0 && Date.now() - quietSince >= quietMs) { + diagnostic?.assertComplete(); return snapshot; + } + await delay(25); + } const snapshot = await request("snapshot"); - if (proxy.errors.length) throw new AggregateError(proxy.errors, "PostgreSQL observer failed"); - if (snapshot.idle.active !== deferredStartupHolds || proxy.pending !== 0 || proxy.queries.length !== previous) quietSince = Date.now(); - previous = proxy.queries.length; - if (snapshot.idle.active === deferredStartupHolds && proxy.pending === 0 && Date.now() - quietSince >= quietMs) return snapshot; - await delay(25); + diagnostic?.sample(snapshot, proxy.queries.length, proxy.pending); + throw new Error(`Maintenance did not settle: ${JSON.stringify({ timeoutMs: settleTimeoutMs, quietMs, queries: proxy.queries.length, pending: proxy.pending, snapshot })}\n${output.slice(-12_000)}`); + } catch (error) { failure = { error }; throw error; } + finally { + if (diagnostic) { + try { await writeFile(path.join(directory, `idle-work-settled-samples-${diagnosticId}.json`), JSON.stringify({ ...diagnostic.report(), settleDeadlineMs: settleTimeoutMs }, null, 2)); } + catch (error) { if (failure) throw new AggregateError([failure.error, error], 'Original settling and passive evidence failed'); throw error; } + } } - throw new Error(`Maintenance did not settle: ${JSON.stringify({ queries: proxy.queries.length, pending: proxy.pending, snapshot: await request("snapshot") })}\n${output.slice(-12_000)}`); }, }; } catch (error) { diff --git a/server/src/__tests__/helpers/native-warm-ingress-fixture.ts b/server/src/__tests__/helpers/native-warm-ingress-fixture.ts new file mode 100644 index 00000000000..15f904de509 --- /dev/null +++ b/server/src/__tests__/helpers/native-warm-ingress-fixture.ts @@ -0,0 +1,64 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import * as tables from "@paperclipai/db"; +import type { NativeExecutionInputV1, NativeExecutionInputV3 } from "@paperclipai/paperclip-runner"; +import { startRecoveryPostgres } from "./recovery-postgres.js"; +import { prepareNativeHeartbeatRun } from "../../services/native-runtime/prepare-native-run.js"; +import { buildNativeCompletionContract } from "../../services/native-runtime/completion-contracts.js"; +import { nativeRuntimeContextFixture } from "../../services/native-runtime/runtime-context.test-fixture.js"; + +export async function createNativeWarmIngressFixture(base: NativeExecutionInputV1, cwd: string) { + const database = await startRecoveryPostgres("shared-ingress-owner-"); + const db = tables.createDb(database.connectionString); + const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), workspaceId = randomUUID(); + const providerLeaseId = `allocation-${randomUUID()}`; + await db.insert(tables.companies).values({ id: companyId, name: "Shared ingress", issuePrefix: `I${companyId.slice(0, 7)}` }); + await db.insert(tables.agents).values({ id: agentId, companyId, name: "Shared worker", adapterType: "paperclip_runner" }); + await db.insert(tables.environments).values({ id: environmentId, name: "Shared allocation", driver: "sandbox" }); + await db.insert(tables.executionWorkspaces).values({ id: workspaceId, companyId, name: "Shared workspace", mode: "shared_workspace", + strategyType: "project_primary", providerType: "cloud_sandbox", cwd }); + const defaultCompanyId = companyId, defaultAgentId = agentId, defaultWorkspaceId = workspaceId, defaultProviderLeaseId = providerLeaseId; + let epoch = 0; + async function admit(options?: { companyId?: string; agentId?: string; workspaceId?: string; providerLeaseId?: string }) { + const ownerCompanyId = options?.companyId ?? defaultCompanyId, ownerAgentId = options?.agentId ?? defaultAgentId; + const ownerWorkspaceId = options?.workspaceId ?? defaultWorkspaceId, ownerProviderLeaseId = options?.providerLeaseId ?? defaultProviderLeaseId; + const companyId = ownerCompanyId, agentId = ownerAgentId, workspaceId = ownerWorkspaceId, providerLeaseId = ownerProviderLeaseId; + const issueId = randomUUID(), runId = randomUUID(), leaseId = randomUUID(); + const [issue] = await db.insert(tables.issues).values({ id: issueId, companyId, title: "Shared handoff", status: "in_progress", + assigneeAgentId: agentId, executionWorkspaceId: workspaceId }).returning(); + const [run] = await db.insert(tables.heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, + status: "running", runtimeMode: "native", contextSnapshot: { issueId } }).returning(); + await db.update(tables.issues).set({ executionRunId: runId }).where(eq(tables.issues.id, issueId)); + await db.insert(tables.environmentLeases).values({ id: leaseId, companyId, environmentId, executionWorkspaceId: workspaceId, + issueId, heartbeatRunId: runId, provider: "daytona", providerLeaseId, leasePolicy: "reuse_by_environment" }); + await db.insert(tables.nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); + await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: leaseId }); + const [contract] = await db.select().from(tables.completionContracts).where(eq(tables.completionContracts.issueId, issueId)); + const execution: NativeExecutionInputV3 = { ...base, schema: "paperclip.native-execution-input.v3", executionMode: "default", planningContext: null, + runtimeContext: nativeRuntimeContextFixture(), binding: { companyId, agentId, issueId, runId, executionWorkspaceId: workspaceId }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", + contract: buildNativeCompletionContract(issue, { revision: contract.revision }) }, + workspace: { cwd, repoUrl: null, repoRef: null, branchName: null }, + session: { ...base.session, normalizedSessionId: `logical-${issueId}`, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 } } }; + await db.update(tables.executionWorkspaces).set({ metadata: { remoteWorkspace: { + schema: "paperclip.remote-workspace.v1", epoch: ++epoch, snapshotId: null, source: { kind: "empty" }, writerRunId: runId, dirty: true, + location: { environmentId, leaseId, providerLeaseId, remoteCwd: cwd }, + } } }).where(eq(tables.executionWorkspaces.id, workspaceId)); + const target = { kind: "remote" as const, transport: "sandbox" as const, providerKey: "daytona", environmentId, leaseId, remoteCwd: cwd, + effectiveCapabilities: { runnerWebSocketIngress: true } as never, + sandboxLeaseAcquisition: { outcome: "resumed" as const, providerLeaseId } }; + return { execution, target, issueId, runId, leaseId }; + } + async function settle(owner: Awaited>, result: Record) { + const resultId = randomUUID(); + await db.insert(tables.nativeRunResults).values({ id: resultId, companyId, issueId: owner.issueId, runId: owner.runId, + completionContractId: owner.execution.completionContract.id, schemaStatus: "accepted", resultJson: result, + serverFingerprint: randomUUID(), canonicalSha256: randomUUID() }); + await db.update(tables.nativeRunFinalizations).set({ phase: "committed", resultId, leaseOwner: null, leaseExpiresAt: null }).where(eq(tables.nativeRunFinalizations.runId, owner.runId)); + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), nativePhase: "committed", costAccountingPending: false }).where(eq(tables.heartbeatRuns.id, owner.runId)); + await db.update(tables.issues).set({ status: "done", executionRunId: null, checkoutRunId: null }).where(eq(tables.issues.id, owner.issueId)); + await db.update(tables.environmentLeases).set({ status: "expired", cleanupStatus: "success", releasedAt: new Date() }).where(eq(tables.environmentLeases.id, owner.leaseId)); + return resultId; + } + return { db, companyId, agentId, environmentId, workspaceId, providerLeaseId, admit, settle, cleanup: () => database.cleanup() }; +} diff --git a/server/src/__tests__/instrumentation-signals.test.ts b/server/src/__tests__/instrumentation-signals.test.ts new file mode 100644 index 00000000000..44e5638b83a --- /dev/null +++ b/server/src/__tests__/instrumentation-signals.test.ts @@ -0,0 +1,30 @@ +import { afterEach, expect, it, vi } from 'vitest'; + +const sdk = vi.hoisted(() => ({ options: [] as Array>, start: vi.fn(), shutdown: vi.fn(async () => {}) })); +vi.mock('../peer-version-check.js', () => ({ checkExactPeerVersions: () => ({ ok: true }) })); +vi.mock('@opentelemetry/sdk-node', () => ({ NodeSDK: class { + constructor(options: Record) { sdk.options.push(options); } + start = sdk.start; + shutdown = sdk.shutdown; +} })); +vi.mock('@opentelemetry/auto-instrumentations-node', () => ({ getNodeAutoInstrumentations: () => [] })); +vi.mock('@opentelemetry/resources', () => ({ resourceFromAttributes: (attributes: unknown) => attributes })); +vi.mock('@opentelemetry/semantic-conventions', () => ({ ATTR_SERVICE_NAME: 'service.name', ATTR_SERVICE_VERSION: 'service.version' })); +vi.mock('@opentelemetry/exporter-trace-otlp-http', () => ({ OTLPTraceExporter: class {} })); + +afterEach(() => { vi.unstubAllEnvs(); vi.restoreAllMocks(); }); +it('configures current plural SDK signal options so an opted-in trace exporter cannot enable default metrics or logs', async () => { + vi.stubEnv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://collector.invalid:4318'); + vi.stubEnv('OTEL_EXPORTER_OTLP_PROTOCOL', 'http/json'); + vi.spyOn(process, 'once').mockReturnValue(process); + vi.resetModules(); + const instrumentation = await import('../instrumentation.js'); + await instrumentation.instrumentationReady; + expect(sdk.options).toHaveLength(1); + expect(sdk.options[0]).toMatchObject({ metricReaders: [], logRecordProcessors: [], traceExporter: expect.any(Object) }); + expect(sdk.options[0]).not.toHaveProperty('metricReader'); + expect(sdk.options[0]).not.toHaveProperty('logRecordProcessor'); + expect(sdk.start).toHaveBeenCalledOnce(); + await instrumentation.shutdownInstrumentation(); + expect(sdk.shutdown).toHaveBeenCalledOnce(); +}); diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index 21553967b5d..85d6bf6afc0 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -90,6 +90,24 @@ function makeRecoveryActionRow(overrides: Record = {}) { const LATEST_RUN_TIMES = { startedAt: new Date(), createdAt: new Date() }; describe("issueRecoveryActionService", () => { + it.each(["unrelated unique constraint", "other SQLSTATE", "cyclic cause"])("does not retry %s errors", async kind => { + const constraint = "issue_recovery_actions_active_source_uq"; + const error: Error & { cause?: unknown } = new Error("Wrapped database failure"); + error.cause = kind === "unrelated unique constraint" ? { code: "23505", constraint_name: "other_unique_constraint" } + : kind === "other SQLSTATE" ? { code: "23503", constraint_name: constraint } : error; + const query = { from() { return this; }, where() { return this; }, orderBy() { return this; }, + limit() { return Promise.resolve([]); } }; + const fakeDb = { + async transaction(fn: (tx: unknown) => Promise): Promise { return fn(fakeDb); }, + select: vi.fn(() => query), + insert: vi.fn(() => ({ values: () => ({ returning: async () => { throw error; } }) })), + }; + await expect(issueRecoveryActionService(fakeDb as never).upsertSourceScoped({ companyId: "company", sourceIssueId: "issue", + kind: "missing_disposition", cause: "saved-debt", fingerprint: "same-source", nextAction: "Restore existing work." })) + .rejects.toBe(error); + expect(fakeDb.select).toHaveBeenCalledOnce(); + expect(fakeDb.insert).toHaveBeenCalledOnce(); + }); it("does not reactivate an action resolved between the active read and update", async () => { const existingRow = makeRecoveryActionRow({ id: "existing-action", attemptCount: 1 }); const createdRow = makeRecoveryActionRow({ id: "new-action", attemptCount: 1 }); @@ -413,6 +431,49 @@ describeEmbeddedPostgres("issue recovery actions", () => { }, ); + it("converges two service instances after their real inserts compete for one active source", async () => { + const { companyId, managerId, sourceIssueId } = await seedCompany(); + const input = { companyId, sourceIssueId, kind: "stranded_assigned_issue" as const, ownerType: "agent" as const, + ownerAgentId: managerId, cause: "stranded_assigned_issue", fingerprint: "concurrent-source", + evidence: { sourceRunId: "concurrent-source-run" }, lastAttemptAt: new Date("2026-11-03T12:00:00Z"), + nextAction: "Restore the existing execution path." }; + let release!: () => void, acquired!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const locked = new Promise(resolve => { acquired = resolve; }); + const lock = db.transaction(async tx => { + await tx.execute(sql`lock table issue_recovery_actions in share mode`); + acquired(); await gate; + }); + await locked; + const outcomes = Promise.allSettled([issueRecoveryActionService(db).upsertSourceScoped(input), + issueRecoveryActionService(db).upsertSourceScoped(input)]); + try { + await vi.waitFor(async () => { + const [waiting] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and state = 'active' and wait_event_type = 'Lock' + and query like 'insert into "issue_recovery_actions"%'`); + expect(waiting.count).toBe(2); + }, { timeout: 5000, interval: 20 }); + } finally { release(); await lock; await outcomes; } + const results = await outcomes; + const active = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, companyId), + eq(issueRecoveryActions.sourceIssueId, sourceIssueId), sql`${issueRecoveryActions.status} in ('active', 'escalated')`)); + expect(active).toHaveLength(1); + expect(results.flatMap(result => result.status === "rejected" ? [{ + name: result.reason?.constructor?.name, code: result.reason?.code, + causeName: result.reason?.cause?.constructor?.name, causeCode: result.reason?.cause?.code, + causeConstraint: result.reason?.cause?.constraint_name, + }] : [])).toEqual([]); + expect(results.map(result => result.status === "fulfilled" ? result.value.id : null)).toEqual([active[0]!.id, active[0]!.id]); + // The lost insert uses the existing upsert merge, just like a second sequential call. + expect(active[0]).toMatchObject({ + status: "active", attemptCount: 2, maxAttempts: null, timeoutAt: null, + ownerType: "agent", ownerAgentId: managerId, ownerUserId: null, + evidence: input.evidence, lastAttemptAt: input.lastAttemptAt, + nextAction: input.nextAction, wakePolicy: null, monitorPolicy: null, + }); + }); + it("upserts one active source-scoped action per issue and keeps company scoping explicit", async () => { const { companyId, managerId, sourceIssueId } = await seedCompany(); const svc = issueRecoveryActionService(db); diff --git a/server/src/__tests__/maintenance-harness-resources.test.ts b/server/src/__tests__/maintenance-harness-resources.test.ts index 179bb9e20eb..6ab58089901 100644 --- a/server/src/__tests__/maintenance-harness-resources.test.ts +++ b/server/src/__tests__/maintenance-harness-resources.test.ts @@ -125,3 +125,11 @@ describe("owned maintenance providers", () => { } }); }); + +describe("configured acceptance boundaries", () => { + it("rejects remote collectors and fake-clock configuration before opening PostgreSQL", async () => { + await expect(startMaintenanceServer("not-a-database-url", { configuration: { telemetryOrigin: "https://telemetry.example.test" } })).rejects.toThrow("loopback"); + await expect(startMaintenanceServer("not-a-database-url", { clock: true, configuration: { announcements: true } })).rejects.toThrow("real clock"); + await expect(startMaintenanceServer("not-a-database-url", { configuration: { automaticBackups: { intervalMinutes: 0, idleCheckpoint: true } } })).rejects.toThrow("positive integer"); + }); +}); diff --git a/server/src/__tests__/slack-conversation-lifecycle.test.ts b/server/src/__tests__/slack-conversation-lifecycle.test.ts index 8644c13282b..6e2f5bf7031 100644 --- a/server/src/__tests__/slack-conversation-lifecycle.test.ts +++ b/server/src/__tests__/slack-conversation-lifecycle.test.ts @@ -206,7 +206,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, f.runId)); await db.transaction(async tx => { await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), { - companyId: f.companyId, agentId: f.agentId, activeExecutionRunId: f.runId, + companyId: f.companyId, agentId: f.agentId, activeExecutionRunId: f.runId, activeExecutionRunStatus: "running", mergedContextSnapshot: { issueId: f.issueId }, source: "automation", triggerDetail: "system", payload: { issueId: f.issueId }, requestedByActorType: "system", requestedByActorId: null, idempotencyKey: key, diff --git a/server/src/__tests__/sleep-backup-observation.test.ts b/server/src/__tests__/sleep-backup-observation.test.ts new file mode 100644 index 00000000000..34bac7bbce2 --- /dev/null +++ b/server/src/__tests__/sleep-backup-observation.test.ts @@ -0,0 +1,61 @@ +import { expect, it, vi } from 'vitest'; +import { mkdtemp, mkdir, open, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { observeIdleBackupCheckpoint } from './helpers/maintenance/backup-observation.js'; + +it('observes complete owned gzip bytes without retaining SQL and refuses missing, truncated, foreign, historical or failed checkpoints', async () => { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-backup-observer-')); + const id = '12345678-1234-1234-1234-123456789012'; + const startedAt = new Date(Date.now() - 1000).toISOString(); + const backupDir = path.join(directory, 'backups'); await mkdir(backupDir); + try { + const observe = () => observeIdleBackupCheckpoint(directory, [], startedAt, id); + await expect(observe()).rejects.toThrow(); + await writeFile(path.join(backupDir, '.idle-backup-wake-required'), 'normal owner marker'); + await expect(observe()).rejects.toThrow('no new archive'); + const archive = path.join(backupDir, 'owned.sql.gz'), bytes = gzipSync(`COPY fixture;\n${id}\n`); + await writeFile(archive, bytes); + const receipt = await observe(); + expect(receipt.archives[0]).toMatchObject({ bytes: bytes.length, gzipVerified: true, fixtureIdentifierFound: true }); + expect(receipt.archives[0].sha256).toMatch(/^[a-f0-9]{64}$/); + expect(JSON.stringify(receipt)).not.toContain(id); + await expect(observeIdleBackupCheckpoint(directory, ['owned.sql.gz'], startedAt, id)).rejects.toThrow('no new archive'); + await expect(observeIdleBackupCheckpoint(directory, [], new Date(Date.now() + 1000).toISOString(), id)).rejects.toThrow('predates'); + await writeFile(archive, bytes.subarray(0, bytes.length - 4)); await expect(observe()).rejects.toThrow(); + await writeFile(archive, gzipSync('foreign fixture')); await expect(observe()).rejects.toThrow('owned fixture'); + await writeFile(archive, bytes); await writeFile(path.join(backupDir, 'db-backup-to-s3.failure'), 'failed'); + await expect(observe()).rejects.toThrow('failure marker'); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it('uses opened-file mtime when birthtime is zero and still rejects old or malformed provenance', async () => { + const directory = await mkdtemp(path.join(tmpdir(), 'sleep-backup-no-birthtime-')); + const id = '12345678-1234-1234-1234-123456789012', startedAt = new Date(Date.now() - 1000).toISOString(); + const backupDir = path.join(directory, 'backups'); await mkdir(backupDir); + const archive = path.join(backupDir, 'owned.sql.gz'); + await writeFile(archive, gzipSync(`COPY fixture;\n${id}\n`)); + await writeFile(path.join(backupDir, '.idle-backup-wake-required'), 'normal owner marker'); + const handle = await open(archive, 'r'), prototype = Object.getPrototypeOf(handle), originalStat = prototype.stat; + await handle.close(); + let override: { birthtimeMs: number; mtimeMs?: number } = { birthtimeMs: 0 }; + const spy = vi.spyOn(prototype, 'stat').mockImplementation(async function (this: unknown, ...args: unknown[]) { + return Object.assign(await originalStat.apply(this, args), override); + }); + try { + const observed = await observeIdleBackupCheckpoint(directory, [], startedAt, id); + expect(observed.archives[0]).toMatchObject({ createdAtSource: 'mtime', gzipVerified: true, fixtureIdentifierFound: true }); + expect(Date.parse(observed.archives[0]!.createdAt)).toBeGreaterThanOrEqual(Date.parse(startedAt)); + for (const invalid of [ + { birthtimeMs: 0, mtimeMs: Date.parse(startedAt) - 1 }, { birthtimeMs: 0, mtimeMs: 0 }, + { birthtimeMs: 0, mtimeMs: NaN }, { birthtimeMs: NaN }, { birthtimeMs: -1 }, + { birthtimeMs: 0, mtimeMs: Date.now() + 60_000 }, + { birthtimeMs: Date.parse(startedAt) - 1, mtimeMs: Date.now() }, + ]) { + override = invalid; + await expect(observeIdleBackupCheckpoint(directory, [], startedAt, id)).rejects.toThrow('predates'); + } + await expect(observeIdleBackupCheckpoint(directory, [], 'invalid', id)).rejects.toThrow('checkpoint timestamp'); + } finally { spy.mockRestore(); await rm(directory, { recursive: true, force: true }); } +}); diff --git a/server/src/__tests__/sleep-configuration-acceptance.test.ts b/server/src/__tests__/sleep-configuration-acceptance.test.ts new file mode 100644 index 00000000000..a6d6a7a7ede --- /dev/null +++ b/server/src/__tests__/sleep-configuration-acceptance.test.ts @@ -0,0 +1,312 @@ +import { createHash, generateKeyPairSync, randomUUID, sign } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; +import { createServer } from 'node:http'; +import { writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { createDb } from '@paperclipai/db'; +import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js'; +import { configuredSleepFeatures, finalizeLocalSleepReceipt, observeConfiguredSignals, observeIncomingHttpWindow, observedNoIncomingHttp, sleepConfigurationDigest } from '../../../scripts/sleep-qualification/evidence.mjs'; +import { idleBackupArchiveNames, observeIdleBackupCheckpoint } from './helpers/maintenance/backup-observation.js'; +import { stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js'; +import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js'; +import { managedSleepConfiguration, managedSleepServerOptions, provisionManagedSleepCompany } from './helpers/maintenance/managed-profile.js'; +import { prepareMaintenanceRuntime } from './helpers/maintenance/runtime-build.js'; +import { produceCompletedToolReview } from './helpers/maintenance/configuration-producers.js'; +import { observeMaintenanceBrowser } from './helpers/maintenance/browser-observation.js'; +import { startMaintenanceServer, withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from '../services/cloud-runtime-identity.js'; + +const databaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL'); +if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !databaseUrl) throw new Error('Configured acceptance requires an explicit dedicated PAPERCLIP_SLEEP_ACCEPTANCE_TEST_DATABASE_URL; embedded allocation is forbidden'); +const cloudSha = process.env.PAPERCLIP_SLEEP_CLOUD_SHA; +if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !/^[a-f0-9]{40}$/.test(cloudSha ?? '')) throw new Error('Configured acceptance requires the exact PAPERCLIP_SLEEP_CLOUD_SHA'); +const cloudCheckout = process.env.PAPERCLIP_SLEEP_CLOUD_CHECKOUT; +if (process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && !cloudCheckout) throw new Error('Configured acceptance requires a reviewed PAPERCLIP_SLEEP_CLOUD_CHECKOUT with its built qualification broker adapter'); +const enabled = process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE === '1' && Boolean(databaseUrl); +const quietMs = 90_000, apiCooldownMs = 600_000; +if (enabled && process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS !== '1') throw new Error('Configured qualification requires retained raw evidence'); + +describe.skipIf(!enabled)('configured assembled sleep qualification (explicit owned external PostgreSQL only)', () => { + beforeAll(async () => { await prepareMaintenanceRuntime(); }, 210_000); + it('retains normal credentials, completed work and a future routine through a real cooldown and quiet window', async () => { + const phase = (name: string) => console.info(JSON.stringify({ qualificationPhase: name, at: new Date().toISOString() })); + phase('starting-owned-fixtures'); + const collectorRequests: Array<{ path: string; body: unknown }> = []; + const collector = createServer(async (req, res) => { + const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); + collectorRequests.push({ path: req.url!, body: JSON.parse(Buffer.concat(chunks).toString('utf8')) }); + res.setHeader('Content-Type', 'application/json'); res.end('{}'); + }); + if (execFileSync('git', ['-C', cloudCheckout!, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() !== cloudSha) throw new Error('Qualification Cloud checkout does not match the exact requested SHA'); + const { startQualificationBroker } = await import(pathToFileURL(path.join(cloudCheckout!, 'scripts/sleep-qualification-broker.mjs')).href); + const broker = await startQualificationBroker(); + const s3 = createMaintenanceS3Boundary(); + try { + await withMaintenanceProvider(s3.server, async s3Origin => { + await withMaintenanceProvider(collector, async origin => { + const pair = generateKeyPairSync('ed25519'); + const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'sleep-qualification', alg: 'EdDSA', use: 'sig' }; + const configuration = managedSleepConfiguration(origin, process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1'); + // Independent intended logical contract; collector ports are observed separately. + const expected = { target: 'owned-local-assembled', appSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), cloudSha: cloudSha!, + configDigest: createHash('sha256').update(JSON.stringify({ automaticBackups: { intervalMinutes: 60, idleCheckpoint: true }, announcements: true, + telemetry: true, otel: { enabled: true, protocol: 'http/json' }, serveUi: process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1', + managedInstance: true, deploymentMode: 'authenticated', storageProvider: 's3' })).digest('hex'), + collectorOrigins: { telemetry: `${origin}/telemetry`, otel: origin } }; + expect(sleepConfigurationDigest(configuration)).toBe(expected.configDigest); + const startedAt = new Date().toISOString(); + const server = await startMaintenanceServer(databaseUrl!, managedSleepServerOptions(configuration, JSON.stringify({ keys: [jwk] }), s3Origin, broker)); + phase('actual-server-ready'); + let fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const control = (action: string, idleCapabilities?: unknown) => { + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url'); + const now = Math.floor(Date.now() / 1000); + const header = encode({ alg: 'EdDSA', typ: CLOUD_CONTROL_JWS_TYPE, kid: jwk.kid }); + const payload = encode({ v: 1, iss: CLOUD_RUNTIME_IDENTITY_ISSUER, aud: CLOUD_CONTROL_AUDIENCE, sub: 'maintenance-stack', action, requestId: randomUUID(), iat: now, exp: now + 60, ...(idleCapabilities ? { idleCapabilities } : {}) }); + return { [CLOUD_CONTROL_HEADER]: `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, 'ascii'), pair.privateKey).toString('base64url')}` }; + }; + let failure: { error: unknown } | undefined; + try { + await writeFile(path.join(server.directory, 'sleep-managed-startup.json'), JSON.stringify(await server.snapshot(), null, 2)); + const company = await provisionManagedSleepCompany(server, fixture); + expect(await server.api('GET', '/announcements/current')).toMatchObject({ id: 'new-projects' }); + const announcement = (await server.snapshot()).announcement; + expect(announcement?.intercepted).toBe(true); + expect(announcement?.requests).toEqual(['https://pages.paperclip.ing/announcements/v1/current.json']); + expect(announcement?.unmatched).toEqual([]); + const agent = await server.api<{ id: string }>('POST', `/companies/${company.id}/agents`, { name: 'Owned process worker', role: 'general', adapterType: 'process', + adapterConfig: { command: process.execPath, args: ['-e', "console.log('sleep-acceptance-useful-work')"], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + const chatHistory = await server.produceChatHistory(); + const toolHistory = await produceCompletedToolReview(server, fixture, company.id); + const github = await server.prepareGitHub(company.id); + await server.api('PATCH', `/tool-connections/${github.connectionId}`, { enabled: true }); + const githubContinuity = await server.settleGitHub(github); + await expect.poll(async () => (await broker.binding())?.bindings.length, { timeout: 30_000 }).toBe(1); + await broker.enqueue(); + await broker.deliver(`http://127.0.0.1:${server.port}`, () => control('connector-events:wake')[CLOUD_CONTROL_HEADER]); + // The normally enabled connection owner leases and acknowledges the real sealed event. + await expect.poll(async () => { + const [delivery] = await fixture.$client`select status from connection_event_deliveries where provider_delivery_id='qualification-delivery'`; + return delivery?.status; + }, { timeout: 30_000 }).toBe('processed'); + expect(await broker.pending()).toEqual([]); + const brokerBinding = await broker.binding(); + expect(brokerBinding?.bindings).toHaveLength(1); + const key = await server.api<{ id: string; token: string }>('POST', `/agents/${agent.id}/keys`, { name: 'Retained qualification credential' }); + const issue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'Owned completed history', status: 'todo', assigneeAgentId: agent.id }); + await expect.poll(async () => { + const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; + return run?.status; + }, { timeout: 30_000 }).toBe('succeeded'); + await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' }); + const routine = await server.api<{ id: string }>('POST', `/companies/${company.id}/routines`, { title: 'Harmless future obligation', assigneeAgentId: agent.id }); + await server.api('POST', `/routines/${routine.id}/triggers`, { kind: 'schedule', cronExpression: (() => { const next = new Date(Date.now() + 24 * 60 * 60_000); return `0 ${next.getUTCHours()} ${next.getUTCDate()} ${next.getUTCMonth() + 1} *`; })(), timezone: 'UTC' }); + await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); + const lastAuthenticatedApiAt = performance.now(); + await writeFile(path.join(server.directory, 'sleep-setup-receipt.json'), JSON.stringify({ ...expected, configuration, company, + agentId: agent.id, keyId: key.id, announcement, chatHistory, toolHistory, githubContinuity, brokerBinding, + brokerBoundary: broker.boundary, s3: s3.snapshot(), snapshot: await server.snapshot(), + boundary: 'Actual managed startup/tenant provisioning/producers; Cloud front door, GitHub HTTP and S3 service are owned simulated transports' }, null, 2)); + expect(s3.snapshot().errors).toEqual([]); + let browser; + try { + browser = process.env.PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE === '1' ? await observeMaintenanceBrowser(`http://127.0.0.1:${server.port}`, 'Organize your work.', server.tenantHeaders) : undefined; + await writeFile(path.join(server.directory, 'sleep-browser-receipt.json'), JSON.stringify({ browser, observedAt: new Date().toISOString() }, null, 2)); + } catch (error) { + await writeFile(path.join(server.directory, 'sleep-browser-error.json'), JSON.stringify({ observedAt: new Date().toISOString(), error: error instanceof Error ? error.message : String(error) }, null, 2)); + throw error; + } + // Setup-only owner settling never substitutes for the later full quiet window. + await server.settled(500, 0, { managedSetupRecovery: true }); + const earlyDrain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + try { + const earlyCapabilities = { incomingWake: await broker.capability(earlyDrain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const earlyRead = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${earlyDrain.ownerId}`, undefined, control('task-drain:read', earlyCapabilities)); + const early = await stabilizeEarlyDiagnostics(server, { ownerId: earlyDrain.ownerId, apiSucceededAt: lastAuthenticatedApiAt, read: earlyRead, + persist: observations => writeFile(path.join(server.directory, 'sleep-early-diagnostic-attempts.json'), JSON.stringify(observations, null, 2)) }); + await writeFile(path.join(server.directory, 'sleep-early-diagnostics.json'), JSON.stringify(early, null, 2)); + } finally { await server.api('DELETE', `/instance/task-drain?ownerId=${earlyDrain.ownerId}`, undefined, control('task-drain:stop')); } + // Anchor the full cooldown to a genuine successful opaque-key request. + await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); + const lastApiActivity = performance.now(); + await fixture.$client.end({ timeout: 1 }); + await server.trace(); + phase('first-real-ten-minute-api-cooldown'); + // The real authenticated-use cooldown is part of ordinary behavior. + // No fake clock, production timing override, tenant health poll or SQL observer. + await delay(apiCooldownMs + 100); + await server.settled(); + const activeSignals = observeConfiguredSignals(collectorRequests); + expect(activeSignals.qualificationSpans).toBeGreaterThan(0); + expect(activeSignals.telemetryRequests).toBeGreaterThan(0); + const before = await server.snapshot(), queries = server.proxy.queries.length, transport = server.proxy.transport; + const wallStart = performance.now(), quietStartedAt = new Date().toISOString(); + phase('first-real-ninety-second-quiet-window'); + await delay(quietMs); + const elapsedMs = performance.now() - wallStart, quietFinishedAt = new Date().toISOString(); + const after = await server.snapshot(), finalTransport = server.proxy.transport; + const quiet = { startedAt: quietStartedAt, finishedAt: quietFinishedAt, elapsedMs, clock: 'real', + sqlQueries: server.proxy.queries.length - queries, + connectionsOpened: finalTransport.connectionsOpened - transport.connectionsOpened, + connectionsRemaining: finalTransport.connectionsOpened - finalTransport.connectionsClosed, + nonDatabaseRequests: after.outgoingHttpRequests - before.outgoingHttpRequests, + nonDatabaseBytes: after.network.nonDatabaseBytesWritten - before.network.nonDatabaseBytesWritten, + observerRequests: observeIncomingHttpWindow(before.incomingHttp, after.incomingHttp) }; + await writeFile(path.join(server.directory, 'sleep-first-frozen-quiet.json'), JSON.stringify({ quiet, announcementBefore: before.announcement, announcementAfter: after.announcement }, null, 2)); + const signalObservation = observeConfiguredSignals(collectorRequests); + await writeFile(path.join(server.directory, 'sleep-first-signal-observation.json'), JSON.stringify(signalObservation, null, 2)); + expect(signalObservation.qualificationSpans).toBeGreaterThan(0); + expect(signalObservation.telemetryRequests).toBeGreaterThan(0); + expect(signalObservation.metricRequests).toBe(0); expect(signalObservation.logRequests).toBe(0); + expect(signalObservation.malformedTraceRequests).toBe(0); expect(signalObservation.unexpectedRequests).toBe(0); + expect(quiet.elapsedMs).toBeGreaterThanOrEqual(quietMs); + expect(quiet.sqlQueries).toBe(0); + expect(quiet.connectionsOpened).toBe(0); + expect(quiet.connectionsRemaining).toBe(0); + expect(quiet.nonDatabaseRequests).toBe(0); + expect(quiet.nonDatabaseBytes).toBe(0); + // The installed listener must have observed the actual setup HTTP traffic. + expect(quiet.observerRequests.before.requests).toBeGreaterThan(0); + expect(observedNoIncomingHttp(quiet.observerRequests, quiet)).toBe(true); + expect(after.announcement).toEqual(before.announcement); + const backupNamesBefore = await idleBackupArchiveNames(server.directory); + const checkpointStartedAt = new Date().toISOString(); + const drain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + const capabilities = { incomingWake: await broker.capability(drain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const read = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities)); + const initial = await read(); + await writeFile(path.join(server.directory, 'sleep-initial-safety.json'), JSON.stringify(initial, null, 2)); + const final = await read(); + phase('first-owned-eligibility-reports-read'); + // Persist raw refusal evidence before any assertion can terminate the case. + await writeFile(path.join(server.directory, 'sleep-initial-final-safety.json'), JSON.stringify({ initial, final }, null, 2)); + if (initial.idleSleepSafety.backgroundWork !== 'none' || final.idleSleepSafety.backgroundWork !== 'none') { + const diagnostic = await server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities)); + await writeFile(path.join(server.directory, 'sleep-refusal-diagnostics.json'), JSON.stringify(diagnostic, null, 2)); + } + const { parseApiWakeAdmission, authorizeApiWakeAdmission } = await import('../services/idle-api-wake-admission.js'); + const assertIncomingProof = (report: any, incomingWake: any, ownerId: string) => { + expect(report.idleSleepSafety.incomingWake).toEqual(incomingWake); + const admission = parseApiWakeAdmission(report.idleSleepSafety.apiWakeAdmission); + expect(admission).toMatchObject({ stackId: 'maintenance-stack', ownerId, + capabilityGeneration: incomingWake.generation, challenge: incomingWake.apiKeyProof.challenge }); + expect(authorizeApiWakeAdmission(admission, { stackId: 'maintenance-stack', token: key.token, now: Date.now() })).toBeDefined(); + }; + assertIncomingProof(initial, capabilities.incomingWake, drain.ownerId); + assertIncomingProof(final, capabilities.incomingWake, drain.ownerId); + expect(final.idleSleepSafety.apiWakeAdmission).toEqual(initial.idleSleepSafety.apiWakeAdmission); + const proof = final.idleSleepSafety; + const backupCheckpoint = await observeIdleBackupCheckpoint(server.directory, backupNamesBefore, checkpointStartedAt, company.id); + const observations = { version: 1, ...expected, startedAt, + configuration, browser, s3: s3.snapshot(), announcement: after.announcement, chatHistory, toolHistory, githubContinuity, brokerBinding, brokerBoundary: broker.boundary, backupCheckpoint, + signalObservation, features: configuredSleepFeatures({ announcement: after.announcement, signalObservation, backupCheckpoint }), + quiet, + eligibility: { observedAt: new Date().toISOString(), backgroundWork: proof.backgroundWork, ownedHold: final.ownerId === drain.ownerId, apiCooldownElapsedMs: performance.now() - lastApiActivity, + unchangedGeneration: /^[a-f0-9]{64}$/.test(proof.deadlineProof?.generation ?? '') && initial.idleSleepSafety.deadlineProof?.generation === proof.deadlineProof?.generation }, initial, final, + live: { databaseSuspension: 'INCONCLUSIVE', hostSuspension: 'INCONCLUSIVE', backupPersistence: 'INCONCLUSIVE', reason: 'Local actual server with simulated Cloud signer; no deployed provider target authorized' }, + coverage: { announcements: 'Actual route, validated feed/cache and UI with one named injected provider transport fetch; DNS/HTTPS guard covered separately, not exercised here.', browser: browser ? 'Actual Chromium foreground/background measured, then closed before server quiet window' : 'Closed; set PAPERCLIP_SLEEP_BROWSER_ACCEPTANCE=1 with built UI and headed Chromium for background/active evidence', broker: 'Retained enabled connection, actual encrypted OAuth vault, actual continuity/binding owners and signed/sealed production broker delivery; host capability from actual ConnectorStore', history: 'Normal task/credential/routine, actual tool review rejection/outbox retirement, actual signed chat ingress completion and endpoint archival' } }; + await writeFile(path.join(server.directory, 'sleep-configuration-observations.json'), JSON.stringify(observations, null, 2)); + const { receipt, report } = finalizeLocalSleepReceipt(observations, expected); + await writeFile(path.join(server.directory, 'sleep-configuration-receipt.json'), JSON.stringify(receipt, null, 2)); + await writeFile(path.join(server.directory, 'sleep-configuration-evaluation.json'), JSON.stringify(report, null, 2)); + expect(proof.version).toBe(3); expect(proof.backgroundWork).toBe('none'); + expect(proof.deadlineProof).toMatchObject({ catalog: 'domain_deadlines_v1', generation: expect.stringMatching(/^[a-f0-9]{64}$/) }); + fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const [retained] = await fixture.$client`select (select count(*)::int from agent_api_keys where id=${key.id}) as keys, (select count(*)::int from issues where id=${issue.id}) as issues, (select count(*)::int from routines where id=${routine.id}) as routines, (select enabled from tool_connections where id=${github.connectionId}) as github_enabled`; + expect(retained).toEqual({ keys: 1, issues: 1, routines: 1, github_enabled: true }); + await server.api('DELETE', `/instance/task-drain?ownerId=${drain.ownerId}`, undefined, control('task-drain:stop')); + const nextIssue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'First ordinary producer after quiet', status: 'todo', assigneeAgentId: agent.id }, { Authorization: `Bearer ${key.token}` }); + const reactivationApiAt = performance.now(); + await expect.poll(async () => { + const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${nextIssue.id} order by created_at desc limit 1`; + return run?.status; + }, { timeout: 30_000 }).toBe('succeeded'); + await server.api('PATCH', `/issues/${nextIssue.id}`, { status: 'done' }); + const [executions] = await fixture.$client`select count(*)::int as count from heartbeat_runs where issue_id=${nextIssue.id}`; + expect(executions.count).toBe(1); + await fixture.$client.end({ timeout: 1 }); + phase('reactivation-real-ten-minute-api-cooldown'); + await delay(apiCooldownMs + 100); + await server.settled(); + const resumed = await server.snapshot(), resumedQueries = server.proxy.queries.length, resumedTransport = server.proxy.transport; + const resumedStart = performance.now(), resumedStartedAt = new Date().toISOString(); + phase('reactivation-real-ninety-second-quiet-window'); + await delay(quietMs); + const resumedElapsedMs = performance.now() - resumedStart, resumedFinishedAt = new Date().toISOString(); + const resumedEnd = await server.snapshot(), resumedFinalTransport = server.proxy.transport; + // Freeze every measurement before signed drain/proof requests do SQL. + const resumedQuiet = { startedAt: resumedStartedAt, finishedAt: resumedFinishedAt, elapsedMs: resumedElapsedMs, clock: 'real', + sqlQueries: server.proxy.queries.length - resumedQueries, + connectionsOpened: resumedFinalTransport.connectionsOpened - resumedTransport.connectionsOpened, + connectionsRemaining: resumedFinalTransport.connectionsOpened - resumedFinalTransport.connectionsClosed, + nonDatabaseRequests: resumedEnd.outgoingHttpRequests - resumed.outgoingHttpRequests, + nonDatabaseBytes: resumedEnd.network.nonDatabaseBytesWritten - resumed.network.nonDatabaseBytesWritten, + observerRequests: observeIncomingHttpWindow(resumed.incomingHttp, resumedEnd.incomingHttp) }; + await writeFile(path.join(server.directory, 'sleep-resumed-frozen-quiet.json'), JSON.stringify({ quiet: resumedQuiet, announcementBefore: resumed.announcement, announcementAfter: resumedEnd.announcement }, null, 2)); + const resumedSignals = observeConfiguredSignals(collectorRequests); + await writeFile(path.join(server.directory, 'sleep-resumed-signal-observation.json'), JSON.stringify(resumedSignals, null, 2)); + expect(resumedSignals.qualificationSpans).toBeGreaterThan(0); + expect(resumedSignals.telemetryRequests).toBeGreaterThan(0); + expect(resumedSignals.metricRequests).toBe(0); expect(resumedSignals.logRequests).toBe(0); + expect(resumedSignals.malformedTraceRequests).toBe(0); expect(resumedSignals.unexpectedRequests).toBe(0); + expect(resumedQuiet.elapsedMs).toBeGreaterThanOrEqual(quietMs); + expect(resumedQuiet.sqlQueries).toBe(0); + expect(resumedQuiet.connectionsOpened).toBe(0); + expect(resumedQuiet.connectionsRemaining).toBe(0); + expect(resumedQuiet.nonDatabaseRequests).toBe(0); + expect(resumedQuiet.nonDatabaseBytes).toBe(0); + expect(resumedQuiet.observerRequests.before.requests).toBeGreaterThan(0); + expect(observedNoIncomingHttp(resumedQuiet.observerRequests, resumedQuiet)).toBe(true); + expect(resumedEnd.announcement).toEqual(resumed.announcement); + expect(resumedEnd.announcement?.unmatched).toEqual([]); + const resumedDrain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + const resumedCapabilities = { incomingWake: await broker.capability(resumedDrain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const readResumed = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&scheduledWake=domain-deadlines-v1&ownerId=${resumedDrain.ownerId}`, undefined, control('task-drain:read', resumedCapabilities)); + const resumedInitial = await readResumed(); + await writeFile(path.join(server.directory, 'sleep-resumed-initial-safety.json'), JSON.stringify(resumedInitial, null, 2)); + const resumedProof = await readResumed(); + await writeFile(path.join(server.directory, 'sleep-resumed-initial-final-safety.json'), JSON.stringify({ initial: resumedInitial, final: resumedProof }, null, 2)); + if (resumedInitial.idleSleepSafety.backgroundWork !== 'none' || resumedProof.idleSleepSafety.backgroundWork !== 'none') { + const diagnostic = await server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${resumedDrain.ownerId}`, undefined, control('task-drain:read', resumedCapabilities)); + await writeFile(path.join(server.directory, 'sleep-resumed-refusal-diagnostics.json'), JSON.stringify(diagnostic, null, 2)); + } + assertIncomingProof(resumedInitial, resumedCapabilities.incomingWake, resumedDrain.ownerId); + assertIncomingProof(resumedProof, resumedCapabilities.incomingWake, resumedDrain.ownerId); + expect(resumedProof.idleSleepSafety.apiWakeAdmission).toEqual(resumedInitial.idleSleepSafety.apiWakeAdmission); + expect(resumedProof.idleSleepSafety.deadlineProof?.generation).toMatch(/^[a-f0-9]{64}$/); + expect(resumedProof.idleSleepSafety.deadlineProof?.generation).toBe(resumedInitial.idleSleepSafety.deadlineProof?.generation); + expect(resumedProof.idleSleepSafety.backgroundWork).toBe('none'); + await writeFile(path.join(server.directory, 'sleep-local-reactivation-receipt.json'), JSON.stringify({ + version: 1, actualProducer: 'POST company issues with process assignee using the retained opaque key after owned drain release', executions: executions.count, + realApiCooldownElapsedMs: performance.now() - reactivationApiAt, + quiet: resumedQuiet, + signalObservation: resumedSignals, + realElapsedMs: resumedQuiet.elapsedMs, sqlQueries: resumedQuiet.sqlQueries, + networkRequests: resumedQuiet.nonDatabaseRequests, + initial: resumedInitial, + eligibility: resumedProof.idleSleepSafety, + boundary: 'Actual local application reactivation, real cooldown and fresh eligibility; no provider suspension or deployed inbound-wake claim', + }, null, 2)); + const s3Evidence = s3.snapshot(); + await writeFile(path.join(server.directory, 'sleep-s3-boundary.json'), JSON.stringify(s3Evidence, null, 2)); + expect(s3Evidence.errors).toEqual([]); + phase('configured-qualification-receipts-complete'); + } catch (error) { failure = { error }; throw error; } + finally { + const errors: unknown[] = []; + try { await writeFile(path.join(server.directory, 'sleep-s3-before-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + try { await server.stop(); } catch (error) { errors.push(error); } + try { await fixture.$client.end(); } catch (error) { errors.push(error); } + // Shutdown can flush storage; capture and validate only after child exit. + try { await writeFile(path.join(server.directory, 'sleep-s3-after-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + if (s3.snapshot().errors.length) errors.push(new Error(`S3 fixture boundary failed: ${JSON.stringify(s3.snapshot().errors)}`)); + if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Configured work or owned cleanup failed'); + } + }); + }); + } finally { await broker.stop(); } + }, 1_800_000); +}); diff --git a/server/src/__tests__/sleep-early-diagnostics.test.ts b/server/src/__tests__/sleep-early-diagnostics.test.ts new file mode 100644 index 00000000000..b9db5979378 --- /dev/null +++ b/server/src/__tests__/sleep-early-diagnostics.test.ts @@ -0,0 +1,15 @@ +import { expect, it } from 'vitest'; +import { assertEarlySleepDiagnostics } from './helpers/maintenance/early-sleep-diagnostics.js'; +const fixture = () => ({ idleSleepSafety: { diagnostics: { complete: true, checks: [ + { reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' }, + { reason: 'backup.checkpoint', state: 'not_checked' }, ...['inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog', 'issues'].map(reason => ({ reason, state: 'none' })), +] } } }); +it('retains only explicit early cooldown/diagnostic-backup reasons and refuses every unexpected or unknown blocker', () => { + expect(() => assertEarlySleepDiagnostics(fixture(), 90_000)).not.toThrow(); + for (const [reason, state] of [['issues', 'present'], ['deadlines.catalog', 'unknown'], ['inspection.hold', 'not_checked'], ['local.registered_probes', 'unknown']]) { + const f = fixture(); f.idleSleepSafety.diagnostics.checks.find(check => check.reason === reason)!.state = state; + expect(() => assertEarlySleepDiagnostics(f, 90_000)).toThrow('Unexpected'); + } + const missing = fixture(); missing.idleSleepSafety.diagnostics.checks.shift(); expect(() => assertEarlySleepDiagnostics(missing, 90_000)).toThrow('Missing'); + for (const duration of [NaN, Infinity, -1, 600_000]) expect(() => assertEarlySleepDiagnostics(fixture(), duration)).toThrow('cooldown'); +}); diff --git a/server/src/__tests__/sleep-idle-work-capture.test.ts b/server/src/__tests__/sleep-idle-work-capture.test.ts new file mode 100644 index 00000000000..114b04057bb --- /dev/null +++ b/server/src/__tests__/sleep-idle-work-capture.test.ts @@ -0,0 +1,90 @@ +import { afterEach, expect, it } from 'vitest'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { createMaintenanceIdleWorkCapture as createCapture, createSettledWorkSamples } from './helpers/maintenance/idle-work-capture.js'; + +const captures: ReturnType[] = []; +function createMaintenanceIdleWorkCapture(...args: Parameters) { + const capture = createCapture(...args); captures.push(capture); return capture; +} +afterEach(() => { for (const capture of captures.splice(0)) capture.dispose(); }); + +const stack = () => 'Error\n at begin (/checkout/server/src/services/owned-work.ts:12:3)\n at owner (/checkout/server/src/index.ts:100:5)'; +function original() { + let active = 0, generation = 0, finishes = 0, tracked: Promise | undefined; + const beginIdleTrackedWork = () => { + active++; generation++; let done = false; + return () => { finishes++; if (!done) { done = true; active--; generation++; } }; + }; + const trackIdleWork = (work: Promise) => { + const finish = beginIdleTrackedWork(); const result = work.finally(finish); tracked = result; return result; + }; + return { beginIdleTrackedWork, trackIdleWork, idleWorkSnapshot: () => ({ active, generation }), + finishes: () => finishes, lastTracked: () => tracked }; +} +it('delegates original finish calls/idempotence and returns the exact original tracked promise and rejection', async () => { + const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack }); + const finish = capture.beginIdleTrackedWork(); expect(actual.idleWorkSnapshot().active).toBe(1); + finish(); finish(); expect(actual.finishes()).toBe(2); expect(actual.idleWorkSnapshot().active).toBe(0); + let resolve!: (value: string) => void; + const tracked = capture.trackIdleWork(new Promise(done => { resolve = done; })); + expect(tracked).toBe(actual.lastTracked()); expect(capture.snapshot().observedActive).toBe(1); + resolve('same'); await expect(tracked).resolves.toBe('same'); expect(capture.snapshot().observedActive).toBe(0); + const failure = new Error('original rejection'), rejected = capture.trackIdleWork(Promise.reject(failure)); + expect(rejected).toBe(actual.lastTracked()); await expect(rejected).rejects.toBe(failure); + expect(capture.snapshot().observedActive).toBe(0); expect(actual.idleWorkSnapshot().active).toBe(0); + capture.dispose(); capture.dispose(); expect(capture.snapshot().disposed).toBe(true); + await expect(capture.trackIdleWork(Promise.resolve('after-dispose'))).resolves.toBe('after-dispose'); + expect(actual.idleWorkSnapshot().active).toBe(0); expect(capture.snapshot().observedActive).toBe(0); +}); +it('cannot throw from passive metadata into actual work and makes missing observation explicit', () => { + const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack, now: () => { throw new Error('private observer detail'); } }); + let finish!: () => void; expect(() => { finish = capture.beginIdleTrackedWork(); }).not.toThrow(); + expect(actual.idleWorkSnapshot().active).toBe(1); expect(capture.snapshot().missingPaths).toBeGreaterThan(0); + expect(() => finish()).not.toThrow(); expect(actual.idleWorkSnapshot().active).toBe(0); + expect(capture.snapshot().observerErrors).toBeGreaterThan(0); expect(JSON.stringify(capture.summary())).not.toContain('private observer detail'); +}); +it('bounds active entries, completion tail and filtered callsites without clearing original work', () => { + const actual = original(); + const callers = () => 'private outside value\n' + Array.from({ length: 8 }, (_, i) => ` at work (/root/server/src/services/owned-${i}.ts:1:2)`).join('\n'); + const capture = createMaintenanceIdleWorkCapture(actual, { stack: callers }); + const finish = Array.from({ length: 513 }, () => capture.beginIdleTrackedWork()); + const busy = capture.snapshot(); expect(busy.active).toHaveLength(512); expect(busy.overflow).toBe(1); + expect(busy.newSources[0]!.callers).toHaveLength(4); expect(JSON.stringify(busy)).not.toContain('private outside value'); + expect(actual.idleWorkSnapshot().active).toBe(513); expect(busy.missingPaths).toBeGreaterThan(0); + finish.forEach(done => done()); expect(actual.idleWorkSnapshot().active).toBe(0); expect(capture.snapshot().completionTail).toHaveLength(64); +}); +it('bounds unique source metadata and preserves original finisher failures', () => { + const actual = original(); let source = 0; + const capture = createMaintenanceIdleWorkCapture(actual, { stack: () => `Error\n at owner (/checkout/server/src/services/source-${source++}.ts:1:1)` }); + for (let i = 0; i < 513; i++) capture.beginIdleTrackedWork()(); + const state = capture.snapshot(); expect(capture.summary().sourceCatalog).toHaveLength(512); expect(state.sourceOverflow).toBe(1); + expect(actual.idleWorkSnapshot().active).toBe(0); + const failure = new Error('original finisher'), begin = actual.beginIdleTrackedWork; + actual.beginIdleTrackedWork = () => { const finish = begin(); return () => { finish(); throw failure; }; }; + const observed = createMaintenanceIdleWorkCapture(actual, { stack }); const finish = observed.beginIdleTrackedWork(); + expect(() => finish()).toThrow(failure); expect(actual.idleWorkSnapshot().active).toBe(0); expect(observed.snapshot().observedActive).toBe(0); +}); +it('bounds settle samples and stores callsites once, while missing paths forbid diagnostic completion', () => { + const actual = original(), capture = createMaintenanceIdleWorkCapture(actual, { stack }); const finish = capture.beginIdleTrackedWork(); + const state = capture.snapshot(), catalog = state.newSources; + const samples = createSettledWorkSamples(() => catalog); + for (let i = 0; i < 2442; i++) samples.sample({ now: 1000 + i, idle: actual.idleWorkSnapshot(), workCapture: state }, 20 + i, 0); + const report = samples.report(); expect(report.samples).toHaveLength(2441); expect(report.sampleLimit).toBe(2441); expect(report.overflowSamples).toBe(1); + expect(report.samples[1]!.queryDelta).toBe(1); expect(report.sourceCatalog).toHaveLength(1); + expect(JSON.stringify(report.samples)).not.toContain('owned-work'); expect(() => samples.assertComplete()).toThrow('incomplete'); + finish(); + const missing = createSettledWorkSamples(() => []); missing.sample({ now: 0, idle: actual.idleWorkSnapshot() }, 0, 0); + expect(() => missing.assertComplete()).toThrow('incomplete'); +}); +it('intercepts the actual Node24 TS module and preserves every other original export and existing state', async () => { + const result = await promisify(execFile)(process.execPath, ['--experimental-test-module-mocks', '--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/idle-work-capture-probe.ts', import.meta.url))], { timeout: 15_000, maxBuffer: 128 * 1024 }); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ identity: true, originalStatePreserved: true, finalActive: 0 }); +}); +it('preserves actual Node unhandledRejection with the same original tracked promise and reason', async () => { + const result = await promisify(execFile)(process.execPath, ['--experimental-test-module-mocks', '--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/idle-work-capture-probe.ts', import.meta.url)), 'unhandled'], { timeout: 15_000, maxBuffer: 128 * 1024 }); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ unhandledOriginalPromise: true, unhandledOriginalReason: true, finalActive: 0 }); +}); diff --git a/server/src/__tests__/sleep-incoming-http-observation.test.ts b/server/src/__tests__/sleep-incoming-http-observation.test.ts new file mode 100644 index 00000000000..e42d215eb54 --- /dev/null +++ b/server/src/__tests__/sleep-incoming-http-observation.test.ts @@ -0,0 +1,100 @@ +import { createServer, request } from "node:http"; +import { setImmediate } from "node:timers/promises"; +import { expect, it } from "vitest"; +import { observeIncomingHttpWindow, observedNoIncomingHttp } from "../../../scripts/sleep-qualification/evidence.mjs"; +import { observeMaintenanceIncomingHttp } from "./helpers/maintenance/incoming-http-observation.js"; +import { withMaintenanceProvider } from "./helpers/maintenance/server-harness.js"; + +it("counts a direct SQL-free health fetch without retaining credentials or changing its response", async () => { + // This fixture has no database or outgoing client; its only work is answering HTTP. + const server = createServer((_req, res) => { res.setHeader("Content-Type", "application/json"); res.end('{"status":"ok"}'); }); + server.on("upgrade", (_req, socket) => socket.destroy()); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async origin => { + const before = capture.snapshot(), startedAt = new Date().toISOString(); + expect(before).toMatchObject({ installed: true, valid: true, requests: 0 }); + expect(observedNoIncomingHttp(observeIncomingHttpWindow(before, before), { startedAt: before.observedAt, finishedAt: before.observedAt })).toBe(true); + const response = await fetch(`${origin}/api/health?fixture=private-query`, { signal: AbortSignal.timeout(5_000), headers: { Authorization: "Bearer fixture-private-token" } }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ status: "ok" }); + const finishedAt = new Date().toISOString(), after = capture.snapshot(); + expect(after).toMatchObject({ installed: true, valid: true, captureId: before.captureId }); + const observed = observeIncomingHttpWindow(before, after); + expect(observed.requests).toBe(1); + expect(observedNoIncomingHttp(observed, { startedAt, finishedAt })).toBe(false); + expect(JSON.stringify(observed)).not.toMatch(/private-query|fixture-private-token|Authorization|\/api\/health/); + }); + } finally { capture.dispose(); } +}); + +it("observes a genuinely request-free interval without making snapshot reads into HTTP", async () => { + const server = createServer((_req, res) => res.end()); + server.on("upgrade", (_req, socket) => socket.destroy()); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async () => { + const before = capture.snapshot(), startedAt = new Date().toISOString(); + capture.snapshot(); await setImmediate(); capture.snapshot(); + const finishedAt = new Date().toISOString(), after = capture.snapshot(); + expect(observeIncomingHttpWindow(before, after).requests).toBe(0); + expect(observedNoIncomingHttp(observeIncomingHttpWindow(before, after), { startedAt, finishedAt })).toBe(true); + // This short boundary test does not claim the configured case's 90-second quiet proof. + }); + } finally { capture.dispose(); } +}); + +it("counts an actual HTTP upgrade as incoming activity", async () => { + const server = createServer((_req, res) => res.end()); + const upgrade = (_req: unknown, socket: import("node:stream").Duplex) => { + socket.end("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n"); + }; + server.on("upgrade", upgrade); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async origin => { + const before = capture.snapshot(); + const client = request(origin, { signal: AbortSignal.timeout(5_000), headers: { Connection: "Upgrade", Upgrade: "fixture" } }); + try { + await new Promise((resolve, reject) => { + client.once("error", reject); + client.once("upgrade", (_response, socket) => { socket.destroy(); resolve(); }); + client.end(); + }); + } finally { client.destroy(); } + expect(observeIncomingHttpWindow(before, capture.snapshot()).requests).toBe(1); + }); + } finally { capture.dispose(); server.off("upgrade", upgrade); } +}); + +it("invalidates a detached listener and disposes only its own listeners", async () => { + const app = (_req: unknown, res: import("node:http").ServerResponse) => res.end("app"); + const server = createServer(app); + const unrelatedUpgrade = () => {}; + server.on("upgrade", unrelatedUpgrade); + const originalRequest = server.listeners("request"), originalUpgrade = server.listeners("upgrade"); + const capture = observeMaintenanceIncomingHttp(server); + try { + await withMaintenanceProvider(server, async origin => { + const counter = server.listeners("request")[0]! as (...args: unknown[]) => void; + server.off("request", counter); + expect(capture.snapshot()).toMatchObject({ installed: true, valid: false }); + // Reattaching cannot erase an unobserved interval. + server.prependListener("request", counter); + expect(capture.snapshot().valid).toBe(false); + capture.dispose(); capture.dispose(); + expect(capture.snapshot()).toMatchObject({ installed: false, valid: false }); + expect(server.listeners("request")).toEqual(originalRequest); + expect(server.listeners("upgrade")).toEqual(originalUpgrade); + expect(await (await fetch(origin, { signal: AbortSignal.timeout(5_000) })).text()).toBe("app"); + }); + } finally { capture.dispose(); } +}); + +it("leaves Node's unsupported-upgrade behavior alone when the app has no upgrade handler", () => { + const server = createServer((_req, res) => res.end()); + const requestHandlers = server.listeners("request"), upgradeHandlers = server.listeners("upgrade"); + expect(() => observeMaintenanceIncomingHttp(server)).toThrow("requires existing app request and upgrade handlers"); + expect(server.listeners("request")).toEqual(requestHandlers); + expect(server.listeners("upgrade")).toEqual(upgradeHandlers); +}); diff --git a/server/src/__tests__/sleep-managed-fixture.test.ts b/server/src/__tests__/sleep-managed-fixture.test.ts new file mode 100644 index 00000000000..0f7a3ada9f5 --- /dev/null +++ b/server/src/__tests__/sleep-managed-fixture.test.ts @@ -0,0 +1,69 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { createMaintenanceS3Boundary, decodeMaintenanceAwsChunks } from './helpers/maintenance/s3-boundary.js'; +import { withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { createS3StorageProvider } from '../storage/s3-provider.js'; +import { classifyEarlyDiagnostic, stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js'; + +afterEach(() => vi.unstubAllEnvs()); +it('uses real small-object SDK HTTP with conditional bytes, reads and deletion; unknown operations fail closed', async () => { + vi.stubEnv('AWS_ACCESS_KEY_ID', 'owned-fixture'); vi.stubEnv('AWS_SECRET_ACCESS_KEY', 'owned-fixture'); vi.stubEnv('AWS_SESSION_TOKEN', undefined); + const fixture = createMaintenanceS3Boundary(); + await withMaintenanceProvider(fixture.server, async endpoint => { + const provider = createS3StorageProvider({ bucket: 'maintenance-test', region: 'us-east-1', endpoint, forcePathStyle: true }); + const input = { objectKey: 'owned/value', contentType: 'text/plain', contentLength: 3, body: Buffer.from('one') }; + const first = await provider.putObjectConditional!({ ...input, ifNoneMatch: '*' }); + expect(first.etag).toBeTruthy(); + await expect(provider.putObjectConditional!({ ...input, ifNoneMatch: '*' })).rejects.toMatchObject({ name: 'PreconditionFailed' }); + await expect(provider.putObjectConditional!({ ...input, ifMatch: 'wrong' })).rejects.toMatchObject({ name: 'PreconditionFailed' }); + await provider.putObjectConditional!({ ...input, body: Buffer.from('two'), ifMatch: first.etag }); + const object = await provider.getObject({ objectKey: input.objectKey }); const chunks: Buffer[] = []; + for await (const chunk of object.stream) chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe('two'); expect((await provider.headObject({ objectKey: input.objectKey })).exists).toBe(true); + expect((await provider.listObjects!({ prefix: 'owned/' })).keys).toEqual([input.objectKey]); + await provider.deleteObject({ objectKey: input.objectKey }); expect((await provider.headObject({ objectKey: input.objectKey })).exists).toBe(false); + expect(fixture.snapshot().errors).toEqual([]); + const unknown = await fetch(`${endpoint}/maintenance-test/key?uploads`, { method: 'POST' }); expect(unknown.status).toBe(501); + expect(fixture.snapshot().errors).toHaveLength(1); + }); +}); +it('decodes only complete bounded AWS checksum chunks and refuses malformed framing', () => { + expect(decodeMaintenanceAwsChunks(Buffer.from('3\r\none\r\n3\r\ntwo\r\n0\r\nx-amz-checksum-crc32:YWJj\r\n\r\n')).toString()).toBe('onetwo'); + for (const wire of ['3\r\none', '3;signature=unknown\r\none\r\n0\r\n\r\n', '0\r\nunknown:secret\r\n\r\n']) expect(() => decodeMaintenanceAwsChunks(Buffer.from(wire))).toThrow(); +}); +it('retries only observed transient inspection states and rejects durable blockers even alongside them', () => { + const checks = [ + { reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' }, + { reason: 'backup.checkpoint', state: 'not_checked' }, { reason: 'inspection.hold', state: 'none' }, + { reason: 'inspection.generation', state: 'unknown' }, { reason: 'local.tracked_work', state: 'present' }, + { reason: 'plugins.runtime_drain', state: 'not_checked' }, { reason: 'durable.snapshot', state: 'none' }, + { reason: 'deadlines.catalog', state: 'none' }, + ]; + const report = { ownerId: 'owned', draining: true, quiescent: true, expiresAt: '2099-01-01T00:00:00Z', idleSleepSafety: { diagnostics: { complete: false, checks } } }; + expect(classifyEarlyDiagnostic(report, 90_000)).toBe('transient'); + const durable = structuredClone(report); durable.idleSleepSafety.diagnostics.checks.push({ reason: 'chat_endpoints', state: 'present' }); + expect(() => classifyEarlyDiagnostic(durable, 90_000)).toThrow('chat_endpoints'); + const unknown = structuredClone(report); unknown.idleSleepSafety.diagnostics.checks.push({ reason: 'deadlines.reader', state: 'unknown' }); + expect(() => classifyEarlyDiagnostic(unknown, 90_000)).toThrow('deadlines.reader'); + for (const check of report.idleSleepSafety.diagnostics.checks) if (['inspection.generation', 'local.tracked_work', 'plugins.runtime_drain'].includes(check.reason)) check.state = 'none'; + report.idleSleepSafety.diagnostics.complete = true; expect(classifyEarlyDiagnostic(report, 90_000)).toBe('strict'); +}); + +it('retains observer generation changes and still rejects a second durable report after a transient first', async () => { + const checks = [ + { reason: 'local.registered_probes', state: 'present' }, { reason: 'local.inspection', state: 'present' }, + { reason: 'backup.checkpoint', state: 'not_checked' }, ...['inspection.hold', 'inspection.generation', 'durable.snapshot', 'deadlines.catalog'].map(reason => ({ reason, state: 'none' })), + ]; + const strict = { ownerId: 'owned', draining: true, quiescent: true, expiresAt: '2099-01-01T00:00:00Z', idleSleepSafety: { diagnostics: { complete: true, checks } } }; + const observations: unknown[][] = []; + const server = { settled: async () => ({ idle: { active: 0, generation: 10 }, managed: { apiActivityActive: true } }), + snapshot: async () => ({ idle: { active: 0, generation: 14 }, managed: { apiActivityActive: true } }) } as unknown as Parameters[0]; + const accepted = await stabilizeEarlyDiagnostics(server, { ownerId: 'owned', apiSucceededAt: performance.now(), read: async () => structuredClone(strict), persist: async values => { observations.push(structuredClone(values)); } }); + expect(accepted.rounds).toBe(1); expect(observations.at(-1)).toHaveLength(4); + const transient = structuredClone(strict); transient.idleSleepSafety.diagnostics.complete = false; + transient.idleSleepSafety.diagnostics.checks.find(check => check.reason === 'inspection.generation')!.state = 'unknown'; + const blocked = structuredClone(transient); blocked.idleSleepSafety.diagnostics.checks.push({ reason: 'chat_endpoints', state: 'present' }); + let reads = 0; + await expect(stabilizeEarlyDiagnostics(server, { ownerId: 'owned', apiSucceededAt: performance.now(), read: async () => reads++ ? blocked : transient, + persist: async values => { observations.push(structuredClone(values)); } })).rejects.toThrow('chat_endpoints'); + expect(reads).toBe(2); +}); diff --git a/server/src/__tests__/sleep-managed-preflight.test.ts b/server/src/__tests__/sleep-managed-preflight.test.ts new file mode 100644 index 00000000000..eef54f4fa15 --- /dev/null +++ b/server/src/__tests__/sleep-managed-preflight.test.ts @@ -0,0 +1,99 @@ +import { execFileSync } from 'node:child_process'; +import { generateKeyPairSync, randomUUID, sign } from 'node:crypto'; +import { createServer } from 'node:http'; +import { writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { createDb } from '@paperclipai/db'; +import { resolveExternalTestDatabaseUrl } from './helpers/external-test-database.js'; +import { prepareMaintenanceRuntime } from './helpers/maintenance/runtime-build.js'; +import { startMaintenanceServer, withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; +import { managedSleepConfiguration, managedSleepServerOptions, managedTenantActor, provisionManagedSleepCompany } from './helpers/maintenance/managed-profile.js'; +import { createMaintenanceS3Boundary } from './helpers/maintenance/s3-boundary.js'; +import { stabilizeEarlyDiagnostics } from './helpers/maintenance/early-diagnostic-stabilization.js'; +import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from '../services/cloud-runtime-identity.js'; + +const requested = process.env.PAPERCLIP_SLEEP_MANAGED_PREFLIGHT === '1'; +const databaseUrl = resolveExternalTestDatabaseUrl('PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_TEST_DATABASE_URL'); +if (requested && !databaseUrl) throw new Error('Managed preflight requires an explicit dedicated external *_test URL; no embedded allocation'); +if (requested && process.env.PAPERCLIP_MAINTENANCE_KEEP_ARTIFACTS !== '1') throw new Error('Managed preflight requires retained raw evidence'); +const cloudCheckout = process.env.PAPERCLIP_SLEEP_CLOUD_CHECKOUT, cloudSha = process.env.PAPERCLIP_SLEEP_CLOUD_SHA; +if (requested && (!cloudCheckout || !/^[a-f0-9]{40}$/.test(cloudSha ?? ''))) throw new Error('Managed preflight requires the exact reviewed private Cloud checkout/SHA'); + +describe.skipIf(!requested)('actual managed fixture preflight (no browser, cooldown or quiet claim)', () => { + beforeAll(async () => { await prepareMaintenanceRuntime(); }, 210_000); + it('provisions through real managed auth and completes a process run before strict activity diagnostics', async () => { + if (execFileSync('git', ['-C', cloudCheckout!, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() !== cloudSha) throw new Error('Cloud source does not match the reviewed preflight SHA'); + const { startQualificationBroker } = await import(pathToFileURL(path.join(cloudCheckout!, 'scripts/sleep-qualification-broker.mjs')).href); + const broker = await startQualificationBroker(); const s3 = createMaintenanceS3Boundary(); + const collectorRequests: string[] = []; + const collector = createServer(async (req, res) => { for await (const _chunk of req) { /* Fully consume actual exports. */ } collectorRequests.push(req.url!); res.setHeader('content-type', 'application/json'); res.end('{}'); }); + try { + await withMaintenanceProvider(s3.server, async s3Origin => withMaintenanceProvider(collector, async origin => { + const pair = generateKeyPairSync('ed25519'); + const jwk = { ...pair.publicKey.export({ format: 'jwk' }), kid: 'managed-preflight', alg: 'EdDSA', use: 'sig' }; + const configuration = managedSleepConfiguration(origin, true); + const server = await startMaintenanceServer(databaseUrl!, { ...managedSleepServerOptions(configuration, JSON.stringify({ keys: [jwk] }), s3Origin, broker), + captureIdleWork: process.env.PAPERCLIP_SLEEP_MANAGED_PREFLIGHT_DIAGNOSTICS === '1' }); + const fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const control = (action: string, idleCapabilities?: unknown) => { + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url'), now = Math.floor(Date.now() / 1000); + const header = encode({ alg: 'EdDSA', typ: CLOUD_CONTROL_JWS_TYPE, kid: jwk.kid }); + const payload = encode({ v: 1, iss: CLOUD_RUNTIME_IDENTITY_ISSUER, aud: CLOUD_CONTROL_AUDIENCE, sub: 'maintenance-stack', action, requestId: randomUUID(), iat: now, exp: now + 60, ...(idleCapabilities ? { idleCapabilities } : {}) }); + return { [CLOUD_CONTROL_HEADER]: `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, 'ascii'), pair.privateKey).toString('base64url')}` }; + }; + let failure: { error: unknown } | undefined; + try { + await writeFile(path.join(server.directory, 'managed-preflight-startup.json'), JSON.stringify({ configuration, snapshot: await server.snapshot(), s3: s3.snapshot() }, null, 2)); + expect(s3.snapshot().errors).toEqual([]); + const company = await provisionManagedSleepCompany(server, fixture); + await expect(server.api('POST', '/companies', { name: 'Must remain platform managed' })).rejects.toThrow('cloud_managed'); + // Reuse the combined case's real vault/continuity/enrollment owners. + // The strict broker capability is unavailable before event-wake enrollment. + const github = await server.prepareGitHub(company.id); + await server.api('PATCH', `/tool-connections/${github.connectionId}`, { enabled: true }); + const githubContinuity = await server.settleGitHub(github); + await writeFile(path.join(server.directory, 'managed-preflight-binding.json'), JSON.stringify({ githubContinuity, brokerBinding: await broker.binding(), boundary: broker.boundary }, null, 2)); + await expect.poll(async () => (await broker.binding())?.bindings.length, { timeout: 30_000 }).toBe(1); + const brokerBinding = await broker.binding(); + await writeFile(path.join(server.directory, 'managed-preflight-binding.json'), JSON.stringify({ githubContinuity, brokerBinding, boundary: broker.boundary }, null, 2)); + expect(brokerBinding?.instanceId).toBe(broker.variables.PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID); + expect(brokerBinding?.bindings).toEqual([[`${github.grantId}_101`, company.id, managedTenantActor.userId, + 'github.code', '101', github.connectionId, github.grantId]]); + const agent = await server.api<{ id: string }>('POST', `/companies/${company.id}/agents`, { name: 'Managed preflight process', role: 'general', adapterType: 'process', + adapterConfig: { command: process.execPath, args: ['-e', "console.log('managed-preflight-useful-work')"], cwd: server.directory }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + const key = await server.api<{ id: string; token: string }>('POST', `/agents/${agent.id}/keys`, { name: 'Actual managed preflight key' }); + const issue = await server.api<{ id: string }>('POST', `/companies/${company.id}/issues`, { title: 'One actual managed process run', status: 'todo', assigneeAgentId: agent.id }, { Authorization: `Bearer ${key.token}` }); + await expect.poll(async () => { const [run] = await fixture.$client`select status from heartbeat_runs where issue_id=${issue.id} order by created_at desc limit 1`; return run?.status; }, { timeout: 30_000 }).toBe('succeeded'); + await server.api('PATCH', `/issues/${issue.id}`, { status: 'done' }); + const [executions] = await fixture.$client`select count(*)::int as count from heartbeat_runs where issue_id=${issue.id}`; expect(executions.count).toBe(1); + await server.api('GET', '/agents/me', undefined, { Authorization: `Bearer ${key.token}` }); const apiSucceededAt = performance.now(); + const activity = await server.snapshot(); + await writeFile(path.join(server.directory, 'managed-preflight-setup.json'), JSON.stringify({ company, githubContinuity, brokerBinding, + agentId: agent.id, keyId: key.id, issueId: issue.id, executions: executions.count, activity, s3: s3.snapshot() }, null, 2)); + expect(activity.managed?.apiActivityActive).toBe(true); + await server.settled(500, 0, { managedSetupRecovery: true }); + const drain = await server.api<{ ownerId: string }>('POST', '/instance/task-drain', { purpose: 'idle', ttlMs: 240_000 }, control('task-drain:start')); + try { + const capabilities = { incomingWake: await broker.capability(drain.ownerId), deadlineCatalog: 'domain_deadlines_v1' }; + const read = () => server.api('GET', `/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=domain-deadlines-v1&ownerId=${drain.ownerId}`, undefined, control('task-drain:read', capabilities)); + const proof = await stabilizeEarlyDiagnostics(server, { ownerId: drain.ownerId, apiSucceededAt, read, + persist: observations => writeFile(path.join(server.directory, 'managed-preflight-diagnostic-attempts.json'), JSON.stringify(observations, null, 2)) }); + await writeFile(path.join(server.directory, 'managed-preflight-receipt.json'), JSON.stringify({ companyId: company.id, proof, s3: s3.snapshot(), collectorRequests, + boundary: 'Actual managed startup/provisioning/key activity/process outcome; no browser, real cooldown/quiet, eligibility, provider suspension or durable S3 claim' }, null, 2)); + } finally { await server.api('DELETE', `/instance/task-drain?ownerId=${drain.ownerId}`, undefined, control('task-drain:stop')); } + } catch (error) { failure = { error }; throw error; } + finally { + const errors: unknown[] = []; + try { await writeFile(path.join(server.directory, 'managed-preflight-s3-before-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + try { await server.stop(); } catch (error) { errors.push(error); } + try { await fixture.$client.end({ timeout: 1 }); } catch (error) { errors.push(error); } + try { await writeFile(path.join(server.directory, 'managed-preflight-s3-after-cleanup.json'), JSON.stringify(s3.snapshot(), null, 2)); } catch (error) { errors.push(error); } + if (s3.snapshot().errors.length) errors.push(new Error(`S3 fixture boundary failed: ${JSON.stringify(s3.snapshot().errors)}`)); + if (errors.length) throw new AggregateError([...(failure ? [failure.error] : []), ...errors], 'Managed preflight or owned cleanup failed'); + } + })); + } finally { await broker.stop(); } + }, 180_000); +}); diff --git a/server/src/__tests__/sleep-otel-export.test.ts b/server/src/__tests__/sleep-otel-export.test.ts new file mode 100644 index 00000000000..f8276760850 --- /dev/null +++ b/server/src/__tests__/sleep-otel-export.test.ts @@ -0,0 +1,43 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { createServer } from 'node:http'; +import { it, expect } from 'vitest'; +import { withMaintenanceProvider } from './helpers/maintenance/server-harness.js'; + +// Opt-in qualification must fail when the SDK falls back to no-op. The ordinary +// bootstrap tests separately retain coverage for the supported missing-peer mode. +it.skipIf(process.env.PAPERCLIP_SLEEP_CONFIGURED_ACCEPTANCE !== '1')('exports an actual named span with no metric or log traffic over 90 real seconds', async () => { + const spans: string[] = [], paths: string[] = []; + const collector = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + paths.push(req.url ?? 'missing-path'); + const envelope = JSON.parse(Buffer.concat(chunks).toString('utf8')); + for (const resource of envelope.resourceSpans ?? []) for (const scope of resource.scopeSpans ?? []) for (const span of scope.spans ?? []) spans.push(span.name); + res.setHeader('Content-Type', 'application/json'); res.end('{}'); + }); + await withMaintenanceProvider(collector, async origin => { + const startedAt = performance.now(); + const { stderr } = await promisify(execFile)(process.execPath, ['--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/otel-child.ts', import.meta.url)), '--observe-exporters'], { + env: { PATH: process.env.PATH, NODE_ENV: 'production', OTEL_EXPORTER_OTLP_ENDPOINT: origin, OTEL_EXPORTER_OTLP_PROTOCOL: 'http/json' }, + timeout: 110_000, maxBuffer: 2 * 1024 * 1024, + }); + expect(performance.now() - startedAt).toBeGreaterThanOrEqual(90_000); + expect(stderr).not.toContain('Continuing without tracing'); + expect(spans).toContain('sleep-qualification-stimulus'); + expect(paths).not.toContain('/v1/metrics'); + expect(paths).not.toContain('/v1/logs'); + }); +}, 120_000); + + +it('keeps the optional SDK and exporters inactive without an endpoint', async () => { + const { stdout, stderr } = await promisify(execFile)(process.execPath, ['--import', fileURLToPath(new URL('../../node_modules/tsx/dist/loader.mjs', import.meta.url)), + fileURLToPath(new URL('./helpers/maintenance/otel-child.ts', import.meta.url))], { + env: { PATH: process.env.PATH, NODE_ENV: 'production' }, timeout: 15_000, maxBuffer: 2 * 1024 * 1024, + }); + expect(stdout).not.toContain('OpenTelemetry service.version='); + expect(stderr).not.toContain('Continuing without tracing'); +}); diff --git a/server/src/__tests__/telemetry-startup.test.ts b/server/src/__tests__/telemetry-startup.test.ts new file mode 100644 index 00000000000..b132405f681 --- /dev/null +++ b/server/src/__tests__/telemetry-startup.test.ts @@ -0,0 +1,113 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const stateFactory = vi.hoisted(() => vi.fn(() => ({ + installId: "test-install", + salt: "test-salt", + createdAt: "2026-01-01T00:00:00Z", + firstSeenVersion: "0.0.0", +}))); + +vi.mock("@paperclipai/shared/telemetry", async (importOriginal) => ({ + ...await importOriginal(), + loadOrCreateState: stateFactory, +})); +vi.mock("../home-paths.js", () => ({ resolvePaperclipInstanceRoot: () => "/unused-test-instance" })); +vi.mock("../version.js", () => ({ serverVersion: "0.0.0-test" })); + +describe("server telemetry startup", () => { + const fixtureDirs: string[] = []; + beforeEach(() => { + vi.resetModules(); + stateFactory.mockClear(); + for (const key of ["CI", "CONTINUOUS_INTEGRATION", "BUILD_NUMBER", "GITHUB_ACTIONS", "GITLAB_CI", "PAPERCLIP_TELEMETRY_DISABLED", "DO_NOT_TRACK"]) { + vi.stubEnv(key, ""); + } + vi.stubEnv("PAPERCLIP_TELEMETRY_ENDPOINT", "http://localhost:9999/ingest"); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); + }); + + afterEach(async () => { + const { getTelemetryClient } = await import("../telemetry.js"); + getTelemetryClient()?.stop(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + for (const dir of fixtureDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + it("emits the OS family exactly once even when initialization is repeated", async () => { + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + const client = initTelemetry({ enabled: true }); + expect(client).not.toBeNull(); + expect(initTelemetry({ enabled: true })).toBe(client); + expect(getTelemetryClient()).toBe(client); + await client!.flush(); + + expect(stateFactory).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledTimes(1); + const body = JSON.parse(String(vi.mocked(fetch).mock.calls[0]?.[1]?.body)); + expect(body.events).toEqual([ + { name: "server.started", occurredAt: expect.any(String), dimensions: { os_platform: process.platform } }, + ]); + }); + + it("disables telemetry after a state failure without aborting startup or retrying", async () => { + stateFactory.mockImplementationOnce(() => { + throw Object.assign(new Error("private telemetry path cannot be written"), { code: "EACCES" }); + }); + const warning = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { TelemetryClient } = await import("@paperclipai/shared/telemetry"); + const periodicFlush = vi.spyOn(TelemetryClient.prototype, "startPeriodicFlush"); + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + + expect(initTelemetry({ enabled: true })).toBeNull(); + expect(getTelemetryClient()).toBeNull(); + expect(initTelemetry({ enabled: true })).toBeNull(); + expect(stateFactory).toHaveBeenCalledTimes(1); + expect(periodicFlush).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + expect(warning.mock.calls).toEqual([ + ["[telemetry] Initialization failed; telemetry is disabled for this process."], + ]); + }); + + it("continues startup when the real state store cannot create its directory", async () => { + const fixtureDir = mkdtempSync(path.join(tmpdir(), "paperclip-telemetry-startup-")); + fixtureDirs.push(fixtureDir); + const blockedStateDir = path.join(fixtureDir, "telemetry"); + writeFileSync(blockedStateDir, "this file prevents directory creation"); + const { loadOrCreateState } = await vi.importActual( + "@paperclipai/shared/telemetry", + ); + stateFactory.mockImplementationOnce(() => loadOrCreateState(blockedStateDir, "0.0.0-test")); + vi.spyOn(console, "warn").mockImplementation(() => {}); + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + + expect(initTelemetry({ enabled: true })).toBeNull(); + expect(getTelemetryClient()).toBeNull(); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("does not initialize state or send when disabled in file configuration", async () => { + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + expect(initTelemetry({ enabled: false })).toBeNull(); + expect(getTelemetryClient()).toBeNull(); + expect(stateFactory).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + }); + + it.each(["PAPERCLIP_TELEMETRY_DISABLED", "DO_NOT_TRACK", "CI", "GITHUB_ACTIONS"])( + "respects %s before creating the client or state", + async (key) => { + vi.stubEnv(key, "1"); + const { initTelemetry, getTelemetryClient } = await import("../telemetry.js"); + expect(initTelemetry({ enabled: true })).toBeNull(); + expect(getTelemetryClient()).toBeNull(); + expect(stateFactory).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/server/src/__tests__/tool-access-service.test.ts b/server/src/__tests__/tool-access-service.test.ts index 5839a505ec2..166ec8f212a 100644 --- a/server/src/__tests__/tool-access-service.test.ts +++ b/server/src/__tests__/tool-access-service.test.ts @@ -9795,6 +9795,7 @@ describeEmbeddedPostgres("tool access service", () => { const revived = await service.connectGalleryApp( company.id, { + reconnectConnectionId: first.connectionId, galleryKey: "google-drive", connectionMethodKey: "paperclip-write", grantKind: "user", @@ -15997,7 +15998,7 @@ describeEmbeddedPostgres("tool access service", () => { ).rejects.toMatchObject({ status: 404 }); }); - it("reuses a removed gallery app while applying the omitted organization identity default", async () => { + it("creates a separate gallery app for the omitted organization default after personal removal", async () => { const company = await createCompany(db); const service = createTestToolAccessService(db); const actor = { actorType: "user" as const, actorId: "local-board" }; @@ -16022,8 +16023,10 @@ describeEmbeddedPostgres("tool access service", () => { actor, ); - expect(second.application.id).toBe(first.application.id); - expect(second.connectionId).toBe(first.connectionId); + expect(second.application.id).not.toBe(first.application.id); + expect(second.connectionId).not.toBe(first.connectionId); + await expect(service.getConnection(first.connectionId, company.id)) + .resolves.toMatchObject({ status: "archived", credentialPolicy: "per_user" }); expect(second.application.status).toBe("draft"); expect(second.connection.status).toBe("draft"); expect(second.connection.credentialPolicy).toBe("shared"); @@ -16038,8 +16041,8 @@ describeEmbeddedPostgres("tool access service", () => { isDefault: true, }), ]); - await expect(db.select().from(toolApplications)).resolves.toHaveLength(1); - await expect(db.select().from(toolConnections)).resolves.toHaveLength(1); + await expect(db.select().from(toolApplications)).resolves.toHaveLength(2); + await expect(db.select().from(toolConnections)).resolves.toHaveLength(2); }); it("restores grants and credential policy when an identity-changing revival fails", async () => { @@ -18414,6 +18417,119 @@ describeEmbeddedPostgres("tool access service", () => { }); }, 15000); + it.each(["local-board", "another-user", "current-user"])( + "creates an independent fresh GitHub identity after archiving a personal connection owned by %s", + async (ownerUserId) => { + const company = await createCompany(db); + const service = createTestToolAccessService(db); + const currentUserId = "current-user"; + await grantBoardUser(db, company.id, currentUserId, [], "member"); + mockToolsList([readTool("get_me", "Read current user")]); + await withGalleryServerUrl("github", PUBLIC_MCP_FIXTURE_URL, async () => { + const first = await service.connectGalleryApp(company.id, { + galleryKey: "github", + connectionMethodKey: "mcp-key", + name: "GitHub", + grantKind: "user", + credentialValues: { "credentials.authorization": "archived-fixture-key" }, + }, { actorType: "user", actorId: ownerUserId }); + // Model a legacy archived row that still has personal credential refs. + // Fresh setup must leave even this retained material untouched. + await db.update(toolConnections).set({ status: "archived", enabled: false }) + .where(eq(toolConnections.id, first.connectionId)); + await db.update(toolApplications).set({ status: "archived", archivedAt: new Date() }) + .where(eq(toolApplications.id, first.application.id)); + await db.insert(activityLog).values({ + companyId: company.id, actorType: "user", actorId: ownerUserId, + action: "tool_connection.archived", entityType: "tool_connection", + entityId: first.connectionId, + }); + const retained = async () => ({ + applications: await db.select().from(toolApplications) + .where(eq(toolApplications.id, first.application.id)), + connections: await db.select().from(toolConnections) + .where(eq(toolConnections.id, first.connectionId)), + grants: await db.select().from(connectionGrants) + .where(eq(connectionGrants.connectionId, first.connectionId)), + secrets: await db.select().from(companySecrets), + versions: await db.select().from(companySecretVersions), + activity: await db.select().from(activityLog) + .where(eq(activityLog.entityId, first.connectionId)), + }); + const before = await retained(); + const app = createRouteApp(db, + boardSessionActor(company.id, "member", currentUserId), + undefined, { deploymentMode: "authenticated" }); + const freshInput = { + galleryKey: "github", + connectionMethodKey: "mcp-key", + name: "GitHub", + grantKind: "user", + credentialValues: { "credentials.authorization": "fresh-fixture-key" }, + }; + + if (ownerUserId !== currentUserId) { + for (const target of [ + { reconnectConnectionId: first.connectionId }, + { applicationId: first.application.id }, + ]) { + await request(app) + .post(`/api/companies/${company.id}/tools/apps/connect`) + .send({ ...freshInput, ...target }).expect(403); + await expect(service.connectGalleryApp(company.id, { + ...freshInput, grantKind: "user", ...target, + }, { actorType: "user", actorId: currentUserId, actorSource: "session" })) + .rejects.toMatchObject({ + status: 403, + message: "Only the existing personal identity can reconnect this connection", + }); + } + expect(await retained()).toEqual(before); + } + + // An omitted key must not borrow the archived personal credential. + await request(app) + .post(`/api/companies/${company.id}/tools/apps/connect`) + .send({ ...freshInput, credentialValues: {} }).expect(400); + expect(await retained()).toEqual(before); + + const fresh = await request(app) + .post(`/api/companies/${company.id}/tools/apps/connect`) + .send(freshInput).expect(201); + expect(fresh.body.connectionId).not.toBe(first.connectionId); + expect(fresh.body.application.id).not.toBe(first.application.id); + expect(fresh.body.application.name).toBe("GitHub (2)"); + expect(fresh.body.connection).toMatchObject({ + credentialPolicy: "per_user", credentialSecretRefs: [], + }); + const after = await retained(); + expect(after.applications).toEqual(before.applications); + expect(after.connections).toEqual(before.connections); + expect(after.grants).toEqual(before.grants); + expect(after.activity).toEqual(before.activity); + for (const secret of before.secrets) expect(after.secrets).toContainEqual(secret); + for (const version of before.versions) expect(after.versions).toContainEqual(version); + const freshGrants = await db.select().from(connectionGrants) + .where(eq(connectionGrants.connectionId, fresh.body.connectionId)); + expect(freshGrants).toEqual([expect.objectContaining({ + kind: "user", subjectUserId: currentUserId, status: "active", + })]); + const oldSecretIds = before.grants.flatMap((grant) => + grant.credentialSecretRefs.map((ref) => ref.secretId)); + expect(freshGrants[0]!.credentialSecretRefs).toHaveLength(1); + expect(oldSecretIds).toHaveLength(1); + const freshSecretId = freshGrants[0]!.credentialSecretRefs[0]!.secretId; + expect(oldSecretIds).not.toContain(freshSecretId); + expect(after.secrets.find((secret) => secret.id === freshSecretId)) + .toMatchObject({ scope: "user", ownerUserId: currentUserId }); + await expect(db.select().from(activityLog) + .where(and(eq(activityLog.entityId, fresh.body.connectionId), + eq(activityLog.action, "tool_app.connected")))) + .resolves.toEqual([expect.objectContaining({ actorId: currentUserId })]); + }, "mcp-key"); + }, + ); + it("reconnects a personal key on the existing user grant without creating an organization credential", async () => { const company = await createCompany(db); const service = createTestToolAccessService(db); diff --git a/server/src/__tests__/voice-session-store.integration.test.ts b/server/src/__tests__/voice-session-store.integration.test.ts index 70903f3b786..8c0bb1f1d6c 100644 --- a/server/src/__tests__/voice-session-store.integration.test.ts +++ b/server/src/__tests__/voice-session-store.integration.test.ts @@ -1892,8 +1892,28 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu }); it("retains one safe report per call and rejects history after access revocation", async () => { const f = await nativeServiceFixture(), first = await f.service.start(f.input); + const foreign = await nativeServiceFixture(); + const [foreignSession] = await db.select().from(chatVoiceSessions).where(eq(chatVoiceSessions.id, foreign.sessionId)); + await db.insert(chatVoiceReports).values({ companyId: foreign.companyId, sessionId: foreign.sessionId, nextCheckAt: new Date(0) }); + const foreignReports = await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, foreign.companyId)); + const reportProviderIds: string[] = []; + const fetchReport = f.transport.callReport as (providerSessionId: string) => ReturnType; + const reports = voiceCallReports(db, session => { + reportProviderIds.push(session.providerSessionId!); + return fetchReport(session.providerSessionId!); + }); + expect(await reports.load({ kind: "company", id: foreign.companyId })).toEqual([ + expect.objectContaining({ id: foreign.sessionId, companyId: foreign.companyId, dueAt: 0 }), + ]); + // This case owns report/history behavior for its company. Global recovery + // also visits every predecessor fixture's live calls, replies and reports. + const reconcileReports = async () => { + const due = (await reports.load({ kind: "company", id: f.companyId })) + .filter(row => row.dueAt <= Date.now()).sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id)); + for (const row of due) await reports.check(row.id); + }; await f.service.end(f.companyId, first.session.id, f.input.caller); - await f.service.reconcile(1000); + await reconcileReports(); const detail = await f.service.callDetail(f.companyId, first.session.id, f.input.caller); expect(detail.report).toMatchObject({ status: "available", costMicroUsd: "120000", transcript: [{ id: "turn1", text: "A test request" }] }); expect(await f.service.history(f.companyId, f.endpointId, f.input.caller)).toEqual(expect.arrayContaining([detail])); @@ -1908,12 +1928,12 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu const baseline = await f.transport.callReport(), latest = new Date(Date.now() + 10000); let response = { ...baseline, costMicroUsd: "240000", providerUpdatedAt: latest }; f.transport.callReport.mockImplementation(async (providerSessionId?: string) => providerSessionId === stored.providerSessionId ? response : baseline); - await f.service.reconcile(1000); + await reconcileReports(); expect((await f.service.callDetail(f.companyId, first.session.id, f.input.caller)).report.costMicroUsd).toBe("240000"); expect((await f.service.callDetail(f.companyId, other.session.id, f.input.caller)).report.costMicroUsd).toBe("120000"); await db.update(chatVoiceReports).set({ nextCheckAt: new Date(0) }).where(eq(chatVoiceReports.sessionId, first.session.id)); response = { ...baseline, costMicroUsd: "1", providerUpdatedAt: new Date(0) }; - await f.service.reconcile(1000); + await reconcileReports(); expect((await f.service.callDetail(f.companyId, first.session.id, f.input.caller)).report.costMicroUsd).toBe("240000"); expect(await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.sessionId, first.session.id))).toHaveLength(1); await expect(f.service.callDetail(f.companyId, first.session.id, { id: randomUUID(), authority: "member" })).rejects.toMatchObject({ status: 403 }); @@ -1922,6 +1942,9 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await expect(f.service.history(f.companyId, f.endpointId, f.input.caller)).rejects.toMatchObject({ status: 403 }); await expect(f.service.callDetail(f.companyId, first.session.id, f.input.caller)).rejects.toMatchObject({ status: 403 }); expect(f.transport.createPhoneSession).not.toHaveBeenCalled(); + expect(await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, foreign.companyId))).toEqual(foreignReports); + expect(reportProviderIds).toContain(stored.providerSessionId); + expect(reportProviderIds).not.toContain(foreignSession.providerSessionId); }); it("does not create provider media when membership is revoked during credential resolution", async () => { const f = await nativeServiceFixture(); diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index b5aba0a7f0e..4e4ca127631 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -14,6 +14,7 @@ import { agents, companies, createDb, + closeRegisteredClients, executionWorkspaces, heartbeatRuns, issueComments, @@ -87,6 +88,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; const RUNTIME_OWNED_GIT_BRANCH_METADATA = { createdByRuntime: true, @@ -134,8 +136,12 @@ function workspaceBranchIncoherenceFingerprintForTest(input: { } const leasedRunIds = new Set(); -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); -const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; +const externalRuntimeControlDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_RUNTIME_CONTROL_TEST_DATABASE_URL"); +const embeddedPostgresSupport: Awaited> = externalRuntimeControlDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +// The explicit fixture belongs only to the service-control suite. Other DB +// suites must not allocate an embedded or admin-owned database in this mode. +const describeEmbeddedPostgres = !externalRuntimeControlDatabaseUrl && embeddedPostgresSupport.supported ? describe : describe.skip; +const describeRuntimeControl = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( @@ -6672,16 +6678,18 @@ describeEmbeddedPostgres("workspace dirty quarantine branch repair", () => { }, 20_000); }); -describeEmbeddedPostgres("workspace runtime service control persistence", () => { +describeRuntimeControl("workspace runtime service control persistence", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-runtime-control-"); - db = createDb(tempDb.connectionString); + if (!externalRuntimeControlDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-runtime-control-"); + db = createDb(externalRuntimeControlDatabaseUrl ?? tempDb!.connectionString, + externalRuntimeControlDatabaseUrl ? { maxConnections: 4 } : undefined); }, 20_000); afterAll(async () => { + if (externalRuntimeControlDatabaseUrl) await closeRegisteredClients(externalRuntimeControlDatabaseUrl, { scope: "database" }); await tempDb?.cleanup(); }); @@ -7284,14 +7292,14 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => return result[0]!; })); - // The two lanes claim ports concurrently in-process. The allocator guarantees - // distinct ports, but not which lane wins the base port. It depends on the - // scheduling order. So assert order-independent invariants, not array index. + // A free-port probe is not a reservation. Both siblings must own distinct + // live ports in the allocator's bounded range even if the preferred port + // becomes unavailable before allocation. const lowerPort = Math.min(first.port!, second.port!); const upperPort = Math.max(first.port!, second.port!); const maxAllocatablePort = basePort + WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS - 1; expect(first.port).not.toBe(second.port); - expect(lowerPort).toBe(basePort); + expect(lowerPort).toBeGreaterThanOrEqual(basePort); expect(upperPort).toBeGreaterThan(basePort); expect(upperPort).toBeLessThanOrEqual(maxAllocatablePort); expect(first.url).toBe(`http://127.0.0.1:${first.port}`); @@ -7333,7 +7341,7 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => it("commits fixed-port reservations across the configured database idle timeout", async () => { const fixture = await createRuntimeFixture(); - const readinessDb = createDb(tempDb!.connectionString, { idleTimeoutSeconds: 1, maxConnections: 1 }); + const readinessDb = createDb(externalRuntimeControlDatabaseUrl ?? tempDb!.connectionString, { idleTimeoutSeconds: 1, maxConnections: 1 }); const [initialConnection] = await readinessDb.$client<{ pid: number }[]>`select pg_backend_pid() as pid`; const cleanupRuntimeHome = await createRuntimeHome(); const workspace = fixture.workspaces[0]!; diff --git a/server/src/instrumentation.ts b/server/src/instrumentation.ts index ac632a57e15..d26b09ceae4 100644 --- a/server/src/instrumentation.ts +++ b/server/src/instrumentation.ts @@ -499,6 +499,10 @@ async function bootstrapOtel(endpoint: string): Promise { console.log(`[paperclip] OpenTelemetry service.version=${serviceVersion}`); const sdk = new NodeSDK({ + // This integration exports traces only. NodeSDK otherwise creates + // default OTLP metric/log exporters even when none were configured. + metricReaders: [], + logRecordProcessors: [], resource: resourceFromAttributes({ [ATTR_SERVICE_NAME]: process.env.OTEL_SERVICE_NAME || "paperclip", [ATTR_SERVICE_VERSION]: serviceVersion, diff --git a/server/src/maintenance/ephemeral-migration-data.ts b/server/src/maintenance/ephemeral-migration-data.ts index 81b54d68f72..a815a51db65 100644 --- a/server/src/maintenance/ephemeral-migration-data.ts +++ b/server/src/maintenance/ephemeral-migration-data.ts @@ -1,5 +1,5 @@ -import { inArray } from "drizzle-orm"; import { heartbeatRuns, type Db } from "@paperclipai/db"; +import { offlineMigrationExecutionWhere } from "../services/offline-migration-execution.js"; import { getStorageService } from "../storage/index.js"; import { migrateAgentFiles, verifyAgentFileObjects } from "../services/agent-file-maintenance.js"; import { readEphemeralServerDataEligibility } from "../services/ephemeral-server-contract.js"; @@ -19,7 +19,7 @@ export async function migrateEphemeralServerData(db: Db, input: { const { signal, phase } = input; const counts: Record = {}; const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) - .where(inArray(heartbeatRuns.status, ["running", "queued"])).limit(1); + .where(offlineMigrationExecutionWhere()).limit(1); if (active) throw new Error("active_execution_requires_drain"); if (await hasUnsettledLocalInstructionCopies(db)) throw new Error("local_instruction_copies_require_settlement"); const before = await readEphemeralServerDataEligibility(db); diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index cfb8919cc80..3a8eae708c0 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -32,6 +32,7 @@ import { import type { WakeQueuePostgresAdapterDeps } from "./postgres.js"; import { createReleaseIssueExecution } from "../application/use-cases.js"; import type { TransactionScope } from "../application/ports.js"; +import { withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../services/issue-queued-comment-queue.js"; // Proves the atomicity and company-scope properties the security review // requires: every mutation names `companyId` in its own SQL `WHERE` clause, @@ -1139,6 +1140,36 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { // owns it. These tests drive the admission writer directly against a // transaction they open themselves, the same way `heartbeat.ts` will. describe("wake admission", () => { + it.each(["running", "cancelled", "receipt-collision", "actor-mismatch"])("preserves both queued input sources after %s", async scenario => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); + const [oldComment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "source-user", body: "Original input" }).returning(); + const [newComment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "source-user", body: "Incoming input" }).returning(); + const runId = await seedRun({ companyId, agentId, status: ["receipt-collision", "actor-mismatch"].includes(scenario) ? "queued" : scenario, + contextSnapshot: withQueuedCommentIdsInRunContext({ issueId }, [oldComment.id]) }); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, runId, source: "assignment", status: "queued", + requestedByActorType: "user", requestedByActorId: "source-user", + payload: withQueuedCommentIdsInWakePayload({ issueId }, [oldComment.id]), + }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, runId)); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const operation = db.transaction(async tx => createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( + createAdmissionTransactionScope(companyId, tx as unknown as Db), { + companyId, agentId, activeExecutionRunId: runId, activeExecutionRunStatus: "queued", + mergedContextSnapshot: withQueuedCommentIdsInRunContext({ issueId }, [oldComment.id, newComment.id]), + source: "on_demand", triggerDetail: "manual", payload: { issueId, commentId: newComment.id }, + requestedByActorType: "user", requestedByActorId: scenario === "actor-mismatch" ? "another-user" : "source-user", idempotencyKey: null, + ...(["receipt-collision", "actor-mismatch"].includes(scenario) + ? { durableReceipt: { id: scenario === "receipt-collision" ? wake.id : randomUUID(), requestedAt: new Date() } } : {}), + }, + )); + if (scenario === "receipt-collision") await expect(operation).rejects.toMatchObject({ cause: { code: "23505" } }); + else await expect(operation).resolves.toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]).toEqual(before); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))).toEqual([wake]); + }); + it("rolls back a deferred merge when its own durable receipt insert fails", async () => { const companyId = await seedCompany(); const agentId = await seedAgent({ companyId }); @@ -1241,7 +1272,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { try { await db.transaction(async tx => { await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(createAdmissionTransactionScope(companyId, tx as unknown as Db), { - companyId, activeExecutionRunId: runId, mergedContextSnapshot: { commentId: "fresh-input" }, + companyId, activeExecutionRunId: runId, activeExecutionRunStatus: "running", mergedContextSnapshot: { commentId: "fresh-input" }, agentId, source: "on_demand", triggerDetail: null, payload: null, requestedByActorType: "user", requestedByActorId: null, idempotencyKey: null, }); @@ -1269,6 +1300,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { await writer.coalesceIntoActiveExecutionRun(scope, { companyId: otherCompanyId, activeExecutionRunId: runId, + activeExecutionRunStatus: "running", mergedContextSnapshot: { taskKey: "issue-1", commentId: "c1" }, agentId, source: "on_demand", diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 5c328e25bce..8cfe44af184 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -49,6 +49,7 @@ import { withRecoveryContext } from "../../../services/recovery/status-only-cont import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js"; import { queuedCommentIdsFromWakePayload, + queuedCommentIdsFromRunContext, withQueuedCommentIdsInWakePayload, } from "../../../services/issue-queued-comment-queue.js"; import { CHAT_CONVERSATION_ID_KEY, extractWakeCommentIds } from "../../run-dispatch/index.js"; @@ -971,7 +972,32 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { async coalesceIntoActiveExecutionRun(scope, input) { const tx = requireAdmissionTx(scope, input.companyId); const now = new Date(); + const [candidate] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, input.activeExecutionRunId), eq(heartbeatRuns.companyId, input.companyId), + )).limit(1); + if (!candidate) throw new Error("wake-queue: the coalesce target run was not found for this company"); + if (candidate.agentId !== input.agentId) return null; + const [primary] = candidate.wakeupRequestId ? await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, candidate.wakeupRequestId), eq(agentWakeupRequests.companyId, input.companyId), + eq(agentWakeupRequests.agentId, input.agentId), + )).for("update").limit(1) : []; + const [current] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, candidate.id), eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.agentId, input.agentId), + )).for("update").limit(1); + if (!current || current.status !== input.activeExecutionRunStatus || current.wakeupRequestId !== candidate.wakeupRequestId) return null; + const commentIds = queuedCommentIdsFromRunContext(input.mergedContextSnapshot); + const synchronizeBatch = current.status === "queued" && commentIds.length > 0 + && primary?.status === "queued" && queuedCommentIdsFromWakePayload(primary.payload).length > 0; + if (current.status === "queued" && commentIds.length && current.wakeupRequestId && + (!primary || !["queued", "claimed"].includes(primary.status) || primary.runId !== current.id + || (input.durableReceipt && (primary.requestedByActorType !== input.requestedByActorType || primary.requestedByActorId !== input.requestedByActorId)) + || primary.payload?.issueId !== input.mergedContextSnapshot.issueId)) return null; await notifyChatRunMilestones(tx, { companyId: input.companyId, runId: input.activeExecutionRunId }); + if (synchronizeBatch && primary) await tx.update(agentWakeupRequests).set({ + payload: withQueuedCommentIdsInWakePayload(primary.payload, commentIds, { preserveSource: true }), updatedAt: now, + }).where(and(eq(agentWakeupRequests.id, primary.id), eq(agentWakeupRequests.companyId, input.companyId), + eq(agentWakeupRequests.agentId, input.agentId), eq(agentWakeupRequests.runId, current.id), eq(agentWakeupRequests.status, primary.status))); const mergedRun = await tx .update(heartbeatRuns) .set({ contextSnapshot: input.mergedContextSnapshot, updatedAt: now }) diff --git a/server/src/modules/wake-queue/adapters/promoted-retry-scope.test.ts b/server/src/modules/wake-queue/adapters/promoted-retry-scope.test.ts new file mode 100644 index 00000000000..f695fd05839 --- /dev/null +++ b/server/src/modules/wake-queue/adapters/promoted-retry-scope.test.ts @@ -0,0 +1,60 @@ +import { agentWakeupRequests, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; +import { createReleaseIssueExecution } from "../application/use-cases.js"; +import { createPostgresWakeQueueAdapter } from "./postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +// A board Retry receipt stores only `{ issueId, taskRetry }`, with no saved +// wake context. Its promoted run takes the issue's execution lock, so it must +// stay issue-scoped for workspace and trust-boundary preparation. +describe.skipIf(!support.supported)("promoted Retry receipt scope", () => { + let database: Awaited>; + let db: ReturnType; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("wake-promoted-retry-scope-"); + db = createDb(database.connectionString); + }, 60_000); + + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + }); + + it("keeps a promoted Retry receipt without saved wake context scoped to its issue", async () => { + const [company] = await db.insert(companies).values({ name: "Retry scope fixture", issuePrefix: "RSF", + defaultResponsibleUserId: "responsible-user" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Retry agent", adapterType: "process", + runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Retry scope fixture issue", + status: "in_review", priority: "medium", assigneeAgentId: agent.id }).returning(); + const [finished] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, + invocationSource: "on_demand", status: "succeeded", contextSnapshot: { issueId: issue.id } }).returning(); + await db.update(issues).set({ executionRunId: finished.id }).where(eq(issues.id, issue.id)); + const [wake] = await db.insert(agentWakeupRequests).values({ + companyId: company.id, agentId: agent.id, source: "on_demand", triggerDetail: "manual", reason: "task_retry", + status: "deferred_issue_execution", requestedByActorType: "user", requestedByActorId: "responsible-user", + payload: { issueId: issue.id, + taskRetry: { schema: "paperclip.task-retry.v1", sourceRunId: finished.id, actorId: "responsible-user" } }, + }).returning(); + const release = createReleaseIssueExecution({ + issueLock: createPostgresWakeQueueAdapter(db, { + resolveResponsibleUserId: async () => "responsible-user", + getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), + resolveSessionBeforeForWakeup: async () => null, + }), + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + }); + + const result = await release({ companyId: company.id, runId: finished.id, now: new Date(), suppressImmediateRecovery: true }); + + expect(result.outcome.kind).toBe("promoted"); + const [promoted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.wakeupRequestId, wake.id)); + expect(promoted).toMatchObject({ scopeKind: "issue", issueId: issue.id, + contextSnapshot: { issueId: issue.id, taskId: issue.id, taskKey: issue.id, wakeReason: "task_retry" } }); + expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0].executionRunId).toBe(promoted.id); + }); +}); diff --git a/server/src/modules/wake-queue/application/ports.ts b/server/src/modules/wake-queue/application/ports.ts index 66c8a631c15..500d1675bc2 100644 --- a/server/src/modules/wake-queue/application/ports.ts +++ b/server/src/modules/wake-queue/application/ports.ts @@ -425,6 +425,7 @@ export interface WakeAdmissionWriter { input: { companyId: string; activeExecutionRunId: string; + activeExecutionRunStatus: WakeAdmissionActiveExecutionRun["status"]; mergedContextSnapshot: Record; durableReceipt?: DurableWakeAdmissionReceipt; agentId: string; @@ -435,7 +436,7 @@ export interface WakeAdmissionWriter { requestedByActorId: string | null; idempotencyKey: string | null; }, - ): Promise>; + ): Promise | null>; /** Merges the wake's context into an already-queued deferred wake, guarded by its current status. */ mergeIntoExistingDeferredWake( scope: TransactionScope, diff --git a/server/src/modules/wake-queue/application/use-cases.test.ts b/server/src/modules/wake-queue/application/use-cases.test.ts index 7f605e70f30..0677818c9c2 100644 --- a/server/src/modules/wake-queue/application/use-cases.test.ts +++ b/server/src/modules/wake-queue/application/use-cases.test.ts @@ -1228,6 +1228,22 @@ describe("admitWakeBehindIssueExecution", () => { expect(reader.findExistingDeferredWake).not.toHaveBeenCalled(); }); + it("retains incoming receipt authority when the locked coalesce target has changed", async () => { + const writer = createFakeAdmissionWriter({ coalesceIntoActiveExecutionRun: vi.fn(async () => null) }); + const reader = createFakeAdmissionReader(); + const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: createFakeAdmissionHelpers() }); + const durableReceipt = { id: "incoming-source", requestedAt: new Date("2026-11-04T00:00:00Z") }; + expect(await admit(SCOPE, admissionInput({ durableReceipt, wakeCommentId: "new-comment", + contextSnapshot: { issueId: "issue-1", wakeCommentId: "new-comment" }, payload: { issueId: "issue-1", commentId: "new-comment" }, + }))).toEqual({ kind: "deferred" }); + expect(writer.insertNewDeferredWake).toHaveBeenCalledWith(SCOPE, expect.objectContaining({ + companyId: "company-1", agentId: "wake-agent", durableReceipt, + requestedByActorType: "user", requestedByActorId: "user-1", + payload: expect.objectContaining({ issueId: "issue-1", commentId: "new-comment" }), + })); + expect(writer.mergeIntoExistingDeferredWake).not.toHaveBeenCalled(); + }); + it("merges into the existing deferred wake when the policy returns a merge target", async () => { const mergeIntoExistingDeferredWake = vi.fn( async (_scope: TransactionScope, _input: Parameters[1]) => {}, diff --git a/server/src/modules/wake-queue/application/use-cases.ts b/server/src/modules/wake-queue/application/use-cases.ts index 79aada9b05f..6484ee18b9a 100644 --- a/server/src/modules/wake-queue/application/use-cases.ts +++ b/server/src/modules/wake-queue/application/use-cases.ts @@ -870,6 +870,7 @@ export function createAdmitWakeBehindIssueExecution(deps: { const run = await deps.writer.coalesceIntoActiveExecutionRun(scope, { companyId: input.companyId, activeExecutionRunId: target.id, + activeExecutionRunStatus: target.status, mergedContextSnapshot, ...(input.durableReceipt ? { durableReceipt: input.durableReceipt } @@ -882,12 +883,9 @@ export function createAdmitWakeBehindIssueExecution(deps: { requestedByActorId: input.requestedByActorId, idempotencyKey: input.idempotencyKey, }); - return { kind: "coalesced", run }; + if (run) return { kind: "coalesced", run }; } - // decision.kind === "defer": only now does the module read for an - // existing deferred wake, so the coalesce path (the common path) never - // pays for this query. const existingDeferred = (interactionReceipt || input.allowRunCoalescing === false) ? null diff --git a/server/src/modules/wake-queue/domain/context.test.ts b/server/src/modules/wake-queue/domain/context.test.ts index d4fe457f716..9dc44b8c099 100644 --- a/server/src/modules/wake-queue/domain/context.test.ts +++ b/server/src/modules/wake-queue/domain/context.test.ts @@ -121,4 +121,26 @@ describe("enrichPromotedWakeContext", () => { expect(result.contextSnapshot.acceptedPlanWakeRouting).toEqual({ targetAgentId: "agent-1" }); expect(result.contextSnapshot.workspaceRefreshReason).toBe("accepted_plan_confirmation"); }); + + it("restores the issue scope of a payload-only receipt without overwriting a queued context", () => { + const payloadOnly = enrichPromotedWakeContext({ + contextSnapshot: {}, + reason: "task_retry", + source: "on_demand", + triggerDetail: "manual", + payload: { issueId: "issue-1", taskRetry: { schema: "paperclip.task-retry.v1" } }, + }); + expect(payloadOnly.contextSnapshot).toMatchObject({ + issueId: "issue-1", taskId: "issue-1", taskKey: "issue-1", wakeReason: "task_retry", + }); + + const queued = enrichPromotedWakeContext({ + contextSnapshot: { issueId: "issue-1", taskId: "task-1", taskKey: "TASK-1" }, + reason: "issue_commented", + source: "automation", + triggerDetail: null, + payload: { issueId: "issue-2" }, + }); + expect(queued.contextSnapshot).toMatchObject({ issueId: "issue-1", taskId: "task-1", taskKey: "TASK-1" }); + }); }); diff --git a/server/src/modules/wake-queue/domain/context.ts b/server/src/modules/wake-queue/domain/context.ts index 4da2346fa63..a4f0df74e52 100644 --- a/server/src/modules/wake-queue/domain/context.ts +++ b/server/src/modules/wake-queue/domain/context.ts @@ -115,15 +115,16 @@ export type EnrichPromotedWakeContextResult = { /** * Fills the promoted run's context snapshot with the fields the original * wake enrichment always fills, without overwriting a field the deferred - * wake already carried. The deferred wake's own context snapshot was - * already enriched once when it was first queued, so most calls only - * normalize the interaction-continuation keys. + * wake already carried. A deferred wake queued through the wake queue was + * already enriched once, so most calls only normalize the + * interaction-continuation keys. */ export function enrichPromotedWakeContext( input: EnrichPromotedWakeContextInput, ): EnrichPromotedWakeContextResult { const contextSnapshot = { ...input.contextSnapshot }; const { reason, source, triggerDetail, payload } = input; + const issueIdFromPayload = readNonEmptyString(payload?.["issueId"]) ?? readNonEmptyString(payload?.["taskId"]); const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]); const taskKey = deriveTaskKey(contextSnapshot, payload); const wakeCommentIds = mergeWakeCommentIds(contextSnapshot, commentIdFromPayload); @@ -131,6 +132,14 @@ export function enrichPromotedWakeContext( if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) { contextSnapshot.wakeReason = reason; } + // Some durable receipts, such as a board Retry, store only the payload and + // never had a queued context snapshot. Keep their issue scope. + if (!readNonEmptyString(contextSnapshot["issueId"]) && issueIdFromPayload) { + contextSnapshot.issueId = issueIdFromPayload; + } + if (!readNonEmptyString(contextSnapshot["taskId"]) && issueIdFromPayload) { + contextSnapshot.taskId = issueIdFromPayload; + } if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) { contextSnapshot.taskKey = taskKey; } diff --git a/server/src/services/agent-file-maintenance.ts b/server/src/services/agent-file-maintenance.ts index 1888e742008..eec32d2fbed 100644 --- a/server/src/services/agent-file-maintenance.ts +++ b/server/src/services/agent-file-maintenance.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { and, asc, eq, gt, inArray } from "drizzle-orm"; import { agentFileRoots, agents, fileStorageDb, heartbeatRuns, type Db } from "@paperclipai/db"; +import { offlineMigrationExecutionWhere } from "./offline-migration-execution.js"; import type { StorageProvider, StorageService } from "../storage/types.js"; import { adoptAgentFiles, inspectAgentDirectory } from "./agent-file-store.js"; import { agentInstructionsBundleMode, deriveBundleState, resolveManagedInstructionsRoot } from "./agent-instructions.js"; @@ -33,7 +34,8 @@ export async function migrateAgentFiles(db: Db, input: { if (agentInstructionsBundleMode(agent) === "external") return "external_instructions"; const [run] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), - inArray(heartbeatRuns.status, ["running", "queued"]), + input.writersStopped && process.env.PAPERCLIP_EPHEMERAL_MIGRATION === "true" + ? offlineMigrationExecutionWhere() : inArray(heartbeatRuns.status, ["running", "queued"]), )).limit(1); if (run) return "active_execution"; const [head] = await tx.select({ generation: agentFileRoots.generation }).from(agentFileRoots) diff --git a/server/src/services/cancelled-native-startup.ts b/server/src/services/cancelled-native-startup.ts index e742a2133e6..88bb022617a 100644 --- a/server/src/services/cancelled-native-startup.ts +++ b/server/src/services/cancelled-native-startup.ts @@ -4,7 +4,7 @@ import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinaliza import { claimedAdapterType } from "./conversation-continuation.js"; import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; -import { canContinueCancelledRun } from "./run-cancellation.js"; +import { canContinueCancelledRun, readRunCancellation } from "./run-cancellation.js"; import { legacyControllerBootId } from "./legacy-controller-lease.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js"; @@ -37,14 +37,28 @@ export async function isFailedNativePreparation(db: Db, run: Run, coordinator: C return !lease && !result && !await nativeStartupHasExecutionEvidence(db, run); } +/** Candidate state only. Admission must also verify ownership, execution records, + * stopped leases, and any saved-file obligation below. */ +export function isInterruptedNativePreparationCandidate(run: Run): boolean { + const cancellation = readRunCancellation(run.resultJson); + // Older shutdown records predate structured cancellation. Current records + // identify unexpected system interruption without relying on an error label. + const systemInterruption = cancellation + ? cancellation.initiator.type === "system" && cancellation.source !== "operator" && + (cancellation.source === "shutdown" || cancellation.expected === false) + : run.errorCode === "server_shutdown_interrupted"; + const recovery = run.resultJson?.executionRecovery as Record | undefined; + return Boolean(systemInterruption && run.status === "interrupted" && run.finishedAt + && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && run.executionStage === "preparing" + && claimedAdapterType(run) === "paperclip_runner" && !run.processPid && !run.processGroupId + && !run.processStartedAt && !run.sessionIdAfter && !run.nativeIssueId && !run.nativeSessionId + && run.runnerProfileJson?.nativeExecutionInput == null && recovery?.providerWorkStarted !== true); +} + export async function isInterruptedNativePreparation(db: Db, run: Run): Promise { const recovery = run.resultJson?.executionRecovery as Record | undefined; const settled = recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false; - if (run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" || !run.finishedAt - || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt || run.executionStage !== "preparing" - || claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId - || run.processStartedAt || run.sessionIdAfter || run.nativeIssueId || run.nativeSessionId - || run.runnerProfileJson?.nativeExecutionInput != null || recovery?.providerWorkStarted === true + if (!isInterruptedNativePreparationCandidate(run) || run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date() || !settled && (!run.controllerBootId || run.controllerBootId === legacyControllerBootId)) return false; const [coordinator] = await db.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and( diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index a6f80f26ba0..5d9d9e3b815 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -7,7 +7,8 @@ import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { randomUUID } from "node:crypto"; import { canonicalExecutionUuid } from "./execution-identity.js"; -import { claimedAdapterType, conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js"; +import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js"; +import { isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js"; import { persistActivity } from "./activity-log.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; import { logger } from "../middleware/logger.js"; @@ -612,8 +613,7 @@ export async function settleUnrecoverableExecutions( if (task.status !== "blocked" && run.runtimeMode === "native") { nativeFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion }; } - if (task.status !== "blocked" && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && - run.errorCode === "server_shutdown_interrupted" && claimedAdapterType(run) === "paperclip_runner") { + if (task.status !== "blocked" && isInterruptedNativePreparationCandidate(run)) { nativeBootstrapFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion, previousStatus: task.status }; } } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 8215d222fc8..8bde92adcda 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3914,7 +3914,8 @@ export function heartbeatService( }); } } - if (latestRun?.status === "interrupted" && latestRun.errorCode === "server_shutdown_interrupted") { + if (latestRun?.status === "interrupted" && latestRun.runtimeMode === "legacy" && + !latestRun.runtimeModeResolvedAt && latestRun.executionStage === "preparing") { latestRun = await settleInterruptedNativeBootstrap(db, { run: latestRun, providerDispatchStarted: legacyAdapterEntered || nativeDispatchStarted || nativeOwnershipHeld, }) ?? latestRun; diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index ef92bf94c65..1dd39dd5067 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -1,13 +1,16 @@ import { claimNativeSessionResumptions } from "../native-runtime/native-finalization-reconciler.js"; import type { Db } from "@paperclipai/db"; -import { authUsers, agents, agentWakeupRequests, caseHandoffs, cases, chatCompletionDeliveries, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueComments, issueThreadInteractions, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { authUsers, agents, agentWakeupRequests, caseHandoffs, cases, chatCompletionDeliveries, companies, companyMemberships, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueComments, issueThreadInteractions, issues, nativeRunFinalizations } from "@paperclipai/db"; import { eq, inArray, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js"; import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; -import { createHeartbeatQueue, type HeartbeatQueueDependencies } from "./queue.js"; +import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js"; +import { createWakeQueue } from "../../modules/wake-queue/index.js"; +import { mergeCoalescedContextSnapshot } from "./run-preparation.js"; +import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js"; import { settleConversationTurn } from "../agent-conversations.js"; import { chatCompletionDeliveryService, recordChatCompletion, recordChatHandoff } from "../chat-completion-delivery.js"; import { instanceSettingsService } from "../instance-settings.js"; @@ -256,6 +259,124 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { return { company, agent, issue, wake, run }; } + it.each((["process", "paperclip_runner"] as const).flatMap(adapterType => (["coalesced", "claimed-primary", "legacy-envelope", "delete-old", "delete-new", "stopped", "running", "system-primary", "foreign-primary", "same-named-agent", "mixed-author"] as const).map(scenario => ({ adapterType, scenario }))))("protects a queued comment batch across $adapterType/$scenario", async ({ adapterType, scenario }) => { + const f = await fixture("queued"); + let admissionAgent: typeof agents.$inferSelect = { ...f.agent, adapterType, adapterConfig: adapterType === "paperclip_runner" ? { provider: "codex" } : {} }; + await db.update(agents).set({ adapterType, adapterConfig: admissionAgent.adapterConfig }).where(eq(agents.id, f.agent.id)); + if (scenario === "same-named-agent") [admissionAgent] = await db.insert(agents).values({ companyId: f.company.id, + name: f.agent.name.toUpperCase(), role: "engineer", status: "idle", adapterType, adapterConfig: admissionAgent.adapterConfig, + }).returning(); + const userId = `source-user-${f.company.id}`; + await db.insert(authUsers).values({ id: userId, name: "Source owner", email: `${f.company.id}@queue.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId: f.company.id, principalType: "user", principalId: userId, status: "active", membershipRole: "operator" }); + await db.update(companies).set({ defaultResponsibleUserId: userId }).where(eq(companies.id, f.company.id)); + const incomingActorId = scenario === "mixed-author" ? `second-user-${f.company.id}` : userId; + if (scenario === "mixed-author") { + await db.insert(authUsers).values({ id: incomingActorId, name: "Incoming source owner", email: `${incomingActorId}@queue.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId: f.company.id, principalType: "user", principalId: incomingActorId, status: "active", membershipRole: "operator" }); + } + const [oldComment] = await db.insert(issueComments).values({ companyId: f.company.id, issueId: f.issue.id, + authorType: "user", authorUserId: userId, body: "Review source A" }).returning(); + const [newComment] = await db.insert(issueComments).values({ companyId: f.company.id, issueId: f.issue.id, + authorType: "user", authorUserId: incomingActorId, body: "Review source B" }).returning(); + await db.update(agentWakeupRequests).set({ status: scenario === "claimed-primary" ? "claimed" : "queued", runId: f.run.id, claimedAt: null, + requestedByActorType: scenario === "system-primary" ? "system" : "user", requestedByActorId: scenario === "system-primary" ? null : userId, + payload: scenario === "legacy-envelope" ? { issueId: f.issue.id, commentId: oldComment.id } + : withQueuedCommentIdsInWakePayload({ issueId: f.issue.id, retainedSource: "original-source" }, [oldComment.id]), + }).where(eq(agentWakeupRequests.id, f.wake.id)); + const [scanned] = await db.update(heartbeatRuns).set({ responsibleUserId: userId, + contextSnapshot: withQueuedCommentIdsInRunContext({ issueId: f.issue.id, wakeReason: "issue_commented" }, [oldComment.id]), + }).where(eq(heartbeatRuns.id, f.run.id)).returning(); + let foreignPrimary: typeof agentWakeupRequests.$inferSelect | undefined; + if (scenario === "foreign-primary") { + const [company] = await db.insert(companies).values({ name: "Foreign source", issuePrefix: "FOREIGN" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Foreign source agent", adapterType: "process" }).returning(); + [foreignPrimary] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", payload: { source: "foreign" } }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: foreignPrimary.id }).where(eq(heartbeatRuns.id, f.run.id)); + } + if (scenario === "delete-old") await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, oldComment.id)); + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(admissionAgent); + deps.canStartNewExternalBotWork.mockResolvedValue(false); + deps.instanceSettings.getExperimental.mockResolvedValue({ enableIsolatedWorkspaces: false, enableNativeRunner: true } as Awaited>); + const admission = createWakeQueue(db, { + resolveResponsibleUserId: async () => userId, + getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), + resolveSessionBeforeForWakeup: async () => null, + wakeAdmissionHelpers: { filterZombieCoalesceTarget: deps.filterZombieCoalesceTarget, + mergeCoalescedContextSnapshot, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake }, + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + }); + deps.wakeQueue.createAdmissionTransactionScope.mockImplementation(admission.createAdmissionTransactionScope); + deps.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(admission.admitWakeBehindIssueExecution); + const queue = createHeartbeatQueue(db, deps); + if (scenario === "running") { + deps.canStartNewExternalBotWork.mockResolvedValue(true); + expect(await queue.claimQueuedRun(scanned)).toMatchObject({ status: "running" }); + deps.canStartNewExternalBotWork.mockResolvedValue(false); + } + const [originalPrimary] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wake.id)); + const coalesced = await queue.enqueueWakeup(admissionAgent.id, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + payload: { issueId: f.issue.id, commentId: newComment.id }, contextSnapshot: { issueId: f.issue.id, wakeCommentId: newComment.id }, + requestedByActorType: "user", requestedByActorId: incomingActorId }); + if (["running", "foreign-primary", "same-named-agent"].includes(scenario)) { + expect(coalesced).toBeNull(); + const [preserved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(preserved.contextSnapshot).toMatchObject({ wakeCommentId: oldComment.id, wakeCommentIds: [oldComment.id] }); + const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.company.id)); + expect(receipts.find(row => row.id !== f.wake.id)).toMatchObject({ status: "deferred_issue_execution", runId: null }); + expect(receipts.find(row => row.id === f.wake.id)).toEqual(originalPrimary); + if (scenario === "same-named-agent") { + expect(preserved).toEqual(scanned); + expect(receipts.find(row => row.id !== f.wake.id)).toMatchObject({ agentId: admissionAgent.id, + requestedByActorType: "user", requestedByActorId: incomingActorId, payload: { issueId: f.issue.id, commentId: newComment.id } }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.company.id))).toEqual([scanned]); + } + if (foreignPrimary) expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, foreignPrimary.id)))[0]).toEqual(foreignPrimary); + expect(deps.executeRun).not.toHaveBeenCalled(); + return; + } + expect(coalesced).toMatchObject({ id: f.run.id, status: "queued", contextSnapshot: { wakeCommentIds: [oldComment.id, newComment.id] } }); + const accepted = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.company.id)); + expect(accepted.find(row => row.id !== f.wake.id)).toMatchObject({ status: "coalesced", runId: f.run.id, finishedAt: expect.any(Date), + requestedByActorType: "user", requestedByActorId: incomingActorId }); + const [updatedPrimary] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wake.id)); + expect(updatedPrimary).toMatchObject({ id: originalPrimary.id, companyId: originalPrimary.companyId, + agentId: originalPrimary.agentId, source: originalPrimary.source, requestedByActorType: originalPrimary.requestedByActorType, + requestedByActorId: originalPrimary.requestedByActorId, idempotencyKey: originalPrimary.idempotencyKey, requestedAt: originalPrimary.requestedAt }); + if (scenario === "legacy-envelope" || scenario === "claimed-primary") expect(updatedPrimary.payload).toEqual(originalPrimary.payload); + else expect(updatedPrimary.payload).toMatchObject({ retainedSource: "original-source", commentId: oldComment.id, + _paperclipWakeContext: { commentId: oldComment.id, wakeCommentId: oldComment.id, wakeCommentIds: [oldComment.id, newComment.id] } }); + if (scenario === "claimed-primary") { + deps.canStartNewExternalBotWork.mockResolvedValue(true); + expect(await queue.claimQueuedRun(scanned)).toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]) + .toMatchObject({ status: "queued", contextSnapshot: { wakeCommentIds: [oldComment.id, newComment.id] } }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wake.id)))[0]).toEqual(originalPrimary); + expect(deps.executeRun).not.toHaveBeenCalled(); + return; + } + if (scenario === "stopped") { + await createHeartbeatRunControl(db, runControlCallbacks(db)).cancelRunInternal(f.run.id, "operator Stop"); + deps.canStartNewExternalBotWork.mockResolvedValue(true); + expect(await queue.claimQueuedRun(scanned)).toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]) + .toMatchObject({ status: "cancelled", contextSnapshot: { wakeCommentIds: [oldComment.id, newComment.id] } }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wake.id)))[0].status).toBe("cancelled"); + expect(deps.executeRun).not.toHaveBeenCalled(); + return; + } + if (scenario === "delete-new") await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, newComment.id)); + deps.canStartNewExternalBotWork.mockResolvedValue(true); + const claimed = await queue.claimQueuedRun(scanned); + expect(claimed).toMatchObject({ id: f.run.id, status: "running", + contextSnapshot: { wakeCommentId: scenario === "delete-new" ? oldComment.id : newComment.id, + wakeCommentIds: scenario === "delete-old" ? [newComment.id] : scenario === "delete-new" ? [oldComment.id] : [oldComment.id, newComment.id] } }); + const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.company.id)); + expect(receipts.find(row => row.id !== f.wake.id)).toMatchObject({ status: "coalesced", runId: f.run.id }); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + async function strandedMention(runtimeMode = "legacy") { const seeded = await fixture("succeeded"); await db.update(heartbeatRuns).set({ runtimeMode, nativeIssueId: runtimeMode === "native" ? seeded.issue.id : null, finishedAt: new Date() }).where(eq(heartbeatRuns.id, seeded.run.id)); diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.ts index 73a665bb53d..d109fcec13c 100644 --- a/server/src/services/heartbeat/recovery.test.ts +++ b/server/src/services/heartbeat/recovery.test.ts @@ -310,7 +310,7 @@ describePostgres("heartbeat recovery module ownership", () => { } finally { try { await f.cleanup(); } finally { await child.stop(); } } }); - async function remoteRetryFixture(pid: number, target: "workspace" | "plugin_task" = "workspace") { + async function remoteRetryFixture(pid: number, target: "workspace" | "remote_workspace" | "plugin_task" = "workspace") { const fixture = await nativeProcessFixture(pid, "remote", "failed"); const sessionId = randomUUID(); const [lease] = await db.insert(environmentLeases).values({ companyId: fixture.run.companyId, @@ -319,13 +319,17 @@ describePostgres("heartbeat recovery module ownership", () => { }).returning(); await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, fixture.run.agentId)); await db.update(issues).set({ executionRunId: fixture.run.id }).where(eq(issues.id, fixture.run.issueId!)); - await db.update(heartbeatRuns).set({ nativeSessionId: sessionId, runnerProfileJson: { + await db.update(heartbeatRuns).set({ nativeSessionId: sessionId, + ...(target === "remote_workspace" ? { processGroupId: null } : {}), runnerProfileJson: { sessionCheckpoint: { identity: { runId: fixture.run.id, companyId: fixture.run.companyId, issueId: fixture.run.issueId, agentId: fixture.run.agentId, sessionId }, providerSessionId: "existing-provider" }, ...(target === "workspace" ? { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared", workspaceId: randomUUID(), leaseId: lease.id, providerLeaseId: lease.providerLeaseId, remoteCwd: "/work", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, resourceDisposition: "keep_running" } } : {}), + ...(target === "remote_workspace" ? { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", + workspaceId: randomUUID(), epoch: 3, leaseId: lease.id, providerLeaseId: lease.providerLeaseId, + remoteCwd: "/work", state: "prepared", snapshotId: randomUUID() } } : {}), } }).where(eq(heartbeatRuns.id, fixture.run.id)); await db.insert(nativeRunnerCheckpoints).values({ runId: fixture.run.id, companyId: fixture.run.companyId, issueId: fixture.run.issueId!, part: "opaque-checkpoint-record", material: { fixture: true } }); @@ -335,9 +339,12 @@ describePostgres("heartbeat recovery module ownership", () => { return { ...fixture, lease }; } - it.each(["failed", "running"] as const)("routes a due remote %s retry through verified recovery without local PID inspection", async status => { + it.each((["workspace", "remote_workspace"] as const).flatMap(target => + (["failed", "running"] as const).map(status => ({ target, status })), + ))("routes a due $target $status retry through verified recovery without local PID inspection", async ({ target, status }) => { const child = await ownedProcess(); - const f = await remoteRetryFixture(child.pid); + const f = await remoteRetryFixture(child.pid, target); + const processGroupId = target === "remote_workspace" ? null : child.pid; const originalKill = process.kill.bind(process); const kill = vi.spyOn(process, "kill").mockImplementation(originalKill); try { @@ -351,7 +358,7 @@ describePostgres("heartbeat recovery module ownership", () => { remote: { providerLeaseId: f.lease.providerLeaseId, remoteCwd: "/work" } }), })); expect(native.claims).toHaveBeenCalledWith(expect.objectContaining({ runIds: [f.run.id] })); - expect(await f.current()).toMatchObject({ status: "running", processPid: child.pid, processGroupId: child.pid }); + expect(await f.current()).toMatchObject({ status: "running", processPid: child.pid, processGroupId }); expect(kill.mock.calls.filter(([pid]) => Math.abs(pid as number) === child.pid)).toEqual([]); expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false); await recovery.reapOrphanedRuns(); @@ -378,10 +385,13 @@ describePostgres("heartbeat recovery module ownership", () => { } finally { try { await f.cleanup(); } finally { await child.stop(); } } }); - it.each(["future_retry", "live_owner", "missing_checkpoint", "active_execution"] as const)( - "does not execute a remote retry without current authority: %s", async condition => { + it.each((["workspace", "remote_workspace"] as const).flatMap(target => + (["future_retry", "live_owner", "missing_checkpoint", "active_execution"] as const).map(condition => ({ target, condition })), + ))( + "does not execute a $target retry without current authority: $condition", async ({ target, condition }) => { const child = await ownedProcess(); - const f = await remoteRetryFixture(child.pid); + const f = await remoteRetryFixture(child.pid, target); + const processGroupId = target === "remote_workspace" ? null : child.pid; try { if (condition === "future_retry") await db.update(nativeRunFinalizations) .set({ nextAttemptAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.run.id)); @@ -397,7 +407,7 @@ describePostgres("heartbeat recovery module ownership", () => { await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns(); expect(f.deps.executeRun).not.toHaveBeenCalled(); expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled(); - expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid }); + expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId }); if (condition === "future_retry") expect(native.claims).not.toHaveBeenCalled(); if (condition === "missing_checkpoint") { const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id)); diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index 2912b041065..739a7584444 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -1862,7 +1862,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD } async function nativeProcessIsLocal(run: HeartbeatRun) { - if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) return false; + const profile = parseObject(run.runnerProfileJson); + if (profile.nativeWorkspaceSync != null || profile.remoteWorkspace != null) return false; const [identity] = await db.select({ eventType: heartbeatRunEvents.eventType, agentId: heartbeatRunEvents.agentId, @@ -2068,7 +2069,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD if (!await nativeProcessIsLocal(run)) { if (nextAttemptAt && nextAttemptAt > now) continue; if (activeRunExecutions.has(run.id) || runningProcesses.has(run.id)) continue; - if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) { + const profile = parseObject(run.runnerProfileJson); + if (profile.nativeWorkspaceSync != null || profile.remoteWorkspace != null) { remoteRetryRunIds.push(run.id); } else { const [taskLease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 2ea6de7c323..07bf4659789 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -429,10 +429,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } const now = opts?.now ?? new Date(); const preparationRecovery = parseObject(run.resultJson?.executionRecovery); + const interruptedPreparation = run.status === "interrupted" && + preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false; const retryReason = opts?.retryReason ?? - (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted" && - preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false - ? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON); + (interruptedPreparation ? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON); const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; const maxAttempts = Math.max( @@ -570,6 +570,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen : baseSchedule; const requiresIssueGate = + interruptedPreparation || (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") || isTransientWorkspacePreparationCode(run.errorCode) || hasConversationContinuationPolicy(run.resultJson) || diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index ead808b6623..461fcffb616 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -572,7 +572,8 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont async function resumeRemoteStopComments(run: NonNullable>>, requestId?: string) { if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return; let interruptedPreparation = false; - if (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") { + if (run.status === "interrupted" && run.runtimeMode === "legacy" && + !run.runtimeModeResolvedAt && run.executionStage === "preparing") { const current = await getRun(run.id, { includeExecutionEvidence: true }); if (current && await isInterruptedNativePreparation(db, current)) { const settled = await settleInterruptedNativeBootstrap(db, { run: current, providerDispatchStarted: false }); diff --git a/server/src/services/issue-queued-comment-queue.ts b/server/src/services/issue-queued-comment-queue.ts index 60ca8efea1e..e50809fe6c7 100644 --- a/server/src/services/issue-queued-comment-queue.ts +++ b/server/src/services/issue-queued-comment-queue.ts @@ -33,15 +33,18 @@ export function queuedCommentIdsFromRunContext(contextValue: unknown): string[] export function withQueuedCommentIdsInWakePayload( payloadValue: unknown, ids: string[], + options: { preserveSource?: boolean } = {}, ): Record { const payload = { ...record(payloadValue) }; const context = { ...record(payload[QUEUE_CONTEXT_KEY]) }; if (ids.length > 0) { const latestId = ids[ids.length - 1]!; context[QUEUE_IDS_KEY] = ids; - context.wakeCommentId = latestId; - context.commentId = latestId; - payload.commentId = latestId; + if (!options.preserveSource) { + context.wakeCommentId = latestId; + context.commentId = latestId; + payload.commentId = latestId; + } } else { delete context[QUEUE_IDS_KEY]; delete context.wakeCommentId; diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index e276dc88f89..ae868406aa6 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -116,19 +116,27 @@ function toReadModel(row: IssueRecoveryActionRow): IssueRecoveryAction { } function isUniqueRecoveryActionConflict(error: unknown) { - const maybe = error as { code?: string; constraint?: string; message?: string } | null; - return Boolean( - maybe && - maybe.code === "23505" && - ( + const seen = new Set(); + let candidate = error; + // Drizzle wraps the driver error in cause. Only this specific uniqueness + // conflict can reuse the existing bounded source-scoped upsert retry. + for (let depth = 0; depth < 8 && candidate && typeof candidate === "object"; depth++) { + if (seen.has(candidate)) break; + seen.add(candidate); + const maybe = candidate as { code?: string; constraint?: string; constraint_name?: string; message?: string; cause?: unknown }; + if (maybe.code === "23505" && ( maybe.constraint === "issue_recovery_actions_active_source_uq" || maybe.constraint === "issue_recovery_actions_active_fingerprint_uq" || + maybe.constraint_name === "issue_recovery_actions_active_source_uq" || + maybe.constraint_name === "issue_recovery_actions_active_fingerprint_uq" || typeof maybe.message === "string" && ( maybe.message.includes("issue_recovery_actions_active_source_uq") || maybe.message.includes("issue_recovery_actions_active_fingerprint_uq") ) - ), - ); + )) return true; + candidate = maybe.cause; + } + return false; } export function issueRecoveryActionService(db: DbOrTransaction) { diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 68138666771..44bdebd1004 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -20,7 +20,7 @@ import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { logActivity } from "./activity-log.js"; import { isSupersededConversationRun } from "./agent-conversations.js"; import { issueService } from "./issues.js"; -import { isInterruptedNativePreparation } from "./cancelled-native-startup.js"; +import { isInterruptedNativePreparation, isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js"; import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -318,10 +318,8 @@ export async function settleInterruptedNativeBootstrap( return db.transaction(async tx => { const [task] = await tx.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, input.run.companyId))).for("update"); const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.run.id), eq(heartbeatRuns.companyId, input.run.companyId))).for("update"); - if (!task || !run || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt || + if (!task || !run || !isInterruptedNativePreparationCandidate(run) || run.contextSnapshot?.issueId !== task.id || - run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" || - claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId || executionFailureRetryCount(run) >= 2) return null; if (hasWorkspaceRestoreFailure(run.resultJson) && !await hasVerifiedWorkspaceRepair(tx as unknown as Db, run)) return null; const [native] = await tx.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)).limit(1); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 6f0cdaf91d8..25613267f68 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -34,12 +34,14 @@ import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js import { readProcessStartedAt } from "../hot-restart.js"; import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; +import { isUniqueViolation } from "../../db-errors.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js"; import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js"; import { nativeRestartInterruptedTurnId, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { retireSupersededNativeRetry } from "./native-superseded-retry.js"; import { legacyWorkspaceRecoveryNotMigrated } from "../legacy-workspace-recovery-migration.js"; +import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; import { requestNativeStartupCleanup, readNativeStartupContext, promoteNativeStartupCleanup } from "./native-startup-cleanup.js"; export type NativeControllerIdentity = { @@ -583,6 +585,7 @@ export async function claimNativeRestartRecoveries(input: { run: heartbeatRuns, coordinator: nativeRunFinalizations, issueExecutionRunId: issues.executionRunId, + issueCheckoutRunId: issues.checkoutRunId, issueAssigneeAgentId: issues.assigneeAgentId, issueStatus: issues.status, }) @@ -630,7 +633,20 @@ export async function claimNativeRestartRecoveries(input: { if (!input.liveRunExecutions?.has(row.run.id) && await retireSupersededNativeRetry(tx as unknown as Db, { run: row.run, coordinator: row.coordinator, issueExecutionRunId: row.issueExecutionRunId, now, })) return { kind: "blocked", runId: row.run.id, reason: "native_retry_superseded" } as const; - if (row.issueExecutionRunId !== row.run.id && !(stopCleanup && row.issueExecutionRunId === null)) { + // A scheduled same-run resume can find its task lock released while no + // other run holds the task. Like the local resume claim, reclaim the + // free lock unless another run is admitted for this task. + const reclaimsExecution = row.issueExecutionRunId === null + && row.coordinator.phase === "retryable_failure" + && (row.issueCheckoutRunId === null || row.issueCheckoutRunId === row.run.id) + && !(await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, row.run.companyId), ne(heartbeatRuns.id, row.run.id), + inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + or(eq(heartbeatRuns.issueId, row.coordinator.issueId), eq(heartbeatRuns.nativeIssueId, row.coordinator.issueId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${row.coordinator.issueId}`), + )).limit(1)).length; + if (row.issueExecutionRunId !== row.run.id && !(stopCleanup && row.issueExecutionRunId === null) + && !reclaimsExecution) { return { kind: "blocked", runId: row.run.id, @@ -1165,6 +1181,23 @@ export async function claimNativeRestartRecoveries(input: { updatedAt: now, }) .where(eq(heartbeatRuns.id, row.run.id)); + if (reclaimsExecution) { + const [agent] = await tx.select({ name: agents.name }).from(agents) + .where(eq(agents.id, row.run.agentId)).limit(1); + await tx.update(issues).set({ + executionRunId: row.run.id, + executionAgentNameKey: agent?.name.trim().toLowerCase() || null, + executionLockedAt: now, + updatedAt: now, + }).where(and(eq(issues.id, row.coordinator.issueId), eq(issues.companyId, row.run.companyId), + isNull(issues.executionRunId))); + await appendHeartbeatRunEvent(tx as unknown as Db, { + companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId, + eventType: "native.recovery.execution_reclaimed", stream: "system", level: "info", + message: "Reclaimed the original task execution for native session recovery.", + payload: { issueId: row.coordinator.issueId }, + }); + } claimScope = { companyId: row.run.companyId, issueId: row.coordinator.issueId, controller, remote: Boolean(durableAuthority) }; const common = { @@ -1204,6 +1237,12 @@ export async function claimNativeRestartRecoveries(input: { kind: claimKind, ...common, } satisfies NativeRestartRecoveryClaim; + }).catch((error: unknown) => { + if (!isUniqueViolation(error, "issues_open_routine_execution_uq")) throw error; + terminalRunToReport = null; + claimScope = undefined; + return { kind: "awaiting_evidence" as const, runId: candidate.runId, + reason: "routine_execution_busy", retryAt: new Date(now.getTime() + 30_000) }; }); if (terminalRunToReport) void reportRunFailure(input.db, terminalRunToReport); diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index c90619e12c7..4083815d215 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -152,6 +152,102 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( expect(next.kind).toBe("bootstrap_incomplete"); }); + it("preserves a retry when another task owns its routine slot and continues startup recovery", async () => { + const candidates = [await seedRun("ROUTINE-COLLISION"), await seedRun("AFTER-ROUTINE-COLLISION")] + .sort((a, b) => a.runId.localeCompare(b.runId)); + const [waiting, unrelated] = candidates; + const holder = await seedRun("ROUTINE-SLOT-OWNER", waiting.db); + const routineId = randomUUID(); + const now = new Date(); + await waiting.db.update(issues).set({ executionRunId: null, originKind: "routine_execution", + originId: routineId, originFingerprint: "scheduled" }).where(eq(issues.id, waiting.issueId)); + await waiting.db.update(issues).set({ originKind: "routine_execution", + originId: routineId, originFingerprint: "scheduled" }).where(eq(issues.id, holder.issueId)); + await waiting.db.update(heartbeatRuns).set({ status: "failed", finishedAt: now, + nativePhase: "retryable_failure", runnerProfileJson: { ...waiting.preparedProfile, + nativeWorkspaceSync: remotePreparedWorkspace(), + sessionCheckpoint: { identity: { companyId, agentId, issueId: waiting.issueId, runId: waiting.runId, + sessionId: waiting.native.normalizedSessionId }, providerSessionId: "retained-routine-session" }, + } }).where(eq(heartbeatRuns.id, waiting.runId)); + await waiting.db.update(nativeRunFinalizations).set({ controllerBootId: successor.bootId, + leaseOwner: "predecessor-routine-controller", leaseExpiresAt: new Date(now.getTime() + 60_000) }) + .where(eq(nativeRunFinalizations.runId, waiting.runId)); + const authority = new DurablePrpControlPlane({ stateDirectory: resolve(runtimeRoot, waiting.runId), + identity: transportOptions(waiting, runtimeRoot).prpIdentity, + expectedRunnerVersion: "routine-recovery-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + try { + await registerNativeRunnerCheckpoint({ db: waiting.db, + binding: { companyId, issueId: waiting.issueId, runId: waiting.runId }, bootId: successor.bootId, authority }); + } finally { await authority.stop(); } + await waiting.db.update(nativeRunFinalizations).set({ phase: "retryable_failure", attempt: 1, + leaseOwner: null, leaseExpiresAt: null, nextAttemptAt: new Date(0), failureCode: "provider_transport_interrupted" }) + .where(eq(nativeRunFinalizations.runId, waiting.runId)); + const state = async () => ({ + run: await waiting.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, waiting.runId)), + coordinator: await waiting.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, waiting.runId)), + checkpoint: await waiting.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, waiting.runId)), + task: await waiting.db.select().from(issues).where(eq(issues.id, waiting.issueId)), + holder: await waiting.db.select().from(issues).where(eq(issues.id, holder.issueId)), + }); + const before = await state(); + expect(before.checkpoint.length).toBeGreaterThan(0); + const onClaim = vi.fn(); + const input = { db: waiting.db, controller: successor, restartKind: "hard" as const, now }; + const dispositions = await claimNativeRestartRecoveries({ ...input, onClaim, + runIds: candidates.map(candidate => candidate.runId) }); + expect(dispositions).toEqual([ + { kind: "awaiting_evidence", runId: waiting.runId, reason: "routine_execution_busy", + retryAt: new Date(now.getTime() + 30_000) }, + expect.objectContaining({ kind: "bootstrap_incomplete", runId: unrelated.runId }), + ]); + expect(onClaim).toHaveBeenCalledOnce(); + expect(onClaim.mock.calls[0][0].runId).toBe(unrelated.runId); + expect(await state()).toEqual(before); + expect(await waiting.db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, waiting.runId), + eq(heartbeatRunEvents.eventType, "native.recovery.execution_reclaimed")))).toHaveLength(0); + expect(await waiting.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, waiting.runId))).toHaveLength(0); + + const heartbeat = heartbeatService(waiting.db); + try { + await expect(heartbeat.recoverNativeRunsAfterRestart(now, [waiting.runId])) + .resolves.toMatchObject({ claims: [], awaitingEvidenceRunIds: [waiting.runId] }); + } finally { await heartbeat.drainActiveRunExecutions(); } + expect(await state()).toEqual({ ...before, run: [{ ...before.run[0], + nextEventSeq: before.run[0].nextEventSeq + 1, updatedAt: expect.any(Date) }] }); + + await waiting.db.update(issues).set({ executionRunId: null }).where(eq(issues.id, holder.issueId)); + const recovered = (await Promise.all([1, 2].map(() => claimNativeRestartRecoveries({ ...input, + runIds: [waiting.runId] })))).flat(); + expect(recovered.filter(row => row.kind === "reattach_remote_runner")).toHaveLength(1); + const after = await state(); + expect(after.run[0].runnerProfileJson).toEqual(before.run[0].runnerProfileJson); + expect(after.checkpoint).toEqual(before.checkpoint); + expect(after.coordinator[0]).toMatchObject({ attempt: 1, controllerGeneration: 1 }); + expect(after.task[0].executionRunId).toBe(waiting.runId); + expect(await waiting.db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, waiting.runId), + eq(heartbeatRunEvents.eventType, "native.recovery.execution_reclaimed")))).toHaveLength(1); + }); + + it("does not turn another database constraint failure into a routine wait", async () => { + const fixture = await seedRun("UNRELATED-DATABASE-FAILURE"); + const db = new Proxy(fixture.db, { get(target, key) { + if (key === "transaction") return () => target.transaction(async tx => { + await tx.execute(sql`create temp table recovery_constraint_control + (id integer constraint recovery_unrelated_unique unique) on commit drop`); + await tx.execute(sql`insert into recovery_constraint_control values (1), (1)`); + }); + const value = Reflect.get(target, key); + return typeof value === "function" ? value.bind(target) : value; + } }); + const onClaim = vi.fn(); + await expect(claimNativeRestartRecoveries({ db, controller: successor, + restartKind: "hard", runIds: [fixture.runId], onClaim })).rejects.toThrow(/recovery_constraint_control/); + expect(onClaim).not.toHaveBeenCalled(); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator).toMatchObject({ phase: "observed", controllerGeneration: 0, leaseOwner: null }); + }); + it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "finalizer-wait", "warm-suspended", "stale-projection", "legacy", "changed", "same-server", "diskless", "diskless-per-turn", "diskless-per-turn-read-retry", "diskless-warm-suspended", "diskless-warm-suspended-read-retry", "diskless-live-inspection"])("preserves remote warm continuity across completed turns: %s", async (scenario) => { const hardRestart = !["warm", "same-server", "warm-suspended"].includes(scenario); const diskless = scenario.startsWith("diskless"); diff --git a/server/src/services/native-runtime/native-scheduled-resume-lock.test.ts b/server/src/services/native-runtime/native-scheduled-resume-lock.test.ts new file mode 100644 index 00000000000..6ddf261b653 --- /dev/null +++ b/server/src/services/native-runtime/native-scheduled-resume-lock.test.ts @@ -0,0 +1,137 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, applyPendingMigrations, companies, createDb, environmentLeases, environments, heartbeatRunEvents, + heartbeatRuns, issues, nativeRunFinalizations, +} from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { recoveryService } from "../recovery/service.js"; +import { claimNativeRestartRecoveries } from "./native-restart-recovery.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TEST_DATABASE_URL"); +const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); + +// A remote per-turn run fails while restoring its provider session. The +// coordinator schedules a same-run resume and keeps the task locks. The +// stale-lock sweeper then sees a failed run, and the remote resume must still +// run when its retry delay ends. +(support.supported ? describe : describe.skip)("scheduled remote native resume and the task execution lock", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + beforeAll(async () => { + if (externalUrl) { + await applyPendingMigrations(externalUrl); + db = createDb(externalUrl); + } else { + temporary = await startEmbeddedPostgresTestDatabase("native-scheduled-resume-lock-"); + db = createDb(temporary.connectionString); + } + }, 30_000); + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + async function seed(options: { lockedBy?: "self" | "none" } = {}) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + const sessionId = randomUUID(); + const failedAt = new Date(Date.now() - 5 * 60_000); + await db.insert(companies).values({ id: companyId, name: "Scheduled resume", issuePrefix: companyId.slice(0, 8) }); + await db.insert(agents).values({ id: agentId, companyId, name: "Engineer", adapterType: "paperclip_runner" }); + const lock = options.lockedBy === "none" ? null : runId; + await db.insert(issues).values({ id: issueId, companyId, title: "Continue after approval", status: "in_progress", + assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, nativeIssueId: issueId, + runtimeMode: "native", status: "failed", nativePhase: "retryable_failure", nativeSessionId: sessionId, + processPid: 4_242, processStartedAt: failedAt, startedAt: failedAt, finishedAt: new Date(), + errorCode: "native_session_interrupted", error: "native_session_recovery_failed: provider session was not restored", + contextSnapshot: { issueId }, + }); + const [environment] = await db.insert(environments).values({ name: `Scheduled resume ${runId}`, driver: "sandbox" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: runId, + environmentId: environment.id, provider: "daytona", providerLeaseId: randomUUID(), status: "active", + leasePolicy: "ephemeral", metadata: { driver: "sandbox", remoteCwd: "/work" }, + }).returning(); + await db.update(heartbeatRuns).set({ runnerProfileJson: { + nativeWorkspaceSync: { + schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64), + baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: runId, + leaseId: lease.id, providerLeaseId: lease.providerLeaseId, remoteCwd: "/work", resourceDisposition: "destroy", + }, + sessionCheckpoint: { + sessionId: "driver-session", providerSessionId: "provider-session", + identity: { runId, companyId, issueId, agentId, sessionId }, + }, + } }).where(eq(heartbeatRuns.id, runId)); + await db.update(issues).set({ executionRunId: lock, checkoutRunId: lock, executionLockedAt: lock ? failedAt : null }) + .where(eq(issues.id, issueId)); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "retryable_failure", attempt: 1, + nextAttemptAt: new Date(Date.now() - 60_000), recoveryState: "resuming_session", + failureCode: "native_session_interrupted", + failureDetail: { recoveryMode: "exact_checkpoint_resume", checkpointExists: true, + providerSessionEstablished: true, providerEventsExist: false }, + }); + return { companyId, agentId, issueId, runId }; + } + + const readIssue = async (id: string) => (await db.select().from(issues).where(eq(issues.id, id)))[0]!; + const readCoordinator = async (runId: string) => + (await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0]!; + const sweep = () => recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks(); + + it("keeps the task locks while the failed run's coordinator owns a scheduled resume", async () => { + const f = await seed(); + const result = await sweep(); + expect(result.issueIds).not.toContain(f.issueId); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: f.runId, checkoutRunId: f.runId }); + }); + + it("keeps the task locks when a resume claim returns the run to running before the clear", async () => { + const f = await seed(); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", nextAttemptAt: null }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + const beforeStaleLockClear = vi.fn(async (issueId: string) => { + if (issueId !== f.issueId) return; + await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId)); + await db.update(heartbeatRuns).set({ status: "running", nativePhase: "observed" }).where(eq(heartbeatRuns.id, f.runId)); + }); + const result = await recoveryService(db, { enqueueWakeup: vi.fn(), beforeStaleLockClear }).sweepStaleIssueLocks(); + expect(beforeStaleLockClear).toHaveBeenCalledWith(f.issueId); + expect(result.issueIds).not.toContain(f.issueId); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: f.runId, checkoutRunId: f.runId }); + }); + + it("still clears a closed task's locks for a failed run with a scheduled resume", async () => { + const f = await seed(); + await db.update(issues).set({ status: "cancelled" }).where(eq(issues.id, f.issueId)); + expect((await sweep()).issueIds).toContain(f.issueId); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: null, checkoutRunId: null }); + }); + + it("claims a due remote resume whose cleared task lock is free and reclaims it", async () => { + const f = await seed({ lockedBy: "none" }); + const dispositions = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.runId] }); + expect(dispositions).toEqual([expect.objectContaining({ kind: "reattach_remote_runner", runId: f.runId })]); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: f.runId }); + expect(await readCoordinator(f.runId)).toMatchObject({ phase: "observed", attempt: 1, nextAttemptAt: null }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]) + .toMatchObject({ status: "running", nativePhase: "observed" }); + const events = await db.select({ eventType: heartbeatRunEvents.eventType }).from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.eventType, "native.recovery.execution_reclaimed"))); + expect(events).toHaveLength(1); + }); + + it("does not reclaim a task that another run owns or has admitted", async () => { + for (const competitor of ["lock", "queued"] as const) { + const f = await seed({ lockedBy: "none" }); + const otherRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: otherRunId, companyId: f.companyId, agentId: f.agentId, + issueId: f.issueId, status: competitor === "lock" ? "running" : "queued", contextSnapshot: { issueId: f.issueId } }); + if (competitor === "lock") await db.update(issues).set({ executionRunId: otherRunId }).where(eq(issues.id, f.issueId)); + expect(await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.runId] })).toEqual([ + { kind: "blocked", runId: f.runId, reason: "issue_execution_lock_changed" }, + ]); + expect((await readIssue(f.issueId)).executionRunId).toBe(competitor === "lock" ? otherRunId : null); + expect(await readCoordinator(f.runId)).toMatchObject({ phase: "retryable_failure", attempt: 1 }); + } + }); +}); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 86df326e85f..e3611e34991 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -84,6 +84,7 @@ import * as noLaunchProofModule from "./native-maintenance-no-launch.js"; import { NativeSessionCleanupQuarantinedError, NativeSessionCloseUnrecoverableError, + NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionProtocolIntegrityError, } from "../../vendor/paperclip-runner/index.js"; @@ -10010,6 +10011,138 @@ describe("native warm session supervision", () => { expect(state.execute).toHaveBeenCalledTimes(2); }); + it.each(["completed", "different-allocation", "foreign-company", "different-workspace", "live-owner", "unaccounted-owner", "changed-controller", "forged-allocation", "foreign-writer", "foreign-active-lease", "reassigned-child", "parent-already-started", "writer-lock-race", "stop-after-claim", "controller-change-during-close", "stop-during-close", "writer-change-during-close", "retirement-fails", "foreign-logical-session", "stopped-parent"] as const)("retires the completed child listener before a different parent task uses the same sandbox: %s", async scenario => { + const { createServer } = await import("node:net"); + const { createNativeWarmIngressFixture } = await import("../../__tests__/helpers/native-warm-ingress-fixture.js"); + const { eq } = await import("drizzle-orm"); + const tables = await import("@paperclipai/db"); + const f = await createNativeWarmIngressFixture(execution, join(isolatedHome, "shared-workspace")); + await mkdir(join(isolatedHome, "shared-workspace")); + const listener = createServer(); + await new Promise((resolve, reject) => { listener.once("error", reject); listener.listen(0, "127.0.0.1", resolve); }); + const port = (listener.address() as import("node:net").AddressInfo).port; + let failClose = scenario === "retirement-fails"; + let unlockWriter: (() => void) | undefined; + let writerTransaction: Promise | undefined; + let incomingRunId: string | undefined; + const close = vi.fn(async () => { + if (failClose) throw new Error("owned close is unconfirmed"); + if (scenario === "controller-change-during-close" && incomingRunId) await f.db.update(nativeRunFinalizations) + .set({ leaseOwner: "replacement-owner", controllerBootId: "replacement-controller", controllerGeneration: 999 }) + .where(eq(nativeRunFinalizations.runId, incomingRunId)); + if (scenario === "stop-during-close" && incomingRunId) await f.db.update(heartbeatRuns) + .set({ status: "cancelled" }).where(eq(heartbeatRuns.id, incomingRunId)); + if (scenario === "writer-change-during-close" && incomingRunId) { + const [workspace] = await f.db.select().from(tables.executionWorkspaces).where(eq(tables.executionWorkspaces.id, f.workspaceId)); + await f.db.update(tables.executionWorkspaces).set({ metadata: { ...workspace.metadata, + remoteWorkspace: { ...(workspace.metadata!.remoteWorkspace as Record), writerRunId: randomUUID() } } }) + .where(eq(tables.executionWorkspaces.id, f.workspaceId)); + } + if (listener.listening) await new Promise((resolve, reject) => listener.close(error => error ? reject(error) : resolve())); + }); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); return result; + }).mockImplementationOnce(async options => { + expect(options.existingSession).toBeUndefined(); + const parentListener = createServer(); + try { + await new Promise((resolve, reject) => { parentListener.once("error", reject); parentListener.listen(scenario === "different-allocation" ? 0 : port, "127.0.0.1", resolve); }); + } finally { + if (parentListener.listening) await new Promise(resolve => parentListener.close(() => resolve())); + } + return result; + }); + try { + const child = await f.admit(); + await executePaperclipNativeSession({ db: f.db, execution: child.execution, runnerInstanceId: "child-runner", useRunnerd: true, + backend: {} as never, runnerExecutionTarget: child.target }); + const resultId = await f.settle(child, result); + const options: Parameters[0] = {}; + if (scenario === "different-allocation") options.providerLeaseId = `different-${f.providerLeaseId}`; + if (scenario === "different-workspace" || scenario === "foreign-company") { + options.workspaceId = randomUUID(); + if (scenario === "foreign-company") { + options.companyId = randomUUID(); options.agentId = randomUUID(); + await f.db.insert(tables.companies).values({ id: options.companyId, name: "Foreign tenant", issuePrefix: `F${options.companyId.slice(0, 7)}` }); + await f.db.insert(tables.agents).values({ id: options.agentId, companyId: options.companyId, name: "Foreign worker", adapterType: "paperclip_runner" }); + } + await f.db.insert(tables.executionWorkspaces).values({ id: options.workspaceId, companyId: options.companyId ?? f.companyId, + name: "Other workspace", mode: "shared_workspace", strategyType: "project_primary", providerType: "cloud_sandbox", cwd: child.execution.workspace.cwd }); + } + const parent = await f.admit(options); + incomingRunId = parent.runId; + if (scenario === "live-owner") await f.db.update(heartbeatRuns).set({ status: "running", nativePhase: "observed", finishedAt: null }).where(eq(heartbeatRuns.id, child.runId)); + if (scenario === "unaccounted-owner") await f.db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, child.runId)); + if (scenario === "changed-controller") await f.db.update(nativeRunFinalizations).set({ controllerBootId: "foreign-controller" }).where(eq(nativeRunFinalizations.runId, child.runId)); + if (scenario === "forged-allocation") await f.db.update(environmentLeases).set({ providerLeaseId: "canonical-other-allocation" }).where(eq(environmentLeases.id, parent.leaseId)); + if (scenario === "foreign-writer") { + const [workspace] = await f.db.select().from(tables.executionWorkspaces).where(eq(tables.executionWorkspaces.id, f.workspaceId)); + await f.db.update(tables.executionWorkspaces).set({ metadata: { ...workspace.metadata, + remoteWorkspace: { ...(workspace.metadata!.remoteWorkspace as Record), writerRunId: randomUUID() } } }).where(eq(tables.executionWorkspaces.id, f.workspaceId)); + } + if (scenario === "reassigned-child") await f.db.update(tables.issues).set({ assigneeAgentId: null }).where(eq(tables.issues.id, child.issueId)); + if (scenario === "parent-already-started") await f.db.update(heartbeatRuns).set({ processPid: 2147483000 }).where(eq(heartbeatRuns.id, parent.runId)); + if (scenario === "foreign-active-lease") { + const issueId = randomUUID(), runId = randomUUID(); + await f.db.insert(tables.issues).values({ id: issueId, companyId: f.companyId, title: "Live sibling", assigneeAgentId: f.agentId, status: "in_progress" }); + await f.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, nativeIssueId: issueId, runtimeMode: "native", status: "running" }); + await f.db.insert(environmentLeases).values({ companyId: f.companyId, environmentId: f.environmentId, executionWorkspaceId: f.workspaceId, + issueId, heartbeatRunId: runId, provider: "daytona", providerLeaseId: f.providerLeaseId, status: "active" }); + } + if (scenario === "foreign-logical-session") parent.execution.session.normalizedSessionId = child.execution.session.normalizedSessionId; + if (scenario === "stopped-parent") await f.db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, parent.runId)); + let claimed!: () => void; + const claimObserved = new Promise(resolve => { claimed = resolve; }); + if (scenario === "writer-lock-race") { + let locked!: () => void; + const lockObserved = new Promise(resolve => { locked = resolve; }); + const release = new Promise(resolve => { unlockWriter = resolve; }); + writerTransaction = f.db.transaction(async tx => { + const [workspace] = await tx.select().from(tables.executionWorkspaces).where(eq(tables.executionWorkspaces.id, f.workspaceId)).for("update"); + locked(); await release; + await tx.update(tables.executionWorkspaces).set({ metadata: { ...workspace.metadata, + remoteWorkspace: { ...(workspace.metadata!.remoteWorkspace as Record), writerRunId: randomUUID() } } }).where(eq(tables.executionWorkspaces.id, f.workspaceId)); + }); + await lockObserved; + } + const start = executePaperclipNativeSession({ db: f.db, execution: parent.execution, runnerInstanceId: "parent-runner", useRunnerd: true, + backend: {} as never, runnerExecutionTarget: parent.target, onExecutionLeaseAcquired: async () => { + claimed(); + if (scenario === "stop-after-claim") await f.db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, parent.runId)); + } }); + const outcome = start.then(value => ({ value }), error => ({ error })); + if (scenario === "writer-lock-race") { + await claimObserved; + expect(close).not.toHaveBeenCalled(); + unlockWriter!(); await writerTransaction; + } + if (scenario === "completed" || scenario === "different-allocation") { + await start; + expect(close).toHaveBeenCalledTimes(scenario === "completed" ? 1 : 0); + expect(state.execute).toHaveBeenCalledTimes(2); + const admittedParent = state.execute.mock.calls[1][0]; + expect(admittedParent.input.binding.issueId).toBe(parent.issueId); + expect(admittedParent.input.session.normalizedSessionId).toBe(parent.execution.session.normalizedSessionId); + expect(admittedParent.input.session.normalizedSessionId).not.toBe(child.execution.session.normalizedSessionId); + expect(admittedParent.existingSession).toBeUndefined(); + } else { + await expect(start).rejects.toThrow(); + expect(close).toHaveBeenCalledTimes(["retirement-fails", "controller-change-during-close", "stop-during-close", "writer-change-during-close"].includes(scenario) ? 1 : 0); + expect(listener.listening).toBe(!["controller-change-during-close", "stop-during-close", "writer-change-during-close"].includes(scenario)); + expect(state.execute).toHaveBeenCalledOnce(); + } + await outcome; + const [retained] = await f.db.select().from(nativeRunResults).where(eq(nativeRunResults.id, resultId)); + expect(retained.resultJson).toEqual(result); + } finally { + failClose = false; + unlockWriter?.(); await writerTransaction; + await closeWarmNativeSessionsForEnvironment({ environmentId: f.environmentId, reason: "shared ingress fixture cleanup" }); + if (listener.listening) await new Promise(resolve => listener.close(() => resolve())); + await f.cleanup(); + } + }, 60_000); + it("claims native execution authority before retiring a superseded warm identity", async () => { const { base, next, run } = fixture("identity-native-claim-order"); const updates: Array<{ table: unknown; values: Record }> = []; @@ -11752,6 +11885,21 @@ describe("native session bounded recovery", () => { }, ); + it("stops same-run retries when the runner proves the saved provider session incompatible", () => { + const failure = new NativeProviderStateIncompatibleError( + "Error: PRP command run.attach failed: ACPX durable launch profile digest does not match runner startup", + ); + const code = nativeSessionFailureSourceCode(failure); + expect(code).toBe("native_provider_state_incompatible"); + const disposition = nativeSessionFailureDisposition(1, new Date(), code); + expect(disposition).toEqual({ phase: "terminal_failure", failureCode: code, nextAttemptAt: null }); + expect(nativeSessionRecoveryProjection({ ...disposition, agentId: "agent" })).toMatchObject({ + issueStatus: "blocked", recoveryOwner: { kind: "board" }, + }); + // Only the typed runner class is deterministic; the same text stays retryable. + expect(nativeSessionFailureSourceCode(new Error(failure.message))).toBe("native_session_interrupted"); + }); + it("makes only typed operator-required cleanup quarantine terminal on the first attempt", () => { const code = nativeSessionFailureSourceCode( new NativeSessionCleanupQuarantinedError(), @@ -14006,7 +14154,7 @@ describe("runnerd provider runtime wiring", () => { || request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false); }); - it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].filter(mode => policy === "per_turn" || !mode.startsWith("resumed")).map(mode => [provider, policy, mode] as const))))( + it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-late-notice", "local-receipt-late-work", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].filter(mode => policy === "per_turn" || !mode.startsWith("resumed")).map(mode => [provider, policy, mode] as const))))( "recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => { const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude; const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`), @@ -14022,7 +14170,12 @@ describe("runnerd provider runtime wiring", () => { const scoped = await initializeScopedRunnerRoot(first, identity.runnerInstanceId); await mkdir(join(scoped, "control-plane"), { recursive: true }); const journal = JSON.stringify({ ...durableControlPlaneState(identity), commands: [{ status: mode.endsWith("pending-command") ? "pending" : "completed" }], - committedEvents: mode.endsWith("missing-terminal") ? [] : [{ eventType: "run.terminal", envelope: { ...identity, payload: { reportedWorkDisposition: "blocked" } } }] }); + committedEvents: mode.endsWith("missing-terminal") ? [] : [{ eventType: "run.terminal", envelope: { ...identity, payload: { reportedWorkDisposition: "blocked" } } }, + ...(mode === "local-receipt-late-notice" ? [{ eventType: "provider.notice.recorded", envelope: { ...identity, payload: { schema: "paperclip.prp.event.v1", eventType: "provider.notice.recorded", + runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, sourceInstanceId: identity.runnerInstanceId, sourceKind: "runner", + payload: { schema: "paperclip.provider.notice.v1", noticeId: "codex-warning", severity: "warning", category: "warning", scope: "turn", recoverable: true, userActionable: true, + summary: "ignored a notification for a settled Codex turn" } } } }] : []), + ...(mode === "local-receipt-late-work" ? [{ eventType: "turn.started", envelope: { ...identity } }] : [])] }); await writeFile(join(scoped, "control-plane/control-plane-state.json"), journal); if (mode.startsWith("empty-runner") || mode === "nonempty-runner") { await mkdir(join(scoped, "runner")); @@ -14095,7 +14248,7 @@ describe("runnerd provider runtime wiring", () => { expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); } const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch); - if (["canonical", "resumed", "resumed-timeout", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) { + if (["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-late-notice", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) { await expect(backend).resolves.toBeDefined(); expect(state.createBackend).toHaveBeenCalledOnce(); expect(owner).toHaveBeenCalledWith(expect.objectContaining({ priorIdentity: identity, @@ -14105,9 +14258,18 @@ describe("runnerd provider runtime wiring", () => { expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" }); expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); } else { - await expect(backend).rejects.toThrow(); + await expect(backend).rejects.toThrow({ + "local-receipt-late-work": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "local-receipt-missing-terminal": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "local-receipt-pending-command": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "pending-command": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "missing-terminal": "plugin_checkpoint_recovery_unproven: journal_not_settled", + "ambiguous": "plugin_checkpoint_recovery_unproven: quarantine_ambiguous", + "not-suspended": "plugin_checkpoint_recovery_unproven: checkpoint_not_settled", + "pending-outbox": "plugin_checkpoint_recovery_unproven: checkpoint_not_settled", + }[mode as string] ?? ""); expect(state.createBackend).not.toHaveBeenCalled(); - if (["local-receipt-missing-terminal", "local-receipt-pending-command", "active", "foreign-scope", "pending-command", "missing-terminal", "ambiguous", "contradictory", "nonempty-runner", "no-stop-proof"].includes(mode)) { + if (["local-receipt-late-work", "local-receipt-missing-terminal", "local-receipt-pending-command", "active", "foreign-scope", "pending-command", "missing-terminal", "ambiguous", "contradictory", "nonempty-runner", "no-stop-proof"].includes(mode)) { expect(task).not.toHaveBeenCalled(); } if (mode === "ambiguous") expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); @@ -15225,6 +15387,8 @@ describe("runnerd provider runtime wiring", () => { it.each([ "quarantined", "large control plane", + "late provider notice", + "post-terminal work", "empty retry shell", "unsuspended current", "live runner", @@ -15441,6 +15605,35 @@ describe("runnerd provider runtime wiring", () => { })) : []), { eventType: "run.terminal", envelope: identity }, + ...(scenario === "late provider notice" + ? [{ + eventType: "provider.notice.recorded", + envelope: { + ...identity, + payload: { + schema: "paperclip.prp.event.v1", + eventType: "provider.notice.recorded", + runId: identity.runId, + normalizedSessionId: identity.normalizedSessionId, + sourceInstanceId: identity.runnerInstanceId, + sourceKind: "runner", + payload: { + schema: "paperclip.provider.notice.v1", + noticeId: "codex-warning", + severity: "warning", + category: "warning", + scope: "turn", + recoverable: true, + userActionable: true, + summary: "ignored a notification for a settled Codex turn", + }, + }, + }, + }] + : []), + ...(scenario === "post-terminal work" + ? [{ eventType: "turn.started", envelope: identity }] + : []), ], }), ); @@ -15481,6 +15674,7 @@ describe("runnerd provider runtime wiring", () => { "quarantined", "released process identity", "large control plane", + "late provider notice", "empty retry shell", "unsuspended current", "active goal", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 06a4f78b4a2..a2600506bd0 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -140,6 +140,7 @@ import type { PrpStructuredRunResult, } from "../../vendor/paperclip-runner/index.js"; import { + NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionCleanupQuarantinedError, NativeSessionCloseUnrecoverableError, @@ -218,7 +219,7 @@ import { renderNativeRunnerStagedAttachmentPrompt, stageNativeRunnerWakeAttachments, } from "./native-runner-file-handoff.js"; -import { nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; +import { NATIVE_PROVIDER_STATE_INCOMPATIBLE, nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { reportRunFailure } from "../run-failure-report.js"; @@ -228,6 +229,7 @@ import { } from "./remote-runner-recovery-diagnostics.js"; import { readRemoteRunnerRecoveryProbe, REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS } from "./remote-runner-recovery-probe.js"; import { inspectRemoteWarmSessionRecovery, isSettledRemoteWarmControlState, type RemoteWarmSessionRecovery } from "./remote-warm-session-recovery.js"; +import { assertNativeWarmIngressRetirement, nativeWarmIngressAllocation, nativeWarmIngressIdentity, type NativeWarmIngressClaim, type NativeWarmIngressOwner } from "./native-warm-ingress-ownership.js"; import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js"; import { emitStartupReuse, environmentStartupReuse } from "@paperclipai/adapter-utils/startup-reuse"; import { persistActivity, publishActivity } from "../activity-log.js"; @@ -581,6 +583,7 @@ export type NativeInstructionWorkingCopy = { }; type WarmNativeSession = { + ingressOwner?: NativeWarmIngressOwner; onReleased: () => void; sessionCheckpointPersisted?: (snapshot: PersistedNativeSession) => void; pluginTaskId?: string; @@ -1857,6 +1860,7 @@ const sessionToolAuthorityEpochs = new Map(); const initializingSessionToolAuthorities = new Set(); const executingRunnerdSessionScopes = new Map(); const executingNativeOwnerScopes = new Map(); +const executingNativeIngressAllocations = new Map(); export function nativeSessionScopeDigestIsBusy(digest: string): boolean { const scopes = [ @@ -1963,21 +1967,32 @@ async function retireSupersededWarmNativeSessions( ownerScope: string, nextSessionScope: string, nextSessionId: string, + ingress?: { db: Db; claim: NativeWarmIngressClaim }, ): Promise { for (const [scope, entry] of warmNativeSessions) { - if (scope === nextSessionScope || entry.ownerScope !== ownerScope) continue; - if (entry.busy || executingRunnerdSessionScopes.has(scope)) { + if (scope === nextSessionScope) continue; + const logicalOwner = entry.ownerScope === ownerScope; + const priorIngress = entry.ingressOwner; + const physicalOwner = ingress && priorIngress && nativeWarmIngressAllocation(priorIngress.ingress) === nativeWarmIngressAllocation(ingress.claim.ingress); + if (!logicalOwner && !physicalOwner) continue; + if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(scope)) { throw new Error("native_session_supervisor_busy"); } + if (!logicalOwner) { + await assertNativeWarmIngressRetirement(ingress!.db, ingress!.claim, priorIngress!); + if (warmNativeSessions.get(scope) !== entry || entry.ingressOwner !== priorIngress + || entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(scope)) throw new Error("native_session_supervisor_busy"); + } if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); entry.idleTimer = null; entry.busy = true; - entry.closeOnReleaseReason = "warm native session identity changed"; + entry.closeOnReleaseReason = logicalOwner ? "warm native session identity changed" : "warm native ingress requires another session"; try { await closeWarmNativeSession(entry, entry.closeOnReleaseReason, { - discardUnsettledSession: entry.normalizedSessionId !== nextSessionId, + discardUnsettledSession: logicalOwner && entry.normalizedSessionId !== nextSessionId, }); forgetWarmNativeSession(scope, entry); + if (!logicalOwner) await assertNativeWarmIngressRetirement(ingress!.db, ingress!.claim, priorIngress!); } finally { // A failed close retains the owner and prevents launch. A later attempt // must retry retirement, never adopt this partially closed transport. @@ -5048,25 +5063,25 @@ async function recoverPluginRunnerCheckpoint(input: { target.sandboxLeaseAcquisition?.outcome === "resumed") return; let root = scoped; const quarantined = !existsSync(scoped); - const deny = (): never => { throw new Error("runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven"); }; + const deny = (reason: string): never => { throw new Error(`runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven: ${reason}`); }; if (quarantined) { const quarantine = resolve(runnerdStateBase(), "quarantine"); if (!existsSync(quarantine)) return; - if (!isSafeNativeStateDirectory(quarantine)) return deny(); + if (!isSafeNativeStateDirectory(quarantine)) return deny("quarantine_unsafe"); const candidates = readdirSync(quarantine).filter(name => name.startsWith(`${basename(scoped)}.identity_indeterminate.`)); if (candidates.length === 0) return; - if (candidates.length !== 1) return deny(); + if (candidates.length !== 1) return deny("quarantine_ambiguous"); root = resolve(quarantine, candidates[0]!); } if (!existsSync(root)) return; - if (!isSafeNativeStateDirectory(root)) return deny(); + if (!isSafeNativeStateDirectory(root)) return deny("root_unsafe"); if (hasRetainedWarmTransitionEvidence(root)) { - if (quarantined) return deny(); + if (quarantined) return deny("quarantined_warm_transition"); return; } const identity = readRunnerdDurableIdentity(root); if (!durableIdentityMatchesSession(identity, input.execution) || identity.runId === input.execution.binding.runId) { - if (quarantined) return deny(); + if (quarantined) return deny("quarantined_identity_mismatch"); return; } const direct = runnerdAuthorityLifecycle(root, identity); @@ -5074,32 +5089,29 @@ async function recoverPluginRunnerCheckpoint(input: { !existsSync(resolve(root, "runner", "runner-state.json")) && pluginRunnerdAuthorityLifecycle(root, identity) === "suspended"; // Never hide contradictory direct authority behind a remote checkpoint. if (direct !== "absent" && !(quarantined && direct === "suspended") && !verifiedPluginReceipt) { - if (quarantined) return deny(); + if (quarantined) return deny("quarantined_authority_conflict"); return; } await assertCleanupActivationCommitted(input.db, root, input.execution); const prior = await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }); if (prior !== "terminal_state_indeterminate" && !((quarantined || verifiedPluginReceipt) && prior === "verified")) { - if (quarantined) return deny(); + if (quarantined) return deny("quarantined_prior_owner_unverified"); return; } const control = record(JSON.parse(readBoundedNativeFile(resolve(root, "control-plane", "control-plane-state.json"), NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8"))); - const terminal = record(Array.isArray(control.committedEvents) ? control.committedEvents.at(-1) : null); - if (!Array.isArray(control.commands) || control.commands.some(command => record(command).status !== "completed") || - terminal.eventType !== "run.terminal" || record(terminal.envelope).runId !== identity.runId || - record(terminal.envelope).normalizedSessionId !== identity.normalizedSessionId) return deny(); + if (!isSettledRemoteWarmControlState(control, identity)) return deny("journal_not_settled"); const resumeFrom = await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId, leaseId: target.leaseId, priorIdentity: identity, inspectCheckpoint: true }); const checkpoint = environmentTaskResultSchema.parse(await target.runnerTask({ kind: "read_state", identity, resumeFrom })); if (checkpoint.kind !== "state" || checkpoint.state.lifecycle !== "suspended" || !Array.isArray(checkpoint.state.outbox) || checkpoint.state.outbox.length !== 0 || checkpoint.state.pendingTerminalDelivery != null || checkpoint.state.pendingProviderCleanup != null || - checkpoint.state.warmTransition != null) return deny(); + checkpoint.state.warmTransition != null) return deny("checkpoint_not_settled"); savePluginRunnerCheckpoint(root, checkpoint.state, identity); - if (await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }) !== "verified") return deny(); + if (await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }) !== "verified") return deny("checkpoint_unverified"); if (quarantined) { - if (existsSync(scoped)) return deny(); + if (existsSync(scoped)) return deny("quarantine_destination_exists"); renameSync(root, scoped); } } @@ -5516,11 +5528,7 @@ async function recoverQuiescentRunnerdState(input: { const runner = record(JSON.parse(runnerBytes)); const control = record(JSON.parse(controlBytes)); const provider = record(JSON.parse(providerBytes)); - const commands = control.commands; const events = control.committedEvents; - const terminalEnvelope = record( - record(Array.isArray(events) ? events.at(-1) : null).envelope, - ); if ( runner.schema !== RUNNERD_STATE_SCHEMA || !["ready", "suspended"].includes(String(runner.lifecycle)) || @@ -5536,12 +5544,8 @@ async function recoverQuiescentRunnerdState(input: { !Array.isArray(runner.outbox) || runner.outbox.length !== 0 || runner.pendingTerminalDelivery !== null || - !Array.isArray(commands) || - commands.some((command) => record(command).status !== "completed") || !Array.isArray(events) || - record(events.at(-1)).eventType !== "run.terminal" || - terminalEnvelope.runId !== identity.runId || - terminalEnvelope.normalizedSessionId !== identity.normalizedSessionId || + !isSettledRemoteWarmControlState(control, identity) || provider.completedTurnAuthoritative !== true ) continue; @@ -6989,6 +6993,7 @@ export function nativeSessionFailureDisposition( sourceFailureCode === "runner_remote_provider_artifact_incompatible" || sourceFailureCode === "runner_remote_provider_verification_unavailable" || sourceFailureCode === "native_provider_terminal_failed" || + sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE || sourceFailureCode === "native_current_wake_comments_unread" || sourceFailureCode === "native_current_wake_comments_changed_after_read" || sourceFailureCode === "native_session_cleanup_quarantined" || @@ -7039,6 +7044,7 @@ export function nativeSessionFailureSourceCode( ): | "native_permission_declined" | "native_provider_terminal_failed" + | typeof NATIVE_PROVIDER_STATE_INCOMPATIBLE | "native_provider_approval_required" | "native_provider_usage_limit" | "native_session_cleanup_quarantined" @@ -7065,6 +7071,7 @@ export function nativeSessionFailureSourceCode( | "provider_session_open_rejected" | "native_session_interrupted" { if (error instanceof NativePermissionDeclinedError) return "native_permission_declined"; + if (error instanceof NativeProviderStateIncompatibleError) return NATIVE_PROVIDER_STATE_INCOMPATIBLE; if (error instanceof NativeProviderTerminalFailure) { if (error.providerCode === "approval_required") return "native_provider_approval_required"; // Failed terminals retain their security meaning across the provider facade. @@ -7155,6 +7162,9 @@ export function nativeSessionFailureSourceCode( return "native_session_interrupted"; } +const NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION = + "The current runner cannot restore this task's saved provider session, so retrying this run cannot succeed. Retry the task to continue in a fresh provider session with the same workspace. The earlier provider conversation is not carried over."; + const NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE = "Send a new message to continue after Paperclip verifies that the previous provider and its tools have stopped. If cleanup cannot be verified, inspect the run and its environment. Clearing a task session does not resolve this quarantine. Automatic retries are stopped."; @@ -8314,7 +8324,17 @@ export async function executePaperclipNativeSession(input: { input.runnerExecutionTarget?.environmentId ?? null, ); const ownerToken = Symbol("native execution owner"); + const physicalIngress = input.useRunnerd ? nativeWarmIngressIdentity(input.execution, input.runnerExecutionTarget) : undefined; + const physicalIngressKey = physicalIngress ? nativeWarmIngressAllocation(physicalIngress) : undefined; + let ownsPhysicalIngress = false; try { + if (physicalIngressKey) { + const retained = warmNativeSessions.get(nativeSessionScopeKey(input.execution)); + if (retained && retained.issueId !== input.execution.binding.issueId) throw new Error("native_session_supervisor_busy"); + if (executingNativeIngressAllocations.has(physicalIngressKey)) throw new Error("native_session_supervisor_busy"); + executingNativeIngressAllocations.set(physicalIngressKey, ownerToken); + ownsPhysicalIngress = true; + } // Reserve across session identities before retiring or staging anything. if (executingNativeOwnerScopes.has(ownerScope)) { throw new Error("native_session_supervisor_busy"); @@ -8365,7 +8385,8 @@ export async function executePaperclipNativeSession(input: { const workspaceRoot = input.execution.workspace.cwd; if (workspaceRoot === null) throw new Error("native_workspace_required"); const needsWarmRetirement = [...warmNativeSessions].some(([scope, entry]) => - scope !== sessionScopeId && entry.ownerScope === ownerScope); + scope !== sessionScopeId && (entry.ownerScope === ownerScope || physicalIngress && entry.ingressOwner + && nativeWarmIngressAllocation(entry.ingressOwner.ingress) === nativeWarmIngressAllocation(physicalIngress))); const prepareAttachments = async () => { const admissionSha256 = nativeSha256(input.execution); const targetKind = input.runnerExecutionTarget?.kind ?? "local"; @@ -8444,6 +8465,9 @@ export async function executePaperclipNativeSession(input: { startupUnsettled = !(await startup.executionContext.drain()); startup.executionContext.finish(); } + if (ownsPhysicalIngress && physicalIngressKey && executingNativeIngressAllocations.get(physicalIngressKey) === ownerToken) { + executingNativeIngressAllocations.delete(physicalIngressKey); + } if (executingNativeOwnerScopes.get(ownerScope) === ownerToken) { executingNativeOwnerScopes.delete(ownerScope); } @@ -8940,9 +8964,12 @@ async function executePaperclipNativeSessionWithinScope( // Claim current native authority before the old warm checkpoint writer // must prove its same-controller successor. Shared staging stays after // strict predecessor suspension; it cannot run on a failed claim/Stop. + const ingress = nativeWarmIngressIdentity(input.execution, input.runnerExecutionTarget); if (input.useRunnerd && !claimedStopCleanup) await retireSupersededWarmNativeSessions( nativeSessionOwnerScope(input.execution, input.runnerExecutionTarget?.environmentId ?? null), nativeSessionScopeKey(input.execution), nativeSessionKey(input.execution), + ingress ? { db: input.db, claim: { ...input.execution.binding, ingress, leaseOwner, + controllerBootId: controller.bootId, controllerGeneration } } : undefined, ); claimedStartupContext?.assertOpen(); if (prepareAfterClaim) input = await prepareAfterClaim(); @@ -10034,11 +10061,18 @@ async function executePaperclipNativeSessionWithinScope( if (existing.session !== session) { existing.sessionCheckpointPersisted = runnerdBackend?.sessionCheckpointPersisted; } + const ingress = input.useRunnerd ? nativeWarmIngressIdentity(input.execution, input.runnerExecutionTarget) : undefined; + existing.ingressOwner = ingress ? { ...input.execution.binding, ingress, + controllerBootId: controller.bootId, controllerGeneration } : undefined; existing.session = session; existing.runId = input.execution.binding.runId; existing.issueId = input.execution.binding.issueId; } else warmNativeSessions.set(warmSessionId, { + ingressOwner: (() => { + const ingress = input.useRunnerd ? nativeWarmIngressIdentity(input.execution, input.runnerExecutionTarget) : undefined; + return ingress ? { ...input.execution.binding, ingress, controllerBootId: controller.bootId, controllerGeneration } : undefined; + })(), sessionCheckpointPersisted: runnerdBackend?.sessionCheckpointPersisted, onReleased: () => releaseRetainedNativeSessionCleanup(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, @@ -10491,6 +10525,8 @@ async function executePaperclipNativeSessionWithinScope( ? "Provider files could not be verified after two read attempts. Inspect the retained workspace and command transport before retrying the original run." : sourceFailureCode === "native_provider_terminal_failed" ? "The provider session is permanently unusable. Verify stopped execution, completed actions, and task context before starting a linked continuation." + : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE + ? NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and durable events; state is ambiguous and a replacement provider session is forbidden." : integrityFailure @@ -10683,6 +10719,8 @@ async function executePaperclipNativeSessionWithinScope( ? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE : sourceFailureCode === "native_provider_terminal_failed" ? "Verify that the failed provider stopped and reconcile its action outcomes. A linked continuation can proceed only after these checks succeed." + : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE + ? NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and explicitly resolve the ambiguous session state; do not open a replacement provider session." : integrityFailure diff --git a/server/src/services/native-runtime/native-session-resume-projection.test.ts b/server/src/services/native-runtime/native-session-resume-projection.test.ts index 502bcaf68f3..2838347e5bd 100644 --- a/server/src/services/native-runtime/native-session-resume-projection.test.ts +++ b/server/src/services/native-runtime/native-session-resume-projection.test.ts @@ -28,6 +28,7 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport( }); const select = () => findNativeSessionResumeRun(db, input); const source = () => run(0, { runnerInstanceId: randomUUID(), processPid: 17, processGroupId: 19, processStartedAt: at(0), + errorCode: "native_session_interrupted", runnerProfileJson: profile, contextSnapshot: { paperclipWake, paperclipIssue: { ...paperclipIssue, unrelated: "Omit" }, prompt: "Omit" } }); const writeProfile = (id: string, value: unknown) => db.execute(sql`update ${heartbeatRuns} set runner_profile_json = ${JSON.stringify(value)}::jsonb where ${heartbeatRuns.id} = ${id}`); diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index 932260c5640..9765563ee98 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -62,6 +62,7 @@ import { buildNativeExecutionInput } from "./native-execution-input.js"; import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, + NATIVE_PROVIDER_STATE_INCOMPATIBLE, NATIVE_TOOL_CONTRACT_FINGERPRINT, isUnusedLegacyNativeRetryReplacement, nativeToolContractFingerprintForTarget, @@ -278,6 +279,28 @@ describe("capability-gated connection tool refresh", () => { }); }); +describe("a source whose saved provider session the runner proved incompatible", () => { + it("continues the task in a fresh provider session with the same workspace", () => { + const current = execution(currentRunId); + expect(rebindNativeSessionCheckpoint({ previousRun: { ...previousRun(), errorCode: "native_session_interrupted" }, + currentExecution: current })).not.toBeNull(); + const source = { ...previousRun(), errorCode: NATIVE_PROVIDER_STATE_INCOMPATIBLE }; + expect(rebindNativeSessionCheckpoint({ previousRun: source, currentExecution: current })).toBeNull(); + const buildExecution = vi.fn(({ normalizedSessionId: sessionId, resumedSession }) => { + const built = execution(currentRunId); + built.session.normalizedSessionId = sessionId; + built.task.prompt = resumedSession ? "wake delta" : "original goal and complete handoff"; + return built; + }); + const result = buildNativeExecutionWithCheckpoint({ previousRun: source, normalizedSessionId, buildExecution }); + expect(result.checkpoint).toBeNull(); + expect(result.normalizedSessionId).not.toBe(normalizedSessionId); + expect(result.execution.task.prompt).toContain("complete handoff"); + expect(result.execution.binding.executionWorkspaceId).toBe(current.binding.executionWorkspaceId); + expect(result.execution.workspace).toEqual(current.workspace); + }); +}); + it("wires exact-session recovery and guarded selected identity into heartbeat persistence", () => { const source = readFileSync( new URL("../heartbeat/runtime-selection.ts", import.meta.url), @@ -1219,6 +1242,41 @@ const recoveryFakeCodex = resolve( currentExecution: execution(currentRunId), }), ).toBeNull(); + // A run whose saved provider session the runner proved unrestorable + // stays the barrier, and its checkpoint is not resumed. + const unrestorableId = randomUUID(); + const sourceCheckpoint = source.runnerProfileJson.sessionCheckpoint; + await db.insert(heartbeatRuns).values({ + ...unused, + id: unrestorableId, + createdAt: at(8), + errorCode: NATIVE_PROVIDER_STATE_INCOMPATIBLE, + runnerProfileJson: { + ...source.runnerProfileJson, + nativeExecutionInput: execution(unrestorableId), + sessionCheckpoint: { + ...sourceCheckpoint, + identity: { ...sourceCheckpoint.identity, runId: unrestorableId }, + }, + }, + }); + const unrestorable = await findNativeSessionResumeRun(db, selection); + expect(unrestorable).toMatchObject({ + id: unrestorableId, + errorCode: NATIVE_PROVIDER_STATE_INCOMPATIBLE, + }); + expect( + rebindNativeSessionCheckpoint({ + previousRun: { ...unrestorable!, errorCode: null }, + currentExecution: execution(currentRunId), + }), + ).not.toBeNull(); + expect( + rebindNativeSessionCheckpoint({ + previousRun: unrestorable!, + currentExecution: execution(currentRunId), + }), + ).toBeNull(); const fresh = buildNativeExecutionWithCheckpoint({ previousRun: null, diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index 135eb9c27b2..9e4fe58b486 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -94,6 +94,10 @@ export function nativeToolContractFingerprintForTarget( export const NATIVE_TOOL_CONTRACT_FINGERPRINT = nativeToolContractFingerprintForTarget("local"); +/** The runner proved that this run's saved provider session cannot be + * restored by the current runner build. Its checkpoint is not resumable. */ +export const NATIVE_PROVIDER_STATE_INCOMPATIBLE = "native_provider_state_incompatible"; + function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -341,6 +345,7 @@ async function selectNativeSessionResumeBarrier(db: Db, input: NativeSessionResu nativeIssueId: heartbeatRuns.nativeIssueId, runtimeMode: heartbeatRuns.runtimeMode, status: heartbeatRuns.status, + errorCode: heartbeatRuns.errorCode, createdAt: heartbeatRuns.createdAt, processPid: heartbeatRuns.processPid, processGroupId: heartbeatRuns.processGroupId, @@ -596,6 +601,7 @@ export function rebindNativeSessionCheckpoint(input: { companyId: string; agentId: string; nativeSessionId: string | null; + errorCode?: string | null; runnerProfileJson: unknown; }; currentExecution: NativeExecutionInput; @@ -604,6 +610,9 @@ export function rebindNativeSessionCheckpoint(input: { refreshTools?: boolean; }): PersistedNativeSession | null { if (input.refreshTools && input.toolRefreshOnResume !== true) return null; + // The source is still the progress barrier. Its provider session cannot be + // restored, so the next run starts a fresh session in the same workspace. + if (input.previousRun.errorCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE) return null; const previousProfile = record(input.previousRun.runnerProfileJson); if ( previousProfile.nativeToolContractFingerprint !== diff --git a/server/src/services/native-runtime/native-warm-ingress-ownership.ts b/server/src/services/native-runtime/native-warm-ingress-ownership.ts new file mode 100644 index 00000000000..8a046006633 --- /dev/null +++ b/server/src/services/native-runtime/native-warm-ingress-ownership.ts @@ -0,0 +1,99 @@ +import { and, eq, inArray } from "drizzle-orm"; +import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; + +export type NativeWarmIngressIdentity = { + provider: string; + environmentId: string; + providerLeaseId: string; + leaseId: string; + workspaceId: string; + cwd: string; +}; +export type NativeWarmIngressOwner = { + companyId: string; + agentId: string; + issueId: string; + runId: string; + controllerBootId: string; + controllerGeneration: number; + ingress: NativeWarmIngressIdentity; +}; +export type NativeWarmIngressClaim = NativeWarmIngressOwner & { leaseOwner: string }; + +export function nativeWarmIngressIdentity(execution: NativeExecutionInput, target?: AdapterExecutionTarget | null): NativeWarmIngressIdentity | undefined { + if (target?.kind !== "remote" || target.transport !== "sandbox" || target.effectiveCapabilities?.runnerWebSocketIngress !== true + || !target.providerKey || !target.environmentId || !target.leaseId || !target.sandboxLeaseAcquisition?.providerLeaseId + || !("cwd" in execution.workspace) || !execution.workspace.cwd + || execution.binding.executionWorkspaceId === execution.binding.runId) return undefined; + return { provider: target.providerKey, environmentId: target.environmentId, + providerLeaseId: target.sandboxLeaseAcquisition.providerLeaseId, leaseId: target.leaseId, + workspaceId: execution.binding.executionWorkspaceId, cwd: execution.workspace.cwd }; +} + +export function nativeWarmIngressAllocation(identity: NativeWarmIngressIdentity): string { + return JSON.stringify([identity.provider, identity.environmentId, identity.providerLeaseId]); +} + +const busy = () => new Error("native_session_supervisor_busy"); +export async function assertNativeWarmIngressRetirement(db: Db, current: NativeWarmIngressClaim, previous: NativeWarmIngressOwner): Promise { + if (current.companyId !== previous.companyId || current.agentId !== previous.agentId + || current.ingress.workspaceId !== previous.ingress.workspaceId || current.ingress.cwd !== previous.ingress.cwd + || nativeWarmIngressAllocation(current.ingress) !== nativeWarmIngressAllocation(previous.ingress) + || current.runId === previous.runId) throw busy(); + await db.transaction(async tx => { + const [workspace] = await tx.select().from(executionWorkspaces).where(and( + eq(executionWorkspaces.companyId, current.companyId), eq(executionWorkspaces.id, current.ingress.workspaceId), + )).for("update"); + const authority = readRemoteWorkspaceAuthority(workspace?.metadata); + if (!workspace || workspace.status !== "active" || authority?.writerRunId !== current.runId + || authority.location?.environmentId !== current.ingress.environmentId + || authority.location.providerLeaseId !== current.ingress.providerLeaseId + || authority.location.leaseId !== current.ingress.leaseId || authority.location.remoteCwd !== current.ingress.cwd) throw busy(); + const runs = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, current.companyId), + inArray(heartbeatRuns.id, [current.runId, previous.runId]))); + const incoming = runs.find(run => run.id === current.runId), prior = runs.find(run => run.id === previous.runId); + const coordinators = await tx.select().from(nativeRunFinalizations).where(and(eq(nativeRunFinalizations.companyId, current.companyId), + inArray(nativeRunFinalizations.runId, [current.runId, previous.runId]))); + const incomingCoordinator = coordinators.find(row => row.runId === current.runId), priorCoordinator = coordinators.find(row => row.runId === previous.runId); + const tasks = await tx.select().from(issues).where(and(eq(issues.companyId, current.companyId), inArray(issues.id, [current.issueId, previous.issueId]))); + const incomingTask = tasks.find(task => task.id === current.issueId), priorTask = tasks.find(task => task.id === previous.issueId); + const leases = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, current.companyId), + inArray(environmentLeases.id, [current.ingress.leaseId, previous.ingress.leaseId]))); + const incomingLease = leases.find(lease => lease.id === current.ingress.leaseId), priorLease = leases.find(lease => lease.id === previous.ingress.leaseId); + const now = Date.now(); + if (!incoming || incoming.runtimeMode !== "native" || incoming.status !== "running" || incoming.nativePhase !== "observed" + || incoming.processPid !== null || incoming.processGroupId !== null || incoming.agentId !== current.agentId + || incoming.nativeIssueId !== current.issueId || incoming.resultJson?.startupCancellation || incoming.resultJson?.nativeCancellation + || incomingCoordinator?.issueId !== current.issueId || incomingCoordinator.phase !== "observed" || incomingCoordinator.resultId !== null || incomingCoordinator.leaseOwner !== current.leaseOwner + || incomingCoordinator.controllerBootId !== current.controllerBootId || incomingCoordinator.controllerGeneration !== current.controllerGeneration + || !incomingCoordinator.leaseExpiresAt || incomingCoordinator.leaseExpiresAt.getTime() <= now + || incomingTask?.assigneeAgentId !== current.agentId || incomingTask.executionRunId !== current.runId + || incomingTask.executionWorkspaceId !== current.ingress.workspaceId + || !["in_progress", "in_review"].includes(incomingTask.status) + || (incomingTask.checkoutRunId !== null && incomingTask.checkoutRunId !== current.runId) + || !prior || prior.runtimeMode !== "native" || prior.status !== "succeeded" || !prior.finishedAt || prior.costAccountingPending + || prior.agentId !== previous.agentId || prior.nativeIssueId !== previous.issueId || prior.nativePhase !== "committed" + || priorCoordinator?.issueId !== previous.issueId || priorCoordinator.phase !== "committed" || !priorCoordinator.resultId + || priorCoordinator.leaseOwner !== null || priorCoordinator.leaseExpiresAt !== null + || priorCoordinator.controllerBootId !== previous.controllerBootId || priorCoordinator.controllerGeneration !== previous.controllerGeneration + || !priorTask || priorTask.assigneeAgentId !== previous.agentId || priorTask.executionWorkspaceId !== previous.ingress.workspaceId + || priorTask.executionRunId !== null || priorTask.checkoutRunId !== null) throw busy(); + const leaseMatches = (lease: typeof environmentLeases.$inferSelect | undefined, owner: NativeWarmIngressOwner) => lease + && lease.heartbeatRunId === owner.runId && lease.issueId === owner.issueId + && lease.environmentId === owner.ingress.environmentId && lease.provider === owner.ingress.provider + && lease.providerLeaseId === owner.ingress.providerLeaseId && lease.executionWorkspaceId === owner.ingress.workspaceId; + if (!leaseMatches(incomingLease, current) || incomingLease!.status !== "active" + || (incomingLease!.expiresAt !== null && incomingLease!.expiresAt.getTime() <= now) + || !leaseMatches(priorLease, previous) || !["released", "expired"].includes(priorLease!.status) + || priorLease!.cleanupStatus !== "success") throw busy(); + const otherLeases = await tx.select({ runId: environmentLeases.heartbeatRunId }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, current.companyId), eq(environmentLeases.environmentId, current.ingress.environmentId), + eq(environmentLeases.provider, current.ingress.provider), eq(environmentLeases.providerLeaseId, current.ingress.providerLeaseId), + eq(environmentLeases.status, "active"), + )); + if (otherLeases.some(lease => lease.runId !== current.runId)) throw busy(); + }); +} diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.test.ts b/server/src/services/native-runtime/remote-warm-session-recovery.test.ts index 91317d2af4b..91231e07f14 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.test.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.test.ts @@ -7,6 +7,10 @@ const goal = { schema: "paperclip.session_goal.snapshot.v1", goal: null, working const notification = { schema: "paperclip.prp.event.v1", eventType: "session.goal.snapshot", runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, sourceInstanceId: "runner", sourceKind: "runner", payload: goal }; const snapshot = { eventType: "session.goal.snapshot", envelope: { ...identity, payload: notification } }; +const notice = { schema: "paperclip.provider.notice.v1", noticeId: "codex-warning", severity: "warning", category: "warning", scope: "turn", + recoverable: true, userActionable: true, summary: "ignored a notification for a settled Codex turn" }; +const noticeNotification = { ...notification, eventType: "provider.notice.recorded", payload: notice }; +const lateNotice = { eventType: "provider.notice.recorded", envelope: { ...identity, payload: noticeNotification } }; const state = (events: unknown[]) => ({ identity, commands: [{ status: "completed" }], committedEvents: events }); describe("settled remote warm control evidence", () => { @@ -34,6 +38,28 @@ describe("settled remote warm control evidence", () => { expect(isSettledRemoteWarmControlState(state([terminal, suffix]), identity)).toBe(false); }); + it("accepts recoverable provider notices recorded after the terminal", () => { + const info = { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, payload: { ...notice, severity: "info" } } } }; + expect(isSettledRemoteWarmControlState(state([terminal, lateNotice]), identity)).toBe(true); + expect(isSettledRemoteWarmControlState(state([terminal, lateNotice, snapshot, info]), identity)).toBe(true); + }); + + it.each([ + ["error notice", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, payload: { ...notice, severity: "error" } } } }], + ["unrecoverable notice", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, payload: { ...notice, recoverable: false } } } }], + ["unknown notice schema", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, payload: { ...notice, schema: "paperclip.provider.notice.v2" } } } }], + ["notification type mismatch", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, eventType: "session.goal.snapshot" } } }], + ["foreign notice source", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, sourceInstanceId: "other" } } }], + ["foreign notice lease", { ...lateNotice, envelope: { ...lateNotice.envelope, environmentLeaseId: "other" } }], + ["control plane notice source", { ...lateNotice, envelope: { ...lateNotice.envelope, payload: { ...noticeNotification, sourceKind: "control_plane" } } }], + ])("holds a %s suffix", (_name, suffix) => { + expect(isSettledRemoteWarmControlState(state([terminal, suffix]), identity)).toBe(false); + }); + + it("holds new work recorded after a late provider notice", () => { + expect(isSettledRemoteWarmControlState(state([terminal, lateNotice, { ...snapshot, eventType: "turn.started" }]), identity)).toBe(false); + }); + it("holds missing terminals, pending commands and mixed core identities", () => { expect(isSettledRemoteWarmControlState(state([snapshot]), identity)).toBe(false); expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), commands: [{ status: "dispatched" }] }, identity)).toBe(false); diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.ts b/server/src/services/native-runtime/remote-warm-session-recovery.ts index ec63a72325e..09222910c96 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.ts @@ -14,9 +14,10 @@ export type RemoteWarmSessionRecovery = { revalidate: (leaseOwner: string) => Promise; }; -/** A goal read can publish an idle snapshot after the terminal. It carries no - * new work; every other suffix still requires reconciliation before handoff. */ -export function isSettledRemoteWarmControlState(control: Record, identity: Record): boolean { +/** A goal read can publish an idle snapshot and a provider can publish a + * recoverable notice after the terminal. Neither carries new work; every other + * suffix still requires reconciliation before handoff. */ +export function isSettledRemoteWarmControlState(control: Record, identity: Record): boolean { const record = (value: unknown): Record => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; const commands = control.commands; @@ -34,13 +35,17 @@ export function isSettledRemoteWarmControlState(control: Record const envelope = record(record(event).envelope); const notification = record(envelope.payload); const payload = record(notification.payload); - return record(event).eventType === "session.goal.snapshot" && matchesSession(event) && + const eventType = record(event).eventType; + const inert = eventType === "session.goal.snapshot" + ? payload.schema === "paperclip.session_goal.snapshot.v1" && payload.goal === null && payload.workingNow === false + : eventType === "provider.notice.recorded" && payload.schema === "paperclip.provider.notice.v1" && + payload.recoverable === true && (payload.severity === "info" || payload.severity === "warning"); + return inert && matchesSession(event) && Object.entries(identity).every(([key, value]) => envelope[key] === value) && (notification.schema === "paperclip.prp.event.v1" || notification.schema === "paperclip.prp.event.v2") && - notification.eventType === "session.goal.snapshot" && + notification.eventType === eventType && notification.sourceInstanceId === identity.runnerInstanceId && notification.sourceKind === "runner" && - notification.runId === identity.runId && notification.normalizedSessionId === identity.normalizedSessionId && - payload.schema === "paperclip.session_goal.snapshot.v1" && payload.goal === null && payload.workingNow === false; + notification.runId === identity.runId && notification.normalizedSessionId === identity.normalizedSessionId; }); } diff --git a/server/src/services/offline-migration-execution.ts b/server/src/services/offline-migration-execution.ts new file mode 100644 index 00000000000..2c0b0ed9e8d --- /dev/null +++ b/server/src/services/offline-migration-execution.ts @@ -0,0 +1,18 @@ +import { and, eq, isNotNull, not, or } from "drizzle-orm"; +import { heartbeatRuns } from "@paperclipai/db"; +import { noRetainedExecutionPredicate } from "./pre-dispatch-execution-evidence.js"; + +/** Only for offline maintenance after every application deployment has stopped. + * Never-started queued turns are durable admission intent, not filesystem + * writers. Keep their rows untouched so the replacement can claim them normally. + * A requeued execution or one with retained execution ownership still blocks. */ +export function offlineMigrationExecutionWhere() { + return or( + eq(heartbeatRuns.status, "running"), + and(eq(heartbeatRuns.status, "queued"), or( + not(noRetainedExecutionPredicate()!), + isNotNull(heartbeatRuns.runnerInstanceId), + isNotNull(heartbeatRuns.nativePhase), + )), + ); +} diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index f935b8a2df9..b380f431ed5 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -975,6 +975,7 @@ export function recoveryService( ) => boolean; liveRunExecutions?: Readonly<{ has(id: string): boolean }>; beforeOrphanedRunTerminalWrite?: (runId: string) => Promise; + beforeStaleLockClear?: (issueId: string) => Promise; }, ) { const issuesSvc = issueService(db); @@ -6477,6 +6478,26 @@ export function recoveryService( if (!status) return true; // missing run row → no real claim return TERMINAL_HEARTBEAT_RUN_STATUSES.has(status); }; + // A native failure that schedules a same-run resume marks its run failed + // but keeps both task locks for that resume. Clearing them lets another + // run take the task first, and then the resume can never run. Only a + // closed task releases them here. + const pendingResumeLock = sql`exists (select 1 from ${nativeRunFinalizations} pending_resume + where pending_resume.company_id = ${issues.companyId} + and pending_resume.run_id in (${issues.checkoutRunId}, ${issues.executionRunId}) + and pending_resume.phase = 'retryable_failure' + and pending_resume.result_id is null)`; + const lockRunLive = sql`exists (select 1 from ${heartbeatRuns} lock_run + where lock_run.id in (${issues.checkoutRunId}, ${issues.executionRunId}) + and lock_run.status not in (${sql.join([...TERMINAL_HEARTBEAT_RUN_STATUSES].map(status => sql`${status}`), sql`, `)}))`; + const pendingResumeRunIds = new Set(referencedRunIds.length === 0 ? [] : (await db + .select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations) + .where(and( + inArray(nativeRunFinalizations.runId, referencedRunIds), + eq(nativeRunFinalizations.phase, "retryable_failure"), + isNull(nativeRunFinalizations.resultId), + ))).map(row => row.runId)); for (const issue of candidates) { const originalOwner = issue.executionRunId ? runById.get(issue.executionRunId) : undefined; @@ -6497,7 +6518,13 @@ export function recoveryService( ) { continue; } + const taskClosed = issue.status === "done" || issue.status === "cancelled"; + if (!taskClosed && [issue.checkoutRunId, issue.executionRunId] + .some(runId => runId !== null && pendingResumeRunIds.has(runId))) { + continue; + } + await deps.beforeStaleLockClear?.(issue.id); const updated = await db.transaction(async tx => { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); @@ -6519,6 +6546,10 @@ export function recoveryService( issue.executionRunId ? eq(issues.executionRunId, issue.executionRunId) : isNull(issues.executionRunId), + // The reads above can be stale here. A resume claim can return a + // referenced run to running, and a failure can schedule a resume. + not(lockRunLive), + taskClosed ? undefined : not(pendingResumeLock), ), ) .returning({ id: issues.id }) diff --git a/server/src/services/remote-workspace-migration-tree.ts b/server/src/services/remote-workspace-migration-tree.ts index 3c7deed9113..4685a231f62 100644 --- a/server/src/services/remote-workspace-migration-tree.ts +++ b/server/src/services/remote-workspace-migration-tree.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { isDeepStrictEqual } from "node:util"; import { and, eq, inArray, ne, sql } from "drizzle-orm"; import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, workspaceOperations, type Db } from "@paperclipai/db"; +import { offlineMigrationExecutionWhere } from "./offline-migration-execution.js"; import type { StorageService } from "../storage/types.js"; import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js"; @@ -41,7 +42,7 @@ export function validateImportedWorkspaceChildren(raw: unknown): ImportedWorkspa export async function offlineWorkspaceMigrationBlockers(db: ReadDb) { if (process.env.PAPERCLIP_EPHEMERAL_MIGRATION !== "true") return ["offline_maintenance_required"]; const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) - .where(inArray(heartbeatRuns.status, ["running", "queued"])).limit(1); + .where(offlineMigrationExecutionWhere()).limit(1); return active ? ["active_execution_requires_drain"] : []; } diff --git a/server/src/services/remote-workspace-migration.test.ts b/server/src/services/remote-workspace-migration.test.ts index 8b75b7be4e1..498ac0797d7 100644 --- a/server/src/services/remote-workspace-migration.test.ts +++ b/server/src/services/remote-workspace-migration.test.ts @@ -112,14 +112,43 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true })).rejects.toThrow("offline_maintenance_required"); await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })).rejects.toThrow("offline_maintenance_required"); }); - it.each(["running", "queued"])("refuses offline archived imports while any execution is %s", async status => { + it.each(["running", "queued"])("refuses offline archived imports while any execution is running or requeued (%s)", async status => { const f = await seed(); await retainDirectory(f); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Unrelated writer" }).returning(); - await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, status }); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, status, startedAt: status === "queued" ? new Date() : null }); await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })) .rejects.toThrow("active_execution_requires_drain"); expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); }); + it.each(["startedAt", "runtimeModeResolvedAt", "processPid", "processStartedAt", "runnerInstanceId", "nativeSessionId", "nativePhase", "controllerLeaseExpiresAt", "executionStage", "checkpoint", "finalizer", "lease"])("keeps queued execution with %s as an offline blocker", async evidence => { + const f = await seed(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Retained execution" }).returning(); + const patch = evidence.endsWith("At") ? { [evidence]: new Date() } + : evidence === "processPid" ? { processPid: 123 } + : evidence.endsWith("Id") ? { [evidence]: randomUUID() } + : evidence === "nativePhase" ? { nativePhase: "observed" } + : evidence === "executionStage" ? { executionStage: "dispatch" } + : evidence === "checkpoint" ? { runnerProfileJson: { sessionCheckpoint: {} } } : {}; + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, + nativeIssueId: evidence === "finalizer" ? f.issueId : null, status: "queued", ...patch }).returning(); + if (evidence === "finalizer") await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, + issueId: f.issueId, runId: run!.id, phase: "observed" }); + if (evidence === "lease") await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: run!.id }); + await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })) + .rejects.toThrow("active_execution_requires_drain"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + }); + it("preserves an unstarted queued turn during offline import while live migration still refuses it", async () => { + const f = await seed(); + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Queued turn" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, + issueId: f.issueId, status: "queued", contextSnapshot: { issueId: f.issueId, executionWorkspaceId: f.workspaceId } }).returning(); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("active_execution"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id)))[0]).toEqual(run); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).not.toBeNull(); + }); it("detects overlapping folders reached through symlinked parent directories", async () => { const f = await seed(), alias = path.join(root, "alias"); await fs.symlink(root, alias); await db.insert(executionWorkspaces).values({ companyId: f.companyId, name: "Physical alias", mode: "isolated_workspace", diff --git a/server/src/services/remote-workspace-migration.ts b/server/src/services/remote-workspace-migration.ts index 7ba8b29a79c..f95ed723b3f 100644 --- a/server/src/services/remote-workspace-migration.ts +++ b/server/src/services/remote-workspace-migration.ts @@ -8,6 +8,7 @@ import { heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, workspaceOperations, workspaceRuntimeServices, type Db, } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared"; +import { offlineMigrationExecutionWhere } from "./offline-migration-execution.js"; import type { StorageService } from "../storage/types.js"; import { persistActivity, publishActivity } from "./activity-log.js"; import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; @@ -93,7 +94,7 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri if (workspace.mode === "shared_workspace" && (issuePolicy.sharedWorkspaceConcurrency ?? (projectPolicy.enabled === true ? projectPolicy.sharedWorkspaceConcurrency : undefined)) === "allow") reasons.push("parallel_shared_writers_unsupported"); if (issue?.executionRunId || issue?.checkoutRunId) reasons.push("task_writer_bound"); const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, workspace.companyId), - inArray(heartbeatRuns.status, ["queued", "running"]), sql`(${heartbeatRuns.nativeIssueId} = ${workspace.sourceIssueId}::uuid + input.writersStopped ? offlineMigrationExecutionWhere() : inArray(heartbeatRuns.status, ["queued", "running"]), sql`(${heartbeatRuns.nativeIssueId} = ${workspace.sourceIssueId}::uuid or ${heartbeatRuns.issueId} = ${workspace.sourceIssueId}::uuid or ${heartbeatRuns.contextSnapshot}->>'executionWorkspaceId' = ${workspace.id} or ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'workspaceId' = ${workspace.id})`, diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index 88b13a85991..523e8c40848 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -50,7 +50,11 @@ describe("remote workspace transport safety with real sandbox commands", () => { return root; } - function target(ambientEnv: Record = {}) { + function target(ambientEnv: Record = {}, outputMode: "split" | "one-shot" = "split") { + // Daytona one-shot execution returns the child streams together in stdout. + // Run the real helper first, then apply that provider response contract. + const streams = (stdout: string, stderr: string) => outputMode === "one-shot" + ? { stdout: stdout + stderr, stderr: "" } : { stdout, stderr }; const execute = vi.fn(async input => { try { const result = await exec(input.command, input.args ?? [], { @@ -58,11 +62,12 @@ describe("remote workspace transport safety with real sandbox commands", () => { // Carry only generated fixture values across this execution boundary. env: { PATH: process.env.PATH, ...ambientEnv, ...input.env }, }); - return { ...result, exitCode: 0, timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }; + return { ...streams(result.stdout, result.stderr), exitCode: 0, timedOut: false, signal: null, + pid: process.pid, startedAt: new Date().toISOString() }; } catch (error) { const failure = error as { code?: number; stdout?: string; stderr?: string; signal?: string }; - return { exitCode: typeof failure.code === "number" ? failure.code : 1, stdout: failure.stdout ?? "", - stderr: failure.stderr ?? "", timedOut: false, signal: failure.signal ?? null, + return { exitCode: typeof failure.code === "number" ? failure.code : 1, + ...streams(failure.stdout ?? "", failure.stderr ?? ""), timedOut: false, signal: failure.signal ?? null, pid: process.pid, startedAt: new Date().toISOString() }; } }); @@ -287,7 +292,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { expect(await fs.readdir(outside)).toEqual(["workspace.pcws"]); }); - it("preserves only the fixed unfinished Git operation diagnostic across capture commands", async () => { + it.each(["split", "one-shot"] as const)("preserves only the fixed unfinished Git operation diagnostic with %s output", async outputMode => { await git(workspace, "init", "--initial-branch=main"); await fs.writeFile(path.join(workspace, "work.txt"), "base\n"); await git(workspace, "add", "."); await git(workspace, "commit", "-m", "base"); await git(workspace, "checkout", "-b", "feature"); @@ -295,10 +300,14 @@ describe("remote workspace transport safety with real sandbox commands", () => { await git(workspace, "checkout", "main"); await fs.writeFile(path.join(workspace, "work.txt"), "main\n"); await git(workspace, "commit", "-am", "main"); await expect(git(workspace, "merge", "--no-edit", "feature")).rejects.toThrow(); - const { remote, execute } = target(); + const mergeHead = await fs.readFile(path.join(workspace, ".git", "MERGE_HEAD"), "utf8"); + const conflictedWork = await fs.readFile(path.join(workspace, "work.txt"), "utf8"); + const { remote, execute } = target({}, outputMode); const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace"; const operationId = randomUUID(); await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId })).rejects.toThrow(gitOperation); + expect(await fs.readFile(path.join(workspace, ".git", "MERGE_HEAD"), "utf8")).toBe(mergeHead); + expect(await fs.readFile(path.join(workspace, "work.txt"), "utf8")).toBe(conflictedWork); const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId); for (const name of ["workspace.pcws", "receipt.json"]) await expect(fs.access(path.join(directory, name))).rejects.toMatchObject({ code: "ENOENT" }); const failure = { exitCode: 1, stdout: "", stderr: `${gitOperation}\nSYNTHETIC_PRIVATE_PATH_OR_CREDENTIAL`, @@ -312,9 +321,11 @@ describe("remote workspace transport safety with real sandbox commands", () => { }); it.each([ - { step: "helper install", failing: String.raw`/archive\.cjs\.[^/]+\.tmp$/` }, - { step: "archive write", failing: String.raw`/workspace\.pcws$/` }, - ])("names a full sandbox disk when the $step runs out of space", async ({ failing }) => { + { step: "helper install", failing: String.raw`/archive\.cjs\.[^/]+\.tmp$/`, outputMode: "split" as const }, + { step: "archive write", failing: String.raw`/workspace\.pcws$/`, outputMode: "split" as const }, + { step: "helper install", failing: String.raw`/archive\.cjs\.[^/]+\.tmp$/`, outputMode: "one-shot" as const }, + { step: "archive write", failing: String.raw`/workspace\.pcws$/`, outputMode: "one-shot" as const }, + ])("names a full sandbox disk when the $step runs out of space with $outputMode output", async ({ failing, outputMode }) => { await fs.writeFile(path.join(workspace, "result.txt"), "synthetic saved result\n"); // Every sandbox Node process inherits this preload. A matching write keeps // one byte and then fails with ENOSPC, as a real full disk does. @@ -327,7 +338,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { fs.open=async(file,...rest)=>{const handle=await open(file,...rest);if(failing.test(String(file))){const write=handle.writeFile.bind(handle); handle.writeFile=async data=>{await write(Buffer.from(data).subarray(0,1));throw enospc(file);};}return handle;}; require('node:module').syncBuiltinESMExports();`); - const { remote, execute } = target(); + const { remote, execute } = target({}, outputMode); const run = execute.getMockImplementation()!; execute.mockImplementation(input => run({ ...input, env: { ...input.env, NODE_OPTIONS: `--require ${preload}` } })); const operationId = randomUUID(); @@ -336,6 +347,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId); const leftovers = await fs.readdir(directory).catch(() => []); expect(leftovers.filter(name => name !== "archive.cjs")).toEqual([]); + expect(await fs.readFile(path.join(workspace, "result.txt"), "utf8")).toBe("synthetic saved result\n"); }); it("keeps the full-disk diagnostic exact", async () => { @@ -347,6 +359,48 @@ describe("remote workspace transport safety with real sandbox commands", () => { .rejects.toThrow(/^remote_workspace_operation_failed$/); }); + it.each([ + { label: "private stdout", stdout: "SYNTHETIC_PRIVATE_STDOUT", stderr: "" }, + { label: "private stderr", stdout: "", stderr: "SYNTHETIC_PRIVATE_STDERR" }, + { label: "suffixed stdout diagnostic", stdout: "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace\nSYNTHETIC_PRIVATE_PATH", stderr: "" }, + { label: "prefixed stdout diagnostic", stdout: "SYNTHETIC_PRIVATE_PATH\nremote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace", stderr: "" }, + { label: "unrelated helper diagnostic", stdout: "remote_workspace_capture_failed", stderr: "" }, + { label: "diagnostic split between streams", stdout: "remote_workspace_sandbox_", stderr: "disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace" }, + ])("keeps $label private", async ({ stdout, stderr }) => { + const { remote, execute } = target(); + execute.mockResolvedValueOnce({ exitCode: 1, stdout, stderr, timedOut: false, signal: null, + pid: process.pid, startedAt: new Date().toISOString() }); + await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow(/^remote_workspace_operation_failed$/); + expect(execute).toHaveBeenCalledTimes(1); + }); + + it("keeps the existing safe stderr diagnostic when stdout reports another safe failure", async () => { + const { remote, execute } = target(); + const diskFull = "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace"; + execute.mockResolvedValueOnce({ exitCode: 1, stdout: diskFull, + stderr: "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace", + timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow(/^remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace$/); + }); + + it("captures and restores verified bytes with one-shot combined output", async () => { + await fs.writeFile(path.join(workspace, "saved.txt"), "saved synthetic one-shot bytes\n"); + const { remote, execute } = target({}, "one-shot"); + const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }); + try { + expect(createHash("sha256").update(await fs.readFile(captured.archivePath)).digest("hex")).toBe(captured.sha256); + const restoredRoot = path.join(scratch, "restored-one-shot"); + const restored = await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: restoredRoot, + archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, restorationIdentity: "c".repeat(64) }); + expect(await fs.readFile(path.join(restoredRoot, "saved.txt"), "utf8")).toBe("saved synthetic one-shot bytes\n"); + expect(await fs.readFile(path.join(workspace, "saved.txt"), "utf8")).toBe("saved synthetic one-shot bytes\n"); + await cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: restored.stagingRoot, operationId: restored.operationId }); + expect(execute.mock.calls.every(([input]) => input.bypassSession === true)).toBe(true); + } finally { await captured.cleanup(); } + }); + it("identifies a timed-out preparation without exposing transport output or retrying it", async () => { const { remote, execute } = target(); const resolveGitAuth = vi.fn(async () => null); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index aa9df01d06c..fc95be3b5db 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -55,8 +55,11 @@ async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspace if (result.timedOut) throw new RemoteWorkspaceOperationTimeoutError(operation, timeoutMs); if (result.exitCode !== 0) { const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace"; - const reported = result.stderr.trim(); - throw new Error([gitOperation, REMOTE_WORKSPACE_SANDBOX_DISK_FULL].includes(reported) ? reported : "remote_workspace_operation_failed"); + // One-shot providers can return combined output in stdout. Keep the same + // exact safe diagnostics; never forward arbitrary output from either stream. + const reported = [result.stderr.trim(), result.stdout.trim()] + .find(value => [gitOperation, REMOTE_WORKSPACE_SANDBOX_DISK_FULL].includes(value)); + throw new Error(reported ?? "remote_workspace_operation_failed"); } return result.stdout; } diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index d87caddcc77..0ef9e7ccdf3 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -38,19 +38,30 @@ import { issueService } from "./issues.js"; import { instanceSettingsService } from "./instance-settings.js"; import { resolveRemoteWorkspaceSource } from "./heartbeat/remote-workspace-source.js"; import { toExecutionWorkspace } from "./execution-workspaces.js"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "../__tests__/helpers/recovery-postgres.js"; const exec = promisify(execFile); -const databaseUrl = process.env.PAPERCLIP_REMOTE_WORKSPACE_TEST_DATABASE_URL; +let databaseUrl = process.env.PAPERCLIP_REMOTE_WORKSPACE_TEST_DATABASE_URL; if (databaseUrl && !/^remote_workspace_[a-z_]*test$/.test(new URL(databaseUrl).pathname.slice(1))) throw new Error("Disposable workspace database required"); -const suite = databaseUrl ? describe : describe.skip; +const suite = databaseUrl || recoveryPostgresSupport.supported ? describe : describe.skip; const s3Bucket = process.env.PAPERCLIP_REMOTE_WORKSPACE_TEST_S3_BUCKET; if (s3Bucket && !s3Bucket.startsWith("paperclip-workspace-")) throw new Error("Disposable workspace S3 bucket required"); suite("remote workspace lifecycle with real Postgres and independent filesystem execution", () => { let db: ReturnType, root: string, storage: StorageService; + let temporary: Awaited> | undefined; let ids: { company: string; agent: string; issue: string; run: string; environment: string; lease: string }; - beforeAll(async () => { await applyPendingMigrations(databaseUrl!); db = createDb(databaseUrl!); }, 120_000); - afterAll(async () => { await closeRegisteredClients(databaseUrl!); }); + beforeAll(async () => { + if (!databaseUrl) { + temporary = await startRecoveryPostgres("remote-workspace-lifecycle-"); + databaseUrl = temporary.connectionString; + } + await applyPendingMigrations(databaseUrl); db = createDb(databaseUrl); + }, 120_000); + afterAll(async () => { + try { if (databaseUrl) await closeRegisteredClients(databaseUrl); } + finally { await temporary?.cleanup(); } + }); beforeEach(async () => { vi.stubEnv("DATABASE_URL", databaseUrl!); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "62".repeat(32)); root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-remote-workspace-lifecycle-"))); @@ -173,40 +184,51 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem expect(Boolean((await lease()).metadata?.remoteExecutionTermination)).toBe(receipt); } - it.each(["receipt", "void"] as const)("recovers a lost preparation reply after %s cleanup and retains the resulting work", async cleanup => { + it.each([["receipt", "fresh"], ["void", "fresh"], ["receipt", "reused"]] as const)( + "continues unpublished preparation after %s cleanup on a %s allocation and retains the resulting work", async (cleanup, allocation) => { await db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null, issueId: ids.issue }) .where(eq(heartbeatRuns.id, ids.run)); const reserved = await reserve({ kind: "empty" }), first = await target("lost-bootstrap-reply"); + const interrupted = allocation === "reused"; const actual = first.runner!.execute.bind(first.runner!); const execute = vi.spyOn(first.runner!, "execute").mockImplementationOnce(async input => { const result = await actual(input); expect(result.exitCode).toBe(0); - return { ...result, timedOut: true }; + return { ...result, timedOut: !interrupted }; }); - await expect(prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first })) - .rejects.toMatchObject({ code: "remote_workspace_prepare_timeout" }); - const oldRoot = path.join(first.remoteCwd, ".paperclip-workspaces", reserved.id); + let ownershipChecks = 0; + const preparation = prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first, + assertOwned: async () => { if (interrupted && ++ownershipChecks === 2) throw new Error("lease_released_before_terminal"); } }); + if (interrupted) await expect(preparation).rejects.toThrow("lease_released_before_terminal"); + else await expect(preparation).rejects.toMatchObject({ code: "remote_workspace_prepare_timeout" }); + const oldRoot = execute.mock.calls[0][0].args?.[2]; + if (!oldRoot) throw new Error("Preparation did not record its directory"); const oldReceipt = await fs.readFile(path.join(oldRoot, ".paperclip-runtime", "workspace-ready.json"), "utf8"); const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, ids.run)); - await terminalizeLegacyExecution({ db, run: failed!, status: "failed", patch: { - errorCode: "remote_workspace_prepare_timeout", resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + await terminalizeLegacyExecution({ db, run: failed!, status: interrupted ? "interrupted" : "failed", patch: { + errorCode: interrupted ? "lease_released_before_terminal" : "remote_workspace_prepare_timeout", + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, } }); await releaseBootstrapLease(cleanup === "receipt"); const continueOn = async (name: string) => { const runId = randomUUID(), leaseId = randomUUID(); + const reused = allocation === "reused" && name === "fresh-bootstrap-retry"; + const providerLeaseId = reused ? `allocation-${ids.lease}` : `allocation-${leaseId}`; await db.insert(heartbeatRuns).values({ id: runId, companyId: ids.company, agentId: ids.agent, nativeIssueId: ids.issue, runtimeMode: "native", status: "running" }); const [nextLease] = await db.insert(environmentLeases).values({ id: leaseId, companyId: ids.company, environmentId: ids.environment, heartbeatRunId: runId, issueId: ids.issue, provider: "daytona", - providerLeaseId: `allocation-${leaseId}`, status: "active" }).returning(); + providerLeaseId, status: "active" }).returning(); const next = await reserveRemoteWorkspace({ db, storage, companyId: ids.company, issueId: ids.issue, runId, agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, existingWorkspaceId: reserved.id, source: { kind: "empty" } }); - const remote = await target(name); + const remote = reused ? { ...first, leaseId, sandboxId: providerLeaseId, + runner: { ...first.runner!, execute: actual } } : await target(name); return prepareRemoteWorkspace({ db, storage, workspace: next, runId, lease: nextLease as EnvironmentLease, target: remote }).then(prepared => ({ runId, prepared, remote })); }; const { runId, prepared, remote } = await continueOn("fresh-bootstrap-retry"); + expect(prepared.remoteRoot).not.toBe(oldRoot); await fs.writeFile(path.join(prepared.remoteRoot, "report.md"), "useful recovered result\n"); const { snapshotId } = await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId, lease: prepared.lease, target: remote, assertOwned: async () => undefined }); @@ -597,7 +619,8 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem existingWorkspaceId: reserved.id, source: { kind: "empty" } }); const nextTarget = await target("restore-cleanup-destination"); const sandbox = nextTarget as Extract; - const expectedRemoteRoot = path.join(sandbox.remoteCwd, ".paperclip-workspaces", reserved.id); + const expectedRemoteRoot = path.join(sandbox.remoteCwd, ".paperclip-workspaces", + `${reserved.id}-${readRemoteWorkspaceAuthority(nextWorkspace.metadata)!.epoch}-${nextLease}`); const execute = sandbox.runner!.execute.bind(sandbox.runner!); let failCleanup = true, cleanupAttempts = 0; sandbox.runner!.execute = async input => { diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 18a31f71f5a..4712ae33568 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -344,7 +344,8 @@ export async function prepareRemoteWorkspace(input: { const sameAllocation = initial.location?.providerLeaseId === input.lease.providerLeaseId && initial.location?.environmentId === input.lease.environmentId; if (initial.location && !sameAllocation && initial.dirty) throw conflict("The prior workspace has unsaved work; recover it before replacing its allocation."); - const remoteRoot = sameAllocation ? initial.location!.remoteCwd : path.posix.join(target.remoteCwd, ".paperclip-workspaces", input.workspace.id); + const remoteRoot = sameAllocation ? initial.location!.remoteCwd : path.posix.join(target.remoteCwd, ".paperclip-workspaces", + `${input.workspace.id}-${initial.epoch}-${input.lease.id}`); const preparationIdentity = createHash("sha256").update(JSON.stringify([input.workspace.companyId, input.workspace.id, input.runId, initial.epoch, input.lease.id, input.lease.providerLeaseId, remoteRoot, initial.source])).digest("hex"); let restoredExport: Awaited> | null = null; diff --git a/server/src/services/run-evidence-migration.integration.test.ts b/server/src/services/run-evidence-migration.integration.test.ts index 7a57c72d91b..1d9cfd463ba 100644 --- a/server/src/services/run-evidence-migration.integration.test.ts +++ b/server/src/services/run-evidence-migration.integration.test.ts @@ -35,7 +35,11 @@ describe.skipIf(!endpoint || !databaseUrl)("run evidence migration (Postgres and await db.execute(sql.raw(`create schema ${schema}`)); await db.execute(sql.raw(`set search_path=${schema}`)); await db.execute(sql`create table heartbeat_runs (id uuid primary key, company_id uuid not null, agent_id uuid not null, - status text not null, runtime_mode text default 'legacy', log_store text, log_ref text, log_bytes bigint, log_sha256 text, last_output_seq int not null default 0, controller_lease_expires_at timestamptz)`); + status text not null, runtime_mode text default 'legacy', log_store text, log_ref text, log_bytes bigint, log_sha256 text, last_output_seq int not null default 0, controller_lease_expires_at timestamptz, started_at timestamptz, process_pid int, process_started_at timestamptz, + runner_instance_id uuid, native_session_id uuid, native_phase text, runtime_mode_resolved_at timestamptz, + process_group_id int, native_issue_id uuid, session_id_after text, controller_boot_id uuid, + execution_stage text, result_json jsonb, runner_profile_json jsonb)`); + await db.execute(sql`create table environment_leases(company_id uuid, heartbeat_run_id uuid)`); await db.execute( sql`create table native_run_finalizations(run_id uuid, company_id uuid, lease_expires_at timestamptz)`, ); @@ -123,7 +127,11 @@ describe.skipIf(!endpoint || !databaseUrl)("run evidence migration (Postgres and await db.execute(sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${active},${randomUUID()},${randomUUID()},'running')`); await expect(migrateRunEvidence(db, options)).rejects.toThrow("active_execution_requires_drain"); await db.execute(sql`delete from heartbeat_runs where id=${active}`); + await db.execute(sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${active},${companyId},${agentId},'queued')`); const result = await migrateRunEvidence(db, options); + expect((await db.$client.unsafe("select status,started_at from heartbeat_runs where id=$1", [active]))[0]) + .toMatchObject({ status: "queued", started_at: null }); + await db.execute(sql`delete from heartbeat_runs where id=${active}`); const store = getRunLogStore(); for (const name of cases.slice(0, 3)) { const run = runs.get(name)!; diff --git a/server/src/services/run-evidence-migration.ts b/server/src/services/run-evidence-migration.ts index 66d0f786323..41d636a2f55 100644 --- a/server/src/services/run-evidence-migration.ts +++ b/server/src/services/run-evidence-migration.ts @@ -9,6 +9,7 @@ import { providerTraceRecords, type Db, } from "@paperclipai/db"; +import { offlineMigrationExecutionWhere } from "./offline-migration-execution.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { createObjectByteJournal } from "./object-byte-journal.js"; import { @@ -63,7 +64,7 @@ export async function migrateRunEvidence( async function assertNoActiveRuns() { options.signal?.throwIfAborted(); const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) - .where(inArray(heartbeatRuns.status, ["queued", "running"])).limit(1); + .where(offlineMigrationExecutionWhere()).limit(1); if (active) throw new Error("active_execution_requires_drain"); } if (preserveHistoricalFiles) { diff --git a/server/src/services/task-retry.test.ts b/server/src/services/task-retry.test.ts index 363663258be..a05fd80146c 100644 --- a/server/src/services/task-retry.test.ts +++ b/server/src/services/task-retry.test.ts @@ -1,10 +1,12 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { agents, agentWakeupRequests, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueComments, issues, issueRecoveryActions, nativeRunFinalizations, toolApplications, toolConnections, toolInvocations, applyPendingMigrations } from "@paperclipai/db"; +import { agents, agentWakeupRequests, chatConversations, chatEndpoints, companies, createDb, type Db, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueComments, issues, issueRecoveryActions, nativeRunFinalizations, toolApplications, toolConnections, toolInvocations, applyPendingMigrations } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { priorRecoveryApiReceipt, unresolvedRecoveryToolInvocation } from "./recovery-effect-receipts.js"; import { requestTaskRetry } from "./task-retry.js"; +import { HttpError } from "../errors.js"; +import { NativeChatReviewPresentationContentionError } from "./native-runtime/native-chat-review-presentation.js"; import { readFreshWorkspaceReadiness } from "./fresh-workspace-continuation.js"; import { settleUnrecoverableExecutions } from "./execution-recovery-resolution.js"; import { createSystemRecovery, failSystemRecovery, readSystemRecovery, SystemRecoveryFailure } from "./system-recovery-state.js"; @@ -441,4 +443,73 @@ describe("durable task Retry admission", () => { .toMatchObject({ status: "skipped", error: "task_retry_superseded" }); }); + async function seedChatTask() { + const input = await seed(); + const [issue] = await db.select().from(issues).where(eq(issues.id, input.issueId)); + const [application] = await db.insert(toolApplications).values({ companyId: input.companyId, name: "Slack", type: "chat" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: input.companyId, applicationId: application.id, + name: "Slack", uid: randomUUID(), transport: "chat_sdk", connectionPurpose: "channel" }).returning(); + const [endpoint] = await db.insert(chatEndpoints).values({ companyId: input.companyId, connectionId: connection.id, + provider: "slack", publicId: randomUUID(), assignedAgentId: issue.assigneeAgentId!, status: "active" }).returning(); + await db.insert(chatConversations).values({ companyId: input.companyId, endpointId: endpoint.id, issueId: input.issueId, + externalConversationId: "channel", externalThreadId: randomUUID(), externalLabel: "#retry", state: "waiting" }); + return input; + } + const chatIdentityLock = (companyId: string, principalId: string) => + sql`pg_try_advisory_xact_lock(hashtextextended(${`chat-identity:${companyId}:${principalId}`}, 0))`; + + it("waits out brief chat authorization contention without a second request", async () => { + const input = await seedChatTask(); + const principalId = randomUUID(), actionId = randomUUID(); + let calls = 0, locked!: () => void, release!: () => void; + const held = new Promise(resolve => { locked = resolve; }); + const mayRelease = new Promise(resolve => { release = resolve; }); + // Another chat transaction owns this principal's authorization lock. + const holder = db.transaction(async tx => { + expect((await tx.execute<{ locked: boolean }>(sql`select ${chatIdentityLock(input.companyId, principalId)} as locked`))[0].locked).toBe(true); + locked(); await mayRelease; + }); + await held; + const prepareChatRetry = async (tx: Db) => { + calls += 1; + if (calls === 2) release(); + // Mirror the production authorization check, including its rollback. + const [row] = await tx.execute<{ locked: boolean }>(sql`select ${chatIdentityLock(input.companyId, principalId)} as locked`); + if (!row?.locked) throw new NativeChatReviewPresentationContentionError(); + return { actionId }; + }; + try { + const receipt = await requestTaskRetry({ ...input, prepareChatRetry }); + expect(receipt).toMatchObject({ accepted: true, chatActionId: actionId }); + expect(calls).toBeGreaterThan(1); + const rows = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, input.companyId)); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: receipt.requestId, status: "deferred_issue_execution", + payload: { taskRetry: { chatActionId: actionId } } }); + expect((await requestTaskRetry({ ...input, prepareChatRetry })).requestId).toBe(receipt.requestId); + } finally { release(); await holder; } + }); + + it("returns a retryable conflict when chat authorization stays busy", async () => { + const input = await seedChatTask(); + let calls = 0; + const failure = await requestTaskRetry({ ...input, prepareChatRetry: async () => { + calls += 1; + throw new NativeChatReviewPresentationContentionError(); + } }).catch(error => error); + expect(failure).toBeInstanceOf(HttpError); + expect(failure).toMatchObject({ status: 409, details: { code: "task_retry_busy", retryable: true } }); + expect(calls).toBe(10); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, input.companyId))).toHaveLength(0); + }); + + it("does not retry an authorization denial", async () => { + const input = await seedChatTask(); + let calls = 0; + await expect(requestTaskRetry({ ...input, prepareChatRetry: async () => { + calls += 1; + throw new HttpError(403, "denied"); + } })).rejects.toMatchObject({ status: 403 }); + expect(calls).toBe(1); + }); }); diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index 31180616cbb..930e753529c 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -21,6 +21,8 @@ import { deferredChatInputPredicate, resumeDeferredChatWake } from "./durable-ch import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js"; import { hasPendingRunCleanup } from "./execution-blocker.js"; import { PersistedExecutionWaitRefusal } from "./deferred-execution-wait.js"; +import { NativeChatReviewPresentationContentionError } from "./native-runtime/native-chat-review-presentation.js"; +import { isExternalChatWaitAuthorizationContention } from "./native-runtime/chat-attachment-reuse.js"; export async function taskRetryMutationAllowed(db: Db, input: { companyId: string; issueId: string; actorId: string; actorSource: TaskRetryIntent["actorSource"]; @@ -206,14 +208,39 @@ export async function retireSupersededTaskRetry(db: Db, input: { return retired.length > 0; } -/** Persist a deliberate click under dispatch's issue-first lock order. This - * transaction never contacts a provider. The normal wake queue owns execution. */ -export async function requestTaskRetry(input: { +type TaskRetryRequest = { db: Db; companyId: string; issueId: string; actorId: string; idempotencyKey: string; observedRunId: string | null; actorSource?: string; prepareChatRetry?: (tx: Db, scope: { companyId: string; issueId: string; agentId: string; failedRunId: string; initiatedByUserId: string; retryIntentId?: string }) => Promise<{ actionId: string }>; -}) { +}; + +const TASK_RETRY_CONTENTION_ATTEMPTS = 10; +const TASK_RETRY_CONTENTION_DELAY_MS = 100; + +function isTaskRetryContention(error: unknown) { + return error instanceof NativeChatReviewPresentationContentionError || isExternalChatWaitAuthorizationContention(error); +} + +/** Persist a deliberate click under dispatch's issue-first lock order. This + * transaction never contacts a provider. The normal wake queue owns execution. */ +export async function requestTaskRetry(input: TaskRetryRequest) { + for (let attempt = 1; ; attempt += 1) { + try { + return await admitTaskRetry(input); + } catch (error) { + if (!isTaskRetryContention(error)) throw error; + // Chat authorization for the same principal is briefly locked. The whole + // admission rolled back, so a fresh attempt rereads every fact. + if (attempt >= TASK_RETRY_CONTENTION_ATTEMPTS) + throw conflict("Retry is busy because chat authorization is in progress. Try again.", + { code: "task_retry_busy", retryable: true }); + } + await new Promise(resolve => setTimeout(resolve, TASK_RETRY_CONTENTION_DELAY_MS)); + } +} + +function admitTaskRetry(input: TaskRetryRequest) { return writeExecutionControl(input.db, DELIVERY_QUEUES.workspaceRecovery, async tx => { const db = tx as unknown as Db; const [issue] = await tx.select().from(issues).where(and(eq(issues.companyId, input.companyId), diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index 3e3f6292565..5b61834c94c 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -1692,8 +1692,8 @@ export function toolAccessService( // application/connection rows. A fresh gallery click has no // applicationId, so recover that retained identity by its company-unique // name and source instead of inserting a duplicate application that the - // unique index rejects. Active applications are excluded: connecting a - // second account still requires its own name/application identity. + // unique index rejects. Archived personal identities are excluded below. + // Active applications also require a separate application identity. const requestedName = input.name ?? galleryEntry?.name ?? defaultLinkName(input.link ?? ""); const recoverableApplicationStatuses = galleryEntry @@ -1841,6 +1841,19 @@ export function toolAccessService( retainedConnection = undefined; existingApplication = null; } + // A fresh setup must not revive a personal identity found only by name. + // Keep its application, grants, credentials, and history archived, and let + // normal name allocation create a separate application/connection below. + // Explicit reconnects still use the fixed personal-owner check. + if ( + !requestedConnectionId && + !input.applicationId && + retainedConnection?.status === "archived" && + retainedConnection.credentialPolicy === "per_user" + ) { + existingApplication = null; + retainedConnection = undefined; + } let applicationName = existingApplication?.name ?? requestedName; let name = retainedConnection?.name ?? requestedName; if (!existingApplication) { @@ -1876,8 +1889,8 @@ export function toolAccessService( : "organization" : null; // An explicit resume/application reconnect continues the retained identity. - // A fresh gallery connect may still reuse an archived row for stable history - // and company-unique naming, but an explicit Access choice is a new identity + // A fresh gallery connect may still reuse an archived non-personal row for + // stable history, but an explicit Access choice is a new identity // decision and must replace the archived policy. Without this distinction, // removing a dedicated-agent connection and reconnecting the default personal // account leaves `per_agent` behind and the OAuth callback cannot persist its diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 7fa4babe6b5..c93da025b86 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -393,16 +393,20 @@ describe("automatic retained workspace recovery", () => { return { ...f, source, workspaceId, resultId, git, commit }; } - async function interruptedRemotePreparation() { + async function interruptedRemotePreparation(interruption: "shutdown" | "lease" = "shutdown") { const f = await acceptedRemoteExport(); await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); await db.delete(nativeRunResults).where(eq(nativeRunResults.runId, f.runId)); const recovery = { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] }; await db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null, completionContractId: null, - status: "interrupted", executionStage: "preparing", errorCode: "server_shutdown_interrupted", + status: "interrupted", executionStage: "preparing", + errorCode: interruption === "lease" ? "lease_released_before_terminal" : "server_shutdown_interrupted", controllerBootId: randomUUID(), - error: "Interrupted by graceful server shutdown (SIGTERM)", - resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery }, + error: interruption === "lease" ? "Environment lease released before preparation settled" : "Interrupted by graceful server shutdown (SIGTERM)", + resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery, + ...(interruption === "lease" ? { cancellation: { source: "unknown", expected: false, + initiator: { type: "system" }, reason: "Environment lease released before preparation settled", + recordedAt: new Date().toISOString() } } : {}) }, }).where(eq(heartbeatRuns.id, f.runId)); await db.update(issues).set({ statusVersion: 2 }).where(eq(issues.id, f.issueId)); await db.update(issueRecoveryActions).set({ cause: "legacy_execution_requires_reconciliation", @@ -416,8 +420,8 @@ describe("automatic retained workspace recovery", () => { return f; } - it("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace", async () => { - const f = await interruptedRemotePreparation(); + it.each(["shutdown", "lease"] as const)("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace (%s)", async interruption => { + const f = await interruptedRemotePreparation(interruption); await recoverRetainedWorkspace(f.input); expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true); const intent = { observedRunId: f.runId, idempotencyKey: randomUUID() }; @@ -441,8 +445,9 @@ describe("automatic retained workspace recovery", () => { finally { await saved.cleanup(); } }, 30_000); - it.each([false, true])("automatically retries interrupted preparation after preservation, including recovered history at startup=%s", async alreadyRecovered => { - const f = await interruptedRemotePreparation(); + it.each([false, true].flatMap(alreadyRecovered => (["shutdown", "lease"] as const).map(interruption => ({ alreadyRecovered, interruption }))))( + "automatically retries interrupted preparation after preservation ($interruption, startup=$alreadyRecovered)", async ({ alreadyRecovered, interruption }) => { + const f = await interruptedRemotePreparation(interruption); if (alreadyRecovered) await recoverRetainedWorkspace(f.input); const restarted = createDb(connectionString); try { @@ -462,6 +467,60 @@ describe("automatic retained workspace recovery", () => { } finally { await restarted.$client.end(); } }, 30_000); + it("retries unpublished preparation from its preserved ancestor without an old failure-block receipt", async () => { + const f = await seed("legacy", false); + await requestWorkspaceRecovery({ ...f, idempotencyKey: randomUUID(), actorId: "board" }); + await recoverRetainedWorkspace(f.input); + const ancestor = await run(f.runId), leaseId = randomUUID(), providerLeaseId = randomUUID(); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: ancestor.agentId, status: "running" }); + const app = boardRecoveryApp(), heartbeat = heartbeatService(db); + const first = (await request(app).post(`/api/issues/${f.issueId}/retry`).send({ + observedRunId: f.runId, idempotencyKey: randomUUID(), + }).expect(202)).body; + await heartbeat.resumeTaskRetry(f.companyId, first.requestId); + const admitted = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + eq(heartbeatRuns.status, "queued"))); + expect(admitted).toHaveLength(1); + const interruptedId = admitted[0].id; + await db.update(heartbeatRuns).set({ runtimeMode: "legacy", status: "interrupted", executionStage: "preparing", + startedAt: new Date(Date.now() - 60_000), finishedAt: new Date(), controllerBootId: randomUUID(), + errorCode: "lease_released_before_terminal", error: "Lease released before preparation settled", + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, + resultJson: { cancellation: { source: "unknown", expected: false, initiator: { type: "system" }, + reason: "Lease released before preparation settled", recordedAt: new Date().toISOString() } }, + }).where(eq(heartbeatRuns.id, interruptedId)); + await db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId)); + const lease = { id: leaseId, companyId: f.companyId, heartbeatRunId: interruptedId, + provider: "daytona", providerLeaseId }; + await db.insert(environmentLeases).values({ ...lease, issueId: f.issueId, environmentId: f.lease.environmentId, + status: "released", cleanupStatus: "success", releasedAt: new Date(), + metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId, state: "stopped" }) }, + }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", ownerType: "board", returnOwnerAgentId: ancestor.agentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${interruptedId}`, + status: "resolved", outcome: "blocked", nextAction: "Recorded actions require reconciliation", + evidence: { runId: interruptedId, originalFailureCode: "lease_released_before_terminal", + automaticRecovery: { policy: "preserve_without_replay_v1", runId: interruptedId, replay: "blocked", actionOutcome: "unknown" } }, + }); + const intent = { observedRunId: interruptedId, idempotencyKey: randomUUID() }; + const receipt = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body; + await Promise.all([heartbeat.resumeTaskRetry(f.companyId, receipt.requestId), + heartbeat.resumeTaskRetry(f.companyId, receipt.requestId)]); + const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + eq(heartbeatRuns.status, "queued"))); + expect(successors).toHaveLength(1); + expect(successors[0].contextSnapshot).toMatchObject({ previousRunId: interruptedId }); + expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body.requestId).toBe(receipt.requestId); + const restored = path.join(root, randomUUID()); await mkdir(restored); + const copies = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId, + sourceRunIds: [interruptedId], cwd: restored }); + expect(copies).toHaveLength(1); + expect(await readFile(path.join(copies[0]!.path, "files", "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n"); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + }, 30_000); + it.each(["local", "none"])("continues verified interrupted preparation with %s leases", async provider => { const f = await interruptedRemotePreparation(); await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, @@ -476,9 +535,10 @@ describe("automatic retained workspace recovery", () => { }, 30_000); it.each(["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop", + "operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption", "retry_exhausted", "human_reblock", "paused", "reassigned"] as const)( "keeps the interrupted preparation continuation behind %s", async gate => { - const f = await interruptedRemotePreparation(); + const f = await interruptedRemotePreparation("lease"); await recoverRetainedWorkspace(f.input); const source = await run(f.runId); if (gate === "provider_started") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, @@ -492,6 +552,17 @@ describe("automatic retained workspace recovery", () => { remoteWorkspace: { ...(source.runnerProfileJson!.remoteWorkspace as Record), snapshotId: randomUUID() }, } }).where(eq(heartbeatRuns.id, f.runId)); if (gate === "operator_stop") await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "operator_interrupted" }).where(eq(heartbeatRuns.id, f.runId)); + if (gate === "operator_stop_metadata") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson, + cancellation: requestedRunCancellation({ cancelledByActorType: "user", cancelledByUserId: "operator" }, "Stop requested"), + } }).where(eq(heartbeatRuns.id, f.runId)); + if (gate === "unrecorded_interruption") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson, + cancellation: null, + } }).where(eq(heartbeatRuns.id, f.runId)); + if (gate === "missing_stop_receipt") { + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ metadata: { ...lease.metadata, remoteExecutionTermination: null } }) + .where(eq(environmentLeases.id, f.lease.id)); + } if (gate === "retry_exhausted") await db.update(heartbeatRuns).set({ scheduledRetryAttempt: 2 }).where(eq(heartbeatRuns.id, f.runId)); if (gate === "human_reblock") await db.update(issues).set({ statusVersion: 3 }).where(eq(issues.id, f.issueId)); if (gate === "paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, source.agentId)); @@ -500,7 +571,8 @@ describe("automatic retained workspace recovery", () => { await heartbeat.resumeRemoteStopComments(await run(f.runId)); expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(0); - if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop"].includes(gate)) { + if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop", + "operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption"].includes(gate)) { const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board", actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() }); await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!); @@ -1743,7 +1815,9 @@ describe("automatic retained workspace recovery", () => { existingWorkspaceId: kind === "same" ? first.workspace.id : null }); const prepared = await prepareRemoteWorkspace({ db, storage, workspace: nextWorkspace, runId: nextRun.id, lease: nextLease as EnvironmentLease, target: { ...target, remoteCwd: resumed!.metadata!.remoteCwd as string } }); - expect(prepared.remoteRoot).toBe(path.join(allocationRoot, ".paperclip-workspaces", nextWorkspace.id)); + const expectedRemoteRoot = kind === "same" ? first.remoteRoot : path.join(allocationRoot, ".paperclip-workspaces", + `${nextWorkspace.id}-${readRemoteWorkspaceAuthority(nextWorkspace.metadata)!.epoch}-${nextLease.id}`); + expect(prepared.remoteRoot).toBe(expectedRemoteRoot); if (kind === "same") expect(await readFile(path.join(prepared.remoteRoot, "saved.txt"), "utf8")).toBe("prior task work\n"); else expect(prepared.remoteRoot).not.toBe(first.remoteRoot); await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId: nextRun.id, lease: prepared.lease, @@ -2361,7 +2435,7 @@ describe("automatic retained workspace recovery", () => { expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, actor)).toBe(true); expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, { ...actor, requestedByActorId: "other-user" })).toBe(false); await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(scope, { - companyId: f.companyId, agentId: original.agentId, activeExecutionRunId: queued.id, + companyId: f.companyId, agentId: original.agentId, activeExecutionRunId: queued.id, activeExecutionRunStatus: queued.status, mergedContextSnapshot: mergeCoalescedContextSnapshot((await run(queued.id)).contextSnapshot, incoming, { preserveExistingInteractionContinuation: true }), source: "on_demand", triggerDetail: "manual", diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index ad1d8ab4f6c..dc8de2e9f1a 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -447,7 +447,7 @@ export function retainedWorkspaceCandidates(db: Db, cursor?: string, unfinishedO and ${agentWakeupRequests.status} = 'deferred_issue_execution' and ${agentWakeupRequests.requestedByActorType} = 'user' and ${agentWakeupRequests.payload}->>'issueId' = ${issueRecoveryActions.sourceIssueId}::text)`; const interruptedPreparation = sql`(${heartbeatRuns.runtimeMode} = 'legacy' - and ${heartbeatRuns.status} = 'interrupted' and ${heartbeatRuns.errorCode} = 'server_shutdown_interrupted' + and ${heartbeatRuns.status} = 'interrupted' and ${heartbeatRuns.runtimeModeResolvedAt} is null and ${heartbeatRuns.executionStage} = 'preparing' and ${heartbeatRuns.runnerProfileJson}#>>'{adapterDispatch,adapterType}' = 'paperclip_runner' and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,replay}' = 'blocked' diff --git a/server/src/telemetry.ts b/server/src/telemetry.ts index eb2b28749e6..bfa7c1930bb 100644 --- a/server/src/telemetry.ts +++ b/server/src/telemetry.ts @@ -3,26 +3,38 @@ import { TelemetryClient, resolveTelemetryConfig, loadOrCreateState, + trackServerStarted, } from "@paperclipai/shared/telemetry"; import { resolvePaperclipInstanceRoot } from "./home-paths.js"; import { serverVersion } from "./version.js"; let client: TelemetryClient | null = null; +let initializationFailed = false; export function initTelemetry(fileConfig?: { enabled?: boolean }): TelemetryClient | null { - if (client) return client; + if (client || initializationFailed) return client; const config = resolveTelemetryConfig(fileConfig); if (!config.enabled) return null; - const stateDir = path.join(resolvePaperclipInstanceRoot(), "telemetry"); - client = new TelemetryClient( - config, - () => loadOrCreateState(stateDir, serverVersion), - serverVersion, - ); - client.startPeriodicFlush(60_000); - return client; + let pendingClient: TelemetryClient | null = null; + try { + const stateDir = path.join(resolvePaperclipInstanceRoot(), "telemetry"); + pendingClient = new TelemetryClient( + config, + () => loadOrCreateState(stateDir, serverVersion), + serverVersion, + ); + trackServerStarted(pendingClient, { os_platform: process.platform }); + pendingClient.startPeriodicFlush(60_000); + client = pendingClient; + return client; + } catch { + pendingClient?.stop(); + initializationFailed = true; + console.warn("[telemetry] Initialization failed; telemetry is disabled for this process."); + return null; + } } export function getTelemetryClient(): TelemetryClient | null { diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 80b70e5b9f2..51ddeff3ea2 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -182,6 +182,7 @@ export const validatePrpStructuredRunResult = runner.validatePrpStructuredRunResult; export const NativeProviderTerminalFailure = runner.NativeProviderTerminalFailure; +export const NativeProviderStateIncompatibleError = runner.NativeProviderStateIncompatibleError; export const nativeRestartInterruptedTurnId = runner.nativeRestartInterruptedTurnId; export const completeTerminatedRemoteNativeSessionCleanup = runner.completeTerminatedRemoteNativeSessionCleanup; diff --git a/tests/e2e/disconnect-write-recovery.spec.ts b/tests/e2e/disconnect-write-recovery.spec.ts index 1d7cc5d2fe1..1450aec855d 100644 --- a/tests/e2e/disconnect-write-recovery.spec.ts +++ b/tests/e2e/disconnect-write-recovery.spec.ts @@ -178,8 +178,9 @@ test("replaying an accepted comment does not reopen completed work", async ({ pa const activity = await json(await request.get(`/api/issues/${issue.id}/activity`)); expect(activity.filter((entry: { action: string; details?: { commentId?: string } }) => entry.action === "issue.comment_added" && entry.details?.commentId === first.id)).toHaveLength(1); - await page.goto(`/${company.issuePrefix}/issues/${issue.id}`); - await expect(page.getByRole("heading", { name: issue.title, exact: true })).toBeVisible(); + await page.goto(`/${company.issuePrefix}/issues/${issue.identifier}`); + await expect(page.getByTestId("task-chat-history-content")).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId("issue-detail-header").getByRole("heading", { name: issue.title, exact: true })).toBeVisible(); await expect(page.getByText(body, { exact: true })).toBeVisible(); } finally { await request.patch(`/api/companies/${company.id}`, { data: { status: "archived" } }); diff --git a/tests/e2e/task-loading-frame.spec.ts b/tests/e2e/task-loading-frame.spec.ts index f607b245059..e784e61dfc1 100644 --- a/tests/e2e/task-loading-frame.spec.ts +++ b/tests/e2e/task-loading-frame.spec.ts @@ -155,13 +155,14 @@ for (const coordinator of [false, true]) test(`mobile chrome stays fixed when pe let release!: () => void; const preferenceReady = new Promise(resolve => { release = resolve; }); await page.addInitScript(() => { - const frames: number[][] = []; - Object.assign(window, { mobileShellFrames: frames }); + const frames: number[][] = [], frameTimes: number[] = []; + Object.assign(window, { mobileShellFrames: frames, mobileShellFrameTimes: frameTimes }); const sample = () => { const shell = document.querySelector(".paperclip-app-shell"); if (shell && shell.getBoundingClientRect().width > 0) { const rects = [".app-breadcrumb", ".mobile-bottom-nav", "#main-content"].map(selector => document.querySelector(selector)!.getBoundingClientRect()); frames.push([innerWidth, innerHeight, scrollX, scrollY, ...rects.flatMap((rect, index) => index === 2 ? [rect.x, rect.y, rect.width] : [rect.x, rect.y, rect.width, rect.height])]); + frameTimes.push(performance.now()); } requestAnimationFrame(sample); }; @@ -184,9 +185,27 @@ for (const coordinator of [false, true]) test(`mobile chrome stays fixed when pe release(); if (coordinator) await expect(page.getByTestId("task-chat-thread").locator(':scope > [aria-busy="false"]')).toBeVisible(); else await expect(page.getByTestId("issue-detail-header").getByRole("heading", { name: fixture.title, exact: true })).toBeVisible(); - await page.waitForTimeout(300); - const frames = await page.evaluate(() => (window as unknown as { mobileShellFrames: number[][] }).mobileShellFrames); - await info.attach("mobile-preference-chrome", { body: JSON.stringify(frames), contentType: "application/json" }); + const ready = await page.evaluate(() => ({ + count: (window as unknown as { mobileShellFrames: number[][] }).mobileShellFrames.length, + at: performance.now(), + })); + let frames: number[][] = []; + try { + // Observe six real post-ready frames and preserve the original 300ms + // interval. Wall time alone does not establish that rAF sampled it. + await expect.poll(() => page.evaluate(({ count, at }) => { + const state = window as unknown as { mobileShellFrames: number[][]; mobileShellFrameTimes: number[] }; + return state.mobileShellFrames.length >= count + 6 && (state.mobileShellFrameTimes.at(-1) ?? 0) >= at + 300; + }, ready)).toBe(true); + } finally { + const captured = await page.evaluate(() => { + const state = window as unknown as { mobileShellFrames: number[][]; mobileShellFrameTimes: number[] }; + return { frames: state.mobileShellFrames, times: state.mobileShellFrameTimes }; + }); + frames = captured.frames; + await info.attach("mobile-preference-chrome", { body: JSON.stringify(frames), contentType: "application/json" }); + await info.attach("mobile-preference-frame-times", { body: JSON.stringify({ ready, times: captured.times }), contentType: "application/json" }); + } expect(frames.length).toBeGreaterThan(5); for (let dimension = 0; dimension < frames[0].length; dimension++) { const values = frames.map(frame => frame[dimension]); diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 6e653494ad6..ce803748eb6 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -1,5 +1,13 @@ # Runner E2E fixture authoring +The deployment-targeted [saved skill journey](product-journeys/saved-skill-use.md) +creates and assigns one campaign-owned synthetic skill by key through public APIs, then +creates the measured task through the browser. The saved outcome must come from +one native run. Setup never supplies the output document or its native read +receipt. The ordinary Latest assignment, immutable created version and admitted +run version must agree. Skill ownership is journaled before creation and cleaned by the target +provider with the other owned resources. + [Chat feature evals](CHAT-FEATURES.md) load two authored YAML requests and seed only company, agents and project context through public APIs. Both agents use production role instructions; evaluator outcomes are never injected. Real browser diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 42d8c6cde70..eec3849f077 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -26,13 +26,17 @@ never start a server or depend on a tenant filesystem. | `repository-continuation` | Native agent; two work runs; a pinned public read-only repository | Verify independent Git bundle history and workspace bytes across two turns, preserve the original random file, and reopen registered outputs after reload. See [repository-continuation.md](product-journeys/repository-continuation.md) | | `routine-scheduled-run` | Native agent; one work run; an owned project and routine | Browser-create a routine, enable its minute schedule, verify one scheduled invocation and run-attributed saved output, then pause and observe the next schedule boundary with no new work | | `company-access-isolation` | Two registered QA Cloud workspaces; disposable company-bound agent credentials; zero model runs | Switch through the real tenant Cloud menu and return. Require allowed task reads and task creation, foreign read/create denial in both directions, exact canonical creator evidence, and key revocation. See [COMPANY-ACCESS.md](product-journeys/COMPANY-ACCESS.md) | +| `saved-skill-use` | Native agent with skill-write capability; one work run; one synthetic company skill assigned by key using Latest | Load the assigned saved skill, use inputs available only in its source, and verify a same-run-authored task document after reload. Bind the immutable created version to the run's admitted skill-version map. Require a complete bound native source read/load and preserve the exact source and version. Works with Beta skills off. See [saved-skill-use.md](product-journeys/saved-skill-use.md) | | `file-input-answer` | Native agent; one work run; synthetic CSV uploaded through the composer | Verify original uploaded bytes, independently calculated accepted-row count and units, a run-attributed task document after reload, and preservation of the input attachment. The task prompt contains neither the random file reference nor the answers | | `inbox-triage` | Signed-in board user; a separately authenticated QA human author; a fresh self-assigned task; zero model runs | Mark an unread task read, archive it through Mine, reload both views, verify unchanged work status, then observe a real Blocked transition resurface it through the normal archive policy | | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | | `project-task-context` | Company, signed-in board user, task-write capability; no model runs | Create a project and a self-assigned task with the project selected through the production dialogs. Reopen the project, open its task, reload both, and verify exact saved company, project, prompt, assignment, standard work mode and zero runs | +| `task-dependency-resume` | Native agent; one automatic work run; two browser-created tasks | Keep an assigned dependent blocked for a measured ten seconds; complete its human-owned blocker through the UI; require one automatic native wake, run-authored saved document, and reloaded result. See [contract](product-journeys/task-dependency-resume.md). | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | +| `chat-continuity` | Fresh owned native agent with no prior work runs; Agent Chat enabled; fresh owned user chat; three provider turns and one synthetic reset | Recall prompt-hidden context after reopening, then use `/new` to advance the same saved conversation to a fresh native session, preserve history and verify the final reply after reload. See [CHAT-CONTINUITY.md](product-journeys/CHAT-CONTINUITY.md) | | `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload | | `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run | +| `task-edit-lifecycle` | Company, signed-in board user, task-write capability; target support for run-owned labels and exact predeclared `taskEdit.revisedTitle` cleanup; zero model runs | Create a self-assigned task, edit its inline header title and label, reload, complete and reopen it; independently verify exact saved state, unchanged context and zero runs. Open a closed Properties panel through its shipped toggle | | `task-document-persistence` | Company, signed-in board user, task-write capability, prepared run-owned backlog task; no model allowance or runs | Create a general document through Artifacts, edit its title and body, reopen and reload it; independently verify exact canonical content, preserved original and latest revisions, board-user authorship, and no agent work | | `task-search-filter` | Company, signed-in board user, task-write capability; zero model runs | Search three owned target/control tasks through the task list, require exact matching identities, filter Backlog to exclude the completed control, reload and return from the exact saved task with search/filter context intact | | `plan-revise-accept` | Native agent; three work runs | Reuse the classic Plan story: canonical draft, browser revision request, new revision-bound confirmation, approval and exact saved final response | @@ -42,6 +46,7 @@ never start a server or depend on a tenant filesystem. | `stop-redirect` | Native agent; three work runs; qualified Docker oracle | Download a verified checkpoint, observe an active native command with current-turn usage, Stop it, verify cancellation and settled accounting, then send one new direction and complete without original-run activity | | `followup-while-running` | Native agent; a maximum of two work runs | Save a follow-up while an independently observed native shell command is still active, then require one attributed final reply with the late requirement after reload; in-turn steering and one normal successor are accepted | | `hire-delegate-reuse` | Native lead with a fresh user chat and scoped hiring permission; Agent Chat enabled; direct hiring policy; seven work runs | Hire one teammate, delegate a saved checklist, reuse the same teammate for a saved review, verify native run and revision authorship, preserve the original, and follow the lead's saved result link after reload | +| `agent-settings-persistence` | Fresh dedicated native agent with the campaign ID in its name; `agent-settings-write`; no previous runs/proof block; sole paid journey; one work run | Save and reload an instruction edit through the browser, verify a task uses the instruction-only reference, reopen its run-authored document, then restore the original bytes. See [AGENT-SETTINGS.md](product-journeys/AGENT-SETTINGS.md) | | `cloud-onboarding` | Signed-in admitted Cloud account; fresh workspace; qualified bounded provider route | Create workspace in the portal, connect API key and native Codex agent in the wizard, complete a task, download and independently check its artifact | The deployment-targeted Stop journey asks the revision to finish one harmless @@ -488,3 +493,38 @@ within the original journey deadline. Binary attachment and browser downloads remain single attempts; transport failures and digest mismatches fail visibly. Git inspection consumes the exact digest-verified download buffer; browser downloads remain an independent result-access check. The repository journey shares the observation reader and browser helpers above. + + +## Task editing and lifecycle + +`task-edit-lifecycle` creates a self-assigned task through the browser with an +explicit title. It edits the title inline in the task header and adds one +run-owned label through the Properties panel. If the panel starts closed, the +journey opens it through the ordinary Toggle side panel or Show properties +control. It changes the status to In Progress, reloads, marks +the task Done, reloads, reopens it to Todo and reloads again. Independent API +reads bind every observed state to the same company, task and human owner. +The exact description, assignee, project and priority remain unchanged. +Priority editing is not part of this journey because the production UI hides +that control. No agent or model work is allowed. + +Labels use `Journey QA label ${campaignId}`, retaining the complete campaign +identity within the production 48-character limit for ordinary generated +campaigns. The production schema rejects an overlong name before any browser +mutation or ownership intent; names are never truncated or hashed. +The target must journal `label` resources by campaign name before browser label +creation and by exact ID after its 201 response. Before title editing, the owned +issue receipt retains its original name and ID and declares +`taskEdit: { revisedTitle }` for this case. Cleanup may match only that one +predeclared title or the original title on the same company and issue ID. +The private target must reconcile a lost label-creation response, remove only +its proven run-owned label, and verify absence. Targets without these ownership +and cleanup semantics must not select this journey. + +The case uses no private fixture writes. Its limits require zero model calls and +zero task runs. `private-stories/task-edit-lifecycle/task-lifecycle-observations.json` +retains the five allowlisted task observations. The adjacent +`observation-read-failures.json` retains bounded read-transport failures. +Valid, wrong and missing evidence +controls calibrate the independent oracle. Registration and local checks do not +claim a Cloud staging pass or admission to a daily plan. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 49c359a300e..49181a8d3ec 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1,5 +1,7 @@ # Paid runner full-stack E2E +The deployment-targeted [saved agent instructions journey](product-journeys/AGENT-SETTINGS.md) verifies browser save/reload, one instruction-dependent native task, independently attributed output, and restoration of the owned agent’s original instructions. Discovery and local grader controls remain separate from QA staging qualification. + The explicit-only `cursor-cloud` suite uses a local controller and hosted Cursor Cloud work. Its four required cells check a downloaded registered artifact, question/answer continuation on the same Cloud agent, UI Stop of an diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 5abc39cb853..1c6969ebcf7 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -1,5 +1,12 @@ # Runner E2E security for a public repository +The deployment-targeted `saved-skill-use` journey creates one synthetic skill on +the campaign-owned agent. Its random receipt reference and numeric inputs occur +only in the saved skill source. Exact source, immutable version, ordinary assignment, native read +events and run-authored document revisions remain in the private bundle. The +existing public projection exports closed checks and the authenticated owned +task screenshot. The journey changes no shared agent, instance flag or budget. + Native `cursor-cloud` acceptance uses a separate callback-only loopback listener. Its external HTTPS tunnel must never target the local_trusted operator app. Credential-free ingress checks require `/` and `/api/health` to return 404 and diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index f5c240c0df4..3729583453d 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -126,7 +126,33 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["company-access-isolation", "repository-continuation", "routine-scheduled-run", "inbox-triage", "file-input-answer", "task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "hire-delegate-reuse", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual([ + "saved-skill-use", + "task-edit-lifecycle", + "company-access-isolation", + "repository-continuation", + "routine-scheduled-run", + "inbox-triage", + "file-input-answer", + "task-create", + "project-task-context", + "task-dependency-resume", + "ask-question", + "chat-continuity", + "question-answer-resume", + "task-properties", + "task-document-persistence", + "task-search-filter", + "plan-revise-accept", + "build-revise", + "protected-actions", + "create-skill-studio", + "stop-redirect", + "followup-while-running", + "hire-delegate-reuse", + "agent-settings-persistence", + "cloud-onboarding" + ]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.json b/tests/runner-e2e/chat-full-catalog-testops-manifest.json index 66cef83f915..01595aab574 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.json +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.json @@ -1,6 +1,6 @@ { "schema": "paperclip.chat-full-catalog-testops-manifest/v1", - "targetSha": "fd5e3b1e5a379524c0d1651ed8273e75ffd86541", + "targetSha": "b1ea76b286b870bb62a80eebf3fe00feed6f12c1", "metadataSourceSha": "de0941835cc8f0d01c982574ee258e110babdbeb", "profile": "runner-opencode", "model": "openrouter/deepseek/deepseek-v4.1-flash-1017", @@ -1527,8 +1527,8 @@ "condition": "Reviewed synthetic staging input only; per-worker singleton550 enables it; production feed and global index unchanged." }, "sdkPublicProof": { - "previousRegistrySha256": "32c7971e1d9c9d31cd6791045b0265f0475c8f10932ff6dd34681681b6a89342", - "registrySha256": "9169583e54d165dc8d08b9a9010175d11ca87f3590477a7b23032508020ea039", + "previousRegistrySha256": "9169583e54d165dc8d08b9a9010175d11ca87f3590477a7b23032508020ea039", + "registrySha256": "8ecea303821644fa8b7c21d042086096d43dda6c1615b1ca00fbcc4ebb9389c4", "inputCount": 506, "scannerAndCanaryCertificatesUnchanged": true, "plugin558StillHeld": true, @@ -1539,23 +1539,42 @@ "newCandidate115Unqualified": true, "sourceInputsChanged": true, "changedInputPaths": [ - "packages/shared/src/types/work-timeline.ts" + "packages/shared/src/connection-intent-guidance.ts" ], "compileQualification": { - "baselineSourceSha": "124232037eebe73a513907b08f0e96bfdd454c82", - "compiledSourceSha": "6abe5739a713d0a8bb0308b1495846881b82d253", - "acceptedSourceSha": "b734393eff0d7ef85b7ad8abd556d2a66ca3a4e6", + "baselineSourceSha": "abbde061c7ec9181021858f7b1af21a48637f97f", + "compiledSourceSha": "8d19f83b9467bd21d9a1fcd52e10ed372cfa2fbf", + "acceptedSourceSha": "b1ea76b286b870bb62a80eebf3fe00feed6f12c1", "all506InputGitBlobsMatched": true, "owningSharedAndPluginSdkTypeScriptBuildsPassed": true, - "changedRuntimeJavaScriptFiles": 0, - "changedDeclarationFiles": [ - "packages/shared/dist/types/work-timeline.d.ts" + "changedRuntimeJavaScriptFiles": 1, + "changedRuntimeJavaScriptPaths": [ + "packages/shared/dist/connection-intent-guidance.js" ], - "existingAndUnassignedConsumerCompiled": true, - "unknownEventKindRejected": true, + "changedDeclarationFiles": [ + "packages/shared/dist/connection-intent-guidance.d.ts" + ], + "emittedSecureEntryGuidanceConsumerPassed": true, + "existingConnectionRequestAuthorityRetained": true, "sourceSdkControlsPassed": 51, "inputAndBuildCustodyControlsPassed": 20, - "runtimeHeldCasesRemainHeld": true + "runtimeHeldCasesRemainHeld": true, + "previousQualification": { + "baselineSourceSha": "124232037eebe73a513907b08f0e96bfdd454c82", + "compiledSourceSha": "6abe5739a713d0a8bb0308b1495846881b82d253", + "acceptedSourceSha": "b734393eff0d7ef85b7ad8abd556d2a66ca3a4e6", + "all506InputGitBlobsMatched": true, + "owningSharedAndPluginSdkTypeScriptBuildsPassed": true, + "changedRuntimeJavaScriptFiles": 0, + "changedDeclarationFiles": [ + "packages/shared/dist/types/work-timeline.d.ts" + ], + "existingAndUnassignedConsumerCompiled": true, + "unknownEventKindRejected": true, + "sourceSdkControlsPassed": 51, + "inputAndBuildCustodyControlsPassed": 20, + "runtimeHeldCasesRemainHeld": true + } } }, "toolProposal487": { @@ -1578,7 +1597,7 @@ }, "candidateLauncherSnapshot": { "path": "fixtures/chat-repair-candidate-launcher.txt", - "targetSha": "fd5e3b1e5a379524c0d1651ed8273e75ffd86541", + "targetSha": "b1ea76b286b870bb62a80eebf3fe00feed6f12c1", "sourcePath": "tests/runner-e2e/launch.ts", "gitBlobOid": "dba1467e211611f37937e9e1fc4c0c5be4329cbc" } diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index 9dca3e5607f..e2411b41388 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -103,7 +103,7 @@ test('repair source metadata binds the corrected candidate and explicit changed const repair = manifest.source.repairMetadata; assert.equal(repair.baseTargetSha, '3b66c3c144360ddf32714a0742fd774b8ceafb16'); assert.equal(repair.previousTargetSha, 'ab2019e976076ec0eb60b026844f3d9211cb0208'); - assert.equal(manifest.targetSha, 'fd5e3b1e5a379524c0d1651ed8273e75ffd86541'); + assert.equal(manifest.targetSha, 'b1ea76b286b870bb62a80eebf3fe00feed6f12c1'); assert.equal(repair.definitionSourceSha, manifest.metadataSourceSha); assert.equal(repair.candidateDescriptorSnapshot, 'fixtures/chat-repair-candidate-descriptors.json'); const descriptorBytes = readFileSync(new URL(repair.candidateDescriptorSnapshot, import.meta.url)); @@ -317,7 +317,7 @@ test('repair source metadata binds the corrected candidate and explicit changed assert.equal(repair.runtimeConditions.toolGuidance106.corePRUnmerged, false); assert.equal(repair.runtimeConditions.sdkPublicProof.scannerAndCanaryCertificatesUnchanged, true); assert.equal(repair.runtimeConditions.sdkPublicProof.sourceInputsChanged, true); - assert.equal(repair.runtimeConditions.sdkPublicProof.registrySha256, "9169583e54d165dc8d08b9a9010175d11ca87f3590477a7b23032508020ea039"); + assert.equal(repair.runtimeConditions.sdkPublicProof.registrySha256, "8ecea303821644fa8b7c21d042086096d43dda6c1615b1ca00fbcc4ebb9389c4"); assert.equal(repair.runtimeConditions.sdkPublicProof.plugin558StillHeld, true); assert.equal(repair.runtimeConditions.sdkPublicProof.expectedLockSha256, "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8"); assert.equal(repair.runtimeConditions.sdkPublicProof.runtimeCompilerParserZodIdentitiesUnchanged, true); diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 962117f2ab1..157f2ffce49 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,15 +19,20 @@ function associate(suites: readonly string[], cases: Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); diff --git a/tests/runner-e2e/product-journeys/agent-settings-evidence.test.ts b/tests/runner-e2e/product-journeys/agent-settings-evidence.test.ts new file mode 100644 index 00000000000..9fc728ea592 --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-evidence.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { AGENT_SETTINGS_CHECKS, gradeAgentSettings, settingsProofHeading, settingsTaskPrompt, settingsResultMatches } from "./agent-settings-evidence.js"; +import { settingsEvidenceFixture } from "./agent-settings-test-fixture.js"; +import { agentJourneyRunCountValid, journeyRequiredChecks } from "./success-contract.js"; +import { parseTargetConfig } from "./catalog.js"; + +describe("saved agent settings independent evidence", () => { + it.each([["ask-question", "agent-settings-persistence"], ["agent-settings-persistence", "ask-question"]])( + "rejects a shared paid campaign before target acquisition: %j", (...journeys) => { + expect(() => parseTargetConfig({ version: 1, id: "settings-target", mode: "managed-self-hosted", journeys, + executionEnvironment: "local", baseline: { features: {}, connections: [] }, + limits: { durationMs: 30000, modelBudgetCents: 500, maxRuns: 2 } })).toThrow("agent_settings_requires_dedicated_campaign"); + }); + it("permits browser-only journeys with the dedicated settings campaign", () => { + expect(parseTargetConfig({ version: 1, id: "settings-target", mode: "managed-self-hosted", + journeys: ["task-create", "agent-settings-persistence"], executionEnvironment: "local", + baseline: { features: {}, connections: [] }, limits: { durationMs: 30000, modelBudgetCents: 500, maxRuns: 1 } }).journeys) + .toEqual(["task-create", "agent-settings-persistence"]); + }); + it("requires every independent check and exactly one work run in retained results", () => { + expect(journeyRequiredChecks["agent-settings-persistence"]).toEqual(AGENT_SETTINGS_CHECKS); + expect(agentJourneyRunCountValid("agent-settings-persistence", 1)).toBe(true); + for (const count of [undefined, 0, 2, -1, 1.5, "1"]) { + expect(agentJourneyRunCountValid("agent-settings-persistence", count)).toBe(false); + } + }); + it("accepts the actual content-token contract, board save audits and one bound result", () => { + const f = settingsEvidenceFixture(); + expect(gradeAgentSettings(f.scope, f.evidence)).toEqual(AGENT_SETTINGS_CHECKS.map(id => ({ id, passed: true }))); + expect(f.evidence.restored!.file.revision.id).toBe(f.evidence.original!.file.revision.id); + expect(settingsTaskPrompt).not.toContain(f.scope.reference); + }); + it.each([null, {}, { result: "bounded output" }])("accepts committed current-run evidence when resultJson is %j", resultJson => { + const f = settingsEvidenceFixture(); + Object.assign(f.run, { resultJson }); + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(true); + }); + it("accepts committed current-run evidence when resultJson is omitted", () => { + const f = settingsEvidenceFixture(); + delete (f.run as Record).resultJson; + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(true); + }); + it("rejects an internally consistent baseline containing a previous QA proof block", () => { + const f = settingsEvidenceFixture(undefined, `# Agent\n${settingsProofHeading}\nStale previous reference\n`); + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(false); + }); + const changes: Array<[string, (f: ReturnType) => void]> = [ + ["foreign company", f => { f.evidence.original!.agent.companyId = "foreign"; }], + ["missing campaign-bearing target name", f => { f.scope.agentName = "Agent without campaign name"; f.evidence.original!.agent.name = f.scope.agentName; }], + ["past work", f => { f.evidence.original!.runs.push(f.run); }], + ["wrong instruction entry", f => { f.evidence.saved!.bundle.entryFile = "OTHER.md"; }], + ["false content hash", f => { f.evidence.saved!.file.contentHash = "0".repeat(64); }], + ["wrong saved bytes", f => { f.evidence.saved!.file.content += "wrong"; }], + ["missing save audit", f => { f.evidence.saved!.activity = []; }], + ["unscoped activity response", f => { f.evidence.original!.activity.push({ ...f.firstSave, entityId: "foreign-agent" }); }], + ["full owned activity page", f => { f.evidence.original!.activity = Array.from({ length: 200 }, (_, index) => ({ ...f.firstSave, id: `old-${index}` })); }], + ["foreign board actor", f => { f.firstSave.actorId = "someone-else"; }], + ["foreign audit entity", f => { f.firstSave.entityId = "other-agent"; }], + ["duplicate save evidence", f => { f.evidence.saved!.activity.push({ ...f.firstSave, id: "duplicate" }); }], + ["rendered draft only", f => { f.evidence.reloaded!.file.content = f.original; }], + ["missing rendered content", f => { delete f.evidence.reloaded!.renderedContent; }], + ["prompt leaks reference", f => { f.evidence.task!.issue.description += f.scope.reference; }], + ["foreign task", f => { f.evidence.task!.issue.companyId = "other"; }], + ["wrong assignee", f => { f.evidence.task!.issue.assigneeAgentId = "other"; }], + ["incomplete task", f => { f.evidence.task!.issue.status = "in_progress"; }], + ["active checkout", f => { f.evidence.task!.issue.executionRunId = "active"; }], + ["recovery pending", f => { f.evidence.task!.issue.activeRecoveryAction = {}; }], + ["question pending", f => { f.evidence.task!.interactions = [{ status: "pending" }]; }], + ["extra run", f => { f.evidence.task!.runs.push({ ...f.run, id: "extra" }); }], + ["failed run", f => { f.run.status = "failed"; }], + ["uncommitted run", f => { f.run.nativePhase = "applied"; }], + ["missing committed phase", f => { delete (f.run as Record).nativePhase; }], + ["uncommitted run with omitted result", f => { f.run.nativePhase = "applied"; Object.assign(f.run, { resultJson: null }); }], + ["legacy run", f => { f.run.runtimeMode = "legacy"; }], + ["conflicting issue bindings", f => { Object.assign(f.run, { issueId: "foreign" }); }], + ["wrong provider", f => { f.run.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }], + ["wrong model", f => { f.run.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }], + ["wrong environment", f => { f.run.contextSnapshot.paperclipEnvironment.id = "other"; }], + ["wrong nonce", f => { f.evidence.task!.document.body = JSON.stringify({ reference: "wrong", settingsSaved: true }); }], + ["human-authored output", f => { f.evidence.task!.document.createdByUserId = "user"; }], + ["foreign revision", f => { f.evidence.task!.revision.createdByRunId = "other"; }], + ["task changes instructions", f => { f.evidence.afterWork!.file.content = "changed"; }], + ["wrong reopened revision", f => { f.evidence.task!.reopenedDocument!.latestRevisionId = "old"; }], + ["missing rendered result", f => { delete f.evidence.task!.renderedReference; }], + ["unrestored instructions", f => { f.evidence.restored!.file.content = f.saved; }], + ["missing restore audit", f => { f.evidence.restored!.activity = [f.firstSave]; }], + ["extra work during restore", f => { f.evidence.restored!.runs.push({ ...f.run, id: "later" }); }], + ]; + it.each(changes)("rejects %s", (_name, change) => { + const f = settingsEvidenceFixture(); change(f); + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(false); + }); + it.each(["original", "saved", "reloaded", "afterWork", "restored", "task"] as const)("rejects missing %s evidence", key => { + const f = settingsEvidenceFixture(); delete f.evidence[key]; + expect(gradeAgentSettings(f.scope, f.evidence).every(row => row.passed)).toBe(false); + }); + it("rejects plausible malformed result bodies and accepts one optional JSON fence", () => { + expect(settingsResultMatches('```json\n{"reference":"r","settingsSaved":true}\n```', "r")).toBe(true); + for (const body of [null, "", "not json", "[]", '{"reference":"r"}', '{"reference":"r","settingsSaved":"true"}', + '{"reference":"r","settingsSaved":true,"extra":1}']) expect(settingsResultMatches(body, "r")).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/agent-settings-evidence.ts b/tests/runner-e2e/product-journeys/agent-settings-evidence.ts new file mode 100644 index 00000000000..9d1be4505e5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-evidence.ts @@ -0,0 +1,125 @@ +import { createHash } from "node:crypto"; + +type Row = Record; +export const AGENT_SETTINGS_CHECKS = [ + "agent-settings-browser-save", "agent-settings-reload-persisted", "agent-settings-instruction-only-input", + "agent-settings-result-attributed", "agent-settings-result-reopened", "agent-settings-original-restored", + "configured-model-and-environment", +] as const; +export const settingsDocumentKey = "settings-proof"; +export const settingsProofHeading = "## QA settings persistence proof"; +export const settingsTaskPrompt = "Follow the QA settings proof procedure in your saved agent instructions. Save the requested task document, then finish the task. No other tasks or agents are needed. Do not change your instruction files."; +export const instructionDigest = (content: string) => createHash("sha256").update(content).digest("hex"); +export function settingsInstructionContent(original: string, reference: string) { + return original + `\n\n${settingsProofHeading}\nWhen a task asks for the QA settings proof, save a task document with key ${settingsDocumentKey} and title Settings proof. Its body must be JSON with exactly two fields: reference equal to ${reference}, and settingsSaved equal to true. Keep these instructions unchanged.\n`; +} +export function settingsResultMatches(body: unknown, reference: string) { + if (typeof body !== "string") return false; + const text = body.trim(), json = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(text)?.[1] ?? text; + try { + const parsed = JSON.parse(json); + return parsed && !Array.isArray(parsed) && Object.keys(parsed).sort().join(",") === "reference,settingsSaved" + && parsed.reference === reference && parsed.settingsSaved === true; + } catch { return false; } +} +export interface AgentSettingsScope { + campaignId: string; companyId: string; userId: string; agentId: string; agentName: string; + provider: string; model: string; environmentId: string; reference: string; +} +export interface SettingsSnapshot { agent: Row; bundle: Row; file: Row; runs: Row[]; activity: Row[]; renderedContent?: string } +export interface SettingsTaskSnapshot { + issue: Row; runs: Row[]; interactions: Row[]; document: Row; revision: Row; + reopenedDocument?: Row; reopenedRevision?: Row; renderedReference?: string; +} +export interface AgentSettingsEvidence { + original?: SettingsSnapshot; saved?: SettingsSnapshot; reloaded?: SettingsSnapshot; + afterWork?: SettingsSnapshot; restored?: SettingsSnapshot; task?: SettingsTaskSnapshot; +} +const present = (value: unknown): value is string => typeof value === "string" && value.length > 0; +const revisionId = (point?: SettingsSnapshot) => point?.file?.revision?.id; +function snapshotBound(point: SettingsSnapshot | undefined, scope: AgentSettingsScope, entry?: string) { + return Boolean(point && point.agent?.id === scope.agentId && point.agent.companyId === scope.companyId + && point.agent.name === scope.agentName && point.agent.adapterType === "paperclip_runner" + && point.agent.adapterConfig?.provider === scope.provider && point.agent.adapterConfig?.model === scope.model + && point.bundle?.agentId === scope.agentId && point.bundle.companyId === scope.companyId + && point.bundle.mode === "managed" && point.bundle.editable === true && present(point.bundle.entryFile) + && (!entry || point.bundle.entryFile === entry) && point.file?.path === point.bundle.entryFile + && point.file.isEntryFile === true && point.file.editable === true && point.file.binary !== true + && typeof point.file.content === "string" && point.file.contentHash === instructionDigest(point.file.content) + && point.file.size === Buffer.byteLength(point.file.content) + && present(revisionId(point)) && point.file.revision.companyId === scope.companyId + && point.file.revision.agentId === scope.agentId && point.file.revision.entryFile === point.bundle.entryFile + && point.file.revision.contentHash === point.file.contentHash + && point.file.revision.byteLength === point.file.size && Array.isArray(point.runs) + && Array.isArray(point.activity) && point.activity.length < 200 + && point.activity.every(row => present(row.id) && row.companyId === scope.companyId + && row.entityType === "agent" && row.entityId === scope.agentId)); +} +export function matchedBoardSaveAudit(point: SettingsSnapshot | undefined, before: SettingsSnapshot | undefined, scope: AgentSettingsScope) { + const old = new Set(before?.activity?.map(row => row.id)); + const added = point?.activity?.filter(row => !old.has(row.id) && row.action === "agent.files_updated" + && row.entityType === "agent" && row.entityId === scope.agentId && row.details?.path === point.bundle.entryFile); + if (!(Array.isArray(added) && added.length === 1 && present(added[0]?.id) + && added[0].companyId === scope.companyId && added[0].actorType === "user" && added[0].actorId === scope.userId + && added[0].agentId === null && added[0].runId === null && added[0].details.contentHash === point!.file.contentHash + && added[0].details.source === "board")) return null; + const row = added[0]; + return { id: row.id, companyId: row.companyId, actorType: row.actorType, actorId: row.actorId, + agentId: row.agentId, runId: row.runId, action: row.action, entityType: row.entityType, entityId: row.entityId, + path: row.details.path, contentHash: row.details.contentHash, source: row.details.source }; +} +function boundIssue(run: Row) { + const ids = [run.issueId, run.nativeIssueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].filter(value => value != null); + return ids.length && ids.every(present) && new Set(ids).size === 1 ? ids[0] : undefined; +} +export function gradeAgentSettings(scope: AgentSettingsScope, value: AgentSettingsEvidence) { + const { original, saved, reloaded, afterWork, restored, task } = value; + const entry = original?.bundle?.entryFile; + const scoped = [scope.campaignId, scope.companyId, scope.userId, scope.agentId, scope.agentName, + scope.provider, scope.model, scope.environmentId, scope.reference].every(present) + && scope.agentName.includes(scope.campaignId) && snapshotBound(original, scope) + && original!.runs.length === 0 && !original!.file.content.includes(scope.reference) + && !original!.file.content.includes(settingsProofHeading); + const expected = scoped ? settingsInstructionContent(original!.file.content, scope.reference) : undefined; + const save = scoped && snapshotBound(saved, scope, entry) && saved!.file.content === expected + && revisionId(saved) !== revisionId(original) && matchedBoardSaveAudit(saved, original, scope) && saved!.runs.length === 0; + const reload = save && snapshotBound(reloaded, scope, entry) && reloaded!.file.content === expected + && revisionId(reloaded) === revisionId(saved) && reloaded!.renderedContent === expected && reloaded!.runs.length === 0; + const issue = task?.issue, runs = task?.runs; + const inputOnly = reload && issue && present(issue.id) && issue.companyId === scope.companyId + && issue.assigneeAgentId === scope.agentId && issue.assigneeUserId === null && issue.createdByUserId === scope.userId + && issue.createdByAgentId === null && issue.workMode === "standard" && issue.description === settingsTaskPrompt + && !issue.description.includes(scope.reference); + const run = Array.isArray(runs) && runs.length === 1 ? runs[0] : undefined; + const provider = run?.runnerProfileJson?.nativeExecutionInput?.provider; + const configured = Boolean(inputOnly && run && present(run.id) && run.companyId === scope.companyId + && run.agentId === scope.agentId && boundIssue(run) === issue.id && run.status === "succeeded" + && run.runtimeMode === "native" && run.nativePhase === "committed" + && provider?.kind === scope.provider && provider.model === scope.model + && run.contextSnapshot?.paperclipEnvironment?.id === scope.environmentId); + const document = task?.document, revision = task?.revision; + const output = configured && issue && issue.status === "done" && !issue.executionRunId && !issue.scheduledRetry && !issue.activeRecoveryAction + && Array.isArray(task!.interactions) && task!.interactions.every(row => row.status !== "pending") + && document && present(document.id) && document.companyId === scope.companyId && document.issueId === issue.id + && document.key === settingsDocumentKey && document.createdByAgentId === scope.agentId && document.createdByUserId === null + && present(document.latestRevisionId) && settingsResultMatches(document.body, scope.reference) + && revision && revision.id === document.latestRevisionId && revision.documentId === document.id + && revision.createdByAgentId === scope.agentId && revision.createdByUserId === null && revision.createdByRunId === run!.id + && settingsResultMatches(revision.body, scope.reference) + && snapshotBound(afterWork, scope, entry) && afterWork!.file.content === expected && revisionId(afterWork) === revisionId(saved) + && afterWork!.runs.length === 1 && afterWork!.runs[0]?.id === run!.id; + const reopened = task?.reopenedDocument, reopenedRevision = task?.reopenedRevision; + const resultReload = output && issue && reopened && reopened.id === document.id && reopened.companyId === scope.companyId + && reopened.issueId === issue.id && reopened.key === settingsDocumentKey && reopened.latestRevisionId === document.latestRevisionId + && reopened.createdByAgentId === scope.agentId && reopened.createdByUserId === null && settingsResultMatches(reopened.body, scope.reference) + && reopenedRevision && reopenedRevision.id === revision.id && reopenedRevision.documentId === document.id + && reopenedRevision.createdByRunId === run!.id && reopenedRevision.createdByAgentId === scope.agentId + && reopenedRevision.createdByUserId === null && settingsResultMatches(reopenedRevision.body, scope.reference) + && task!.renderedReference === scope.reference; + const restore = resultReload && snapshotBound(restored, scope, entry) && restored!.file.content === original!.file.content + && restored!.renderedContent === original!.file.content && restored!.file.contentHash === original!.file.contentHash + && revisionId(restored) === revisionId(original) && revisionId(restored) !== revisionId(saved) && matchedBoardSaveAudit(restored, afterWork, scope) + && restored!.runs.length === 1 && restored!.runs[0]?.id === run!.id; + return [save, reload, inputOnly, output, resultReload, restore, configured] + .map((passed, index) => ({ id: AGENT_SETTINGS_CHECKS[index]!, passed: Boolean(passed) })); +} diff --git a/tests/runner-e2e/product-journeys/agent-settings-flow.test.ts b/tests/runner-e2e/product-journeys/agent-settings-flow.test.ts new file mode 100644 index 00000000000..4569b771fbf --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-flow.test.ts @@ -0,0 +1,180 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; +import { settingsEvidenceFixture } from "./agent-settings-test-fixture.js"; +import { settingsTaskPrompt } from "./agent-settings-evidence.js"; + +const controls = vi.hoisted(() => ({ createTask: vi.fn(), openDocument: vi.fn() })); +vi.mock("../user-actions.js", () => ({ createTaskThroughUi: controls.createTask })); +vi.mock("./task-document-flow.js", () => ({ openDocumentAuthoring: controls.openDocument })); +vi.mock("@playwright/test", () => ({ expect: (locator: any) => ({ + async toBeVisible() { expect(locator.visible).toBe(true); }, + async toBeEnabled() { expect(locator.enabled()).toBe(true); }, + async toBeDisabled() { expect(locator.enabled()).toBe(false); }, + async toHaveValue(value: string) { expect(locator.value()).toBe(value); }, + async toContainText(value: string) { expect(await locator.textContent()).toContain(value); }, +}) })); +import { runAgentSettings } from "./agent-settings-flow.js"; + +const directories: string[] = []; +afterEach(async () => { await Promise.all(directories.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); }); + +async function fixture(nativeFailure = false, unscopedActivity = false, delayedRestore = false, original?: string, entryFile = "AGENTS.md") { + let f = settingsEvidenceFixture(undefined, original, entryFile), content = f.original, rendered = content, draft = content, dirty = false, dispatched = false; + const originalContent = f.original; + const availablePaths = new Set(f.evidence.original!.bundle.files.map((file: any) => file.path)); + const neighbor = "# AGENTS.md sibling must remain unchanged\n"; + let selectedFile = "AGENTS.md"; + const expandedDirs = new Set(), selectedPaths: string[] = [], savedPaths: string[] = []; + let pendingContent: string | undefined; + const saves: string[] = [], owns: any[] = [], apiWrites: string[] = [], activityReads: string[] = []; + const unrelatedActivity = Array.from({ length: 600 }, (_, index) => ({ id: `unrelated-${index}`, + companyId: f.scope.companyId, entityType: "agent", entityId: "another-agent", details: {} })); + const current = () => content === f.original + ? dispatched ? f.evidence.restored! : f.evidence.original! + : dispatched ? f.evidence.afterWork! : f.evidence.saved!; + const editor = { visible: true, value: () => draft, async inputValue() { return draft; }, async fill(value: string) { draft = value; dirty = true; } }; + const raw = { get visible() { return Boolean(rendered.trim()); }, async textContent() { return rendered; } }; + const edit = { get visible() { return selectedFile === entryFile; }, async click() { draft = content; } }; + const save = { enabled: () => dirty, async click() { + expect(selectedFile).toBe(entryFile); savedPaths.push(selectedFile); + saves.push(draft); dirty = false; + if (delayedRestore && saves.length === 2) pendingContent = draft; + else content = draft; + const nonce = /reference equal to ([a-f0-9]+),/.exec(draft)?.[1]; + if (nonce) f = settingsEvidenceFixture(nonce, originalContent, entryFile); + } }; + const document = { visible: true, async textContent() { return f.evidence.task!.document.body; } }; + const page = { async goto() { + if (pendingContent !== undefined) throw new Error("navigation_before_save_committed"); + selectedFile = "AGENTS.md"; expandedDirs.clear(); rendered = entryFile === "AGENTS.md" ? content : neighbor; + }, async reload() { selectedFile = "AGENTS.md"; expandedDirs.clear(); rendered = entryFile === "AGENTS.md" ? content : neighbor; }, + getByTestId(id: string) { expect(id).toBe("instructions-raw-source"); return raw; }, + getByRole(role: string, options: { name: string }) { + if (role === "button" && options.name === `Edit ${entryFile}`) return edit; + if (role === "button" && options.name === `Stop editing ${entryFile}`) return { async click() {} }; + if (role === "button" && options.name === "Save changes") return save; + if (role === "textbox" && options.name === "Instruction file editor") return editor; + throw new Error("unexpected_locator"); + }, locator(selector: string) { + if (selector === "#document-settings-proof") return document; + const match = /^\[role="treeitem"\]\[data-file-tree-path=("(?:\\.|[^"\\])*")\]$/.exec(selector); + if (!match) throw new Error("unexpected_selector:" + selector); + const filePath: string = JSON.parse(match[1]!); + const directory = entryFile.startsWith(`${filePath}/`); + const parents = filePath.split("/").slice(0, -1); + return { get visible() { return (directory || availablePaths.has(filePath)) + && parents.every((_, index) => expandedDirs.has(parents.slice(0, index + 1).join("/"))); }, + async getAttribute(name: string) { expect(name).toBe("aria-expanded"); return directory ? String(expandedDirs.has(filePath)) : null; }, + async click() { + if (directory) expandedDirs.add(filePath); + else { selectedFile = filePath; selectedPaths.push(filePath); rendered = filePath === entryFile ? content : neighbor; } + } }; + } }; + const api = { origin: "https://qa.example.test", async get(route: string) { + if (route === "/api/agents/agent") return structuredClone(current().agent); + if (route === "/api/agents/agent/instructions-bundle") return structuredClone(current().bundle); + if (route === `/api/agents/agent/instructions-bundle/file?path=${encodeURIComponent(entryFile)}`) { + if (pendingContent !== undefined) { content = pendingContent; pendingContent = undefined; } + return structuredClone(current().file); + } + if (route.includes("/heartbeat-runs?agentId=agent")) return structuredClone(current().runs); + if (route.startsWith("/api/companies/company/activity?")) { + activityReads.push(route); + const query = new URL(route, api.origin).searchParams; + expect(query.get("entityType")).toBe("agent"); expect(query.get("entityId")).toBe("agent"); + expect(query.get("limit")).toBe("200"); + if (unscopedActivity) return [unrelatedActivity[0]]; + return structuredClone([...unrelatedActivity, ...current().activity] + .filter(row => row.entityType === query.get("entityType") && row.entityId === query.get("entityId")) + .slice(0, Number(query.get("limit"))).map(row => ({ ...row, + details: { ...row.details, privateFixturePayload: "must-not-be-retained" } }))); + } + if (route === "/api/issues/task") return structuredClone(f.evidence.task!.issue); + if (route === "/api/issues/task/runs") return [{ runId: "run" }]; + if (route === "/api/issues/task/interactions") return []; + if (route === "/api/heartbeat-runs/run") return { ...structuredClone(f.run), ...(nativeFailure ? { status: "failed" } : {}) }; + if (route === "/api/issues/task/documents/settings-proof") return structuredClone(f.evidence.task!.document); + if (route === "/api/issues/task/documents/settings-proof/revisions/document-revision") return structuredClone(f.evidence.task!.revision); + throw new Error("unexpected_read:" + route); + }, async put() { apiWrites.push("put"); throw new Error("tested save must use browser"); }, + async patch() { apiWrites.push("patch"); throw new Error("tested save must use browser"); } }; + const context = { page, api, company: { id: "company", issuePrefix: "QA" }, user: { id: "user", name: "QA" }, + agent: { id: "agent", name: "Owned agent campaign-1", provider: "opencode", model: "model", runtimeMode: "native" }, + environmentId: "environment", capabilities: ["company", "board-user", "task-write", "agent", "agent-settings-write"] } as unknown as JourneyContext; + controls.createTask.mockReset(); controls.openDocument.mockReset(); + controls.createTask.mockImplementation(async (input: any) => { + expect(saves).toHaveLength(1); expect(input.prompt).toBe(settingsTaskPrompt); + expect(input.prompt).not.toContain(f.scope.reference); + await input.beforeDispatch(); dispatched = true; return { issueId: "task" }; + }); + const session = { async own(resource: any) { owns.push(resource); } } as TargetSession; + const evidence: JourneyEvidence = { outcome: "passed", startingState: "recipe", phases: [], checks: [], resourceDisposition: "retained", privacy: "private" }; + const directory = await mkdtemp(path.join(os.tmpdir(), "qa-settings-flow-")); directories.push(directory); + return { context, session, evidence, directory, saves, savedPaths, selectedPaths, availablePaths, owns, apiWrites, activityReads, unrelatedActivity, original: f.original, + currentContent: () => content, options: { directory, deadlineAt: Date.now() + 30000 } }; +} + +describe("agent settings production-surface orchestration", () => { + it.each(["WORKFLOW.md", "team guide/qa [settings].md", "team/nested/INSTRUCTIONS.md"])( + "selects configured entry %s when AGENTS.md also exists and is initially selected", async entryFile => { + const f = await fixture(false, false, false, undefined, entryFile); + expect(f.availablePaths).toEqual(new Set(["AGENTS.md", entryFile])); + await runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options); + expect(f.evidence.checks.every(row => row.passed)).toBe(true); + expect(f.savedPaths).toEqual([entryFile, entryFile]); + expect(f.selectedPaths.length).toBeGreaterThan(2); expect(new Set(f.selectedPaths)).toEqual(new Set([entryFile])); + expect(f.currentContent()).toBe(f.original); expect(f.apiWrites).toEqual([]); + const receipt = JSON.parse(await readFile(path.join(f.directory, "agent-settings-observations.json"), "utf8")); + expect(receipt.saveAudits.saved.path).toBe(entryFile); expect(receipt.saveAudits.restored.path).toBe(entryFile); + }); + it.each(["", " \n\t\n"])("reads and restores the exact empty-file editor value %j", async original => { + const f = await fixture(false, false, false, original); + await runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options); + expect(f.saves).toHaveLength(2); expect(f.saves[1]).toBe(original); expect(f.currentContent()).toBe(original); + expect(f.evidence.checks.every(row => row.passed)).toBe(true); + }); + it("saves and restores only through browser controls, then leaves the result ready for screenshot", async () => { + const f = await fixture(); + await runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options); + expect(f.evidence.checks).toHaveLength(7); expect(f.evidence.checks.every(row => row.passed)).toBe(true); + expect(f.saves).toHaveLength(2); expect(f.saves[1]).toBe(f.original); expect(f.apiWrites).toEqual([]); + expect(f.owns.map(row => row.id ?? "intent")).toEqual(["intent", "task"]); + const artifact = await readFile(path.join(f.directory, "agent-settings-observations.json"), "utf8"); + expect(artifact).not.toContain(f.original); expect(artifact).not.toContain("adapterConfig"); + expect(artifact).not.toContain("must-not-be-retained"); + const saved = JSON.parse(artifact); expect(saved.checks.every((row: any) => row.passed)).toBe(true); + expect(saved.checkpoints.original.contentHash).toBe(saved.checkpoints.restored.contentHash); + expect(saved.saveAudits).toEqual({ saved: { id: "save", companyId: "company", actorType: "user", actorId: "user", + agentId: null, runId: null, action: "agent.files_updated", entityType: "agent", entityId: "agent", path: "AGENTS.md", + contentHash: saved.checkpoints.saved.contentHash, source: "board" }, restored: { id: "restore", companyId: "company", + actorType: "user", actorId: "user", agentId: null, runId: null, action: "agent.files_updated", entityType: "agent", + entityId: "agent", path: "AGENTS.md", contentHash: saved.checkpoints.restored.contentHash, source: "board" } }); + expect(f.unrelatedActivity).toHaveLength(600); expect(f.activityReads.length).toBeGreaterThan(0); + }); + it("rejects a wrongly unscoped activity response before any settings edit", async () => { + const f = await fixture(false, true); + await expect(runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options)).rejects.toThrow("agent_settings_read_coverage_unavailable"); + expect(f.saves).toEqual([]); expect(f.owns).toEqual([]); + }); + it("confirms a delayed restore audit and bytes before navigating away from the editor", async () => { + const f = await fixture(false, false, true); + await runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options); + expect(f.saves).toHaveLength(2); expect(f.currentContent()).toBe(f.original); + expect(f.evidence.checks.every(row => row.passed)).toBe(true); + }); + it("requires scoped settings-write capability before any configuration edit", async () => { + const f = await fixture(); f.context.capabilities = ["company", "board-user", "task-write", "agent"]; + await expect(runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options)).rejects.toThrow("owned_agent_settings_capability_required"); + expect(f.saves).toEqual([]); expect(f.owns).toEqual([]); + }); + it("preserves a native failure and does not restore instructions while failed work may require cleanup", async () => { + const f = await fixture(true); + await expect(runAgentSettings(f.context, f.session, "campaign-1", f.evidence, f.options)).rejects.toThrow("agent_settings_native_run_failed"); + expect(f.saves).toHaveLength(1); expect(f.currentContent()).not.toBe(f.original); + expect(JSON.parse(await readFile(path.join(f.directory, "agent-settings-observations.json"), "utf8")).checks.every((row: any) => row.passed)).toBe(false); + expect(f.apiWrites).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/product-journeys/agent-settings-flow.ts b/tests/runner-e2e/product-journeys/agent-settings-flow.ts new file mode 100644 index 00000000000..5480ebc163c --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-flow.ts @@ -0,0 +1,174 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createTaskThroughUi } from "../user-actions.js"; +import { waitFor } from "./agent-flows.js"; +import { createObservedReader } from "./observation-read.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { AGENT_SETTINGS_CHECKS, gradeAgentSettings, instructionDigest, matchedBoardSaveAudit, settingsInstructionContent, + settingsDocumentKey, settingsProofHeading, settingsTaskPrompt, type AgentSettingsEvidence, type AgentSettingsScope, + type SettingsSnapshot } from "./agent-settings-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "agent-settings-persistence"; + +export async function runAgentSettings(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId || !context.capabilities.includes("agent-settings-write")) { + throw new JourneyError("unsupported", "owned_agent_settings_capability_required"); + } + const scope: AgentSettingsScope = { campaignId, companyId: context.company.id, userId: context.user.id, + agentId: agent.id, agentName: agent.name, provider: agent.provider, model: agent.model, + environmentId: context.environmentId, reference: randomBytes(16).toString("hex") }; + const observations: AgentSettingsEvidence = {}; + const read = createObservedReader(context.api, options.deadlineAt), agentRoute = `/api/agents/${agent.id}`; + const agentRunsRoute = `/api/companies/${scope.companyId}/heartbeat-runs?agentId=${agent.id}&limit=100`; + const activityRoute = `/api/companies/${scope.companyId}/activity?entityType=agent&entityId=${agent.id}&limit=200`; + const snapshot = async (): Promise => { + const [current, bundle, runs, activity] = await Promise.all([ + read.get(agentRoute), read.get(`${agentRoute}/instructions-bundle`), + read.get(agentRunsRoute), read.get(activityRoute), + ]); + if (typeof bundle?.entryFile !== "string" || !bundle.entryFile || !Array.isArray(runs) || runs.length >= 100 + || !Array.isArray(activity) || activity.length >= 200 + || activity.some(row => row.companyId !== scope.companyId || row.entityType !== "agent" || row.entityId !== agent.id)) { + throw new JourneyError("failed", "agent_settings_read_coverage_unavailable"); + } + const file = await read.get(`${agentRoute}/instructions-bundle/file?path=${encodeURIComponent(bundle.entryFile)}`); + return { agent: current, bundle, file, runs, activity }; + }; + const openInstructions = async (entry: string, expectedContent: string, reload = false) => { + await context.page.goto(`${context.api.origin}/${context.company.issuePrefix}/agents/${agent.id}/instructions`, { waitUntil: "domcontentloaded" }); + if (reload) await context.page.reload(); + const segments = entry.split("/"); + const treeRow = (filePath: string) => context.page.locator(`[role="treeitem"][data-file-tree-path=${JSON.stringify(filePath)}]`); + for (let depth = 1; depth < segments.length; depth++) { + const directory = treeRow(segments.slice(0, depth).join("/")); + await expect(directory).toBeVisible(); + if (await directory.getAttribute("aria-expanded") !== "true") await directory.click(); + } + const entryRow = treeRow(entry); + await expect(entryRow).toBeVisible(); + await entryRow.click(); + const edit = context.page.getByRole("button", { name: `Edit ${entry}`, exact: true }); + await expect(edit).toBeVisible(); + if (!expectedContent.trim()) { + await edit.click(); + const editor = context.page.getByRole("textbox", { name: "Instruction file editor", exact: true }); + await expect(editor).toHaveValue(expectedContent); + const renderedContent = await editor.inputValue(); + await context.page.getByRole("button", { name: `Stop editing ${entry}`, exact: true }).click(); + return { edit, renderedContent }; + } + const raw = context.page.getByTestId("instructions-raw-source"); + await expect(raw).toBeVisible(); + const renderedContent = await raw.textContent(); + if (renderedContent !== expectedContent) throw new JourneyError("failed", "agent_settings_rendered_source_mismatch"); + return { edit, renderedContent }; + }; + const saveThroughUi = async (beforePoint: SettingsSnapshot, content: string) => { + const entry = beforePoint.bundle.entryFile, before = beforePoint.file.content; + const { edit } = await openInstructions(entry, before); + await edit.click(); + const editor = context.page.getByRole("textbox", { name: "Instruction file editor", exact: true }); + await expect(editor).toHaveValue(before); + await editor.fill(content); + const save = context.page.getByRole("button", { name: "Save changes", exact: true }); + await expect(save).toBeEnabled(); + await save.click(); + await expect(save).toBeDisabled({ timeout: 30_000 }); + // Disabled can also mean saving. Confirm persisted bytes and the new audit + // before navigating, including during restoration. + return waitFor(options.deadlineAt, snapshot, point => point.file.content === content + && Boolean(matchedBoardSaveAudit(point, beforePoint, scope)), "agent_settings_save_not_persisted"); + }; + const check = (name: typeof AGENT_SETTINGS_CHECKS[number]) => { + const result = gradeAgentSettings(scope, observations).find(row => row.id === name)!; + evidence.checks.push(result); + if (!result.passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + let failed = false; + try { + observations.original = await snapshot(); + const original = observations.original, entry = original.bundle.entryFile; + if (original.agent.id !== agent.id || original.agent.companyId !== scope.companyId || original.agent.name !== agent.name + || !agent.name.includes(campaignId) || original.agent.adapterType !== "paperclip_runner" + || original.agent.adapterConfig?.provider !== agent.provider || original.agent.adapterConfig?.model !== agent.model + || original.bundle.mode !== "managed" || original.bundle.editable !== true || original.file.editable !== true + || typeof original.file.content !== "string" || original.file.content.includes("\r") + || original.file.content.includes(settingsProofHeading) || original.runs.length !== 0) { + throw new JourneyError("failed", "agent_settings_fixture_not_fresh_owned"); + } + const edited = settingsInstructionContent(original.file.content, scope.reference); + observations.saved = await saveThroughUi(original, edited); + check("agent-settings-browser-save"); + const reopenedSource = await openInstructions(entry, edited, true); + observations.reloaded = { ...await snapshot(), renderedContent: reopenedSource.renderedContent }; + check("agent-settings-reload-persisted"); + + const title = `Journey QA agent settings ${campaignId}`; + const task = await createTaskThroughUi({ page: context.page, origin: context.api.origin, issuePrefix: context.company.issuePrefix, + agentName: agent.name, title, prompt: settingsTaskPrompt, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: task.issueId, journeyId: id }); + const route = `/api/issues/${task.issueId}`, documentRoute = `${route}/documents/${settingsDocumentKey}`; + const state = await waitFor(options.deadlineAt, async () => { + const [issue, summaries, interactions] = await Promise.all([ + read.get(route), read.get(`${route}/runs`), read.get(`${route}/interactions`), + ]); + if (!Array.isArray(summaries) || summaries.length > 1 || !Array.isArray(interactions)) throw new JourneyError("failed", "agent_settings_run_coverage_mismatch"); + const runs = await Promise.all(summaries.map(row => read.get(`/api/heartbeat-runs/${row.runId}`))); + if (runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status))) throw new JourneyError("failed", "agent_settings_native_run_failed"); + return { issue, runs, interactions }; + }, point => point.issue.status === "done" && point.runs.length === 1 && point.runs[0]!.status === "succeeded" + && point.runs[0]!.nativePhase === "committed" && point.interactions.every(row => row.status !== "pending"), "agent_settings_task_not_completed"); + const document = await read.get(documentRoute); + const revision = await read.get(`${documentRoute}/revisions/${document.latestRevisionId}`); + observations.task = { ...state, document, revision }; + observations.afterWork = await snapshot(); + check("agent-settings-instruction-only-input"); + check("configured-model-and-environment"); + check("agent-settings-result-attributed"); + await context.page.goto(`${context.api.origin}/${context.company.issuePrefix}/issues/${state.issue.identifier ?? task.issueId}`, { waitUntil: "domcontentloaded" }); + await context.page.reload(); + await openDocumentAuthoring(context); + const displayed = context.page.locator(`#document-${settingsDocumentKey}`); + await expect(displayed).toContainText(scope.reference); + observations.task.reopenedDocument = await read.get(documentRoute); + observations.task.reopenedRevision = await read.get(`${documentRoute}/revisions/${document.latestRevisionId}`); + observations.task.renderedReference = (await displayed.textContent())?.includes(scope.reference) ? scope.reference : undefined; + check("agent-settings-result-reopened"); + + await saveThroughUi(observations.afterWork, original.file.content); + const restoredSource = await openInstructions(entry, original.file.content); + observations.restored = { ...await snapshot(), renderedContent: restoredSource.renderedContent }; + check("agent-settings-original-restored"); + // The shared screenshot policy permits only the task result, not settings. + await context.page.goto(`${context.api.origin}/${context.company.issuePrefix}/issues/${state.issue.identifier ?? task.issueId}`, { waitUntil: "domcontentloaded" }); + await openDocumentAuthoring(context); + await expect(context.page.locator(`#document-${settingsDocumentKey}`)).toContainText(scope.reference); + } catch (error) { failed = true; throw error; } + finally { + try { + await read.preserve(options.directory); + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + await writeFile(path.join(options.directory, "agent-settings-observations.json"), JSON.stringify({ + companyId: scope.companyId, agentId: scope.agentId, reference: scope.reference, + checkpoints: Object.fromEntries(["original", "saved", "reloaded", "afterWork", "restored"].map(key => { + const point = observations[key as Exclude]; + return [key, point ? { entryFile: point.file.path, contentHash: point.file.contentHash, + independentlyComputedHash: typeof point.file.content === "string" ? instructionDigest(point.file.content) : null, + revisionId: point.file.revision?.id ?? null, runIds: point.runs.map(row => row.id ?? row.runId) } : null]; + })), + issueId: observations.task?.issue.id ?? null, runIds: observations.task?.runs.map(row => row.id) ?? [], + documentId: observations.task?.document.id ?? null, documentRevisionId: observations.task?.revision.id ?? null, + saveAudits: { saved: matchedBoardSaveAudit(observations.saved, observations.original, scope), + restored: matchedBoardSaveAudit(observations.restored, observations.afterWork, scope) }, + checks: gradeAgentSettings(scope, observations), + }, null, 2), { mode: 0o600 }); + } catch (error) { if (!failed) throw error; } + } +} diff --git a/tests/runner-e2e/product-journeys/agent-settings-test-fixture.ts b/tests/runner-e2e/product-journeys/agent-settings-test-fixture.ts new file mode 100644 index 00000000000..1467eeb3db7 --- /dev/null +++ b/tests/runner-e2e/product-journeys/agent-settings-test-fixture.ts @@ -0,0 +1,40 @@ +import { instructionDigest, settingsInstructionContent, settingsDocumentKey, settingsTaskPrompt, type AgentSettingsEvidence, type AgentSettingsScope } from "./agent-settings-evidence.js"; + +export function settingsEvidenceFixture(reference = "1a2b3c4d5e6f", original = "# Ordinary instructions\nFollow the assigned task.\n", entryFile = "AGENTS.md") { + const scope: AgentSettingsScope = { campaignId: "campaign-1", companyId: "company", userId: "user", agentId: "agent", + agentName: "Owned agent campaign-1", environmentId: "environment", provider: "opencode", model: "model", reference }; + const saved = settingsInstructionContent(original, scope.reference); + const run = { id: "run", companyId: scope.companyId, agentId: scope.agentId, nativeIssueId: "task", status: "succeeded", + runtimeMode: "native", nativePhase: "committed", resultJson: { finalizationPhase: "committed" }, + contextSnapshot: { issueId: "task", paperclipEnvironment: { id: scope.environmentId } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: scope.provider, model: scope.model } } } }; + const snapshot = (content: string, runs: any[] = [], activity: any[] = []) => { + const hash = instructionDigest(content); + return { agent: { id: scope.agentId, companyId: scope.companyId, name: scope.agentName, adapterType: "paperclip_runner", + adapterConfig: { provider: scope.provider, model: scope.model } }, + bundle: { agentId: scope.agentId, companyId: scope.companyId, entryFile, mode: "managed", editable: true, + files: [{ path: entryFile, isEntryFile: true, editable: true }, + ...(entryFile !== "AGENTS.md" ? [{ path: "AGENTS.md", isEntryFile: false, editable: true }] : [])] }, + file: { path: entryFile, content, contentHash: hash, size: Buffer.byteLength(content), isEntryFile: true, editable: true, + revision: { id: `agent-files:${hash}`, companyId: scope.companyId, agentId: scope.agentId, entryFile, + contentHash: hash, byteLength: Buffer.byteLength(content), parentRevisionId: null, baseRevisionId: null, source: "api" } }, + runs, activity, renderedContent: content }; + }; + const audit = (id: string, content: string) => ({ id, companyId: scope.companyId, actorType: "user", actorId: scope.userId, + agentId: null, runId: null, action: "agent.files_updated", entityType: "agent", entityId: scope.agentId, + details: { path: entryFile, contentHash: instructionDigest(content), source: "board" } }); + const firstSave = audit("save", saved), restore = audit("restore", original); + const body = JSON.stringify({ reference: scope.reference, settingsSaved: true }); + const document = { id: "document", companyId: scope.companyId, issueId: "task", key: settingsDocumentKey, + body, latestRevisionId: "document-revision", createdByAgentId: scope.agentId, createdByUserId: null }; + const revision = { id: "document-revision", documentId: document.id, body, createdByRunId: run.id, + createdByAgentId: scope.agentId, createdByUserId: null }; + const evidence: AgentSettingsEvidence = { + original: snapshot(original), saved: snapshot(saved, [], [firstSave]), reloaded: snapshot(saved, [], [firstSave]), + afterWork: snapshot(saved, [run], [firstSave]), restored: snapshot(original, [run], [firstSave, restore]), + task: { issue: { id: "task", companyId: scope.companyId, assigneeAgentId: scope.agentId, assigneeUserId: null, + createdByUserId: scope.userId, createdByAgentId: null, workMode: "standard", description: settingsTaskPrompt, status: "done" }, + runs: [run], interactions: [], document, revision, reopenedDocument: { ...document }, reopenedRevision: { ...revision }, renderedReference: scope.reference }, + }; + return { scope, evidence, original, saved, run, firstSave, restore }; +} diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 1e1f2cabf36..95e51b6f072 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -2,6 +2,8 @@ import { agentJourneyRuns } from "./success-contract.js"; export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ + { id: "saved-skill-use", label: "Use a saved assigned skill and verify its run-authored result", requires: ["company", "board-user", "task-write", "agent", "skill-write"], capabilities: ["skills/policy", "skills/versions", "documents-artifacts/documents"], paid: true }, + { id: "task-edit-lifecycle", label: "Edit task properties, complete and reopen with saved context", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/properties", "tasks/completion"], paid: false }, { id: "company-access-isolation", label: "Switch QA workspaces and verify restricted agent isolation", requires: ["company", "board-user", "task-write", "company-access"], capabilities: ["companies/selection", "access/resources"], paid: false }, { id: "repository-continuation", label: "Continue repository work with the same saved workspace", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["projects/task-context", "documents-artifacts/attachments"], paid: true }, { id: "routine-scheduled-run", label: "Run a scheduled routine once and verify pause and saved history", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["routines/definition", "routines/triggers", "routines/history", "routines/management"], paid: true }, @@ -9,7 +11,9 @@ export const productJourneys = [ { id: "file-input-answer", label: "Upload a reference file and verify the agent uses its contents", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["documents-artifacts/attachments", "documents-artifacts/documents"], paid: true }, { id: "task-create", label: "Create a task for yourself and reopen it", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/create"], paid: false }, { id: "project-task-context", label: "Create a project and reopen its self-assigned task", requires: ["company", "board-user", "task-write"], capabilities: ["projects/lifecycle", "projects/task-context", "tasks/create"], paid: false }, + { id: "task-dependency-resume", label: "Complete a blocker and automatically resume its dependent task", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["tasks/properties", "tasks/completion", "documents-artifacts/documents"], paid: true }, { id: "ask-question", label: "Ask an agent a question and read its answer", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true }, + { id: "chat-continuity", label: "Reopen a chat, recall its context, then start a fresh session", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["agent-chat/history", "runs-adapters/session"], paid: true }, { id: "question-answer-resume", label: "Answer agent questions and verify the saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true }, { id: "task-properties", label: "Assign a task to the signed-in user", requires: ["company", "board-user", "task-write", "backlog-task"], capabilities: ["tasks/properties"], paid: false }, { id: "task-document-persistence", label: "Create, edit, and reopen a task document", requires: ["company", "board-user", "task-write", "backlog-task"], capabilities: ["documents-artifacts/documents"], paid: false }, @@ -21,6 +25,7 @@ export const productJourneys = [ { id: "stop-redirect", label: "Stop active work and send a new direction once", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/stop-and-pause"], paid: true }, { id: "followup-while-running", label: "Send a follow-up during active work and verify its saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/queue"], paid: true }, { id: "hire-delegate-reuse", label: "Hire, delegate and reuse one teammate", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["agents/hire", "documents-artifacts/documents"], paid: true }, + { id: "agent-settings-persistence", label: "Save a fresh dedicated agent's instructions and verify its first task", requires: ["company", "board-user", "task-write", "agent", "agent-settings-write"], capabilities: ["agents/instructions", "tasks/completion"], paid: true }, { id: "cloud-onboarding", label: "Create a Cloud workspace and complete a first task", requires: ["cloud-account", "fresh-workspace", "provider-key"], capabilities: ["onboarding/first-task"], paid: true }, ] as const; @@ -56,6 +61,8 @@ export function parseTargetConfig(value: unknown): TargetConfig { if (c.adapterModule !== undefined && (typeof c.adapterModule !== "string" || !c.adapterModule)) throw new Error("invalid_adapter_module"); if (c.adapterConfig !== undefined && !object(c.adapterConfig)) throw new Error("invalid_adapter_configuration"); if (c.modelProfileId !== undefined && !["runner-codex", "runner-codex-mini"].includes(c.modelProfileId)) throw new Error("unsupported_model_profile"); + if (c.journeys.includes("agent-settings-persistence") && c.journeys.some(id => id !== "agent-settings-persistence" + && productJourneys.find(journey => journey.id === id)!.paid)) throw new Error("agent_settings_requires_dedicated_campaign"); if (c.mode === "new-cloud" && c.journeys[0] !== "cloud-onboarding") throw new Error("new_cloud_requires_onboarding_first"); if (c.mode === "new-cloud" && c.baseline.connections.length) throw new Error("onboarding_requires_ui_connection_setup"); if (c.mode !== "new-cloud" && c.journeys.includes("cloud-onboarding")) throw new Error("onboarding_requires_fresh_cloud_workspace"); diff --git a/tests/runner-e2e/product-journeys/chat-continuity-evidence.test.ts b/tests/runner-e2e/product-journeys/chat-continuity-evidence.test.ts new file mode 100644 index 00000000000..c95f93cbe53 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-evidence.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; +import { CHAT_CONTINUITY_CHECKS, chatContinuityIssueId, chatContinuityScenario, gradeChatContinuity, + } from "./chat-continuity-evidence.js"; + +import { chatContinuityFixture, compactChatContinuityFixture } from "./chat-continuity.test-fixture.js"; +import { parseTargetConfig } from "./catalog.js"; +import { journeyRequiredChecks } from "./success-contract.js"; + +type Row = Record; +const allPass = (f: ReturnType) => gradeChatContinuity(f.scope, f.points).every(check => check.passed); + +describe("chat reload and new-session evidence", () => { + it("requires the actual three-turn outcome, one reset and run-bound reloaded replies", () => { + const f = chatContinuityFixture(); + expect(gradeChatContinuity(f.scope, f.points).map(row => row.id)).toEqual(CHAT_CONTINUITY_CHECKS); + expect(allPass(f)).toBe(true); + }); + it("accepts normal compact run details only with bound native-final events and the original reset summary", () => { + expect(allPass(compactChatContinuityFixture())).toBe(true); + }); + it("rejects an unscoped activity response even below the truncation limit", () => { + const f = chatContinuityFixture(); + f.points[5]!.activity.push({ id: "foreign-event", companyId: "foreign", agentId: "other-agent", action: "project.created", + entityType: "project", entityId: "foreign-project", runId: "other-run" }); + expect(allPass(f)).toBe(false); + }); + it("requires the fresh agent's pre-send empty-history receipt", () => { + const f = chatContinuityFixture(); delete f.scope.agentHistory; + expect(allPass(f)).toBe(false); + }); + it("never supplies the old phrase in the recall or fresh-context request", () => { + const s = chatContinuityScenario("random-123"); + expect(s.remember).toContain(s.remembered); + expect(s.recall).not.toContain(s.remembered); + expect(s.fresh).not.toContain(s.remembered); + expect(s.reset).toBe("/new"); + }); + it("registers the exact measured checks and counts all four persisted runs", () => { + expect(journeyRequiredChecks["chat-continuity"]).toEqual(CHAT_CONTINUITY_CHECKS); + const config = { version: 1, id: "qa-chat", mode: "existing-cloud", adapterModule: "/private/provider.mjs", + executionEnvironment: "daytona", journeys: ["chat-continuity"], baseline: { features: {}, connections: [] }, + limits: { durationMs: 900_000, maxRuns: 4, modelBudgetCents: 500 } }; + expect(parseTargetConfig(config).limits.maxRuns).toBe(4); + for (const maxRuns of [0, 3, 5]) expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns } })).toThrow("agent_journeys_require_bounded_runs"); + }); + const mutants: ReadonlyArray) => void]> = [ + ["missing point", (f: ReturnType) => { f.points.pop(); }], + ["duplicate phase", (f) => { f.points[5]!.phase = "fresh"; }], + ["foreign chat", (f) => { f.points[1]!.chat.companyId = "foreign"; }], + ["different user", (f) => { f.points[3]!.chat.conversationUserId = "foreign"; }], + ["replaced backing issue", (f) => { f.points[5]!.chat.id = "replacement"; }], + ["active conversation", (f) => { f.points[5]!.chat.conversationState = "running"; }], + ["scheduled retry", (f) => { f.points[5]!.chat.scheduledRetry = { state: "pending" }; }], + ["duplicate user send", (f) => { f.points[5]!.comments.push({ ...f.points[5]!.comments[0], id: "duplicate" }); }], + ["foreign author", (f) => { f.points[5]!.comments[0]!.authorUserId = "foreign"; }], + ["wrong remembered phrase", (f) => { f.points[2]!.reply!.body = "I do not remember"; }], + ["unbound first reply", (f) => { f.points[0]!.reply!.anchor = "unbound"; }], + ["first reply missing after reopen", (f) => { delete f.points[1]!.reply; }], + ["stale recall response", (f) => { f.points[2]!.reply!.runId = "run-0"; }], + ["foreign recall run", (f) => { f.points[2]!.runs[1]!.agentId = "foreign"; }], + ["different provider", (f) => { f.points[2]!.runs[1]!.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }], + ["different model", (f) => { f.points[5]!.runs[3]!.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }], + ["different environment", (f) => { f.points[5]!.runs[3]!.contextSnapshot.paperclipEnvironment.id = "other"; }], + ["uncommitted native phase", (f) => { f.points[5]!.runs[3]!.nativePhase = "finalizing"; }], + ["missing session proof", (f) => { delete f.points[0]!.runs[0]!.nativeSessionId; }], + ["unintended session replacement on reload", (f) => { f.points[2]!.runs[1]!.nativeSessionId = "replacement"; }], + ["reset generation absent", (f) => { f.points[3]!.chat.conversationSessionGeneration = 0; }], + ["fresh run generation stale", (f) => { f.points[4]!.runs[3]!.contextSnapshot.conversationSessionGeneration = 0; }], + ["fresh turn resumes old session", (f) => { f.points[4]!.runs[3]!.nativeSessionId = "original-session"; }], + ["previous session supplied on fresh admission", (f) => { f.points[4]!.runs[3]!.sessionIdBefore = "original-session"; }], + ["missing fresh-admission session evidence", (f) => { delete f.points[4]!.runs[3]!.sessionIdBefore; }], + ["undefined fresh-admission session evidence", (f) => { f.points[4]!.runs[3]!.sessionIdBefore = undefined; }], + ["empty fresh-admission session evidence", (f) => { f.points[4]!.runs[3]!.sessionIdBefore = ""; }], + ["old phrase leaked in native context", (f) => { f.points[4]!.runs[3]!.contextSnapshot.paperclipTaskMarkdown += f.scenario.remembered; }], + ["old phrase returned after reset", (f) => { f.points[4]!.reply!.body = f.scenario.remembered; }], + ["fresh marker absent", (f) => { f.points[4]!.reply!.body = "I have no phrase"; }], + ["extra run", (f) => { f.points[5]!.runs.push({ ...f.points[5]!.runs[3], id: "extra" }); }], + ["duplicate run ID", (f) => { f.points[5]!.runs[3]!.id = "run-0"; }], + ["missing reset", (f) => { f.points[5]!.runs.splice(2, 1); }], + ["missing synthetic reset receipt", (f) => { f.points[5]!.resetReceipts.splice(0); }], + ["foreign reset summary agent", (f) => { f.points[5]!.resetReceipts[0]!.agentId = "foreign"; }], + ["foreign reset summary company", (f) => { f.points[5]!.resetReceipts[0]!.companyId = "foreign"; }], + ["foreign reset summary issue", (f) => { f.points[5]!.resetReceipts[0]!.contextIssueId = "foreign"; }], + ["wrong reset summary run", (f) => { f.points[5]!.resetReceipts[0]!.runId = "run-0"; }], + ["wrong reset summary wake", (f) => { f.points[5]!.resetReceipts[0]!.wakeCommentId = "user-0"; }], + ["reset summary not successful", (f) => { f.points[5]!.resetReceipts[0]!.status = "running"; }], + ["contradictory reset detail", (f) => { f.points[5]!.runs[2]!.resultJson.conversationReset = false; }], + ["provider work disguised as reset", (f) => { f.points[5]!.runs[2]!.runnerProfileJson = { nativeExecutionInput: { provider: { kind: f.scope.provider, model: f.scope.model } } }; }], + ["native session attached to synthetic reset", (f) => { f.points[5]!.runs[2]!.nativeSessionId = "unexpected-provider-session"; }], + ["reset publishes an answer", (f) => { f.points[5]!.comments.push({ id: "reset-answer", authorAgentId: f.scope.agentId, createdByRunId: "run-2", body: "Hi" }); }], + ["unrequested completion callback", (f) => { f.points[5]!.runs[3]!.contextSnapshot.source = "automation"; }], + ["wrong wake comment", (f) => { f.points[5]!.runs[3]!.contextSnapshot.wakeCommentId = "user-0"; }], + ["run has conflicting issue IDs", (f) => { f.points[5]!.runs[3]!.issueId = "foreign"; }], + ["changed original history", (f) => { f.points[5]!.comments[0]!.body = "changed"; }], + ["missing original history", (f) => { f.points[5]!.comments.splice(1, 1); }], + ["reopen repeats the send", (f) => { f.points[5]!.runs.push({ ...f.points[5]!.runs[3], id: "repeated" }); }], + ["reply not observed after reload", (f) => { f.points[5]!.reply!.observedAfterReload = false; }], + ["hidden final reply", (f) => { f.points[5]!.reply!.visible = false; }], + ["created work", (f) => { f.points[5]!.tasks.push({ id: "unexpected" }); }], + ["created project", (f) => { f.points[5]!.activity.push({ id: "created-project", companyId: f.scope.companyId, agentId: f.scope.agentId, + runId: "run-1", action: "project.created", entityType: "project", entityId: "unexpected" }); }], + ["truncated creation history", (f) => { f.points[5]!.activity = Array.from({ length: 500 }, () => ({})); }], + ["missing run collection", (f) => { f.points[5]!.runs = undefined as unknown as Row[]; }], + ["malformed history row", (f) => { f.points[5]!.comments.push(null as unknown as Row); }], + ]; + it.each(mutants)("rejects %s", (_name, mutate) => { + const f = chatContinuityFixture(); + mutate(f); + expect(allPass(f)).toBe(false); + }); + it("rejects another agent's activity rather than filtering an unscoped response", () => { + const f = chatContinuityFixture(); + f.points[5]!.activity.push({ companyId: f.scope.companyId, agentId: "other-agent", runId: "other-run", action: "project.created" }); + expect(allPass(f)).toBe(false); + }); + it("accepts complete owned activity and rejects unaccounted creation by that same agent", () => { + const f = chatContinuityFixture(); + const row = { id: "owned-event", companyId: f.scope.companyId, agentId: f.scope.agentId, + runId: "run-0", action: "heartbeat.run_completed", entityType: "heartbeat_run", entityId: "run-0" }; + f.points[5]!.activity.push(row); + expect(allPass(f)).toBe(true); + row.action = "project.created"; row.runId = "unaccounted-owned-run"; + expect(allPass(f)).toBe(false); + }); + it("does not treat a compact run as sufficient without actual native output proof", () => { + const f = compactChatContinuityFixture(); + f.points[5]!.events = []; + expect(allPass(f)).toBe(false); + }); + it("retains backward-compatible issue aliases only when they agree", () => { + expect(chatContinuityIssueId({ nativeIssueId: "chat", contextSnapshot: { issueId: "chat" } })).toBe("chat"); + expect(chatContinuityIssueId({ issueId: "foreign", nativeIssueId: "chat" })).toBeUndefined(); + expect(chatContinuityIssueId({})).toBeUndefined(); + }); +}); diff --git a/tests/runner-e2e/product-journeys/chat-continuity-evidence.ts b/tests/runner-e2e/product-journeys/chat-continuity-evidence.ts new file mode 100644 index 00000000000..598d76f3ff5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-evidence.ts @@ -0,0 +1,157 @@ +import { chatMarker } from "../chat-cases.js"; +import { taskReplyBelongsToRun, type TaskReplyObservation } from "../native-task-reply.js"; + +type Row = Record; +export const CHAT_CONTINUITY_CHECKS = [ + "chat-fresh-owned-conversation", "chat-user-messages-stored-once", "chat-reload-preserved-context", + "chat-new-session-boundary", "chat-fresh-context-answer", "chat-history-preserved-after-reload", + "chat-no-unrequested-work", "chat-exact-turn-accounting", "configured-model-and-environment", +] as const; + +export function chatContinuityScenario(campaignId: string) { + const remembered = chatMarker("OLDCONTEXT", campaignId), marker = chatMarker("CHAT", campaignId); + return { + remembered, marker, + remember: `For this conversation only, remember the phrase ${remembered}. Just acknowledge briefly; no project or task is needed.`, + recall: "What phrase did I just ask you to remember? Reply with the phrase only. No project or task is needed.", + reset: "/new", + fresh: `Without reading older history or files, if you have a remembered phrase in your current context return it; otherwise reply exactly ${marker}. Do not look it up. No project or task is needed.`, + }; +} + +export interface ChatContinuityScope { + companyId: string; userId: string; agentId: string; agentName: string; + environmentId: string; provider: string; model: string; campaignId: string; + agentHistory?: { source: "company-agent-runs"; companyId: string; agentId: string; limit: 1; runIds: string[]; observedBeforeSend: true }; +} +export interface ChatContinuityResetReceipt { + source: "issue-runs"; companyId: string; issueId: string; + runId: string; agentId: string; status: string; contextIssueId: string; wakeCommentId: string; + resultJson: { conversationReset: true }; +} +export interface ChatContinuityPoint { + phase: "initial" | "returned" | "recalled" | "reset" | "fresh" | "reopened"; + chat: Row; runs: Row[]; comments: Row[]; tasks: Row[]; activity: Row[]; + reply?: TaskReplyObservation & { runId: string; agentId: string; observedAfterReload: boolean }; + events?: Row[]; + resetReceipts: ChatContinuityResetReceipt[]; +} +const nonempty = (value: unknown): value is string => typeof value === "string" && value.length > 0; +export const chatContinuityResetRun = (run: Row) => run.contextSnapshot?.conversationReset === true || run.resultJson?.conversationReset === true; +export const chatContinuityIssueId = (run: Row): string | undefined => { + const ids = [run.issueId, run.nativeIssueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].filter(nonempty); + return ids.length && new Set(ids).size === 1 ? ids[0] : undefined; +}; +const userBody = (row: Row) => typeof row.body === "string" ? row.body.replaceAll("\\_", "_").trim() : ""; +const sameIds = (left: Row[], right: Row[]) => left.length === right.length && left.every(row => right.some(other => row.id === other.id)); +const commentContent = (row: Row) => [row.id, row.companyId, row.issueId, row.body, row.authorAgentId, + row.authorUserId, row.createdByRunId, row.conversationSessionGeneration]; +const sameRows = (left: Row[], right: Row[]) => left.every(row => right.some(other => other.id === row.id + && JSON.stringify(commentContent(other)) === JSON.stringify(commentContent(row)))); + +export function chatContinuityActivityComplete(scope: Pick, rows: Row[]): boolean { + return Array.isArray(rows) && rows.length < 500 && new Set(rows.map(row => row?.id)).size === rows.length + && rows.every(row => row && nonempty(row.id) && row.companyId === scope.companyId && row.agentId === scope.agentId + && nonempty(row.action) && nonempty(row.entityType) && nonempty(row.entityId)); +} + +export function chatContinuityResetReceiptMatches(scope: Pick, + chatId: string, run: Row, receipt: ChatContinuityResetReceipt): boolean { + const fullResult = run.resultJson; + const optionalFlagValid = fullResult == null || typeof fullResult === "object" && !Array.isArray(fullResult) + && (!Object.hasOwn(fullResult, "conversationReset") || fullResult.conversationReset === true); + return receipt?.source === "issue-runs" && receipt.companyId === scope.companyId && receipt.issueId === chatId + && receipt.runId === run.id && receipt.agentId === scope.agentId && receipt.status === "succeeded" + && receipt.contextIssueId === chatId && nonempty(receipt.wakeCommentId) + && receipt.wakeCommentId === run.contextSnapshot?.wakeCommentId && receipt.resultJson?.conversationReset === true + && run.companyId === scope.companyId && run.agentId === scope.agentId && chatContinuityIssueId(run) === chatId + && run.status === "succeeded" && run.contextSnapshot?.conversationReset === true && optionalFlagValid + && !run.runnerProfileJson?.nativeExecutionInput && !run.nativeSessionId; +} + +export function gradeChatContinuity(scope: ChatContinuityScope, points: ChatContinuityPoint[]) { + const scenario = chatContinuityScenario(scope.campaignId); + const phases = ["initial", "returned", "recalled", "reset", "fresh", "reopened"] as const; + const complete = points.length === phases.length && points.every((point, index) => point?.phase === phases[index] + && point.chat && [point.runs, point.comments, point.tasks, point.activity, point.resetReceipts].every(rows => Array.isArray(rows) + && rows.every(row => row && typeof row === "object" && !Array.isArray(row)))); + if (!complete) return CHAT_CONTINUITY_CHECKS.map(id => ({ id, passed: false })); + const [initial, returned, recalled, reset, fresh, reopened] = complete ? points : []; + const chatId = initial?.chat?.id; + const scoped = complete && [scope.companyId, scope.userId, scope.agentId, scope.agentName, scope.environmentId, + scope.provider, scope.model, scope.campaignId].every(nonempty) && nonempty(chatId) + && scope.agentHistory?.source === "company-agent-runs" && scope.agentHistory.companyId === scope.companyId + && scope.agentHistory.agentId === scope.agentId && scope.agentHistory.limit === 1 && scope.agentHistory.observedBeforeSend === true + && Array.isArray(scope.agentHistory.runIds) && scope.agentHistory.runIds.length === 0 + && points.every(point => point.chat?.id === chatId && point.chat.companyId === scope.companyId + && point.chat.assigneeAgentId === scope.agentId && point.chat.conversationAgentId === scope.agentId + && point.chat.conversationUserId === scope.userId && point.chat.createdByUserId === scope.userId + && point.chat.title === `Chat with ${scope.agentName}` && point.chat.status === "in_review" + && point.chat.conversationState === "waiting" && !point.chat.executionRunId && !point.chat.scheduledRetry && !point.chat.activeRecoveryAction); + const requested = [scenario.remember, scenario.recall, scenario.reset, scenario.fresh]; + const finalComments = reopened?.comments ?? []; + const board = finalComments.filter(row => row.authorUserId === scope.userId && !row.authorAgentId); + const userMessages = scoped && board.length === requested.length && new Set(board.map(row => row.id)).size === requested.length + && requested.every(body => board.filter(row => userBody(row) === body && row.companyId === scope.companyId && row.issueId === chatId).length === 1) + && finalComments.every(row => !row.authorUserId || row.authorUserId === scope.userId); + const runs = reopened?.runs ?? [], providerRuns = runs.filter(run => !chatContinuityResetRun(run)), resetRuns = runs.filter(chatContinuityResetRun); + const runBinding = scoped && points.every(point => point.runs.every(run => nonempty(run.id) && run.companyId === scope.companyId + && run.agentId === scope.agentId && chatContinuityIssueId(run) === chatId && run.status === "succeeded")); + const providerBinding = runBinding && providerRuns.length === 3 && points.flatMap(point => point.runs) + .filter(run => !chatContinuityResetRun(run)).every(run => { + const provider = run.runnerProfileJson?.nativeExecutionInput?.provider; + return run.runtimeMode === "native" && provider?.kind === scope.provider && provider?.model === scope.model + && run.contextSnapshot?.paperclipEnvironment?.id === scope.environmentId + && run.nativePhase === "committed" + && nonempty(run.nativeSessionId); + }); + const expectedCounts = [1, 1, 2, 3, 4, 4]; + const exactTurns = runBinding && providerBinding && resetRuns.length === 1 && runs.length === 4 && new Set(runs.map(run => run.id)).size === 4 + && points.every(point => point.resetReceipts.length === point.runs.filter(chatContinuityResetRun).length + && point.runs.filter(chatContinuityResetRun).every(run => point.resetReceipts + .filter(receipt => chatContinuityResetReceiptMatches(scope, chatId, run, receipt)).length === 1)) + && points.every((point, index) => point.runs.length === expectedCounts[index] + && point.runs.every(run => runs.some(final => final.id === run.id && final.status === run.status))) + && runs.every(run => { + const request = board.find(comment => comment.id === run.contextSnapshot?.wakeCommentId); + return request && run.contextSnapshot?.source === "issue.comment" + && (chatContinuityResetRun(run) ? userBody(request) === scenario.reset : userBody(request) !== scenario.reset); + }) && new Set(runs.map(run => run.contextSnapshot?.wakeCommentId)).size === 4; + const pointReply = (point: ChatContinuityPoint | undefined) => { + const receipt = point?.reply, run = point?.runs.find(run => run.id === receipt?.runId); + return receipt && run && receipt.agentId === scope.agentId && receipt.observedAfterReload === true + && taskReplyBelongsToRun(receipt, run, scope.agentId, point!.comments, point!.events ?? []) ? receipt : undefined; + }; + const originalReply = pointReply(initial), returnedReply = pointReply(returned), recalledReply = pointReply(recalled), freshReply = pointReply(fresh), reopenedReply = pointReply(reopened); + const firstRuns = initial?.runs.filter(run => !chatContinuityResetRun(run)) ?? []; + const recallRuns = recalled?.runs.filter(run => !chatContinuityResetRun(run) && run.id !== firstRuns[0]?.id) ?? []; + const freshRuns = fresh?.runs.filter(run => !chatContinuityResetRun(run) && !recalled?.runs.some(old => old.id === run.id)) ?? []; + const first = firstRuns[0], second = recallRuns[0], last = freshRuns[0], resetRun = resetRuns[0]; + const recall = exactTurns && Boolean(originalReply && returnedReply && recalledReply && originalReply.runId === first?.id + && returnedReply.runId === originalReply.runId && returnedReply.body === originalReply.body && recalledReply.runId === second?.id + && recalledReply.body.includes(scenario.remembered) && first?.nativeSessionId === second?.nativeSessionId + && initial!.chat.conversationSessionGeneration === 0 && returned!.chat.conversationSessionGeneration === 0 + && recalled!.chat.conversationSessionGeneration === 0 && first?.contextSnapshot?.conversationSessionGeneration === 0 + && second?.contextSnapshot?.conversationSessionGeneration === 0 && sameIds(initial!.runs, returned!.runs) + && sameRows(initial!.comments, returned!.comments)); + const boundary = exactTurns && Boolean(first && second && last && resetRun && freshRuns.length === 1 + && reset!.chat.conversationSessionGeneration === 1 && fresh!.chat.conversationSessionGeneration === 1 + && reopened!.chat.conversationSessionGeneration === 1 && last.contextSnapshot?.conversationSessionGeneration === 1 + && resetRun.contextSnapshot?.conversationSessionGeneration === 1 + && Object.hasOwn(last, "sessionIdBefore") && last.sessionIdBefore === null + && last.nativeSessionId !== first.nativeSessionId && last.nativeSessionId !== second.nativeSessionId + && typeof last.contextSnapshot?.paperclipTaskMarkdown === "string" + && !last.contextSnapshot.paperclipTaskMarkdown.includes(scenario.remembered) + && finalComments.every(row => row.createdByRunId !== resetRun.id)); + const freshAnswer = boundary && Boolean(freshReply && freshReply.runId === last?.id + && freshReply.body.trim() === scenario.marker && !freshReply.body.includes(scenario.remembered)); + const history = scoped && Boolean(reopenedReply && freshReply && reopenedReply.runId === freshReply.runId + && reopenedReply.body === freshReply.body && sameIds(fresh!.runs, reopened!.runs) + && sameRows(initial!.comments, reopened!.comments) && sameRows(recalled!.comments, reopened!.comments) + && sameRows(fresh!.comments, reopened!.comments)); + const noWork = scoped && points.every(point => Array.isArray(point.tasks) && point.tasks.length === 0 + && chatContinuityActivityComplete(scope, point.activity) + && !point.activity.some(row => ["issue.created", "issue.child_created", "project.created", "agent.created", "agent.hire_created"].includes(row.action))); + const values = [scoped, userMessages, recall, boundary, freshAnswer, history, noWork, exactTurns, providerBinding]; + return CHAT_CONTINUITY_CHECKS.map((id, index) => ({ id, passed: Boolean(values[index]) })); +} diff --git a/tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts b/tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts new file mode 100644 index 00000000000..f69c7f69bcf --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts @@ -0,0 +1,180 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; +import { chatContinuityFixture, compactChatContinuityFixture } from "./chat-continuity.test-fixture.js"; + +const mocks = vi.hoisted(() => ({ send: vi.fn(), reply: vi.fn(), capture: vi.fn() })); +vi.mock("../chat-flow.js", () => ({ sendChatMessage: mocks.send })); +vi.mock("../native-task-reply.js", async importOriginal => ({ + ...await importOriginal(), readBoundTaskFinalReply: mocks.reply, +})); +vi.mock("./product-screenshot.js", () => ({ captureProductScreenshot: mocks.capture })); +vi.mock("@playwright/test", () => ({ expect: (locator: { isVisible(): Promise }) => ({ + async toBeVisible() { expect(await locator.isVisible()).toBe(true); }, +}) })); +import { runChatContinuity } from "./chat-continuity-flow.js"; + +const directories: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + vi.resetAllMocks(); + for (const directory of directories.splice(0)) await rm(directory, { recursive: true, force: true }); +}); +async function fixture(options: { preexisting?: boolean; intentFails?: boolean; sendFails?: boolean; foreignRun?: boolean; + compactRuns?: boolean; unrelatedActivity?: number; ignoreActivityScope?: boolean; priorAgentStatus?: string; mixedAgentHistory?: boolean; + provisioningActor?: "board" | "owned-agent" } = {}) { + const f = options.compactRuns ? compactChatContinuityFixture() : chatContinuityFixture(); + const directory = await mkdtemp(path.join(os.tmpdir(), "chat-continuity-flow-")); + directories.push(directory); + let stage = options.preexisting ? 0 : -1; + const messages: string[] = [], actions: string[] = []; + const realTimeout = globalThis.setTimeout; + vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[]) => + realTimeout(callback, delay === 1000 ? 0 : delay, ...args)) as typeof setTimeout); + const current = () => f.points[stage]!; + const companyActivity = [ + ...Array.from({ length: options.unrelatedActivity ?? 0 }, (_, index) => ({ id: `unrelated-${index}`, + companyId: f.scope.companyId, agentId: "other-agent", runId: "other-run", action: "project.created", entityType: "project", entityId: `other-project-${index}` })), + ...(options.provisioningActor ? [{ id: "provisioned-agent", companyId: f.scope.companyId, + agentId: options.provisioningActor === "board" ? null : f.scope.agentId, runId: null, + action: "agent.created", entityType: "agent", entityId: f.scope.agentId }] : []), + ]; + const companyRuns = Array.from({ length: options.unrelatedActivity ?? 0 }, (_, index) => ({ id: `unrelated-run-${index}`, + companyId: f.scope.companyId, agentId: "other-agent", status: "succeeded" })); + if (options.priorAgentStatus) companyRuns.push({ id: "prior-owned-run", companyId: f.scope.companyId, agentId: f.scope.agentId, status: options.priorAgentStatus }); + const get = vi.fn(async (route: string) => { + if (route === "/api/instance/settings/experimental") return { enableAgentChat: true }; + if (route === "/api/companies/company/heartbeat-runs?agentId=agent&limit=1&summary=true") { + actions.push("history"); + if (options.mixedAgentHistory) return [{ id: "foreign-run", companyId: "foreign-company", agentId: "other-agent", status: "running" }]; + return companyRuns.filter(run => run.agentId === f.scope.agentId).slice(0, 1); + } + if (route === "/api/companies/company/chats/agent") return stage < 0 ? null : structuredClone(current().chat); + if (route === "/api/issues/chat/runs") return current().runs.map(run => ({ runId: run.id, agentId: run.agentId, status: run.status, + contextIssueId: run.contextSnapshot.issueId, wakeCommentId: run.contextSnapshot.wakeCommentId, + resultJson: run.contextSnapshot.conversationReset ? { conversationReset: true } : {} })); + if (route === "/api/issues/chat/comments?order=asc") return structuredClone(current().comments); + if (route === "/api/companies/company/issues?view=index&assigneeAgentId=agent&includeAgentCreatedTasks=true&limit=2") return []; + if (route === "/api/companies/company/activity?agentId=agent&limit=500") { + return (options.ignoreActivityScope ? companyActivity : companyActivity.filter(row => row.agentId === f.scope.agentId)).slice(0, 500); + } + if (/\/api\/heartbeat-runs\/run-\d\/events\?/.test(route)) return structuredClone(current().events ?? []); + const id = route.match(/^\/api\/heartbeat-runs\/(run-\d)$/)?.[1]; + if (id) { + const run = structuredClone(current().runs.find(row => row.id === id)!); + if (options.foreignRun) run.nativeIssueId = "foreign"; + return run; + } + throw new Error(`unexpected GET ${route}`); + }); + const page = { goto: vi.fn(async (_url: string) => { actions.push("navigate"); }), reload: vi.fn(async () => { actions.push("reload"); }), + getByTestId: vi.fn(() => ({ isVisible: async () => true })) }; + const own = vi.fn(async (resource: { id?: string }) => { + actions.push(resource.id ? "receipt" : "intent"); + if (options.intentFails && !resource.id) throw new Error("ownership intent failed"); + }); + const context = { page, api: { get, origin: "https://qa.example.test" }, + company: { id: f.scope.companyId, issuePrefix: "QA" }, user: { id: f.scope.userId, name: "QA" }, + environmentId: f.scope.environmentId, agent: { id: f.scope.agentId, name: f.scope.agentName, + provider: f.scope.provider, model: f.scope.model, runtimeMode: "native" } } as unknown as JourneyContext; + const evidence: JourneyEvidence = { outcome: "passed", startingState: "recipe", phases: [], checks: [], resourceDisposition: "released", privacy: "private" }; + mocks.send.mockImplementation(async (_page, message: string) => { + actions.push("send"); messages.push(message); + if (options.sendFails) throw new Error("uncertain send response"); + const index = [f.scenario.remember, f.scenario.recall, f.scenario.reset, f.scenario.fresh].indexOf(message); + expect(index).toBe(messages.length - 1); + stage = [0, 2, 3, 4][index]!; + }); + mocks.reply.mockImplementation(async input => { + await input.page.reload(); + const data = await input.load(); + expect(data.run.id).toBe(input.runId); + const point = f.points.find(point => point.reply?.runId === input.runId)!; + return structuredClone(point.reply!); + }); + mocks.capture.mockResolvedValue(undefined); + return { ...f, context, evidence, own, directory, messages, actions, page, get, companyActivity, + run: () => runChatContinuity(context, { own } as unknown as TargetSession, f.scope.campaignId, evidence, + { directory, deadlineAt: Date.now() + 10_000 }) }; +} +describe("chat continuity journey controls", () => { + it("uses ordinary user messages once, owns the fresh chat before dispatch, and retains the independently graded result", async () => { + const f = await fixture(); + await f.run(); + expect(f.messages).toEqual([f.scenario.remember, f.scenario.recall, "/new", f.scenario.fresh]); + expect(f.actions.indexOf("intent")).toBeLessThan(f.actions.indexOf("send")); + expect(f.actions.indexOf("history")).toBeLessThan(f.actions.indexOf("intent")); + expect(f.own).toHaveBeenCalledTimes(2); + expect(f.own.mock.calls[0]![0]).toMatchObject({ journeyId: "chat-continuity", freshChat: { absent: true, userId: f.scope.userId } }); + expect(f.own.mock.calls[1]![0]).toMatchObject({ id: "chat", journeyId: "chat-continuity" }); + expect(f.evidence.checks).toHaveLength(9); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(mocks.reply).toHaveBeenCalledTimes(5); + expect(mocks.capture).toHaveBeenCalledTimes(6); + expect(f.page.goto.mock.calls.flat().every(url => typeof url !== "string" || url.startsWith("https://qa.example.test/"))).toBe(true); + const saved = JSON.parse(await readFile(path.join(f.directory, "chat-continuity.json"), "utf8")); + expect(saved.points.map((point: { phase: string }) => point.phase)).toEqual(["initial", "returned", "recalled", "reset", "fresh", "reopened"]); + expect(saved.checks.every((check: { passed: boolean }) => check.passed)).toBe(true); + }); + it("accepts compact details with original reset summaries despite more than 500 unrelated company events", async () => { + const f = await fixture({ compactRuns: true, unrelatedActivity: 750 }); + await f.run(); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const saved = JSON.parse(await readFile(path.join(f.directory, "chat-continuity.json"), "utf8")); + expect(saved.points.every((point: { activity: unknown[] }) => point.activity.length === 0)).toBe(true); + expect(saved.points[5].resetReceipts[0]).toMatchObject({ source: "issue-runs", runId: "run-2", agentId: f.scope.agentId, + contextIssueId: "chat", wakeCommentId: "user-2", resultJson: { conversationReset: true } }); + expect(saved.points[5].runs[2].resultJson).toBeNull(); + }); + it.each([1, 750])("rejects an ignored agent filter instead of filtering %s unrelated global activity rows", async unrelatedActivity => { + const f = await fixture({ unrelatedActivity, ignoreActivityScope: true }); + await expect(f.run()).rejects.toThrow("chat_continuity_observation_incomplete"); + expect(f.messages).toEqual([f.scenario.remember]); + }); + it("accepts board provisioning whose created entity is the owned agent, rather than treating its entity id as the actor", async () => { + const f = await fixture({ provisioningActor: "board", unrelatedActivity: 750 }); + expect(f.companyActivity.at(-1)).toMatchObject({ action: "agent.created", agentId: null, entityId: f.scope.agentId }); + await f.run(); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const saved = JSON.parse(await readFile(path.join(f.directory, "chat-continuity.json"), "utf8")); + expect(saved.points.every((point: { activity: unknown[] }) => point.activity.length === 0)).toBe(true); + }); + it("rejects creation attributed to the fresh owned agent even if that activity predates the first send", async () => { + const f = await fixture({ provisioningActor: "owned-agent" }); + await expect(f.run()).rejects.toThrow("chat_continuity_evidence_failed"); + expect(f.evidence.checks.find(check => check.id === "chat-no-unrequested-work")?.passed).toBe(false); + }); + it.each(["succeeded", "running", "queued", "failed"])("rejects prior %s work on a reused agent before any send", async priorAgentStatus => { + const f = await fixture({ priorAgentStatus, unrelatedActivity: 750 }); + await expect(f.run()).rejects.toThrow("fresh_chat_agent_required"); + expect(f.messages).toEqual([]); + expect(f.own).not.toHaveBeenCalled(); + }); + it("rejects mixed or foreign agent history before any send", async () => { + const f = await fixture({ mixedAgentHistory: true }); + await expect(f.run()).rejects.toThrow("chat_continuity_agent_history_scope_mismatch"); + expect(f.messages).toEqual([]); + expect(f.own).not.toHaveBeenCalled(); + }); + it.each(["preexisting", "intentFails"] as const)("does not dispatch model work when %s rejects ownership", async option => { + const f = await fixture({ [option]: true }); + await expect(f.run()).rejects.toThrow(); + expect(f.messages).toEqual([]); + expect(f.evidence.checks.every(check => !check.passed)).toBe(true); + }); + it("never replays an uncertain send", async () => { + const f = await fixture({ sendFails: true }); + await expect(f.run()).rejects.toThrow("uncertain send response"); + expect(f.messages).toEqual([f.scenario.remember]); + expect(mocks.send).toHaveBeenCalledTimes(1); + expect(f.evidence.checks.some(check => !check.passed)).toBe(true); + }); + it("rejects a foreign native run before sending another message", async () => { + const f = await fixture({ foreignRun: true }); + await expect(f.run()).rejects.toThrow("chat_continuity_run_mismatch"); + expect(f.messages).toEqual([f.scenario.remember]); + expect(f.evidence.checks.some(check => !check.passed)).toBe(true); + }); +}); diff --git a/tests/runner-e2e/product-journeys/chat-continuity-flow.ts b/tests/runner-e2e/product-journeys/chat-continuity-flow.ts new file mode 100644 index 00000000000..fe2a2301371 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-flow.ts @@ -0,0 +1,164 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { sendChatMessage } from "../chat-flow.js"; +import { readBoundTaskFinalReply } from "../native-task-reply.js"; +import { collectRunEvents } from "../run-observations.js"; +import { chatContinuityActivityComplete, chatContinuityIssueId, chatContinuityResetReceiptMatches, + chatContinuityResetRun, chatContinuityScenario, gradeChatContinuity, + type ChatContinuityPoint, type ChatContinuityScope, type ChatContinuityResetReceipt } from "./chat-continuity-evidence.js"; +import { createObservedReader } from "./observation-read.js"; +import { captureProductScreenshot } from "./product-screenshot.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +interface Options { deadlineAt: number; directory: string } + +export async function runChatContinuity(context: JourneyContext, session: TargetSession, + campaignId: string, evidence: JourneyEvidence, options: Options) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !agent.model || !agent.provider || !context.environmentId) { + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + } + const scope: ChatContinuityScope = { companyId: context.company.id, userId: context.user.id, + agentId: agent.id, agentName: agent.name, environmentId: context.environmentId, + provider: agent.provider, model: agent.model, campaignId }; + const scenario = chatContinuityScenario(campaignId), reader = createObservedReader(context.api, options.deadlineAt); + const chatPath = `/api/companies/${scope.companyId}/chats/${agent.id}`; + const chatUrl = `${context.api.origin}/${encodeURIComponent(context.company.issuePrefix)}/chats/${encodeURIComponent(agent.id)}`; + const points: ChatContinuityPoint[] = []; + let chatId: string | undefined, owned = false; + const remaining = () => { + const value = options.deadlineAt - Date.now(); + if (value <= 0) throw new JourneyError("failed", "chat_continuity_deadline"); + return value; + }; + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const retain = async () => writeFile(path.join(options.directory, "chat-continuity.json"), + JSON.stringify({ scope, points, checks: gradeChatContinuity(scope, points) }, null, 2), { mode: 0o600 }); + const freshChat = { companyId: scope.companyId, userId: scope.userId, leadAgentId: agent.id, absent: true as const }; + const read = async (): Promise | undefined> => { + const chat = await reader.get(chatPath); + if (!chat) return; + if (!chat.id || chat.companyId !== scope.companyId || chat.assigneeAgentId !== agent.id + || chat.conversationAgentId !== agent.id || chat.conversationUserId !== scope.userId + || chat.createdByUserId !== scope.userId || chat.title !== `Chat with ${agent.name}` || chatId && chat.id !== chatId) { + throw new JourneyError("failed", "chat_continuity_scope_mismatch"); + } + chatId ??= chat.id; + if (!owned) { + await session.own({ kind: "issue", name: chat.title, id: chat.id, journeyId: "chat-continuity", freshChat }); + owned = true; + } + const [summaries, comments, tasks, activity] = await Promise.all([ + reader.get(`/api/issues/${chat.id}/runs`), reader.get(`/api/issues/${chat.id}/comments?order=asc`), + reader.get(`/api/companies/${scope.companyId}/issues?view=index&assigneeAgentId=${agent.id}&includeAgentCreatedTasks=true&limit=2`), + reader.get(`/api/companies/${scope.companyId}/activity?agentId=${agent.id}&limit=500`), + ]); + if (![summaries, comments, tasks, activity].every(Array.isArray) || summaries.length > 4 || !chatContinuityActivityComplete(scope, activity) + || new Set(summaries.map(row => row.runId)).size !== summaries.length) { + throw new JourneyError("failed", "chat_continuity_observation_incomplete"); + } + const runs = await Promise.all(summaries.map(async row => { + if (typeof row.runId !== "string" || !row.runId) throw new JourneyError("failed", "chat_continuity_run_mismatch"); + if (row.agentId !== agent.id || row.contextIssueId !== chat.id) throw new JourneyError("failed", "chat_continuity_summary_scope_mismatch"); + const run = await reader.get(`/api/heartbeat-runs/${row.runId}`); + if (run.id !== row.runId || run.companyId !== scope.companyId || run.agentId !== agent.id + || chatContinuityIssueId(run) !== chat.id || !["queued", "running", "succeeded"].includes(run.status)) { + throw new JourneyError("failed", "chat_continuity_run_mismatch"); + } + return run; + })); + const resetReceipts: ChatContinuityResetReceipt[] = summaries.flatMap(summary => summary.resultJson?.conversationReset === true ? [{ + source: "issue-runs" as const, companyId: scope.companyId, issueId: chat.id, + runId: summary.runId, agentId: summary.agentId, status: summary.status, + contextIssueId: summary.contextIssueId, wakeCommentId: summary.wakeCommentId, resultJson: { conversationReset: true as const }, + }] : []); + return { chat, runs, comments, tasks, activity, resetReceipts }; + }; + const wait = async (providerCount: number, resetCount: number) => { + let stable = ""; + while (remaining() > 0) { + const state = await read(); + if (state) { + const providerRuns = state.runs.filter(run => !chatContinuityResetRun(run)); + const resetRuns = state.runs.filter(chatContinuityResetRun); + if (providerRuns.length > providerCount || resetRuns.length > resetCount) throw new JourneyError("failed", "chat_continuity_extra_run"); + const ready = providerRuns.length === providerCount && resetRuns.length === resetCount + && state.runs.every(run => run.status === "succeeded") + && providerRuns.every(run => run.nativePhase === "committed") + && resetRuns.every(run => state.resetReceipts.filter(receipt => chatContinuityResetReceiptMatches(scope, state.chat.id, run, receipt)).length === 1) + && state.chat.status === "in_review" && state.chat.conversationState === "waiting" + && !state.chat.executionRunId && !state.chat.scheduledRetry && !state.chat.activeRecoveryAction; + const key = ready ? JSON.stringify([state.chat.id, state.chat.conversationSessionGeneration, + state.runs.map(run => [run.id, run.status]), state.comments.map(row => row.id)]) : ""; + if (key && stable === key) return state; + stable = key; + } + await new Promise(resolve => setTimeout(resolve, Math.min(1000, remaining()))); + } + throw new JourneyError("failed", "chat_continuity_not_settled"); + }; + const snapshot = async (phase: ChatContinuityPoint["phase"], state: Awaited>, runId?: string) => { + let point: ChatContinuityPoint = { phase, ...state }; + if (runId) { + const load = async () => { + const latest = await read(); + const run = latest?.runs.find(run => run.id === runId); + if (!latest || !run) throw new JourneyError("failed", "chat_continuity_reply_run_missing"); + const events = await collectRunEvents((afterSeq, limit) => + reader.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + return { run, comments: latest.comments, events }; + }; + const reply = await readBoundTaskFinalReply({ page: context.page, runId, agentId: agent.id, + deadlineAt: options.deadlineAt, knownCompanyPrefixes: [context.company.issuePrefix], load }); + const latest = await read(), runEvidence = await load(); + if (!latest) throw new JourneyError("failed", "chat_continuity_reply_chat_missing"); + point = { phase, ...latest, reply, events: runEvidence.events }; + } + points.push(point); + await retain(); + await captureProductScreenshot(context.page, context.api.origin, path.join(options.directory, `chat-${phase}.png`), options.deadlineAt); + return point; + }; + const newProviderRun = (state: Awaited>, old: Row[] = []) => { + const added = state.runs.filter(run => !chatContinuityResetRun(run) && !old.some(before => before.id === run.id)); + if (added.length !== 1) throw new JourneyError("failed", "chat_continuity_turn_ambiguous"); + return added[0]!; + }; + try { + const settings = await reader.get("/api/instance/settings/experimental"); + if (settings.enableAgentChat !== true) throw new JourneyError("unsupported", "agent_chat_required"); + const history = await reader.get(`/api/companies/${scope.companyId}/heartbeat-runs?agentId=${agent.id}&limit=1&summary=true`); + if (!Array.isArray(history) || history.some(run => !run || run.companyId !== scope.companyId || run.agentId !== agent.id)) { + throw new JourneyError("failed", "chat_continuity_agent_history_scope_mismatch"); + } + if (history.length) throw new JourneyError("unsupported", "fresh_chat_agent_required"); + scope.agentHistory = { source: "company-agent-runs", companyId: scope.companyId, agentId: agent.id, limit: 1, runIds: [], observedBeforeSend: true }; + if (await reader.get(chatPath) !== null) throw new JourneyError("unsupported", "fresh_agent_chat_required"); + await session.own({ kind: "issue", name: `Chat with ${agent.name}`, journeyId: "chat-continuity", freshChat }); + await context.page.goto(chatUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(context.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() }); + await sendChatMessage(context.page, scenario.remember); + const first = await wait(1, 0), initial = await snapshot("initial", first, newProviderRun(first).id); + await context.page.goto(`${context.api.origin}/${encodeURIComponent(context.company.issuePrefix)}/issues`, { waitUntil: "domcontentloaded", timeout: remaining() }); + await context.page.goto(chatUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await context.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(context.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() }); + await snapshot("returned", await wait(1, 0), initial.reply!.runId); + await sendChatMessage(context.page, scenario.recall); + const second = await wait(2, 0), recalled = await snapshot("recalled", second, newProviderRun(second, initial.runs).id); + await sendChatMessage(context.page, scenario.reset); + await snapshot("reset", await wait(2, 1)); + await sendChatMessage(context.page, scenario.fresh); + const third = await wait(3, 1), freshRun = newProviderRun(third, recalled.runs); + await snapshot("fresh", third, freshRun.id); + await context.page.goto(chatUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await snapshot("reopened", await wait(3, 1), freshRun.id); + } finally { + evidence.checks.push(...gradeChatContinuity(scope, points)); + await retain(); + await reader.preserve(options.directory); + } + if (evidence.checks.some(check => !check.passed)) throw new JourneyError("failed", "chat_continuity_evidence_failed"); +} diff --git a/tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts b/tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts new file mode 100644 index 00000000000..a236106b597 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts @@ -0,0 +1,66 @@ +import { chatContinuityScenario, type ChatContinuityPoint, type ChatContinuityScope } from "./chat-continuity-evidence.js"; + +type Row = Record; +export function chatContinuityFixture() { + const scope: ChatContinuityScope = { companyId: "company", userId: "board", agentId: "agent", agentName: "Owned agent", + environmentId: "environment", provider: "opencode", model: "configured-model", campaignId: "synthetic-continuity-123", + agentHistory: { source: "company-agent-runs", companyId: "company", agentId: "agent", limit: 1, runIds: [], observedBeforeSend: true } }; + const scenario = chatContinuityScenario(scope.campaignId); + const user = (id: string, body: string, generation: number) => ({ id, body, authorUserId: scope.userId, + authorAgentId: null, companyId: scope.companyId, issueId: "chat", conversationSessionGeneration: generation }); + const answer = (id: string, runId: string, body: string, generation: number) => ({ id, body, + authorAgentId: scope.agentId, authorUserId: null, createdByRunId: runId, + companyId: scope.companyId, issueId: "chat", conversationSessionGeneration: generation }); + const prompts = [scenario.remember, scenario.recall, scenario.reset, scenario.fresh]; + const board = prompts.map((body, index) => user(`user-${index}`, body, index >= 2 ? 1 : 0)); + const replies = [answer("reply-0", "run-0", "Remembered.", 0), answer("reply-1", "run-1", scenario.remembered, 0), + answer("reply-3", "run-3", scenario.marker, 1)]; + const run = (index: number): Row => ({ id: `run-${index}`, companyId: scope.companyId, agentId: scope.agentId, + nativeIssueId: "chat", status: "succeeded", runtimeMode: "native", nativePhase: "committed", + nativeSessionId: index === 3 ? "new-session" : "original-session", sessionIdBefore: null, + contextSnapshot: { issueId: "chat", source: "issue.comment", wakeCommentId: board[index]!.id, + conversationSessionGeneration: index >= 2 ? 1 : 0, paperclipTaskMarkdown: index === 3 ? scenario.fresh : scenario.remember, + paperclipEnvironment: { id: scope.environmentId } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: scope.provider, model: scope.model } } }, + resultJson: { finalizationPhase: "committed", presentationDecision: { commentId: `reply-${index}` } } }); + const runs = [run(0), run(1), run(2), run(3)]; + runs[2]!.contextSnapshot.conversationReset = true; + runs[2]!.resultJson = { conversationReset: true }; + delete runs[2]!.runnerProfileJson; + delete runs[2]!.nativeSessionId; + delete runs[2]!.nativePhase; + delete runs[2]!.runtimeMode; + const chat = { id: "chat", companyId: scope.companyId, assigneeAgentId: scope.agentId, conversationAgentId: scope.agentId, + conversationUserId: scope.userId, createdByUserId: scope.userId, title: `Chat with ${scope.agentName}`, + status: "in_review", conversationState: "waiting", conversationSessionGeneration: 0 }; + const comments = [board[0]!, replies[0]!, board[1]!, replies[1]!, board[2]!, board[3]!, replies[2]!]; + const makePoint = (phase: ChatContinuityPoint["phase"], count: number, commentCount: number, generation: number, replyIndex?: number): ChatContinuityPoint => { + const reply = replyIndex === undefined ? undefined : replies[replyIndex]!; + return structuredClone({ phase, chat: { ...chat, conversationSessionGeneration: generation }, runs: runs.slice(0, count), + comments: comments.slice(0, commentCount), tasks: [], activity: [], events: [], resetReceipts: count < 3 ? [] : [{ + source: "issue-runs", companyId: scope.companyId, issueId: "chat", runId: "run-2", agentId: scope.agentId, + status: "succeeded", contextIssueId: "chat", wakeCommentId: "user-2", resultJson: { conversationReset: true }, + }], + ...(reply ? { reply: { runId: reply.createdByRunId, agentId: scope.agentId, observedAfterReload: true, + body: reply.body, anchor: reply.id, visible: true, settled: true, nativeFinal: false } } : {}) }); + }; + const points = [makePoint("initial", 1, 2, 0, 0), makePoint("returned", 1, 2, 0, 0), makePoint("recalled", 2, 4, 0, 1), + makePoint("reset", 3, 5, 1), makePoint("fresh", 4, 7, 1, 2), makePoint("reopened", 4, 7, 1, 2)]; + return { scope, scenario, points }; +} + +export function compactChatContinuityFixture() { + const fixture = chatContinuityFixture(); + for (const point of fixture.points) { + for (const run of point.runs) run.resultJson = null; + if (!point.reply) continue; + point.reply.anchor = `${point.reply.runId}:turn`; + point.reply.nativeFinal = true; + point.events = [{ seq: 1, runId: point.reply.runId, companyId: fixture.scope.companyId, agentId: fixture.scope.agentId, + eventType: "item.completed", protocolSchemaVersion: 1, sourceEventId: "event", sourceInstanceId: "instance", sourceSeq: 1, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: point.reply.runId, sourceKind: "runner", + eventType: "item.completed", sourceEventId: "event", sourceInstanceId: "instance", sourceSeq: 1, + payload: { kind: "agentMessage", channel: "final", text: point.reply.body } } } }]; + } + return fixture; +} diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.ts index b3d3bd7b441..a702406ee61 100644 --- a/tests/runner-e2e/product-journeys/contracts.test.ts +++ b/tests/runner-e2e/product-journeys/contracts.test.ts @@ -39,6 +39,32 @@ function connectReceipt(): OnboardingConnectEvidence { }; } describe("deployment journey contracts", () => { + it("discovers an ordinary assigned-skill journey without an experimental pin prerequisite", () => { + const discovered = productJourneys.find(journey => journey.id === "saved-skill-use")!; + expect(discovered).not.toHaveProperty("requiredSettings"); + expect(journeyRequiredChecks[discovered.id]?.[0]).toBe("saved-skill-assignment-bound"); + const config = parseTargetConfig({ ...local, journeys: ["saved-skill-use"], + limits: { durationMs: 600_000, modelBudgetCents: 100, maxRuns: 1 } }); + expect(preparationPreview(config).journeys).toContainEqual(discovered); + expect(config.baseline.features).toEqual({}); + }); + it("requires one native saved-skill run and every closed source/use/result check", () => { + const config = { ...local, id: "qa", mode: "existing-cloud", adapterModule: "/private/provider.mjs", + executionEnvironment: "daytona", journeys: ["saved-skill-use"], limits: { durationMs: 600_000, modelBudgetCents: 100, maxRuns: 1 } }; + expect(parseTargetConfig(config).journeys).toEqual(["saved-skill-use"]); + for (const maxRuns of [0, 2]) expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns } })).toThrow("agent_journeys_require_bounded_runs"); + const retained = result(); + Object.assign(retained, { caseId: "saved-skill-use", provider: "opencode", model: "configured-model", runtimeMode: "native", runIds: ["run"] }); + retained.productJourney!.checks = journeyRequiredChecks[retained.caseId]!.map(id => ({ id, passed: true })); + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + for (const check of retained.productJourney!.checks) { + const missing = structuredClone(retained); + missing.productJourney!.checks = missing.productJourney!.checks.filter(row => row.id !== check.id); + expect(() => validateRetainedRunnerResult(missing)).toThrow("result.productJourney"); + } + retained.runIds = []; + expect(() => validateRetainedRunnerResult(retained)).toThrow("result.agentJourneyRun"); + }); it("accepts historical results without Connect diagnostics and optional closed receipts", () => { const retained = result(); retained.caseId = "cloud-onboarding"; diff --git a/tests/runner-e2e/product-journeys/fixtures.ts b/tests/runner-e2e/product-journeys/fixtures.ts index 3f33df99bac..de4b3d2e055 100644 --- a/tests/runner-e2e/product-journeys/fixtures.ts +++ b/tests/runner-e2e/product-journeys/fixtures.ts @@ -11,7 +11,7 @@ export function journeyFixtures(session: TargetSession, campaignId?: string) { id: "ready-context", async setup() { return ready ??= await (preparation ??= session.prepare()); }, }); - for (const id of ["company", "board-user", "task-write", "agent", "skill-write", "review-service", "inbox-author", "company-access"]) registry.register({ + for (const id of ["company", "board-user", "task-write", "agent", "agent-settings-write", "skill-write", "review-service", "inbox-author", "company-access"]) registry.register({ id, dependencies: ["ready-context"], async setup(resolved) { const context = resolved.get("ready-context") as JourneyContext; diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 6482fc9580a..930112167c5 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -9,6 +9,7 @@ import { runnerSuites } from "../catalog.js"; import { productJourneys, parseTargetConfig, preparationPreview, isAgentJourney } from "./catalog.js"; import { agentJourneyRunCountValid } from "./success-contract.js"; import { journeyFixtures } from "./fixtures.js"; +import { taskEditLifecycle } from "./task-lifecycle.js"; import { taskProperties } from "./task-properties.js"; import { taskSearchFilter } from "./task-search-filter.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; @@ -17,9 +18,13 @@ import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; import { runActiveFollowup } from "./active-followup-flow.js"; import { runClarificationJourney } from "./clarification-flow.js"; +import { runChatContinuity } from "./chat-continuity-flow.js"; import { runRepositoryContinuation } from "./repository-continuation-flow.js"; import { runScheduledRoutine } from "./routine-schedule-flow.js"; +import { taskDependencyResume } from "./task-dependency-resume.js"; import { runInputFile } from "./input-file-flow.js"; +import { runSavedSkillUse } from "./saved-skill-use-flow.js"; +import { runAgentSettings } from "./agent-settings-flow.js"; import { companyAccessIsolation } from "./company-access-flow.js"; import { inboxTriage } from "./inbox-triage-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; @@ -113,7 +118,83 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["observation-read.ts", "../connection-config.ts", "catalog.ts", "company-access-flow.ts", "company-access-contract.ts", "repository-continuation-flow.ts", "repository-continuation-contract.ts", "repository-continuation-evidence.ts", "../registered-artifact.ts", "../user-actions.ts", "../connection-target.ts", "../../../packages/shared/src/project-mentions.ts", "routine-schedule-flow.ts", "routine-schedule-evidence.ts", "input-file-flow.ts", "input-file-evidence.ts", "inbox-triage-flow.ts", "inbox-triage-evidence.ts", "inbox-triage-fixture.ts", "inbox-author-contract.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts", "../../../ui/src/lib/markdown-reply-content.ts", "../../../ui/src/lib/issue-reference.ts", "../../../ui/src/lib/company-routes.ts", "../../../ui/package.json"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of [ + "saved-skill-use-flow.ts", + "saved-skill-use-evidence.ts", + "task-lifecycle.ts", + "task-lifecycle-evidence.ts", + "../../../packages/shared/src/validators/issue.ts", + "observation-read.ts", + "../connection-config.ts", + "catalog.ts", + "company-access-flow.ts", + "company-access-contract.ts", + "repository-continuation-flow.ts", + "repository-continuation-contract.ts", + "repository-continuation-evidence.ts", + "../registered-artifact.ts", + "../user-actions.ts", + "../connection-target.ts", + "../../../packages/shared/src/project-mentions.ts", + "routine-schedule-flow.ts", + "routine-schedule-evidence.ts", + "input-file-flow.ts", + "input-file-evidence.ts", + "inbox-triage-flow.ts", + "inbox-triage-evidence.ts", + "inbox-triage-fixture.ts", + "inbox-author-contract.ts", + "task-properties.ts", + "../../../packages/shared/src/constants.ts", + "task-dependency-resume.ts", + "task-dependency-resume-evidence.ts", + "task-search-filter.ts", + "task-search-evidence.ts", + "basic-flows.ts", + "basic-evidence.ts", + "task-document-flow.ts", + "task-document-evidence.ts", + "project-context-flow.ts", + "project-context-readback.ts", + "project-context-evidence.ts", + "agent-flows.ts", + "hire-delegate-flow.ts", + "agent-settings-flow.ts", + "agent-settings-evidence.ts", + "hire-delegate-composer.ts", + "hire-delegate-evidence.ts", + "active-followup-flow.ts", + "active-followup-evidence.ts", + "stop-redirect-evidence.ts", + "clarification-flow.ts", + "clarification-evidence.ts", + "../question-resume-ui.ts", + "../question-resume-scoring.ts", + "../runtime-question-readiness.ts", + "../chat-flow.ts", + "chat-continuity-flow.ts", + "chat-continuity-evidence.ts", + "../chat-cases.ts", + "../continuation-readiness.ts", + "../continuation-cases.ts", + "success-contract.ts", + "product-screenshot.ts", + "cloud-onboarding.ts", + "onboarding-connect.ts", + "onboarding-first-task.ts", + "workspace-deferral.ts", + "onboarding-pre-acceptance.ts", + "artifact-evidence.ts", + "fixtures.ts", + "launch.ts", + "types.ts", + "collection.ts", + "local-target.ts", + "../../../ui/src/lib/markdown-reply-content.ts", + "../../../ui/src/lib/issue-reference.ts", + "../../../ui/src/lib/company-routes.ts", + "../../../ui/package.json" + ]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "chat-hardening.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -209,10 +290,16 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.provider = ready.agent.provider; result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; - if (id === "repository-continuation") await runRepositoryContinuation(ready, session!, campaignId, evidence, + if (id === "saved-skill-use") await runSavedSkillUse(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); + else if (id === "repository-continuation") await runRepositoryContinuation(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); + else if (id === "agent-settings-persistence") await runAgentSettings(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "routine-scheduled-run") await runScheduledRoutine(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); + else if (id === "task-dependency-resume") await taskDependencyResume(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "file-input-answer") await runInputFile(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "ask-question") await askQuestion(ready, session!, campaignId, evidence); @@ -225,6 +312,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri await mkdir(directory, { recursive: true, mode: 0o700 }); await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file), deadlineAt); } }); + else if (id === "chat-continuity") await runChatContinuity(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else await runAgentJourney(id, ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id), capture: async (_id, _label, file) => { @@ -234,6 +323,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri } }); } else if (id === "company-access-isolation") await companyAccessIsolation(ready, session!, campaignId, evidence); else if (id === "inbox-triage") await inboxTriage(ready, session!, campaignId, evidence); + else if (id === "task-edit-lifecycle") await taskEditLifecycle(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "task-create") await taskCreate(ready, session!, campaignId, evidence); else if (id === "task-document-persistence") await taskDocumentPersistence(ready, session!, campaignId, evidence); else if (id === "project-task-context") await projectTaskContext(ready, session!, campaignId, evidence); diff --git a/tests/runner-e2e/product-journeys/product-screenshot.test.ts b/tests/runner-e2e/product-journeys/product-screenshot.test.ts index 486b6c19f0e..97b8b5134db 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.test.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.test.ts @@ -13,12 +13,12 @@ function fixture(options: { mounted?: boolean; redirect?: string; initialPath?: return { page, locator, waitFor, screenshot }; } describe("product screenshot boundary", () => { - it.each(["/QA/inbox/unread", "/QA/routines"])("retains an authenticated journey failure on %s", async initialPath => { + it.each(["/QA/inbox/unread", "/QA/routines", "/QA/chats/owned-agent"])("retains an authenticated journey failure on %s", async initialPath => { const f = fixture({ initialPath }); await captureProductScreenshot(f.page, origin, "failed.png"); expect(f.screenshot).toHaveBeenCalledTimes(1); }); - it.each(["/login", "/QA/settings/keys", "/QA/inboxing", "/QA/routines-export"])("excludes unrelated or authentication paths: %s", async initialPath => { + it.each(["/login", "/QA/settings/keys", "/QA/inboxing", "/QA/routines-export", "/QA/chats-export"])("excludes unrelated or authentication paths: %s", async initialPath => { const f = fixture({ initialPath }); await expect(captureProductScreenshot(f.page, origin, "failed.png")).rejects.toThrow("product_screen_unavailable"); expect(f.waitFor).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/product-journeys/product-screenshot.ts b/tests/runner-e2e/product-journeys/product-screenshot.ts index cedfe14a17a..c7d352c4102 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.ts @@ -3,7 +3,7 @@ import { JourneyError } from "./types.js"; export function isProductScreenshotUrl(value: string, origin: string): boolean { const url = new URL(value); - return url.origin === origin && /^\/[^/]+\/(issues|search|inbox|routines)(\/|$)/.test(url.pathname); + return url.origin === origin && /^\/[^/]+\/(issues|search|inbox|routines|chats)(\/|$)/.test(url.pathname); } export async function captureProductScreenshot(page: Page, origin: string, file: string, deadlineAt = Date.now() + 10_000) { diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts new file mode 100644 index 00000000000..1bbf1b32a7f --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-codex.test.ts @@ -0,0 +1,157 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromCodex } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { createPrpSemanticToolInputEnvelope, createPrpSemanticToolResultEnvelope, digestPrpSemanticContent } from "../../../packages/paperclip-runner/src/protocol/semantic-tool-receipts.js"; +import { validatePrpEvent } from "../../../packages/paperclip-runner/src/protocol/replay-contract.js"; +import { redactEventPayload } from "../../../server/src/redaction.js"; +import { readAssignedSkill, RUNNER_BRIDGE_READ_OPERATIONS } from "../../../server/src/services/native-runtime/runner-bridge-tools.js"; +import { savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts } from "./saved-skill-use-evidence.js"; + +type Row = Record; +const binding = { companyId: "company", agentId: "agent", runId: "run" }; +const fixture = savedSkillFixture("codex-campaign", "62ff548dfec748da8fef6d5fbf6cc23d"); +const key = `company/${binding.companyId}/${fixture.slug}`, runtimeName = `${fixture.slug}--0123456789`; +const correlation = { runId: "run", normalizedSessionId: "native-session", turnId: "turn", itemId: "item" }; +const callId = "assigned-read"; +const hash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); + +function event(seq: number, eventType: string, payload: Row, itemId = correlation.itemId): Row { + return { ...binding, seq, eventType, protocolSchemaVersion: 1, sourceSeq: seq, + sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, + payload: redactEventPayload({ prpEvent: { + schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", ...correlation, itemId, + priority: 0, emittedAt: "2026-11-03T12:00:00.000Z", + sourceSeq: seq, sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, + eventType, payload, + } }) }; +} + +// Read the real immutable-file producer, not a hand-written success result. No model or DB is used. +async function observedRead(text = fixture.markdown, rawInput?: Row) { + const directory = await mkdtemp(path.join(tmpdir(), "saved-skill-native-read-")); + try { + const files = [{ path: "SKILL.md", sha256: hash(text), mode: 0o444, size: Buffer.byteLength(text) }]; + const digest = hash(JSON.stringify(files)), rootPath = path.join(directory, "bundles", digest); + const manifest = `${JSON.stringify({ schema: "paperclip.runtime-asset-manifest.v1", digest, + fileCount: 1, totalBytes: Buffer.byteLength(text), files })}\n`; + await mkdir(rootPath, { recursive: true }); + await mkdir(path.join(directory, "manifests")); + await writeFile(path.join(rootPath, "SKILL.md"), text); + await writeFile(path.join(directory, "manifests", `${digest}.json`), manifest); + const bundle = { rootPath, digest, manifestDigest: hash(manifest) }; + const input = rawInput ?? { offset: 0, skill: key, path: "SKILL.md" }; + const result = await readAssignedSkill({ skills: [{ key, runtimeName, versionId: "version", bundle }] } as any, input); + return { input, result }; + } finally { await rm(directory, { recursive: true, force: true }); } +} + +async function observations(text = fixture.markdown, rawInput?: Row) { + const { input, result } = await observedRead(text, rawInput); + const run: Row = { id: binding.runId, companyId: binding.companyId, agentId: binding.agentId, runtimeMode: "native", + runnerInstanceId: "runner-instance", nativeSessionId: correlation.normalizedSessionId, + contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex" }, + runtimeContext: { skills: [{ key, versionId: "version", runtimeName }] } } } }; + const assignment = savedSkillReadAssignment({ key, currentVersionId: "version" }, + { entries: [{ key, versionId: "version", runtimeName }] }, run)!; + // Core091 provider_backend.rs uses these protocol envelopes, adds safe input, + // and names the successful fallback receipt operation_. + const semanticInput = { ...createPrpSemanticToolInputEnvelope({ operationId: "read_assigned_skill", callId, + correlation, content: input }), input }; + const semanticResult = createPrpSemanticToolResultEnvelope({ operationId: "read_assigned_skill", callId, + correlation, content: result, outcome: "succeeded", code: "semantic_tool_succeeded", retryable: false, + authorizationBoundary: "active_task", operationReceiptId: `operation_${callId}` }); + // Genuine converter output has dynamic transport and no invented first-party namespace/readOnly hint. + const dynamic = ["item/started", "item/completed"].map(method => canonicalProviderEventsFromCodex(method, + { item: { id: callId, type: "dynamicToolCall", tool: "read_assigned_skill", status: "completed" } })[0]!); + const events = [event(1, dynamic[0]!.eventType, dynamic[0]!.payload, dynamic[0]!.itemId), + event(2, "semantic_tool.input", { semantic_tool: semanticInput }), + event(3, "semantic_tool.result", { semantic_tool: semanticResult }), + event(4, dynamic[1]!.eventType, dynamic[1]!.payload, dynamic[1]!.itemId)]; + return { events, run, assignment, result, input }; +} + +const read = (value: Awaited>, f = fixture) => + savedSkillReadReceipts(value.events, binding, f, undefined, value.assignment, value.run); +const semantic = (value: Awaited>, index: number) => + value.events[index]!.payload.prpEvent.payload.semantic_tool as Row; + +describe("saved-skill Codex native authority", () => { + it("accepts the complete real read's authoritative input/result commitment after API redaction", async () => { + const value = await observations(); + expect(RUNNER_BRIDGE_READ_OPERATIONS.has("read_assigned_skill")).toBe(true); + expect(Object.keys(value.result).sort()).toEqual(["byteSize", "nextOffset", "path", "sha256", "skill", "text", "versionId"]); + for (const row of value.events) expect(validatePrpEvent(row.payload.prpEvent).ok).toBe(true); + expect(value.events[0]!.payload.prpEvent.payload).toMatchObject({ transport: "dynamic", readOnly: null, namespace: null }); + expect(read(value)).toEqual([{ runId: "run", executionId: callId, seq: 3 }]); + }); + + it("checks UTF-8 source bytes and canonical key order using the real Unicode producer", async () => { + const text = `${fixture.markdown}\nUnicode: café 🌍\n`; + const value = await observations(text); + expect(value.result.byteSize).toBe(Buffer.byteLength(text)); + expect(value.result.byteSize).toBeGreaterThan(text.length); + expect(read(value, { ...fixture, markdown: text })).toHaveLength(1); + }); + + it("accepts Core's omitted defaults and only its exact admitted runtime alias", async () => { + const value = await observations(fixture.markdown, { skill: runtimeName }); + expect(read(value)).toHaveLength(1); + }); + + it("rejects raw dynamic calls even when they copy the complete source and invent readOnly/namespace", async () => { + const value = await observations(); + value.events = [value.events[0]!, value.events[3]!]; + for (const row of value.events) Object.assign(row.payload.prpEvent.payload, + { name: "paperclip_read_assigned_skill", readOnly: true, namespace: "paperclip", + output: JSON.stringify({ type: "inputText", text: JSON.stringify(value.result) }), outputTruncated: false }); + const p = value.events[1]!.payload.prpEvent.payload; p.outputBytes = Buffer.byteLength(p.output); + expect(read(value)).toEqual([]); + }); + + it.each([ + ["missing input", (v: any) => { v.events.splice(1, 1); }], + ["missing result", (v: any) => { v.events.splice(2, 1); }], + ["duplicate input", (v: any) => { v.events.push(structuredClone(v.events[1])); }], + ["duplicate result", (v: any) => { v.events.push(structuredClone(v.events[2])); }], + ["result before input", (v: any) => { v.events[2].seq = 0; }], + ["foreign provider", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.provider.kind = "unknown"; }], + ["foreign runner", (v: any) => { v.run.runnerInstanceId = "foreign"; }], + ["foreign session", (v: any) => { v.run.nativeSessionId = "foreign"; }], + ["missing runner identity", (v: any) => { delete v.run.runnerInstanceId; }], + ["missing session identity", (v: any) => { delete v.run.nativeSessionId; }], + ["foreign run owner", (v: any) => { v.run.companyId = "foreign"; }], + ["missing native assignment", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }], + ["duplicate native assignment", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]); }], + ["foreign native version", (v: any) => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "foreign"; }], + ["missing selection snapshot", (v: any) => { v.run.contextSnapshot.skillSelectionSnapshot = false; }], + ["foreign admitted version", (v: any) => { v.run.contextSnapshot.skillVersions[key] = "foreign"; }], + ["foreign call", (v: any) => { semantic(v, 2).callId = "foreign"; }], + ["foreign turn", (v: any) => { semantic(v, 2).correlation.turnId = "foreign"; }], + ["foreign item", (v: any) => { semantic(v, 2).correlation.itemId = "foreign"; }], + ["foreign source", (v: any) => { v.events[2].sourceInstanceId = v.events[2].payload.prpEvent.sourceInstanceId = "foreign"; }], + ["foreign operation", (v: any) => { for (const i of [1, 2]) semantic(v, i).operationId = "paperclip_read_assigned_skill"; }], + ["MCP carriers", (v: any) => { for (const i of [1, 2]) v.events[i].eventType = v.events[i].payload.prpEvent.eventType = i === 1 ? "mcp_app.tool_input" : "mcp_app.tool_result"; }], + ["failed result", (v: any) => { semantic(v, 2).outcome = "failed"; }], + ["wrong boundary", (v: any) => { semantic(v, 2).authorizationBoundary = "company"; }], + ["wrong receipt", (v: any) => { semantic(v, 2).operationReceiptId = "foreign"; }], + ["wrong input digest", (v: any) => { semantic(v, 1).content.digest = `sha256:${"0".repeat(64)}`; }], + ["foreign input key", (v: any) => { semantic(v, 1).input.skill = "foreign"; semantic(v, 1).content.digest = digestPrpSemanticContent(semantic(v, 1).input); }], + ["wrong source path", (v: any) => { semantic(v, 1).input.path = "other.md"; semantic(v, 1).content.digest = digestPrpSemanticContent(semantic(v, 1).input); }], + ["nonzero offset", (v: any) => { semantic(v, 1).input.offset = 1; semantic(v, 1).content.digest = digestPrpSemanticContent(semantic(v, 1).input); }], + ["wrong result digest", (v: any) => { semantic(v, 2).content.digest = `sha256:${"0".repeat(64)}`; }], + ["truncated disclosure", (v: any) => { semantic(v, 2).content.redactionDisposition = "redacted"; }], + ["truncated marker", (v: any) => { semantic(v, 2).outputTruncated = true; }], + ["partial page", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, nextOffset: 1 }); }], + ["missing page completion", (v: any) => { const result = { ...v.result }; delete result.nextOffset; semantic(v, 2).content.digest = digestPrpSemanticContent(result); }], + ["wrong source bytes", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, byteSize: v.result.byteSize + 1 }); }], + ["wrong source checksum", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, sha256: "0".repeat(64) }); }], + ["wrong version", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, versionId: "foreign" }); }], + ["copied nonce only", (v: any) => { semantic(v, 2).content.digest = digestPrpSemanticContent({ ...v.result, text: fixture.expected.reference }); }], + ] as const)("rejects authoritative-looking evidence with %s", async (_label, mutate) => { + const value = await observations(); mutate(value); expect(read(value)).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts new file mode 100644 index 00000000000..3841766ef97 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.test.ts @@ -0,0 +1,311 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { savedSkillDocumentAttributed, savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts, savedSkillReadRuntimeName, savedSkillResultMatches, savedSkillSourceMatches } from "./saved-skill-use-evidence.js"; + +type Row = Record; +const fixture = savedSkillFixture("campaign-id", "62ff548dfec748da8fef6d5fbf6cc23d"); +const binding = { companyId: "company", agentId: "agent", issueId: "issue", runId: "run" }; +const assignment = { key: `company/${binding.companyId}/${fixture.slug}`, versionId: "version" }; +function native(seq: number, eventType: string, payload: Row): Row { + return { ...binding, seq, eventType, protocolSchemaVersion: 1, sourceSeq: seq, sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: binding.runId, + sourceSeq: seq, sourceInstanceId: "runner-instance", sourceEventId: `event-${seq}`, turnId: "turn", eventType, payload } } }; +} +function readEvents(): Row[] { + const payload = { schema: "paperclip.tool.execution.v1", executionId: "read-skill", name: "read", operation: "read", + transport: "builtin", readOnly: true, target: `/workspace/.agents/skills/${fixture.slug}/SKILL.md` }; + return [native(1, "tool.execution.started", { ...payload, status: "running" }), + native(2, "tool.execution.completed", { ...payload, status: "completed", output: fixture.markdown, outputBytes: Buffer.byteLength(fixture.markdown), outputTruncated: false })]; +} +const runtimeName = `${fixture.slug}--0123456789`; +function wrappedReadEvents(): Row[] { + const output = `/workspace/.claude/skills/${runtimeName}/SKILL.md\nfile\n\n` + + fixture.markdown.trimEnd().split("\n").map((line, index) => `${index + 1}: ${line}`).join("\n") + + "\n(End of file - total 12 lines)\n"; + const events = readEvents(); + for (const row of events) Object.assign(row.payload.prpEvent.payload, { operation: "unknown", target: null }); + Object.assign(events[1]!.payload.prpEvent.payload, { output, outputBytes: Buffer.byteLength(output) }); + return events; +} +function assignedReadEvents(): Row[] { + const result = { skill: `company/${binding.companyId}/${fixture.slug}`, versionId: "version", path: "SKILL.md", + text: fixture.markdown, sha256: createHash("sha256").update(fixture.markdown).digest("hex"), + byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }; + const output = JSON.stringify({ type: "inputText", text: JSON.stringify(result) }); + const events = readEvents(); + for (const row of events) Object.assign(row.payload.prpEvent.payload, + { name: "paperclip_read_assigned_skill", operation: "unknown", target: null }); + Object.assign(events[1]!.payload.prpEvent.payload, { output, outputBytes: Buffer.byteLength(output) }); + return events; +} +function changeAssignedResult(events: Row[], change: (value: Row) => void) { + const payload = events[1]!.payload.prpEvent.payload, wrapper = JSON.parse(payload.output), result = JSON.parse(wrapper.text); + change(result); wrapper.text = JSON.stringify(result); payload.output = JSON.stringify(wrapper); +} +function source() { + const skill: Row = { id: "skill", companyId: binding.companyId, key: `company/${binding.companyId}/${fixture.slug}`, sourceType: "managed", name: fixture.name, slug: fixture.slug, + markdown: fixture.markdown, currentVersionId: "version" }; + const version: Row = { id: "version", companyId: binding.companyId, companySkillId: skill.id, + fileInventory: [{ path: "SKILL.md", kind: "skill", content: fixture.markdown }] }; + const assignments: Row = { desiredSkillEntries: [{ key: skill.key, versionId: null }] }; + return { skill, version, assignments }; +} +function output() { + const document: Row = { id: "document", companyId: binding.companyId, issueId: binding.issueId, key: fixture.key, format: "markdown", + latestRevisionId: "revision", latestRevisionNumber: 1, createdByAgentId: binding.agentId, createdByUserId: null, + updatedByAgentId: binding.agentId, updatedByUserId: null, body: JSON.stringify(fixture.expected) }; + const revision: Row = { id: "revision", documentId: document.id, companyId: binding.companyId, issueId: binding.issueId, + key: fixture.key, format: "markdown", revisionNumber: 1, createdByAgentId: binding.agentId, createdByUserId: null, createdByRunId: binding.runId, + body: document.body }; + return { document, revision }; +} + +describe("saved skill use evidence", () => { + it("keeps the random output reference in the skill, independently of its discoverable name", () => { + expect(fixture.markdown).toContain(fixture.expected.reference); + expect(fixture.name + fixture.slug).not.toContain(fixture.expected.reference); + const owned = savedSkillFixture("journey-1234-5678", fixture.expected.reference); + expect(owned.name).toBe(owned.slug); + expect(owned.name).toContain("journey-1234-5678"); + expect(owned.markdown).toContain(`\nname: ${owned.name}\n`); + expect(fixture.markdown).not.toContain("\\n"); + expect(() => savedSkillFixture("", fixture.expected.reference)).toThrow(); + expect(() => savedSkillFixture(fixture.expected.reference, fixture.expected.reference)).toThrow(); + expect(() => savedSkillFixture("campaign", "short")).toThrow(); + }); + it("requires an ordinary Latest assignment, its exact managed current version and original source bytes", () => { + const good = source(); + expect(savedSkillSourceMatches(good.skill, good.version, good.assignments, fixture, binding.companyId)).toBe(true); + const changes: Array<(value: ReturnType) => void> = [ + v => { v.skill.companyId = "foreign"; }, v => { v.version.companyId = "foreign"; }, + v => { v.version.companySkillId = "foreign"; }, v => { v.skill.slug = "foreign"; }, + v => { v.skill.name = "different saved name"; }, + v => { v.skill.sourceType = "local_path"; }, + v => { v.skill.key = "unknown"; v.assignments.desiredSkillEntries[0].key = "unknown"; }, + v => { v.skill.currentVersionId = "different"; }, + v => { v.skill.markdown += "changed"; }, v => { v.version.id = "stale"; }, + v => { v.version.fileInventory = []; }, v => { v.version.fileInventory[0].content += "changed"; }, + v => { v.version.fileInventory[0].encoding = "base64"; }, v => { v.assignments.desiredSkillEntries = []; }, + v => { v.assignments.desiredSkillEntries[0].versionId = "version"; }, + v => { v.assignments.desiredSkillEntries[0].versionId = "stale"; }, + v => { delete v.assignments.desiredSkillEntries[0].versionId; }, + v => { v.assignments.desiredSkillEntries[0].key = "foreign"; }, + v => { v.assignments.desiredSkillEntries.push(v.assignments.desiredSkillEntries[0]); }, + ]; + for (const change of changes) { + const wrong = source(); change(wrong); + expect(savedSkillSourceMatches(wrong.skill, wrong.version, wrong.assignments, fixture, binding.companyId)).toBe(false); + } + expect(savedSkillSourceMatches(undefined, good.version, good.assignments, fixture, binding.companyId)).toBe(false); + }); + it("accepts the bound successful source read and rejects missing, foreign, claimed and failed reads", () => { + expect(savedSkillReadReceipts(readEvents(), binding, fixture)).toEqual([{ runId: "run", executionId: "read-skill", seq: 2 }]); + const changes: Array<(events: Row[]) => void> = [ + e => { e.splice(0, 1); }, e => { e.pop(); }, e => { e[1].runId = "foreign"; }, + e => { e[1].companyId = "foreign"; }, e => { e[1].agentId = "foreign"; }, + e => { e[1].payload.prpEvent.runId = "foreign"; }, e => { e[1].payload.prpEvent.sourceKind = "provider"; }, + e => { e[1].sourceInstanceId = "foreign"; }, e => { e[1].sourceSeq += 1; }, + e => { e[1].payload.prpEvent.turnId = "foreign"; }, e => { e[1].payload.prpEvent.payload.status = "failed"; }, + e => { for (const row of e) delete row.payload.prpEvent.turnId; }, + e => { e[1].payload.prpEvent.payload.outputBytes = 0; }, e => { e[1].payload.prpEvent.payload.outputTruncated = true; }, + e => { delete e[1].payload.prpEvent.payload.outputTruncated; }, + e => { e[1].payload.prpEvent.payload.output = "I read the skill and followed it."; }, + e => { for (const row of e) row.payload.prpEvent.payload.target = `/workspace/other/${fixture.slug}/../SKILL.md`; }, + e => { for (const row of e) row.payload.prpEvent.payload.target = "/workspace/other-skill/SKILL.md"; }, + e => { for (const row of e) row.payload.prpEvent.payload.target = null; }, + e => { for (const row of e) Object.assign(row.payload.prpEvent.payload, { name: "bash", operation: "execute", transport: "process" }); }, + e => { e[1].payload.prpEvent.payload.executionId = "other-read"; }, + e => { e[1].payload.prpEvent.eventType = e[1].eventType = "item.completed"; }, + ]; + for (const change of changes) { + const wrong = readEvents(); change(wrong); + expect(savedSkillReadReceipts(wrong, binding, fixture)).toEqual([]); + } + expect(savedSkillReadReceipts([], binding, fixture)).toEqual([]); + }); + it("accepts the deployed OpenCode read wrapper for its exact version-bound runtime skill directory", () => { + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture, runtimeName)).toHaveLength(1); + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture)).toEqual([]); + }); + it("accepts the complete first-party assigned-skill receipt with exact immutable source identity and bytes", () => { + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName, assignment)) + .toEqual([{ runId: "run", executionId: "read-skill", seq: 2 }]); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, undefined, assignment)).toHaveLength(1); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName)).toEqual([]); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName, { ...assignment, key: "foreign" })).toEqual([]); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, runtimeName, { ...assignment, versionId: "foreign" })).toEqual([]); + }); + it.each([{ kind: "codex" }, { kind: "acpx", agent: "cursor" }, { kind: "cursor_cloud" }])( + "binds the semantic source to common native skill entries for %j while preserving the OpenCode file guard", provider => { + const { skill } = source(); + const good = () => ({ assignments: { entries: [{ key: skill.key, versionId: "version", runtimeName }] }, + run: { contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [skill.key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider, + runtimeContext: { skills: [{ key: skill.key, versionId: "version", runtimeName }] } } } } }); + const valid = good(), selected = savedSkillReadAssignment(skill, valid.assignments, valid.run); + expect(selected).toEqual(assignment); + expect(savedSkillReadRuntimeName(skill, valid.assignments, valid.run)).toBeUndefined(); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, undefined, selected)).toHaveLength(1); + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture, undefined, selected)).toEqual([]); + const changes: Array<(value: ReturnType) => void> = [ + v => { v.assignments.entries = []; }, + v => { v.assignments.entries[0]!.key = "foreign"; }, + v => { v.assignments.entries[0]!.versionId = "foreign"; }, + v => { v.assignments.entries.push(v.assignments.entries[0]!); }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.key = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.versionId = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!); }, + v => { v.run.contextSnapshot.skillSelectionSnapshot = false; }, + v => { v.run.contextSnapshot.skillVersions = {}; }, + v => { v.run.contextSnapshot.skillVersions[skill.key] = "foreign"; }, + ]; + for (const mutate of changes) { + const wrong = good(); mutate(wrong); + const selected = savedSkillReadAssignment(skill, wrong.assignments, wrong.run); + expect(selected).toBeUndefined(); + expect(savedSkillReadReceipts(assignedReadEvents(), binding, fixture, undefined, selected)).toEqual([]); + } + }); + it.each([ + ["foreign canonical key", (e: Row[]) => changeAssignedResult(e, r => { r.skill = "company/foreign/skill"; })], + ["foreign immutable version", (e: Row[]) => changeAssignedResult(e, r => { r.versionId = "foreign"; })], + ["wrong file", (e: Row[]) => changeAssignedResult(e, r => { r.path = "other/SKILL.md"; })], + ["incomplete page", (e: Row[]) => changeAssignedResult(e, r => { r.nextOffset = r.text.length; })], + ["missing pagination receipt", (e: Row[]) => changeAssignedResult(e, r => { delete r.nextOffset; })], + ["wrong checksum", (e: Row[]) => changeAssignedResult(e, r => { r.sha256 = "0".repeat(64); })], + ["wrong source byte count", (e: Row[]) => changeAssignedResult(e, r => { r.byteSize++; })], + ["partial procedure", (e: Row[]) => changeAssignedResult(e, r => { r.text = fixture.expected.reference; })], + ["changed full source", (e: Row[]) => changeAssignedResult(e, r => { r.text += "extra source"; })], + ["semantic failure with copied source", (e: Row[]) => changeAssignedResult(e, r => { r.outcome = "failed"; r.code = "runner_skill_not_assigned"; })], + ["shell copied receipt", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.name = "bash"; }], + ["another first-party tool", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.name = "paperclip_read_document"; }], + ["non-builtin transport", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.transport = "process"; }], + ["mutating operation", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.operation = "edit"; }], + ["not read only", (e: Row[]) => { e[1]!.payload.prpEvent.payload.readOnly = false; }], + ["start not read only", (e: Row[]) => { e[0]!.payload.prpEvent.payload.readOnly = false; }], + ["failed completion", (e: Row[]) => { e[1]!.payload.prpEvent.payload.status = "failed"; }], + ["truncated completion", (e: Row[]) => { e[1]!.payload.prpEvent.payload.outputTruncated = true; }], + ["missing start", (e: Row[]) => { e.splice(0, 1); }], + ["call mismatch", (e: Row[]) => { e[0]!.payload.prpEvent.payload.executionId = "another-call"; }], + ["foreign runner", (e: Row[]) => { e[0]!.sourceInstanceId = e[0]!.payload.prpEvent.sourceInstanceId = "another-runner"; }], + ["foreign run", (e: Row[]) => { e[1]!.runId = e[1]!.payload.prpEvent.runId = "another-run"; }], + ["foreign company", (e: Row[]) => { e[1]!.companyId = "another-company"; }], + ["foreign turn", (e: Row[]) => { e[1]!.payload.prpEvent.turnId = "another-turn"; }], + ["invalid wrapper", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = fixture.markdown; }], + ["wrong wrapper type", (e: Row[]) => { const p = e[1]!.payload.prpEvent.payload; p.output = p.output.replace("inputText", "outputText"); }], + ] as const)("rejects a first-party assigned-skill receipt with %s", (_label, mutate) => { + const events = assignedReadEvents(); mutate(events); + const completed = events.find(row => row.eventType === "tool.execution.completed")!; + completed.payload.prpEvent.payload.outputBytes = Buffer.byteLength(completed.payload.prpEvent.payload.output); + expect(savedSkillReadReceipts(events, binding, fixture, runtimeName, assignment)).toEqual([]); + }); + it.each([ + ["foreign path", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace(runtimeName, "foreign"); }], + ["relative path", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("/workspace/", "workspace/"); }], + ["path traversal", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("/workspace/", "/workspace/../"); }], + ["wrong file type", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("file", "directory"); }], + ["missing wrapper", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = fixture.procedure; }], + ["partial procedure", (e: Row[]) => { e[1]!.payload.prpEvent.payload.output = e[1]!.payload.prpEvent.payload.output.replace("Do not create other tasks or deliverables.", ""); }], + ["copied shell output", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.name = "bash"; }], + ["execute operation", (e: Row[]) => { for (const row of e) row.payload.prpEvent.payload.operation = "execute"; }], + ["not read only", (e: Row[]) => { e[1]!.payload.prpEvent.payload.readOnly = false; }], + ["start not read only", (e: Row[]) => { e[0]!.payload.prpEvent.payload.readOnly = false; }], + ["failed read", (e: Row[]) => { e[1]!.payload.prpEvent.payload.status = "failed"; }], + ["truncated read", (e: Row[]) => { e[1]!.payload.prpEvent.payload.outputTruncated = true; }], + ["missing start", (e: Row[]) => { e.shift(); }], + ["call mismatch", (e: Row[]) => { e[0]!.payload.prpEvent.payload.executionId = "another-call"; }], + ["foreign runner", (e: Row[]) => { e[0]!.sourceInstanceId = e[0]!.payload.prpEvent.sourceInstanceId = "another-runner"; }], + ["foreign run", (e: Row[]) => { e[1]!.runId = e[1]!.payload.prpEvent.runId = "another-run"; }], + ["foreign turn", (e: Row[]) => { e[1]!.payload.prpEvent.turnId = "another-turn"; }], + ] as const)("rejects a wrapped native read with %s", (_label, mutate) => { + const events = wrappedReadEvents(); mutate(events); + const completed = events.find(row => row.eventType === "tool.execution.completed")!; + completed.payload.prpEvent.payload.outputBytes = Buffer.byteLength(completed.payload.prpEvent.payload.output); + expect(savedSkillReadReceipts(events, binding, fixture, runtimeName)).toEqual([]); + }); + it("requires both API and admitted native input to bind the same immutable skill runtime name", () => { + const { skill } = source(); + const good = () => ({ assignments: { entries: [{ key: skill.key, runtimeName, versionId: "version" }] }, + run: { contextSnapshot: { skillSelectionSnapshot: true, skillVersions: { [skill.key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode" }, + runtimeContext: { skills: [{ key: skill.key, runtimeName, versionId: "version" }] } } } } }); + const valid = good(); + expect(savedSkillReadRuntimeName(skill, valid.assignments, valid.run)).toBe(runtimeName); + const changes: Array<(value: ReturnType) => void> = [ + v => { v.assignments.entries = []; }, + v => { v.assignments.entries[0]!.key = "foreign"; }, + v => { v.assignments.entries[0]!.versionId = "stale"; }, + v => { v.assignments.entries[0]!.runtimeName = "foreign"; }, + v => { v.assignments.entries.push(v.assignments.entries[0]!); }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.key = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.versionId = "stale"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.runtimeName = "foreign"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.push(v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!); }, + v => { v.run.contextSnapshot.skillSelectionSnapshot = false; }, + v => { v.run.contextSnapshot.skillVersions[skill.key] = "stale"; }, + v => { v.run.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }, + v => { v.assignments.entries[0]!.runtimeName = v.run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0]!.runtimeName = "../foreign"; }, + ]; + for (const mutate of changes) { + const wrong = good(); mutate(wrong); + const name = savedSkillReadRuntimeName(skill, wrong.assignments, wrong.run); + expect(name).toBeUndefined(); + expect(savedSkillReadReceipts(wrappedReadEvents(), binding, fixture, name)).toEqual([]); + } + }); + it("accepts the real OpenCode native skill-loader mapping, including its completed provider output", () => { + const part = (status: string) => ({ id: "load-skill", type: "tool", tool: "skill", state: { + status, output: `\n${fixture.procedure}\n`, + } }); + const events = ["pending", "completed"].map((status, index) => { + const event = canonicalProviderEventsFromOpenCodePart(part(status))[0]!; + return native(index + 1, event.eventType, event.payload); + }); + expect(savedSkillReadReceipts(events, binding, fixture)).toHaveLength(1); + for (const output of [fixture.procedure, `\n${fixture.procedure}\n`]) { + const wrong = structuredClone(events); wrong[1]!.payload.prpEvent.payload.output = output; + wrong[1]!.payload.prpEvent.payload.outputBytes = Buffer.byteLength(output); + expect(savedSkillReadReceipts(wrong, binding, fixture)).toEqual([]); + } + }); + it("allows known file-reader line prefixes without treating partial output as a complete read", () => { + const events = readEvents(); + events[1]!.payload.prpEvent.payload.output = fixture.markdown.split("\n").map((line, i) => `${i + 1}: ${line}`).join("\n"); + events[1]!.payload.prpEvent.payload.outputBytes = Buffer.byteLength(events[1]!.payload.prpEvent.payload.output); + expect(savedSkillReadReceipts(events, binding, fixture)).toHaveLength(1); + events[1]!.payload.prpEvent.payload.output = fixture.expected.reference; + events[1]!.payload.prpEvent.payload.outputBytes = Buffer.byteLength(fixture.expected.reference); + expect(savedSkillReadReceipts(events, binding, fixture)).toEqual([]); + }); + it("requires the exact derived result and a saved revision authored by this run", () => { + const good = output(); + expect(savedSkillResultMatches(`\u0060\u0060\u0060json\n${good.document.body}\n\u0060\u0060\u0060`, fixture.expected)).toBe(true); + expect(savedSkillDocumentAttributed(good.document, good.revision, fixture, binding)).toBe(true); + for (const body of ["I finished", "{}", JSON.stringify({ ...fixture.expected, extra: true }), + JSON.stringify({ ...fixture.expected, total: fixture.expected.total + 1 }), JSON.stringify({ ...fixture.expected, reference: "foreign" })]) { + expect(savedSkillResultMatches(body, fixture.expected)).toBe(false); + } + const changes: Array<(value: ReturnType) => void> = [ + v => { v.document.companyId = "foreign"; }, v => { v.document.issueId = "foreign"; }, + v => { v.document.createdByAgentId = "foreign"; }, v => { v.document.updatedByAgentId = "foreign"; }, + v => { v.document.createdByUserId = "owner"; }, v => { v.document.updatedByUserId = "owner"; }, + v => { v.document.key = "other"; }, v => { v.document.body = "{}"; }, + v => { v.document.latestRevisionId = null; }, v => { v.revision.id = "old"; }, + v => { delete v.document.latestRevisionNumber; delete v.revision.revisionNumber; }, + v => { v.revision.documentId = "foreign"; }, v => { v.revision.companyId = "foreign"; }, + v => { v.revision.issueId = "foreign"; }, v => { v.revision.key = "other"; }, + v => { v.revision.format = "unknown"; }, + v => { v.revision.createdByRunId = "foreign"; }, v => { v.revision.createdByAgentId = "foreign"; }, + v => { v.revision.createdByUserId = "owner"; }, v => { v.revision.revisionNumber = 2; }, + v => { v.revision.body = "{}"; }, + ]; + for (const change of changes) { + const wrong = output(); change(wrong); + expect(savedSkillDocumentAttributed(wrong.document, wrong.revision, fixture, binding)).toBe(false); + } + expect(savedSkillDocumentAttributed(undefined, good.revision, fixture, binding)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts new file mode 100644 index 00000000000..0f4ede10092 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-evidence.ts @@ -0,0 +1,238 @@ +import { createHash } from "node:crypto"; +import { isBoundNativePrpEventRow, type NativePrpRunBinding } from "../native-event-envelope.js"; + +type Row = Record; +export interface SavedSkillFixture { + name: string; slug: string; key: "saved-skill-result"; markdown: string; procedure: string; + expected: { reference: string; total: number }; +} + +/** The public request names the skill; only its saved version contains these inputs. */ +export function savedSkillFixture(identity: string, reference: string): SavedSkillFixture { + if (!/^[a-f0-9]{32}$/.test(reference)) throw new Error("invalid_saved_skill_reference"); + if (!/^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/.test(identity) || identity.length > 48 || identity.includes(reference)) + throw new Error("invalid_saved_skill_identity"); + const slug = `qa-saved-skill-${identity}`, key = "saved-skill-result"; + const seed = createHash("sha256").update(reference).digest(); + const first = seed[0]! + 1, second = seed[1]! + 1; + const procedure = `# Saved receipt procedure\n\nThe receipt reference is ${reference}.\n` + + `Add the two amounts ${first} and ${second}.\n` + + `Save exactly one Paperclip task document with key ${key}. Its body must be JSON with exactly ` + + `two fields: reference (the receipt reference) and total (the sum of the two amounts).\n` + + "Check the saved document and finish the task. Do not create other tasks or deliverables.\n"; + return { name: slug, slug, key, procedure, + markdown: `---\nname: ${slug}\ndescription: Make a receipt using this saved procedure.\n---\n\n${procedure}`, + expected: { reference, total: first + second } }; +} + +export function savedSkillSourceMatches(skill: Row | undefined, version: Row | undefined, assignments: Row | undefined, + fixture: SavedSkillFixture, companyId: string): boolean { + if (!skill || !version || !assignments) return false; + const files = version.fileInventory; + const entries = assignments.desiredSkillEntries; + return Boolean(skill.id && skill.companyId === companyId && skill.sourceType === "managed" + && skill.name === fixture.name && skill.slug === fixture.slug && skill.key === `company/${companyId}/${fixture.slug}` + && skill.markdown === fixture.markdown && skill.currentVersionId + && version.id === skill.currentVersionId && version.companyId === companyId && version.companySkillId === skill.id + && Array.isArray(files) && files.length === 1 && files[0].path === "SKILL.md" && files[0].content === fixture.markdown + && (files[0].encoding === undefined || files[0].encoding === "utf8") + && Array.isArray(entries) && entries.filter(row => row.key === skill.key).length === 1 + && entries.find(row => row.key === skill.key)?.versionId === null); +} + +export function savedSkillResultMatches(body: unknown, expected: SavedSkillFixture["expected"]): boolean { + if (typeof body !== "string") return false; + const text = body.trim(), json = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(text)?.[1] ?? text; + try { + const value = JSON.parse(json); + return Boolean(value && typeof value === "object" && !Array.isArray(value) + && Object.keys(value).sort().join(",") === "reference,total" + && value.reference === expected.reference && value.total === expected.total); + } catch { return false; } +} + +export interface SavedSkillReadAssignment { key: string; versionId: string } + +/** All native providers share the canonical skill/version selection contract. */ +export function savedSkillReadAssignment(skill: Row, assignments: Row, run: Row): SavedSkillReadAssignment | undefined { + const input = run.runnerProfileJson?.nativeExecutionInput; + const assigned = Array.isArray(assignments.entries) ? assignments.entries.filter((row: Row) => row.key === skill.key) : []; + const native = Array.isArray(input?.runtimeContext?.skills) + ? input.runtimeContext.skills.filter((row: Row) => row.key === skill.key) : []; + const versions = run.contextSnapshot?.skillVersions; + if (typeof skill.key !== "string" || !skill.key.trim() || assigned.length !== 1 || native.length !== 1 + || typeof skill.currentVersionId !== "string" || !skill.currentVersionId + || run.contextSnapshot?.skillSelectionSnapshot !== true + || !versions || typeof versions !== "object" || Array.isArray(versions) || !Object.hasOwn(versions, skill.key) + || versions[skill.key] !== skill.currentVersionId + || assigned[0].versionId !== skill.currentVersionId || native[0].versionId !== skill.currentVersionId) return undefined; + return { key: skill.key, versionId: skill.currentVersionId }; +} + +/** OpenCode's file wrapper also needs the exact materialized directory identity. */ +export function savedSkillReadRuntimeName(skill: Row, assignments: Row, run: Row): string | undefined { + if (run.runnerProfileJson?.nativeExecutionInput?.provider?.kind !== "opencode" + || !savedSkillReadAssignment(skill, assignments, run)) return undefined; + const assigned = assignments.entries.find((row: Row) => row.key === skill.key); + const native = run.runnerProfileJson.nativeExecutionInput.runtimeContext.skills.find((row: Row) => row.key === skill.key); + if (assigned.runtimeName !== native.runtimeName + || typeof native.runtimeName !== "string" || !/^[a-z0-9][a-z0-9_-]*$/i.test(native.runtimeName)) return undefined; + return native.runtimeName; +} + +/** Core's assigned-skill read verifies the pinned manifest and returns one UTF-8 page. */ +function assignedSkillReadText(output: string, binding: NativePrpRunBinding, fixture: SavedSkillFixture, versionId: string): string | undefined { + try { + const wrapper = JSON.parse(output); + if (wrapper?.type !== "inputText" || typeof wrapper.text !== "string" + || Object.keys(wrapper).sort().join(",") !== "text,type") return undefined; + const result = JSON.parse(wrapper.text); + if (Object.keys(result).sort().join(",") !== "byteSize,nextOffset,path,sha256,skill,text,versionId" + || result.skill !== `company/${binding.companyId}/${fixture.slug}` || result.versionId !== versionId + || result.path !== "SKILL.md" || result.nextOffset !== null || result.text !== fixture.markdown + || result.byteSize !== Buffer.byteLength(fixture.markdown) + || result.sha256 !== createHash("sha256").update(fixture.markdown).digest("hex")) return undefined; + return result.text; + } catch { return undefined; } +} + +// Core091 provider_bridge.rs hashes UTF-8 JSON with object keys ordered by UTF-16. +// This oracle reconstructs the complete pinned read result, not provider display text. +function semanticDigest(value: unknown): string { + const canonical = (value: unknown): string => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => + `${JSON.stringify(key)}:${canonical((value as Row)[key])}`).join(",")}}` : JSON.stringify(value) ?? "null"; + return `sha256:${createHash("sha256").update(canonical(value), "utf8").digest("hex")}`; +} + +/** Codex's raw dynamic name, namespace and optional readOnly hint grant no authority. + * Its admitted first-party read instead has a controller-bound semantic input/result. + * Core's read_assigned_skill operation is read-only and verifies the immutable file. + */ +function codexAssignedSkillReceipts(bound: Row[], binding: NativePrpRunBinding, fixture: SavedSkillFixture, + assignment: SavedSkillReadAssignment | undefined, run: Row | undefined) { + const input = run?.runnerProfileJson?.nativeExecutionInput; + const skills = Array.isArray(input?.runtimeContext?.skills) + ? input.runtimeContext.skills.filter((skill: Row) => skill.key === assignment?.key) : []; + if (!run || run.runtimeMode !== "native" || input?.provider?.kind !== "codex" + || run.id !== binding.runId || run.companyId !== binding.companyId || run.agentId !== binding.agentId + || typeof run.runnerInstanceId !== "string" || !run.runnerInstanceId.trim() + || typeof run.nativeSessionId !== "string" || !run.nativeSessionId.trim() + || !assignment || assignment.key !== `company/${binding.companyId}/${fixture.slug}` || !assignment.versionId + || skills.length !== 1 || skills[0].versionId !== assignment.versionId + || run.contextSnapshot?.skillSelectionSnapshot !== true + || run.contextSnapshot?.skillVersions?.[assignment.key] !== assignment.versionId + || !fixture.markdown.includes(fixture.procedure.trim()) || !fixture.markdown.includes(fixture.expected.reference)) return []; + const expected = { skill: assignment.key, versionId: assignment.versionId, path: "SKILL.md", text: fixture.markdown, + sha256: createHash("sha256").update(fixture.markdown, "utf8").digest("hex"), + byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }; + const semantic = (row: Row): Row | undefined => row.payload.prpEvent.payload?.semantic_tool; + const content = (value: Row, digest: string) => value.content?.digest === digest + && value.content.redactionDisposition === "digest_only" && Array.isArray(value.content.references) + && value.content.references.length === 0 + && Object.keys(value.content).sort().join(",") === "digest,redactionDisposition,references"; + const correlated = (row: Row, phase: "input" | "result") => { + const event = row.payload.prpEvent, value = semantic(row), correlation = value?.correlation; + const keys = ["schema", "schemaVersion", "phase", "operationId", "callId", "correlation", "idempotencyKey", "content", + ...(phase === "input" ? ["input"] : ["outcome", "code", "retryable", "authorizationBoundary", "operationReceiptId"])]; + return Boolean(value && event.eventType === `semantic_tool.${phase}` && value.phase === phase + && value.schema === "paperclip.prp.semantic_tool.v1" && value.schemaVersion === 1 + && Object.keys(value).sort().join(",") === keys.sort().join(",") + && value.operationId === "read_assigned_skill" && typeof value.callId === "string" && value.callId.trim() + && value.idempotencyKey === null && Number.isSafeInteger(row.seq) + && event.sourceInstanceId === run.runnerInstanceId && event.normalizedSessionId === run.nativeSessionId + && typeof event.turnId === "string" && event.turnId.trim() && typeof event.itemId === "string" && event.itemId.trim() + && correlation && Object.keys(correlation).sort().join(",") === "itemId,normalizedSessionId,runId,turnId" + && correlation.runId === binding.runId && correlation.normalizedSessionId === run.nativeSessionId + && correlation.turnId === event.turnId && correlation.itemId === event.itemId); + }; + return bound.flatMap(start => { + if (!correlated(start, "input")) return []; + const initial = semantic(start)!, args = initial.input; + if (!args || typeof args !== "object" || Array.isArray(args) + || Object.keys(args).some(key => !["skill", "path", "offset"].includes(key)) + || !(args.skill === assignment.key || typeof skills[0].runtimeName === "string" + && skills[0].runtimeName.trim() && args.skill === skills[0].runtimeName) + || Object.hasOwn(args, "path") && args.path !== "SKILL.md" + || Object.hasOwn(args, "offset") && args.offset !== 0 || !content(initial, semanticDigest(args))) return []; + // Duplicates, reconciled alternatives and foreign same-call carriers cannot supply a second interpretation. + const pair = bound.filter(row => semantic(row)?.callId === initial.callId); + const end = pair.find(row => correlated(row, "result")); + if (pair.length !== 2 || !end || start.seq >= end.seq + || start.payload.prpEvent.sourceSeq >= end.payload.prpEvent.sourceSeq) return []; + const result = semantic(end)!; + if (semanticDigest(initial.correlation) !== semanticDigest(result.correlation) + || result.outcome !== "succeeded" || result.code !== "semantic_tool_succeeded" || result.retryable !== false + || result.authorizationBoundary !== "active_task" || result.operationReceiptId !== `operation_${initial.callId}` + || !content(result, semanticDigest(expected))) return []; + return [{ runId: binding.runId, executionId: String(initial.callId), seq: Number(end.seq) }]; + }); +} + +/** Successful built-in reads and native skill loads expose the actual skill text. + * A model claim, discovered name, shell echo, failed read or foreign run cannot qualify. */ +export function savedSkillReadReceipts(events: Row[], binding: NativePrpRunBinding, fixture: SavedSkillFixture, + runtimeName?: string, assignment?: SavedSkillReadAssignment, run?: Row) { + const bound = events.filter(row => isBoundNativePrpEventRow(row, binding)); + const builtin = bound.filter(row => { + const event = row.payload.prpEvent, payload = event.payload; + if (event.eventType !== "tool.execution.completed" || payload?.schema !== "paperclip.tool.execution.v1" + || payload.transport !== "builtin" || payload.status !== "completed" + || typeof event.turnId !== "string" || !event.turnId.trim() + || typeof payload.executionId !== "string" || !payload.executionId.trim() + || payload.outputTruncated !== false || typeof payload.output !== "string" || payload.outputBytes <= 0 + || payload.outputBytes !== Buffer.byteLength(payload.output)) return false; + const name = String(payload.name ?? "").toLowerCase(); + const target = typeof payload.target === "string" ? payload.target.replaceAll("\\", "/") : ""; + const reading = payload.operation === "read" && payload.readOnly === true && /^(read|read_file|file_read)$/.test(name) + && !target.split("/").includes("..") && target.endsWith(`/${fixture.slug}/SKILL.md`); + // OpenCode's skill tool names the loaded skill in its provider-visible result. + const loading = payload.operation === "execute" && name === "skill" + && payload.output.includes(``); + // Deployed OpenCode reports this read with operation unknown and target null. + // Its successful file-result wrapper supplies the exact materialized source path. + const wrapped = /^([^<>\r\n]+)<\/path>\r?\nfile<\/type>\r?\n\r?\n([\s\S]*)\r?\n<\/content>\s*$/.exec(payload.output); + const wrappedPath = wrapped?.[1]; + const wrappedRead = name === "read" && payload.readOnly === true && payload.target === null + && (payload.operation === "unknown" || payload.operation === "read") + && typeof runtimeName === "string" && /^[a-z0-9][a-z0-9_-]*$/i.test(runtimeName) + && wrappedPath?.startsWith("/") && !wrappedPath.includes("\\") && !wrappedPath.includes("\0") + && !wrappedPath.split("/").some(part => part === "." || part === "..") + && wrappedPath.endsWith(`/.claude/skills/${runtimeName}/SKILL.md`); + // The ordinary semantic tool is another native read path, not a filesystem wrapper. + // Its canonical key/version binding does not depend on a provider's directory name. + const assignedText = name === "paperclip_read_assigned_skill" && payload.readOnly === true && payload.target === null + && (payload.operation === "unknown" || payload.operation === "read") + && assignment?.key === `company/${binding.companyId}/${fixture.slug}` + && typeof assignment.versionId === "string" && assignment.versionId.trim() + ? assignedSkillReadText(payload.output, binding, fixture, assignment.versionId) : undefined; + if (!reading && !loading && !wrappedRead && assignedText === undefined) return false; + // File readers may prefix each source line with its line number. + const output = (assignedText ?? (wrappedRead ? wrapped![2]! : payload.output)).split("\n").map((line: string) => line.replace(/^\s*\d+(?:[:|]|→)\s?/, "")).join("\n"); + if (!output.includes(fixture.procedure.trim())) return false; + return bound.some(start => { + const initial = start.payload.prpEvent, p = initial.payload; + return initial.eventType === "tool.execution.started" && start.seq < row.seq + && initial.sourceInstanceId === event.sourceInstanceId && initial.turnId === event.turnId + && p?.schema === payload.schema && p.executionId === payload.executionId && p.name === payload.name + && p.transport === payload.transport && p.operation === payload.operation && p.status === "running" + && (!(wrappedRead || assignedText !== undefined) || p.readOnly === true); + }); + }).map(row => ({ runId: binding.runId, executionId: String(row.payload.prpEvent.payload.executionId), seq: Number(row.seq) })); + return [...builtin, ...codexAssignedSkillReceipts(bound, binding, fixture, assignment, run)]; +} + +export function savedSkillDocumentAttributed(document: Row | undefined, revision: Row | undefined, fixture: SavedSkillFixture, + binding: NativePrpRunBinding & { issueId: string }): boolean { + if (!document || !revision) return false; + return Boolean(document.id && document.companyId === binding.companyId && document.issueId === binding.issueId + && document.key === fixture.key && document.format === "markdown" && document.latestRevisionId + && Number.isSafeInteger(document.latestRevisionNumber) && document.latestRevisionNumber > 0 + && document.createdByAgentId === binding.agentId && document.createdByUserId === null + && document.updatedByAgentId === binding.agentId && document.updatedByUserId === null + && savedSkillResultMatches(document.body, fixture.expected) + && revision.id === document.latestRevisionId && revision.documentId === document.id + && revision.companyId === binding.companyId && revision.issueId === binding.issueId && revision.key === fixture.key && revision.format === document.format + && revision.createdByRunId === binding.runId && revision.createdByAgentId === binding.agentId && revision.createdByUserId === null + && revision.revisionNumber === document.latestRevisionNumber && savedSkillResultMatches(revision.body, fixture.expected)); +} diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts b/tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts new file mode 100644 index 00000000000..266d07514f5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-flow.test.ts @@ -0,0 +1,370 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { createPrpSemanticToolInputEnvelope, createPrpSemanticToolResultEnvelope } from "../../../packages/paperclip-runner/src/protocol/semantic-tool-receipts.js"; +import { parseFrontmatterMarkdown } from "../../../packages/shared/src/frontmatter.js"; +import { agentSkillSyncSchema } from "../../../packages/shared/src/validators/adapter-skills.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { runSavedSkillUse } from "./saved-skill-use-flow.js"; +import { savedSkillFixture, type SavedSkillFixture } from "./saved-skill-use-evidence.js"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; + +type Row = Record; +vi.mock("../user-actions.js", async original => ({ ...await original(), createTaskThroughUi: vi.fn() })); +vi.mock("./task-document-flow.js", async original => ({ ...await original(), openDocumentAuthoring: vi.fn() })); +vi.mock("@playwright/test", () => ({ expect: (locator: { textContent(): Promise }) => ({ + async toContainText(text: string) { expect(await locator.textContent()).toContain(text); }, +}) })); +const directories: string[] = []; +afterEach(async () => { vi.clearAllMocks(); await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))); }); + +async function setup(change: (kind: string, row: any) => any = (_kind, row) => row, campaignId = "campaign", betaSkillsEnabled = false, + wrappedRead: boolean | "assigned" | "codex" = false) { + const directory = await mkdtemp(path.join(os.tmpdir(), "saved-skill-flow-")); directories.push(directory); + let fixture: SavedSkillFixture; + let skill: Row, assigned = false, issueCreated = false, reloaded = false; + const savedSkills: Row[] = []; + const own = vi.fn(async (_resource: Row) => {}); + const evidence = { checks: [] } as unknown as JourneyEvidence; + const api = { + origin: "https://qa.example.test", + post: vi.fn(async (route: string, body: Row) => { + if (route === "/api/companies/company/skills") { + const reference = /receipt reference is ([a-f0-9]{32})/.exec(body.markdown)![1]!; + fixture = savedSkillFixture(campaignId, reference); + // createLocalSkill gives the parsed frontmatter name precedence over the API input name. + const savedName = parseFrontmatterMarkdown(body.markdown).frontmatter.name; + skill = { id: "skill", companyId: "company", key: `company/company/${fixture.slug}`, sourceType: "managed", slug: fixture.slug, + name: typeof savedName === "string" && savedName.trim() ? savedName.trim() : body.name, + markdown: body.markdown, currentVersionId: "version" }; + savedSkills.push(structuredClone(skill)); + return change("create-skill", structuredClone(skill)); + } + if (route === "/api/agents/agent/skills/sync?companyId=company") { + const parsed = agentSkillSyncSchema.parse(body); + // Core normalizes a string selection to Latest; only a non-null pin requires Beta skills. + const selections = parsed.desiredSkills.map(value => typeof value === "string" ? { key: value, versionId: null } : value); + if (!betaSkillsEnabled && selections.some(entry => entry.versionId !== null)) throw new Error("Beta skill version pins require Beta skills"); + expect(parsed).toEqual({ desiredSkills: [skill.key], mode: "add" }); + assigned = true; return {}; + } + throw new Error(`Unexpected mutation ${route}`); + }), + get: vi.fn(async (route: string) => { + if (route === "/api/instance/settings/experimental") return { enableBetaSkills: betaSkillsEnabled }; + if (route === "/api/companies/company/skills/skill") return change(reloaded ? "retained-skill" : "skill", structuredClone(skill)); + if (route === "/api/companies/company/skills/skill/versions/version") return change(reloaded ? "retained-version" : "version", { + id: "version", companyId: "company", companySkillId: "skill", fileInventory: [{ path: "SKILL.md", content: fixture.markdown }], + }); + if (route === "/api/agents/agent/skills?companyId=company") return change("assignment", { + desiredSkillEntries: assigned ? [{ key: skill.key, versionId: null }] : [], + entries: assigned ? [{ key: skill.key, versionId: "version", runtimeName: `${fixture.slug}--0123456789` }] : [], + }); + if (!issueCreated) throw new Error("Measured result was read before browser dispatch"); + if (route === "/api/issues/issue") return { id: "issue", identifier: "QA-1", companyId: "company", assigneeAgentId: "agent", status: "done", + description: change("description", vi.mocked(createTaskThroughUi).mock.calls.at(-1)![0].prompt) }; + if (route === "/api/issues/issue/runs") return change("runs", [{ runId: "run" }]); + if (route === "/api/issues/issue/comments?order=asc" || route === "/api/issues/issue/interactions") return []; + if (route === "/api/heartbeat-runs/run") return change("run", { id: "run", companyId: "company", agentId: "agent", issueId: "issue", + status: "succeeded", runtimeMode: "native", nativePhase: "committed", resultJson: null, + runnerInstanceId: "instance", nativeSessionId: "session", + contextSnapshot: { paperclipEnvironment: { id: "environment" }, skillSelectionSnapshot: true, skillVersions: { [skill.key]: "version" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode", model: "model" }, + runtimeContext: { skills: [{ key: skill.key, versionId: "version", runtimeName: `${fixture.slug}--0123456789` }] } } } }); + if (route.startsWith("/api/heartbeat-runs/run/events?")) { + if (wrappedRead === "codex") { + const correlation = { runId: "run", normalizedSessionId: "session", turnId: "turn", itemId: "item" }; + const input = { skill: skill.key }; + const result = change("assigned-read", { skill: skill.key, versionId: skill.currentVersionId, + path: "SKILL.md", text: fixture.markdown, sha256: createHash("sha256").update(fixture.markdown).digest("hex"), + byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }); + const semantic = [{ ...createPrpSemanticToolInputEnvelope({ operationId: "read_assigned_skill", callId: "load", + correlation, content: input }), input }, createPrpSemanticToolResultEnvelope({ operationId: "read_assigned_skill", + callId: "load", correlation, content: result, outcome: "succeeded", code: "semantic_tool_succeeded", + retryable: false, authorizationBoundary: "active_task", operationReceiptId: "operation_load" })]; + return semantic.map((value, i) => change("event", { runId: "run", companyId: "company", agentId: "agent", + seq: i + 1, sourceSeq: i + 1, sourceEventId: `event-${i}`, sourceInstanceId: "instance", protocolSchemaVersion: 1, + eventType: `semantic_tool.${value.phase}`, payload: { prpEvent: { ...correlation, + schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceSeq: i + 1, + sourceEventId: `event-${i}`, sourceInstanceId: "instance", eventType: `semantic_tool.${value.phase}`, + payload: { semantic_tool: value } } } })); + } + return ["pending", "completed"].map((status, i) => { + const p = canonicalProviderEventsFromOpenCodePart({ id: "load", type: "tool", tool: "skill", state: { + status, output: `\n${fixture.procedure}\n`, + } })[0]!; + if (wrappedRead === true) { + const output = `/workspace/.claude/skills/${fixture.slug}--0123456789/SKILL.md\nfile\n\n` + + fixture.markdown.trimEnd().split("\n").map((line, index) => `${index + 1}: ${line}`).join("\n") + + "\n(End of file - total 12 lines)\n"; + Object.assign(p.payload, { name: "read", operation: "unknown", readOnly: true, target: null, + output: status === "completed" ? output : "", outputBytes: status === "completed" ? Buffer.byteLength(output) : 0 }); + } + if (wrappedRead === "assigned") { + const result = { skill: skill.key, versionId: skill.currentVersionId, path: "SKILL.md", text: fixture.markdown, + sha256: createHash("sha256").update(fixture.markdown).digest("hex"), byteSize: Buffer.byteLength(fixture.markdown), nextOffset: null }; + const output = JSON.stringify({ type: "inputText", text: JSON.stringify(change("assigned-read", result)) }); + Object.assign(p.payload, { name: "paperclip_read_assigned_skill", operation: "unknown", readOnly: true, target: null, + output: status === "completed" ? output : "", outputBytes: status === "completed" ? Buffer.byteLength(output) : 0 }); + } + return change("event", { runId: "run", companyId: "company", agentId: "agent", seq: i + 1, eventType: p.eventType, + sourceEventId: `event-${i}`, sourceInstanceId: "instance", sourceSeq: i + 1, protocolSchemaVersion: 1, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", runId: "run", turnId: "turn", + eventType: p.eventType, sourceEventId: `event-${i}`, sourceInstanceId: "instance", sourceSeq: i + 1, payload: p.payload } } }); + }); + } + if (route === "/api/issues/issue/documents") return change("documents", [{ key: fixture.key }]); + if (route === `/api/issues/issue/documents/${fixture.key}`) return { id: "document", companyId: "company", issueId: "issue", + key: fixture.key, format: "markdown", body: JSON.stringify(fixture.expected), latestRevisionId: "revision", latestRevisionNumber: 1, + createdByAgentId: "agent", createdByUserId: null, updatedByAgentId: "agent", updatedByUserId: null }; + if (route === `/api/issues/issue/documents/${fixture.key}/revisions/revision`) return change("revision", { + id: "revision", companyId: "company", issueId: "issue", documentId: "document", key: fixture.key, format: "markdown", revisionNumber: 1, + createdByRunId: "run", createdByAgentId: "agent", createdByUserId: null, body: JSON.stringify(fixture.expected), + }); + throw new Error(`Unexpected observation ${route}`); + }), + }; + const page = { goto: vi.fn(async () => {}), reload: vi.fn(async () => { reloaded = true; }), + locator: vi.fn(() => ({ async textContent() { return JSON.stringify(fixture.expected); } })) }; + vi.mocked(createTaskThroughUi).mockImplementation(async input => { + expect(assigned).toBe(true); expect(input.prompt).not.toContain(fixture.expected.reference); + await input.beforeDispatch?.(); issueCreated = true; return { issueId: "issue", submittedAtMs: Date.now() }; + }); + const context = { api, page, company: { id: "company", issuePrefix: "QA" }, user: { id: "owner", name: "Owner" }, + environmentId: "environment", capabilities: ["company", "board-user", "task-write", "agent", "skill-write"], + agent: { id: "agent", name: "Owned agent", provider: "opencode", model: "model", runtimeMode: "native" } } as unknown as JourneyContext; + return { context, api, page, own, evidence, directory, savedSkills, + run: (timeoutMs = 5000) => runSavedSkillUse(context, { own } as unknown as TargetSession, campaignId, evidence, + { deadlineAt: Date.now() + timeoutMs, directory }) }; +} + +describe("saved skill journey orchestration", () => { + it.each([{ kind: "codex" }, { kind: "acpx", agent: "cursor" }, { kind: "cursor_cloud" }])( + "binds a common semantic read for provider %j without an OpenCode directory", async provider => { + const f = await setup((kind, row) => { + if (kind === "run") Object.assign(row.runnerProfileJson.nativeExecutionInput.provider, provider); + return row; + }, "campaign", false, provider.kind === "codex" ? "codex" : "assigned"); + f.context.agent!.provider = provider.kind; + await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.page.reload).toHaveBeenCalledOnce(); + }); + it.each(["codex", "acpx", "cursor_cloud"])("rejects missing or foreign semantic assignment evidence for %s", async provider => { + const changes: Array<{ target: string; mutate(row: Row): void }> = [ + { target: "assignment", mutate: row => { row.entries = []; } }, + { target: "run", mutate: row => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; } }, + { target: "run", mutate: row => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "foreign"; } }, + { target: "assigned-read", mutate: row => { row.versionId = "foreign"; } }, + ]; + for (const { target, mutate } of changes) { + const f = await setup((kind, row) => { + if (kind === "run") Object.assign(row.runnerProfileJson.nativeExecutionInput.provider, { kind: provider, + ...(provider === "acpx" ? { agent: "cursor" } : {}) }); + if (kind === target) mutate(row); + return row; + }, "campaign", false, provider === "codex" ? "codex" : "assigned"); + f.context.agent!.provider = provider; + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + } + }); + it("verifies the ordinary semantic assigned-skill read through all seven checks with Beta skills off", async () => { + const f = await setup(undefined, "campaign", false, "assigned"); await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.page.reload).toHaveBeenCalledOnce(); + expect(f.api.post).toHaveBeenCalledTimes(2); + expect(createTaskThroughUi).toHaveBeenCalledOnce(); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/settings/"))).toBe(false); + const receipt = JSON.parse(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")); + expect(receipt.receipts).toEqual([{ runId: "run", executionId: "load", seq: 2 }]); + expect(receipt.reopened).toEqual(receipt.document); + expect(receipt.retainedSkill).toEqual(receipt.skill); + }); + it.each([ + ["wrong receipt key", "assigned-read", (r: Row) => { r.skill = "foreign"; }], + ["wrong receipt version", "assigned-read", (r: Row) => { r.versionId = "stale"; }], + ["partial source page", "assigned-read", (r: Row) => { r.nextOffset = r.text.length; }], + ["semantic failure", "assigned-read", (r: Row) => { r.outcome = "failed"; }], + ["wrong assigned version", "assignment", (r: Row) => { r.entries[0].versionId = "stale"; }], + ["missing admitted skill", "run", (r: Row) => { r.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }], + ["wrong admitted version", "run", (r: Row) => { r.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "stale"; }], + ] as const)("rejects a semantic assigned-skill read with %s before reading its document", async (_label, target, mutate) => { + const f = await setup((kind, row) => { if (kind === target) mutate(row); return row; }, "campaign", false, "assigned"); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + }); + it("assigns the input before browser dispatch and verifies native use, saved result and reload without setup supplying output", async () => { + const f = await setup(); await f.run(); + expect(f.own.mock.calls.map(call => call[0])).toEqual([ + expect.objectContaining({ kind: "skill", journeyId: "saved-skill-use" }), + expect.objectContaining({ kind: "skill", id: "skill" }), + expect.objectContaining({ kind: "issue", journeyId: "saved-skill-use" }), + expect.objectContaining({ kind: "issue", id: "issue" }), + ]); + expect(f.api.post).toHaveBeenCalledTimes(2); + expect(createTaskThroughUi).toHaveBeenCalledTimes(1); + expect(f.page.reload).toHaveBeenCalledTimes(1); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const receipt = JSON.parse(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")); + expect(receipt.receipts).toEqual([{ runId: "run", executionId: "load", seq: 2 }]); + expect(receipt.reopened).toEqual(receipt.document); + expect(receipt.retainedSkill).toEqual(receipt.skill); + expect(receipt.retainedVersion).toEqual(receipt.version); + expect(receipt.retainedAssignments).toEqual(receipt.assignments); + }); + it("verifies the deployed wrapped native read and then independently checks the saved result and reload", async () => { + const f = await setup(undefined, "campaign", false, true); await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.page.reload).toHaveBeenCalledOnce(); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents/saved-skill-result/revisions/"))).toBe(true); + }); + it.each([ + ["missing native skill", "run", (row: Row) => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills = []; }], + ["wrong native version", "run", (row: Row) => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].versionId = "stale"; }], + ["wrong native name", "run", (row: Row) => { row.runnerProfileJson.nativeExecutionInput.runtimeContext.skills[0].runtimeName = "foreign"; }], + ["wrong assigned version", "assignment", (row: Row) => { row.entries[0].versionId = "stale"; }], + ["wrong assigned name", "assignment", (row: Row) => { row.entries[0].runtimeName = "foreign"; }], + ] as const)("rejects a complete wrapped read with %s before inspecting its document", async (_label, target, mutate) => { + const f = await setup((kind, row) => { if (kind === target) mutate(row); return row; }, "campaign", false, true); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + }); + it.each([false, true])("uses ordinary Latest assignment with Beta skills %s and no setting read or write", async betaSkillsEnabled => { + const f = await setup(undefined, "campaign", betaSkillsEnabled); await f.run(); + expect(f.api.post.mock.calls[1]).toEqual(["/api/agents/agent/skills/sync?companyId=company", + { desiredSkills: [f.savedSkills[0]!.key], mode: "add" }]); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/settings/"))).toBe(false); + expect(f.api.post.mock.calls.some(([route]) => route.includes("/settings/"))).toBe(false); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + }); + it.each([ + ["missing snapshot", (row: Row) => { delete row.contextSnapshot.skillSelectionSnapshot; }], + ["false snapshot", (row: Row) => { row.contextSnapshot.skillSelectionSnapshot = false; }], + ["missing versions", (row: Row) => { delete row.contextSnapshot.skillVersions; }], + ["null versions", (row: Row) => { row.contextSnapshot.skillVersions = null; }], + ["array versions", (row: Row) => { row.contextSnapshot.skillVersions = []; }], + ["foreign key", (row: Row) => { row.contextSnapshot.skillVersions = { foreign: "version" }; }], + ["wrong version", (row: Row) => { Object.keys(row.contextSnapshot.skillVersions).forEach(key => { row.contextSnapshot.skillVersions[key] = "stale"; }); }], + ] as const)("rejects an admitted run with %s before trusting its read or result", async (_label, mutate) => { + const f = await setup((kind, row) => { if (kind === "run") mutate(row); return row; }); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_run_version_mismatch" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/events?") || route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + }); + it.each([undefined, "stale"])("rejects a created-version receipt %s that cannot bind the immutable input", async currentVersionId => { + const f = await setup((kind, row) => kind === "create-skill" ? { ...row, currentVersionId } : row); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_created_version_mismatch" }); + expect(f.api.post).toHaveBeenCalledTimes(1); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + expect(f.own).toHaveBeenCalledTimes(2); + }); + it("rejects Latest source drift between creation and dispatch", async () => { + let reads = 0; + const f = await setup((kind, row) => kind === "skill" && ++reads === 2 ? { ...row, currentVersionId: "changed" } : row); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_assignment_bound" }); + expect(reads).toBe(2); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + }); + it("rejects another skill's complete native output even when the receipt document would be correct", async () => { + const f = await setup((kind, row) => { + if (kind === "event" && row.eventType === "tool.execution.completed") { + const payload = row.payload.prpEvent.payload; + payload.output = payload.output.replace(//, ''); + payload.outputBytes = Buffer.byteLength(payload.output); + } + return row; + }); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_native_read_attributed" }); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/documents"))).toBe(false); + }); + it.each([null, { presentationDecision: { commentAction: "none" } }])( + "qualifies authoritative committed finalization with optional resultJson %j", async resultJson => { + const f = await setup((kind, row) => kind === "run" ? { ...row, resultJson } : row); + await f.run(); + expect(f.evidence.checks).toHaveLength(7); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + }); + it.each([ + [null, null], + [undefined, { presentationDecision: { commentAction: "none" } }], + ["workspace_finalizing", null], + ["retryable_failure", { finalizationPhase: "committed" }], + ])("does not qualify Done and succeeded before native phase %s is committed", async (nativePhase, resultJson) => { + const f = await setup((kind, row) => kind === "run" ? { ...row, nativePhase, resultJson } : row); + await expect(f.run(250)).rejects.toMatchObject({ outcome: "failed", code: "saved_skill_task_not_completed" }); + expect(f.api.get.mock.calls.some(([route]) => route === "/api/heartbeat-runs/run")).toBe(true); + expect(f.api.get.mock.calls.some(([route]) => route.includes("/events?") || route.includes("/documents"))).toBe(false); + expect(f.page.reload).not.toHaveBeenCalled(); + const receipt = JSON.parse(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")); + expect(receipt.state.taskRuns[0].nativePhase).toBe(nativePhase); + }); + it("waits for native commit before qualifying the already saved current-run result", async () => { + let runReads = 0; + const f = await setup((kind, row) => { + if (kind === "run") return { ...row, nativePhase: ++runReads === 1 ? "workspace_finalizing" : "committed", + resultJson: { finalizationPhase: "committed" } }; + if (kind === "documents") expect(runReads).toBe(2); + return row; + }); + await f.run(); + expect(runReads).toBe(2); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + }); + it.each([ + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: [{ key: "unknown", versionId: null }] })], + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: row.desiredSkillEntries.map((entry: Row) => ({ ...entry, versionId: "stale" })) })], + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: row.desiredSkillEntries.map(({ key }: Row) => ({ key })) })], + ["assignment", "saved_skill_assignment_bound", (row: Row) => ({ ...row, desiredSkillEntries: [] })], + ["skill", "saved_skill_assignment_bound", (row: Row) => ({ ...row, sourceType: "local_path" })], + ["version", "saved_skill_assignment_bound", (row: Row) => ({ ...row, fileInventory: [{ path: "SKILL.md", content: "changed source" }] })], + ["runs", "saved_skill_run_limit_exceeded", (row: Row[]) => [...row, { runId: "extra" }]], + ["event", "saved_skill_native_read_attributed", (row: Row) => ({ ...row, companyId: "foreign" })], + ["documents", "saved_skill_unexpected_documents", (row: Row[]) => [...row, { key: "unrelated" }]], + ["revision", "saved_skill_result_authored_by_run", (row: Row) => ({ ...row, createdByRunId: "foreign" })], + ["retained-skill", "saved_skill_source_preserved", (row: Row) => ({ ...row, currentVersionId: "changed" })], + ["retained-version", "saved_skill_source_preserved", (row: Row) => ({ ...row, fileInventory: [] })], + ] as const)("fails closed on changed %s evidence and retains its failure checkpoint", async (kind, code, change) => { + const f = await setup((which, row) => which === kind ? change(row as never) : row); + await expect(f.run()).rejects.toMatchObject({ outcome: "failed", code }); + expect(await readFile(path.join(f.directory, "saved-skill-use-checks.json"), "utf8")).toContain("fixture"); + if (kind === "assignment" || kind === "version" || kind === "skill") expect(createTaskThroughUi).not.toHaveBeenCalled(); + else expect(createTaskThroughUi).toHaveBeenCalledTimes(1); + }); + it("journals uncertain creation once without replaying it or dispatching a task", async () => { + const f = await setup(); f.api.post.mockRejectedValueOnce(new Error("Response lost after POST")); + await expect(f.run()).rejects.toThrow("Response lost after POST"); + expect(f.api.post).toHaveBeenCalledTimes(1); + expect(f.own).toHaveBeenCalledTimes(1); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + }); + it("recovers the exact persisted frontmatter name after a successful creation response is lost", async () => { + const campaignId = "journey-1793701020304-a1b2c3d4"; + const f = await setup((kind, row) => { + if (kind === "create-skill") throw new Error("Response lost after successful creation"); + return row; + }, campaignId); + await expect(f.run()).rejects.toThrow("Response lost after successful creation"); + expect(f.savedSkills).toHaveLength(1); + expect(f.api.post).toHaveBeenCalledTimes(1); + expect(f.own).toHaveBeenCalledTimes(1); + expect(createTaskThroughUi).not.toHaveBeenCalled(); + const intent = f.own.mock.calls[0]![0]; + const recovered = f.savedSkills.filter(row => row.companyId === f.context.company.id && row.name === intent.name); + expect(recovered).toEqual([f.savedSkills[0]]); + expect(intent.name).toContain(campaignId); + expect(f.savedSkills[0]!.name).toContain(campaignId); + expect(f.api.post.mock.calls[0]![1].name).toBe(f.savedSkills[0]!.name); + }); +}); diff --git a/tests/runner-e2e/product-journeys/saved-skill-use-flow.ts b/tests/runner-e2e/product-journeys/saved-skill-use-flow.ts new file mode 100644 index 00000000000..750de80f293 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use-flow.ts @@ -0,0 +1,129 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { collectRunEvents } from "../run-observations.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { validateAgentTaskState, waitFor } from "./agent-flows.js"; +import { createObservedReader } from "./observation-read.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { savedSkillDocumentAttributed, savedSkillFixture, savedSkillReadAssignment, savedSkillReadReceipts, savedSkillReadRuntimeName, savedSkillSourceMatches } from "./saved-skill-use-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "saved-skill-use"; + +export async function runSavedSkillUse(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + if (!context.capabilities.includes("skill-write")) throw new JourneyError("unsupported", "missing_skill_write"); + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const read = createObservedReader(context.api, options.deadlineAt); + const fixture = savedSkillFixture(campaignId, randomBytes(16).toString("hex")); + const title = `Journey QA saved skill ${campaignId}`; + const prompt = `Use /${fixture.slug} for this task. Follow that saved skill's procedure, verify its saved result, and finish. ` + + "Do not create child tasks or unrelated deliverables."; + const check = (name: string, passed: boolean) => { + evidence.checks.push({ id: name, passed }); + if (!passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + let checkpoint: Row = { fixture }; + let primaryFailure: unknown; + try { + const companyRoute = `/api/companies/${context.company.id}`; + await session.own({ kind: "skill", name: fixture.name, journeyId: id }); + const createdSkill = await context.api.post(`${companyRoute}/skills`, { + name: fixture.name, slug: fixture.slug, description: "Make a receipt using this saved procedure.", + markdown: fixture.markdown, idempotencyKey: `saved-skill-${campaignId}`, + }); + if (!createdSkill?.id) throw new JourneyError("failed", "saved_skill_creation_receipt_missing"); + await session.own({ kind: "skill", name: fixture.name, id: createdSkill.id, journeyId: id }); + checkpoint = { ...checkpoint, createdSkill }; + const skillRoute = `${companyRoute}/skills/${createdSkill.id}`; + const skill = await read.get(skillRoute); + if (!skill?.key || !skill.currentVersionId) throw new JourneyError("failed", "saved_skill_version_missing"); + if (skill.id !== createdSkill.id || !createdSkill.currentVersionId || skill.currentVersionId !== createdSkill.currentVersionId) + throw new JourneyError("failed", "saved_skill_created_version_mismatch"); + await context.api.post(`/api/agents/${agent.id}/skills/sync?companyId=${context.company.id}`, { + desiredSkills: [skill.key], mode: "add", + }); + const assignmentRoute = `/api/agents/${agent.id}/skills?companyId=${context.company.id}`; + const versionRoute = `${skillRoute}/versions/${skill.currentVersionId}`; + const [version, assignments, beforeDispatchSkill] = await Promise.all([ + read.get(versionRoute), read.get(assignmentRoute), read.get(skillRoute), + ]); + checkpoint = { ...checkpoint, skill, version, assignments, beforeDispatchSkill }; + check("saved-skill-assignment-bound", beforeDispatchSkill.id === createdSkill.id + && beforeDispatchSkill.currentVersionId === createdSkill.currentVersionId + && savedSkillSourceMatches(beforeDispatchSkill, version, assignments, fixture, context.company.id)); + + const created = await createTaskThroughUi({ page: context.page, origin: context.api.origin, + issuePrefix: context.company.issuePrefix, agentName: agent.name, title, prompt, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: id }); + const route = `/api/issues/${created.issueId}`; + const state = await waitFor(options.deadlineAt, async () => { + const [currentIssue, summaries, comments, interactions] = await Promise.all([ + read.get(route), read.get(`${route}/runs`), + read.get(`${route}/comments?order=asc`), read.get>(`${route}/interactions`), + ]); + if (!Array.isArray(summaries) || summaries.length > 1) throw new JourneyError("failed", "saved_skill_run_limit_exceeded"); + const taskRuns = await Promise.all(summaries.map(row => read.get(`/api/heartbeat-runs/${row.runId}`))); + const value = { currentIssue, taskRuns, comments, interactions, attachments: [] }; + validateAgentTaskState(context, created.issueId, value, 1); + checkpoint = { ...checkpoint, state: value }; + return value; + }, value => value.currentIssue.status === "done" && value.taskRuns.length === 1 && value.taskRuns[0]!.status === "succeeded" + && value.taskRuns[0]!.nativePhase === "committed" + && value.interactions.every(row => row.status !== "pending"), "saved_skill_task_not_completed"); + const admittedVersions = state.taskRuns[0]!.contextSnapshot?.skillVersions; + if (state.taskRuns[0]!.contextSnapshot?.skillSelectionSnapshot !== true || !admittedVersions + || typeof admittedVersions !== "object" || Array.isArray(admittedVersions) + || !Object.hasOwn(admittedVersions, skill.key) || admittedVersions[skill.key] !== version.id) + throw new JourneyError("failed", "saved_skill_run_version_mismatch"); + const binding = { companyId: context.company.id, agentId: agent.id, runId: state.taskRuns[0]!.id, issueId: created.issueId }; + const request = String(state.currentIssue.description ?? ""); + check("saved-skill-input-only-in-source", request.includes(`/${fixture.slug}`) && !request.includes(fixture.expected.reference) + && !state.comments.some(row => row.authorUserId && String(row.body ?? "").includes(fixture.expected.reference))); + const events = await collectRunEvents( + (afterSeq, limit) => read.get(`/api/heartbeat-runs/${binding.runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + const runtimeName = savedSkillReadRuntimeName(skill, assignments, state.taskRuns[0]!); + const assignment = savedSkillReadAssignment(skill, assignments, state.taskRuns[0]!); + const receipts = savedSkillReadReceipts(events, binding, fixture, runtimeName, assignment, state.taskRuns[0]!); + checkpoint = { ...checkpoint, events, receipts }; + check("saved-skill-native-read-attributed", receipts.length > 0); + const documents = await read.get(`${route}/documents`); + if (documents.length !== 1 || documents[0]?.key !== fixture.key) throw new JourneyError("failed", "saved_skill_unexpected_documents"); + const documentRoute = `${route}/documents/${fixture.key}`; + const document = await read.get(documentRoute); + const revision = await read.get(`${documentRoute}/revisions/${document.latestRevisionId}`); + checkpoint = { ...checkpoint, document, revision }; + check("saved-skill-result-authored-by-run", savedSkillDocumentAttributed(document, revision, fixture, binding)); + const url = `${context.api.origin}/${context.company.issuePrefix}/issues/${state.currentIssue.identifier ?? created.issueId}`; + await context.page.goto(url, { waitUntil: "domcontentloaded" }); + await context.page.reload(); + await openDocumentAuthoring(context); + await expect(context.page.locator(`#document-${fixture.key}`)).toContainText(fixture.expected.reference); + const [reopened, retainedSkill, retainedVersion, retainedAssignments] = await Promise.all([ + read.get(documentRoute), read.get(skillRoute), read.get(versionRoute), read.get(assignmentRoute), + ]); + checkpoint = { ...checkpoint, reopened, retainedSkill, retainedVersion, retainedAssignments }; + check("saved-skill-result-reopened", reopened.latestRevisionId === document.latestRevisionId + && savedSkillDocumentAttributed(reopened, revision, fixture, binding)); + check("saved-skill-source-preserved", savedSkillSourceMatches(retainedSkill, retainedVersion, retainedAssignments, fixture, context.company.id) + && retainedSkill.id === skill.id && retainedSkill.currentVersionId === skill.currentVersionId); + check("configured-model-and-environment", true); // All observed runs were independently validated above. + } catch (error) { + primaryFailure = error; + throw error; + } finally { + try { + await writeFile(path.join(options.directory, "saved-skill-use-checks.json"), JSON.stringify(checkpoint, null, 2), { mode: 0o600 }); + await read.preserve(options.directory); + } catch (error) { + if (!primaryFailure) throw error; + } + } +} diff --git a/tests/runner-e2e/product-journeys/saved-skill-use.md b/tests/runner-e2e/product-journeys/saved-skill-use.md new file mode 100644 index 00000000000..1ea01e75855 --- /dev/null +++ b/tests/runner-e2e/product-journeys/saved-skill-use.md @@ -0,0 +1,97 @@ +# Use a saved assigned skill + +`saved-skill-use` is a reusable Product E2E journey for an existing target. It +uses one native work run on the target provider's campaign-owned agent. It does +not enroll itself in a daily plan. Local calibration and catalog discovery are +separate from a real staging pass. + +The target supplies ordinary company skill-write access. Assignment uses +`desiredSkills: [canonicalSkillKey]`, which selects Latest and returns a stored +`versionId: null`. This path works with Beta skills disabled. No experimental +setting is read or changed. Experimental version pins need separate qualification. + +The fixture creates one synthetic company skill through the public skill API. +The journal name, API name and Markdown frontmatter name equal its slug. Each +contains the exact campaign ID, including its hyphens. Recovery can therefore +find the persisted skill by name if a successful creation response is lost. +It assigns the skill on the owned agent through the normal skill sync API. +The created skill must be managed, with the exact canonical company key. Its +creation receipt and separate reads must agree on the immutable current version. +A version read must return the exact `SKILL.md` bytes. The skill's +random receipt reference and two numeric inputs occur only in those saved bytes. +The slug and task prompt do not contain the reference. The task is created +through the production browser composer and explicitly names that skill. + +A pass requires all seven closed checks: + +- `saved-skill-assignment-bound`: exact company, managed skill, creation version, + source bytes and one ordinary Latest assignment before dispatch. The successful + run must have `contextSnapshot.skillSelectionSnapshot: true` and an admitted + `skillVersions[canonicalSkillKey]` equal to that immutable created version. +- `saved-skill-input-only-in-source`: the persisted request names the skill but + does not expose its receipt reference; no human comment supplies that reference. +- `saved-skill-native-read-attributed`: a complete successful native skill load or + source-file read, with start/completion identity bound to this company, agent, + run, turn and runner. Its native receipt must contain or cryptographically + commit to the complete saved procedure. +- `saved-skill-result-authored-by-run`: exactly one saved task document contains + the reference and independently calculated sum. Its revision belongs to the + successful native run and agent, without human authorship. The run's + authoritative `nativePhase` must be `committed` before the result qualifies. +- `saved-skill-result-reopened`: the browser reload shows that document, and its + independently read content and revision are unchanged. +- `saved-skill-source-preserved`: the original skill version, source bytes and + assignment remain unchanged after execution. +- `configured-model-and-environment`: the one successful run matches the owned + company, task, agent, configured native provider/model and execution environment. + +The read oracle accepts a completed built-in file read with the exact skill +directory target, or OpenCode's native `skill` load with its exact named +`skill_content` wrapper. Deployed OpenCode can instead report a native `read` +with an unknown operation and null target. That path requires its complete file +result wrapper: the file path must name the exact materialized skill directory. +Both the saved assignment entry and the admitted native execution input must +bind that runtime name to the same canonical key and immutable created version. +The native read must remain read-only, successful, paired by call/turn/runner, +untruncated and byte-complete, and its content must contain the full procedure. +A targetless filesystem read without this bound file wrapper cannot qualify. Discovery, +shell output, model prose, failed or truncated reads, foreign run receipts and +marker-only output cannot pass. + +The first-party `paperclip_read_assigned_skill` tool can also supply the read. +Its complete `inputText` result must contain the exact canonical skill key, +immutable created version, `SKILL.md` path, original Markdown, byte size and +SHA-256 checksum. `nextOffset` must be null. The same dual assignment and +admitted canonical key/version binding is required for every native provider. +The OpenCode materialized directory name is required only for its filesystem +wrapper; it is not a condition for the common first-party semantic read. +A semantic read failure can have a +completed tool status; it does not qualify. A partial page, copied tool result, +foreign version, missing start or unbound native event also fails closed. + +Codex's dynamic tool display is not first-party authority. Its raw name, +optional namespace or read-only hint cannot qualify a read. For Codex, the +grader also accepts Core's retained `semantic_tool.input` and `semantic_tool.result` +for the closed read-only operation `read_assigned_skill`. Both must name the +actual run's runner, native session, call, turn and item. The input must match +its own digest and select the canonical skill or its exact admitted alias, +`SKILL.md` and offset zero (including Core's defaults). The successful +`active_task` result must commit to the exact complete immutable read object: +canonical key, version, file, all UTF-8 source text, SHA-256, byte size and a +null next-page offset. The digest uses Core's canonical JSON key ordering. +Missing, duplicate, reordered, failed, foreign or differently hashed carriers +fail. This path does not relax the existing OpenCode built-in or filesystem guards. + +The existing campaign deadline bounds observations. Only failed read transport +operations can be retried; task input, skill creation and execution are not +replayed. Done status and a successful run alone cannot qualify incomplete native +finalization. The optional `resultJson.finalizationPhase` presentation field is +not required or accepted in place of `nativePhase`; compact or null result data +can qualify when the authoritative phase is committed. +The provider journals skill and task ownership before creation, then +collects evidence and stops/cleans only campaign-owned resources. The fixed skill +source, native events and saved document receipts stay in the private bundle. +The public report uses the existing closed checks and authenticated task capture. + +No instance-wide experimental setting, shared source agent, global model, +company budget or production workspace is changed by this journey. diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index 32c2eca9b47..d730a85f21b 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,6 +1,22 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ -export const agentJourneyRuns = { "repository-continuation": 2, "routine-scheduled-run": 1, "file-input-answer": 1, "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, - "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3, "hire-delegate-reuse": 7 } as const; +export const agentJourneyRuns = { + "saved-skill-use": 1, + "repository-continuation": 2, + "routine-scheduled-run": 1, + "file-input-answer": 1, + "task-dependency-resume": 1, + "ask-question": 1, + "chat-continuity": 4, + "plan-revise-accept": 3, + "build-revise": 2, + "followup-while-running": 2, + "protected-actions": 4, + "create-skill-studio": 1, + "stop-redirect": 3, + "agent-settings-persistence": 1, + "question-answer-resume": 3, + "hire-delegate-reuse": 7 +} as const; export function agentJourneyRunCountValid(id: string, count: unknown): boolean { if (!Object.hasOwn(agentJourneyRuns, id) || !Number.isSafeInteger(count)) return false; @@ -9,6 +25,8 @@ export function agentJourneyRunCountValid(id: string, count: unknown): boolean { return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { + "saved-skill-use": ["saved-skill-assignment-bound", "saved-skill-input-only-in-source", "saved-skill-native-read-attributed", "saved-skill-result-authored-by-run", "saved-skill-result-reopened", "saved-skill-source-preserved", "configured-model-and-environment"], + "task-edit-lifecycle": ["task-lifecycle-browser-created", "task-lifecycle-visible-edits-persisted", "task-lifecycle-edits-reloaded", "task-lifecycle-completed-and-reopened", "task-lifecycle-unedited-context-preserved", "no-agent-work"], "company-access-isolation": ["isolation-owner-resource-controls", "cloud-workspace-switch-and-return", "workspace-data-scoped", "restricted-agent-positive-controls", "cross-workspace-agent-read-write-denied", "isolation-credential-revoked", "no-agent-work"], "repository-continuation": ["repository-configured-source-bound", "repository-first-commit-verified", "repository-original-files-saved", "repository-followup-stored-once", "repository-same-workspace-retained", "repository-second-commit-verified", "repository-original-and-result-reopened", "configured-model-and-environment"], "routine-scheduled-run": ["routine-browser-created", "routine-scheduler-created-execution", "routine-result-attributed", "routine-history-preserved", "routine-pause-prevented-next-run", "routine-result-reopened", "configured-model-and-environment"], @@ -18,8 +36,10 @@ export const journeyRequiredChecks: Record = { "task-create": ["browser-created-task-persisted", "no-agent-work"], "task-document-persistence": ["browser-document-created", "document-revision-history-preserved", "document-reopened-after-reload", "no-agent-work"], "project-task-context": ["browser-created-project-persisted", "browser-created-project-task-persisted", "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"], + "task-dependency-resume": ["dependency-browser-created-and-linked", "dependency-unresolved-prevented-execution", "dependency-blocker-completed-by-board", "dependency-one-automatic-resume", "dependency-output-authored-by-run", "dependency-result-reopened", "dependency-history-preserved", "configured-model-and-environment"], "task-search-filter": ["task-search-results-exact", "task-status-filter-excludes-controls", "task-search-filter-reloaded", "task-search-result-reopened", "no-agent-work"], "ask-question": ["question-answer-persisted", "one-bound-successful-run", "configured-model-and-environment"], + "chat-continuity": ["chat-fresh-owned-conversation", "chat-user-messages-stored-once", "chat-reload-preserved-context", "chat-new-session-boundary", "chat-fresh-context-answer", "chat-history-preserved-after-reload", "chat-no-unrequested-work", "chat-exact-turn-accounting", "configured-model-and-environment"], "question-answer-resume": ["question-resume.forms", "question-resume.choice.answer", "question-resume.choice.path", "question-resume.text.answer", "question-resume.text.path", "question-resume.complete-evidence", "question-no-premature-output", "question-answers-owned-by-board", "question-document-attributed", "question-document-reloaded", "configured-model-and-environment"], @@ -32,5 +52,6 @@ export const journeyRequiredChecks: Record = { "stopped-run-no-later-events", "stop-preserved-download", "redirect-consumed-once", "user-reply-stored-once", "configured-model-and-environment"], "followup-while-running": ["followup-native-command-active", "followup-stored-once", "followup-consumed-once", "followup-final-answer-persisted", "followup-one-or-two-native-runs", "configured-model-and-environment"], "hire-delegate-reuse": ["one-attributed-native-hire", "first-delegated-task-complete", "checklist-authored-by-hired-run", "same-teammate-reused", "review-authored-by-same-hired-run", "original-checklist-preserved", "hire-user-messages-stored-once", "delegated-result-visible-after-reload", "configured-model-and-environment"], + "agent-settings-persistence": ["agent-settings-browser-save", "agent-settings-reload-persisted", "agent-settings-instruction-only-input", "agent-settings-result-attributed", "agent-settings-result-reopened", "agent-settings-original-restored", "configured-model-and-environment"], "cloud-onboarding": ["fresh-workspace", "ui-connection-and-native-agent", "first-task-complete", "downloadable-artifact-verified", "native-trace-verified"], }; diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts new file mode 100644 index 00000000000..5c997d832be --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-contract.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { parseTargetConfig, productJourneys } from "./catalog.js"; +import { agentJourneyRunCountValid, journeyRequiredChecks } from "./success-contract.js"; +import { TASK_DEPENDENCY_RESUME_CHECKS } from "./task-dependency-resume-evidence.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import type { RunnerE2EResult } from "../types.js"; + +const caseId = "task-dependency-resume"; +const configuration = () => ({ version: 1, id: "qa-fixture", mode: "existing-cloud", executionEnvironment: "daytona", + adapterModule: "/private/qa-provider.mjs", journeys: [caseId], baseline: { features: {}, connections: [] }, + limits: { durationMs: 600_000, modelBudgetCents: 500, maxRuns: 1 } }); +function retained(): RunnerE2EResult { + return { schema: "paperclip.runner-e2e.result/v2", suiteId: "product-journeys", suiteDefinitionHash: "fixture", caseId, + executionId: `product-journeys.qa-fixture.configured-agent.daytona.${caseId}`, profileId: "configured-agent", environmentId: "daytona", + provider: "opencode", model: "fixture-model", runtimeMode: "native", status: "passed", cleanup: "passed", attempt: 1, + startedAt: "2020-01-01T00:00:00Z", finishedAt: "2020-01-01T00:01:00Z", durationMs: 60_000, runIds: ["run"], + productJourney: { outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: (["authentication", "setup", "journey", "collection", "cleanup"] as const).map(phase => ({ phase, outcome: "passed" })), + checks: TASK_DEPENDENCY_RESUME_CHECKS.map(id => ({ id, passed: true })), deployment: { kind: "cloud", targetId: "qa-fixture", + origin: "https://qa.example.test", commit: "a".repeat(40), cloudRelease: "b".repeat(40), runtimeImage: null, configurationHash: "fixture" } } }; +} +describe("dependency journey registration and retained acceptance", () => { + it("is explicit and requires exactly one bounded configured native run", () => { + expect(productJourneys.find(row => row.id === caseId)).toMatchObject({ paid: true, requires: ["company", "board-user", "task-write", "agent"] }); + expect(parseTargetConfig(configuration()).journeys).toEqual([caseId]); + expect(journeyRequiredChecks[caseId]).toEqual(TASK_DEPENDENCY_RESUME_CHECKS); + expect(agentJourneyRunCountValid(caseId, 1)).toBe(true); + for (const count of [0, 2]) { + const value = configuration(); value.limits.maxRuns = count; + expect(() => parseTargetConfig(value)).toThrow("agent_journeys_require_bounded_runs"); + expect(agentJourneyRunCountValid(caseId, count)).toBe(false); + } + expect(() => parseTargetConfig({ ...configuration(), limits: { ...configuration().limits, modelBudgetCents: 0 } })).toThrow(); + }); + it("accepts a complete retained measurement and rejects partial checks, runtime or cleanup", () => { + expect(() => validateRetainedRunnerResult(retained())).not.toThrow(); + for (const change of [ + (r: RunnerE2EResult) => { r.productJourney!.checks.pop(); }, + (r: RunnerE2EResult) => { r.productJourney!.checks[0]!.passed = false; }, + (r: RunnerE2EResult) => { r.runIds = []; }, + (r: RunnerE2EResult) => { r.runIds = ["run", "extra"]; }, + (r: RunnerE2EResult) => { r.runtimeMode = "none"; }, + (r: RunnerE2EResult) => { r.cleanup = "failed"; }, + ]) { const row = retained(); change(row); expect(() => validateRetainedRunnerResult(row)).toThrow(); } + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts new file mode 100644 index 00000000000..c10c27f6896 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import { dependencyOutputMatches, gradeTaskDependencyResume, TASK_DEPENDENCY_RESUME_CHECKS, type DependencyEvidence } from "./task-dependency-resume-evidence.js"; +import { dependencyEvidenceFixture } from "./task-dependency-resume-fixture.js"; + +describe("task dependency resume evidence", () => { + it("accepts a finite blocked observation followed by one automatic, attributed native result", () => { + const checks = gradeTaskDependencyResume(dependencyEvidenceFixture()); + expect(checks).toEqual(TASK_DEPENDENCY_RESUME_CHECKS.map(id => ({ id, passed: true }))); + }); + const changeRuns = (f: DependencyEvidence, mutate: (run: Record) => void) => { + for (const point of [f.finished, f.reopened]) for (const run of [...point.dependentRuns, ...point.agentRuns]) mutate(run); + }; + it.each([ + ["provider", (f: DependencyEvidence) => changeRuns(f, run => { run.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }), "configured-model-and-environment"], + ["model", (f: DependencyEvidence) => changeRuns(f, run => { run.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }), "configured-model-and-environment"], + ["environment", (f: DependencyEvidence) => changeRuns(f, run => { run.contextSnapshot.paperclipEnvironment.id = "other"; }), "configured-model-and-environment"], + ["runtime mode", (f: DependencyEvidence) => changeRuns(f, run => { run.runtimeMode = "legacy"; }), "configured-model-and-environment"], + ["uncommitted phase", (f: DependencyEvidence) => changeRuns(f, run => { run.nativePhase = "saving"; }), "configured-model-and-environment"], + ["document author", (f: DependencyEvidence) => { f.finished.document!.createdByUserId = "user"; }, "dependency-output-authored-by-run"], + ["revision run", (f: DependencyEvidence) => { f.finished.revision!.createdByRunId = "foreign"; }, "dependency-output-authored-by-run"], + ["reload visibility", (f: DependencyEvidence) => { f.reopened.resultVisible = false; }, "dependency-result-reopened"], + ["activity coverage", (f: DependencyEvidence) => { f.reopened.activityCoverage.agent = false; }, "dependency-history-preserved"], + ["extra child work", (f: DependencyEvidence) => { f.reopened.activity.push({ runId: "run", action: "issue.created" }); }, "dependency-history-preserved"], + ["retained completion", (f: DependencyEvidence) => { f.reopened.blocker.completedAt = "2020-01-01T00:00:22.000Z"; }, "dependency-history-preserved"], + ] as const)("attributes the %s defect to its own evidence check", (_name, mutate, failedId) => { + const f = dependencyEvidenceFixture(); mutate(f); + expect(gradeTaskDependencyResume(f).filter(check => !check.passed).map(check => check.id)).toEqual([failedId]); + }); + const mutants: Array<[string, (value: DependencyEvidence) => void]> = [ + ["missing observation", f => { f.waiting = []; }], + ["short quiet window", f => { f.waiting[1]!.observedAt--; }], + ["non-monotonic observation", f => { f.waiting.reverse(); }], + ["quiet window before creation", f => { f.created.observedAt = f.waiting[0]!.observedAt + 1; }], + ["release before final quiet read", f => { f.releaseRequestedAt = f.waiting[1]!.observedAt - 1; }], + ["blocker not completed", f => { f.released.blocker.status = "todo"; }], + ["wrong board actor", f => { f.reopened.activity[0]!.actorId = "foreign"; }], + ["agent claimed board completion", f => { f.reopened.activity[0]!.actorType = "agent"; }], + ["missing board completion audit", f => { f.reopened.activity = []; }], + ["duplicate board completion", f => { f.reopened.activity.push({ ...f.reopened.activity[0] }); }], + ["wrong blocker relation", f => { f.waiting[0]!.dependent.blockedByIssueIds = ["foreign"]; }], + ["removed relation after completion", f => { f.reopened.dependent.blockedByIssueIds = []; }], + ["parked backlog instead of blocked", f => { f.waiting[0]!.dependent.status = "backlog"; }], + ["premature execution", f => { f.waiting[0]!.dependentRuns = [f.finished.dependentRuns[0]!]; }], + ["unknown work on owned agent", f => { f.waiting[0]!.agentRuns = [f.finished.agentRuns[0]!]; }], + ["blocker ran an agent", f => { f.reopened.blockerRuns = [f.finished.dependentRuns[0]!]; }], + ["dependency never resumed", f => { f.finished.dependent.status = "blocked"; }], + ["duplicate native run", f => { f.reopened.dependentRuns.push({ ...f.reopened.dependentRuns[0], id: "extra" }); }], + ["native run preceded blocker completion", f => { f.finished.dependentRuns[0]!.startedAt = "2020-01-01T00:00:19.000Z"; }], + ["manual run instead of automatic resume", f => { f.finished.dependentRuns[0]!.invocationSource = "on_demand"; }], + ["wrong wake reason", f => { f.finished.dependentRuns[0]!.contextSnapshot.wakeReason = "issue_assigned"; }], + ["mutation name used as wake source", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) r.contextSnapshot.source = "issue.blocker_done"; }], + ["missing saved wake source", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) delete r.contextSnapshot.source; }], + ["wrong resolved blocker", f => { f.finished.dependentRuns[0]!.contextSnapshot.resolvedBlockerIssueId = "foreign"; }], + ["conflicting native issue binding", f => { f.finished.dependentRuns[0]!.issueId = "foreign"; }], + ["reopened run changed status", f => { f.reopened.dependentRuns[0]!.status = "running"; }], + ["reopened native work still owns checkout", f => { f.reopened.dependent.executionRunId = "run"; }], + ["wrong provider", f => { f.finished.dependentRuns[0]!.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }], + ["wrong model", f => { f.finished.dependentRuns[0]!.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }], + ["wrong environment", f => { f.finished.dependentRuns[0]!.contextSnapshot.paperclipEnvironment.id = "other"; }], + ["uncommitted native phase", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) r.nativePhase = "saving"; }], + ["missing native phase", f => { for (const p of [f.finished, f.reopened]) for (const r of [...p.dependentRuns, ...p.agentRuns]) delete r.nativePhase; }], + ["incomplete activity coverage", f => { f.reopened.activityCoverage.agent = false; }], + ["wrong saved reference", f => { f.finished.document!.body = '{"reference":"wrong","blockerIdentifier":"QA-1"}'; }], + ["wrong document task", f => { f.finished.document!.issueId = "foreign"; }], + ["board-authored document", f => { f.finished.document!.createdByUserId = "user"; }], + ["revision from another run", f => { f.finished.revision!.createdByRunId = "foreign"; }], + ["missing saved result", f => { delete f.finished.document; }], + ["missing revision", f => { delete f.finished.revision; }], + ["result not visible after reload", f => { f.reopened.resultVisible = false; }], + ["replaced document after reload", f => { f.reopened.document!.id = "replacement"; }], + ["reference leaked in task prompt", f => { f.scope.prompt += f.scope.reference; }], + ["foreign creator", f => { f.created.dependent.createdByUserId = "foreign"; }], + ["unrequested child work", f => { f.reopened.activity.push({ runId: "run", action: "issue.created" }); }], + ]; + it.each(mutants)("rejects %s", (_name, mutate) => { + const f = dependencyEvidenceFixture(); mutate(f); + expect(gradeTaskDependencyResume(f).some(check => !check.passed)).toBe(true); + }); + it("missing input never becomes a pass", () => { expect(gradeTaskDependencyResume(undefined).every(c => !c.passed)).toBe(true); }); + it.each([null, {}, { summary: "Compacted result" }])("accepts the public run projection without duplicate finalization metadata: %j", resultJson => { + const f = dependencyEvidenceFixture(); + for (const point of [f.finished, f.reopened]) for (const run of [...point.dependentRuns, ...point.agentRuns]) run.resultJson = resultJson; + expect(gradeTaskDependencyResume(f).every(check => check.passed)).toBe(true); + }); + const withHistory = () => { + const f = dependencyEvidenceFixture(); + const prior = { ...structuredClone(f.finished.agentRuns[0]!), id: "prior", nativeIssueId: "prior-task", + contextSnapshot: { issueId: "prior-task" }, status: "failed", startedAt: "2019-12-31T23:00:00Z" }; + f.startingAgentRuns = [prior]; + for (const p of [f.created, ...f.waiting, f.released, f.finished, f.reopened]) p.agentRuns.push(structuredClone(prior)); + return f; + }; + it("retains completed earlier campaign work while measuring only new runs", () => { + const f = withHistory(); + expect(gradeTaskDependencyResume(f).every(check => check.passed)).toBe(true); + expect(f.reopened.agentRuns.map(run => run.id)).toEqual(["run", "prior"]); + }); + it.each([ + ["missing baseline", (f: DependencyEvidence) => { delete (f as Partial).startingAgentRuns; }], + ["foreign baseline", (f: DependencyEvidence) => { f.startingAgentRuns[0]!.agentId = "foreign"; }], + ["active baseline", (f: DependencyEvidence) => { f.startingAgentRuns[0]!.status = "running"; }], + ["baseline run disappeared", (f: DependencyEvidence) => { f.waiting[0]!.agentRuns = []; }], + ["duplicate baseline ID", (f: DependencyEvidence) => { f.startingAgentRuns.push(f.startingAgentRuns[0]!); }], + ["premature run hidden as prior work", (f: DependencyEvidence) => { f.startingAgentRuns.push(structuredClone(f.finished.dependentRuns[0]!)); }], + ["new unrelated run while blocked", (f: DependencyEvidence) => { f.waiting[0]!.agentRuns.push({ ...structuredClone(f.startingAgentRuns[0]!), id: "extra" }); }], + ] as const)("rejects %s", (_name, mutate) => { + const f = withHistory(); mutate(f); + expect(gradeTaskDependencyResume(f).some(check => !check.passed)).toBe(true); + }); + it("has no global activity length cap after complete scoped observation", () => { + const f = dependencyEvidenceFixture(); + f.reopened.activity.push(...Array.from({ length: 600 }, (_, i) => ({ id: `prior-${i}`, companyId: f.scope.companyId, + entityType: "issue", entityId: f.scope.dependentId, action: "issue.observed" }))); + expect(gradeTaskDependencyResume(f).every(check => check.passed)).toBe(true); + }); + it("accepts plain or fenced JSON but rejects extra keys and prose", () => { + const s = dependencyEvidenceFixture().scope, body = JSON.stringify({ reference: s.reference, blockerIdentifier: s.blockerIdentifier }); + expect(dependencyOutputMatches(body, s)).toBe(true); + expect(dependencyOutputMatches(`\`\`\`json\n${body}\n\`\`\``, s)).toBe(true); + expect(dependencyOutputMatches(JSON.stringify({ reference: s.reference, blockerIdentifier: s.blockerIdentifier, extra: true }), s)).toBe(false); + expect(dependencyOutputMatches(`Completed ${body}`, s)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts new file mode 100644 index 00000000000..d212a1f3ce2 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-evidence.ts @@ -0,0 +1,131 @@ +type Row = Record; +export const DEPENDENCY_IDLE_MS = 10_000; +export const TASK_DEPENDENCY_RESUME_CHECKS = [ + "dependency-browser-created-and-linked", "dependency-unresolved-prevented-execution", + "dependency-blocker-completed-by-board", "dependency-one-automatic-resume", + "dependency-output-authored-by-run", "dependency-result-reopened", + "dependency-history-preserved", "configured-model-and-environment", +] as const; + +export interface DependencyScope { + companyId: string; userId: string; agentId: string; environmentId: string; provider: string; model: string; + blockerId: string; blockerIdentifier: string; blockerTitle: string; blockerDescription: string; + dependentId: string; dependentTitle: string; prompt: string; reference: string; documentKey: string; +} +export interface DependencyPoint { + observedAt: number; blocker: Row; dependent: Row; + blockerRuns: Row[]; dependentRuns: Row[]; agentRuns: Row[]; activity: Row[]; + activityCoverage: { blocker: boolean; dependent: boolean; agent: boolean }; + document?: Row; revision?: Row; resultVisible?: boolean; +} +export interface DependencyEvidence { + scope: DependencyScope; startingAgentRuns: Row[]; created: DependencyPoint; waiting: DependencyPoint[]; + releaseRequestedAt: number; released: DependencyPoint; finished: DependencyPoint; reopened: DependencyPoint; +} +const nonempty = (value: unknown): value is string => typeof value === "string" && value.length > 0; +const date = (value: unknown) => typeof value === "string" ? Date.parse(value) : NaN; +const exactId = (rows: unknown, id: string) => Array.isArray(rows) && rows.length === 1 && rows[0] === id; +const runFacts = (run: Row) => JSON.stringify([run.id, run.companyId, run.agentId, dependencyRunIssueId(run), run.status, + run.runtimeMode, run.nativePhase, run.invocationSource, run.triggerDetail, run.startedAt, + run.contextSnapshot?.wakeReason, run.contextSnapshot?.source, run.contextSnapshot?.resolvedBlockerIssueId, + run.contextSnapshot?.blockerIssueIds, run.contextSnapshot?.paperclipEnvironment?.id, + run.runnerProfileJson?.nativeExecutionInput?.provider?.kind, run.runnerProfileJson?.nativeExecutionInput?.provider?.model]); +export function dependencyNewAgentRuns(runs: Row[], startingRuns: Row[]) { + const priorIds = new Set(startingRuns.map(run => run.id)); + return runs.filter(run => !priorIds.has(run.id)); +} +export function dependencyRunIssueId(run: Row) { + const ids = [run.issueId, run.nativeIssueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].filter(value => value != null); + return ids.length && ids.every(nonempty) && new Set(ids).size === 1 ? ids[0] as string : undefined; +} +export function dependencyOutputMatches(body: unknown, scope: Pick) { + if (typeof body !== "string") return false; + const text = body.trim(), json = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(text)?.[1] ?? text; + try { + const value = JSON.parse(json); + return Boolean(value && !Array.isArray(value) && Object.keys(value).sort().join(",") === "blockerIdentifier,reference" + && value.reference === scope.reference && value.blockerIdentifier === scope.blockerIdentifier); + } catch { return false; } +} + +export function gradeTaskDependencyResume(value: DependencyEvidence | undefined) { + const fail = () => TASK_DEPENDENCY_RESUME_CHECKS.map(id => ({ id, passed: false })); + if (!value?.scope || !Array.isArray(value.startingAgentRuns) || !value.created || !Array.isArray(value.waiting) || value.waiting.length < 2 + || !value.released || !value.finished || !value.reopened) return fail(); + const { scope: s, startingAgentRuns, created, waiting, released, finished, reopened } = value; + const points = [created, ...waiting, released, finished, reopened]; + if (Object.values(s).some(item => !nonempty(item)) || s.blockerId === s.dependentId || s.prompt.includes(s.reference) + || !s.blockerDescription.includes(s.reference) + || points.some(p => !p?.blocker || !p.dependent || !Number.isFinite(p.observedAt) + || [p.blockerRuns, p.dependentRuns, p.agentRuns, p.activity].some(rows => !Array.isArray(rows)))) return fail(); + const startingIds = startingAgentRuns.map(run => run.id); + const historyComplete = startingIds.every(nonempty) && new Set(startingIds).size === startingIds.length + && startingAgentRuns.every(run => run.companyId === s.companyId && run.agentId === s.agentId + && ["succeeded", "failed", "cancelled", "timed_out"].includes(run.status) + && ![s.blockerId, s.dependentId].includes(dependencyRunIssueId(run) ?? "")) + && points.every(p => new Set(p.agentRuns.map(run => run.id)).size === p.agentRuns.length + && p.agentRuns.every(run => nonempty(run.id) && run.companyId === s.companyId && run.agentId === s.agentId) + && startingAgentRuns.every(prior => p.agentRuns.some(run => runFacts(run) === runFacts(prior)))); + const newRuns = (point: DependencyPoint) => dependencyNewAgentRuns(point.agentRuns, startingAgentRuns); + const scoped = historyComplete && points.every(p => p.blocker.id === s.blockerId && p.dependent.id === s.dependentId + && p.blocker.companyId === s.companyId && p.dependent.companyId === s.companyId + && p.blocker.title === s.blockerTitle && p.dependent.title === s.dependentTitle + && p.blocker.description === s.blockerDescription && p.dependent.description === s.prompt + && p.blocker.createdByUserId === s.userId && p.dependent.createdByUserId === s.userId + && !p.blocker.createdByAgentId && !p.dependent.createdByAgentId + && p.blocker.assigneeUserId === s.userId && p.blocker.assigneeAgentId === null); + const linked = scoped && [created.blocker, created.dependent].every(task => task.status === "todo") + && created.dependent.assigneeUserId === s.userId && created.dependent.assigneeAgentId === null + && [created.blockerRuns, created.dependentRuns, newRuns(created)].every(rows => rows.length === 0) + && points.slice(1).every(p => p.dependent.assigneeAgentId === s.agentId && p.dependent.assigneeUserId === null + && exactId(p.dependent.blockedByIssueIds, s.blockerId)); + const held = linked && waiting.every(p => p.blocker.status === "todo" && p.dependent.status === "blocked" + && !p.dependent.executionRunId && !p.dependent.checkoutRunId + && [p.blockerRuns, p.dependentRuns, newRuns(p)].every(rows => rows.length === 0)) + && waiting.every((p, index) => index === 0 || p.observedAt >= waiting[index - 1]!.observedAt) + && waiting[waiting.length - 1]!.observedAt - waiting[0]!.observedAt >= DEPENDENCY_IDLE_MS + && waiting[0]!.observedAt >= created.observedAt + && Number.isFinite(value.releaseRequestedAt) && value.releaseRequestedAt >= waiting[waiting.length - 1]!.observedAt + && released.observedAt >= value.releaseRequestedAt && finished.observedAt >= released.observedAt && reopened.observedAt >= finished.observedAt; + const completedByBoard = held && [released, finished, reopened].every(p => p.blocker.status === "done" + && Number.isFinite(date(p.blocker.completedAt))) + && reopened.activity.filter(row => row.companyId === s.companyId && row.entityType === "issue" + && row.entityId === s.blockerId && row.action === "issue.updated" && row.actorType === "user" + && row.actorId === s.userId && row.details?.status === "done").length === 1; + const run = finished.dependentRuns[0]; + const oneRun = completedByBoard && finished.dependent.status === "done" && reopened.dependent.status === "done" + && !finished.dependent.executionRunId && !finished.dependent.checkoutRunId + && !reopened.dependent.executionRunId && !reopened.dependent.checkoutRunId + && finished.dependentRuns.length === 1 && reopened.dependentRuns.length === 1 + && newRuns(finished).length === 1 && newRuns(reopened).length === 1 + && nonempty(run?.id) && [reopened.dependentRuns[0], newRuns(finished)[0], newRuns(reopened)[0]].every(row => runFacts(row) === runFacts(run)) + && run.companyId === s.companyId && run.agentId === s.agentId && dependencyRunIssueId(run) === s.dependentId + && run.status === "succeeded" && run.invocationSource === "automation" && run.triggerDetail === "system" + && run.contextSnapshot?.wakeReason === "issue_blockers_resolved" && run.contextSnapshot?.source === "issue.blockers_resolved" + && run.contextSnapshot?.resolvedBlockerIssueId === s.blockerId && exactId(run.contextSnapshot?.blockerIssueIds, s.blockerId) + && Number.isFinite(date(run.startedAt)) && date(run.startedAt) >= date(released.blocker.completedAt); + const boundRun = scoped && [run, reopened.dependentRuns[0], newRuns(finished)[0], newRuns(reopened)[0]].every(row => + nonempty(row?.id) && row.id === run?.id && row.companyId === s.companyId && row.agentId === s.agentId + && dependencyRunIssueId(row) === s.dependentId); + const runtime = boundRun && run?.runtimeMode === "native" && run.nativePhase === "committed" + && run.runnerProfileJson?.nativeExecutionInput?.provider?.kind === s.provider + && run.runnerProfileJson?.nativeExecutionInput?.provider?.model === s.model + && run.contextSnapshot?.paperclipEnvironment?.id === s.environmentId; + const documentMatches = (p: DependencyPoint) => p.document && p.revision + && nonempty(p.document.id) && p.document.companyId === s.companyId && p.document.issueId === s.dependentId + && p.document.key === s.documentKey && p.document.createdByAgentId === s.agentId && p.document.createdByUserId === null + && p.document.latestRevisionId === p.revision.id && p.revision.documentId === p.document.id + && p.revision.createdByAgentId === s.agentId && p.revision.createdByUserId === null && p.revision.createdByRunId === run?.id + && dependencyOutputMatches(p.document.body, s) && dependencyOutputMatches(p.revision.body, s); + const authored = boundRun && documentMatches(finished); + const resultReopened = boundRun && documentMatches(reopened) && finished.document && reopened.resultVisible === true + && reopened.document!.id === finished.document.id && reopened.document!.latestRevisionId === finished.document.latestRevisionId + && reopened.document!.body === finished.document.body; + const history = boundRun && points.every(p => p.blockerRuns.length === 0) + && points.every(p => p.activityCoverage?.blocker === true && p.activityCoverage?.dependent === true && p.activityCoverage?.agent === true) + && !reopened.activity.some(row => row.runId === run?.id + && ["issue.created", "issue.child_created", "project.created", "agent.created", "agent.hire_created"].includes(row.action)) + && reopened.blocker.completedAt === released.blocker.completedAt; + const values = [linked, held, completedByBoard, oneRun, authored, resultReopened, history, runtime]; + return TASK_DEPENDENCY_RESUME_CHECKS.map((id, index) => ({ id, passed: Boolean(values[index]) })); +} diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts b/tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts new file mode 100644 index 00000000000..984459110b0 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume-fixture.ts @@ -0,0 +1,38 @@ +import type { DependencyEvidence, DependencyPoint } from "./task-dependency-resume-evidence.js"; + +export function dependencyEvidenceFixture(): DependencyEvidence { + const start = Date.parse("2020-01-01T00:00:00.000Z"); + const scope = { companyId: "company", userId: "user", agentId: "agent", environmentId: "environment", provider: "opencode", model: "fixture-model", + blockerId: "blocker", blockerIdentifier: "QA-1", blockerTitle: "Review gate", blockerDescription: "The review reference is secret-reference.", + dependentId: "dependent", dependentTitle: "Continue after review", prompt: "Read QA-1 after its review and write dependency-proof.", reference: "secret-reference", documentKey: "dependency-proof" }; + const point = (observedAt: number): DependencyPoint => ({ observedAt, + blocker: { id: scope.blockerId, identifier: scope.blockerIdentifier, companyId: scope.companyId, title: scope.blockerTitle, + description: scope.blockerDescription, status: "todo", createdByUserId: scope.userId, createdByAgentId: null, + assigneeUserId: scope.userId, assigneeAgentId: null }, + dependent: { id: scope.dependentId, identifier: "QA-2", companyId: scope.companyId, title: scope.dependentTitle, + description: scope.prompt, status: "blocked", createdByUserId: scope.userId, createdByAgentId: null, + assigneeUserId: null, assigneeAgentId: scope.agentId, blockedByIssueIds: [scope.blockerId], executionRunId: null, checkoutRunId: null }, + blockerRuns: [], dependentRuns: [], agentRuns: [], activity: [], activityCoverage: { blocker: true, dependent: true, agent: true } }); + const created = point(start); + Object.assign(created.dependent, { status: "todo", assigneeAgentId: null, assigneeUserId: scope.userId, blockedByIssueIds: [] }); + const released = point(start + 20_000); Object.assign(released.blocker, { status: "done", completedAt: "2020-01-01T00:00:20.000Z" }); + const run = { id: "run", companyId: scope.companyId, agentId: scope.agentId, nativeIssueId: scope.dependentId, + status: "succeeded", runtimeMode: "native", nativePhase: "committed", invocationSource: "automation", triggerDetail: "system", + startedAt: "2020-01-01T00:00:21.000Z", + contextSnapshot: { issueId: scope.dependentId, taskId: scope.dependentId, wakeReason: "issue_blockers_resolved", source: "issue.blockers_resolved", + resolvedBlockerIssueId: scope.blockerId, blockerIssueIds: [scope.blockerId], paperclipEnvironment: { id: scope.environmentId } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: scope.provider, model: scope.model } } } }; + const document = { id: "document", companyId: scope.companyId, issueId: scope.dependentId, key: scope.documentKey, + body: JSON.stringify({ reference: scope.reference, blockerIdentifier: scope.blockerIdentifier }), latestRevisionId: "revision", + createdByAgentId: scope.agentId, createdByUserId: null }; + const revision = { id: "revision", documentId: document.id, body: document.body, createdByAgentId: scope.agentId, + createdByUserId: null, createdByRunId: run.id }; + const finished = structuredClone(released); finished.observedAt = start + 30_000; finished.dependent.status = "done"; + finished.dependentRuns = [structuredClone(run)]; finished.agentRuns = [structuredClone(run)]; + finished.document = document; finished.revision = revision; + const reopened = structuredClone(finished); reopened.observedAt = start + 31_000; reopened.resultVisible = true; + reopened.activity = [{ id: "completion", companyId: scope.companyId, actorType: "user", actorId: scope.userId, + action: "issue.updated", entityType: "issue", entityId: scope.blockerId, details: { status: "done" } }]; + return { scope, startingAgentRuns: [], created, waiting: [point(start + 1_000), point(start + 11_000)], releaseRequestedAt: start + 19_000, + released, finished, reopened }; +} diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume.md b/tests/runner-e2e/product-journeys/task-dependency-resume.md new file mode 100644 index 00000000000..f8f18dee135 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume.md @@ -0,0 +1,64 @@ +# Blocked task resumes after its human-owned blocker + +`task-dependency-resume` is an explicit Product E2E journey for an authenticated +existing Cloud target or a target provider with a qualified native agent. It uses +the ordinary task UI and public observation APIs. Its registration is not evidence +of a local or deployed pass. + +The browser creates two tasks assigned to the signed-in person. The blocker holds +a random review reference absent from the dependent's prompt. On the dependent, +the person adds the blocker, selects Blocked, and assigns the disposable agent. +Closed properties panes are opened through their visible toolbar control; mounted +inert or accessibility-hidden rows cannot substitute for an interactive pane. +Status choices are bound to the matching labeled glyph inside the open picker. +The saved dependency, blocked status, and empty task/agent run histories must +remain observable for at least ten seconds. This is a finite measured wait, +not an unlimited claim that no execution can ever occur. + +The person completes the blocker through the task's status control. Paperclip must +then start exactly one native run automatically. Its durable wake must name the +same resolved blocker, reason `issue_blockers_resolved`, and saved source +`issue.blockers_resolved`; `blocker_done` is the producer's separate mutation +payload. Manual +Run or a follow-up message cannot substitute. The run starts after the saved +blocker completion, uses the configured provider/model/environment, commits its +result through the durable committed native phase, and completes the dependent. +The public API may omit or compact `resultJson`; duplicate finalization metadata +in that optional object is not required. A task document named `dependency-proof` +contains the review reference and blocker identifier. Its current revision must +be authored by this agent and run. Reloading the task must show that same saved +revision and preserve the two tasks' descriptions, dependency, and board-owned +blocker completion audit. No agent work is permitted on the human-owned blocker. + +The case requires `company`, `board-user`, `task-write`, and `agent`. It declares +one work run, uses the existing bounded case budget, and shares the original +campaign deadline. The native agent must have nonblank provider and model +identifiers before any observation read or task creation. An incomplete execution +identity returns `unsupported` without task or agent work. +It saves the agent's starting run history and retains those +records while checking only newly appearing runs for this case. Earlier terminal +campaign work stays in the accounting and does not count as new execution. +Owned-task activity reads are complete and uncapped; the separate agent-filtered +activity read fails closed if its 500-row bound is reached. The agent run list +likewise fails closed at the public API's 1000-row cap. Unrelated company history +does not enter these completeness checks. +Agent setup usage is outside task-run accounting; missing +provider prices remain unknown. The owning target provider controls normal +cancellation, termination, retained history, and evidence cleanup. It never +changes a shared source agent, company budget, or execution environment. + +The private evidence folder contains `task-dependency-observations.json` and +`observation-read-failures.json`, with allowlisted owned task, run, audit, and +document facts. The normal launcher captures the final owned task screenshot. +Both evidence writes are attempted. A write failure retains an existing journey +error and its outcome; a previously successful journey fails if required evidence +cannot be saved. Diagnostics name only the affected evidence file. +Runtime identity, document authorship, reload visibility, and retained history +use their own observations bound to the same company, task, agent, and run. One +failed control does not mark unrelated controls failed. All eight checks remain +required for acceptance. +There are no direct database writes, private runner hooks, manually dispatched +resumption wakes, retry loops for failed model work, or fixture-authored outputs. +Independent controls reject premature or duplicate execution, a never-resumed +or manually started task, foreign/changed runtime identity, missing completion +audit, wrong output or revision authorship, and missing or shortened observations. diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume.test.ts b/tests/runner-e2e/product-journeys/task-dependency-resume.test.ts new file mode 100644 index 00000000000..fa694af7d1d --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume.test.ts @@ -0,0 +1,285 @@ +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; +import { dependencyEvidenceFixture } from "./task-dependency-resume-fixture.js"; +import * as observedReader from "./observation-read.js"; + +const controls = vi.hoisted(() => ({ create: vi.fn() })); +vi.mock("../user-actions.js", () => ({ createTaskThroughUi: (...args: unknown[]) => controls.create(...args) })); +vi.mock("./task-document-flow.js", () => ({ openDocumentAuthoring: vi.fn(async () => {}) })); +vi.mock("@playwright/test", () => ({ expect: Object.assign((locator: any) => ({ + async toBeVisible() { expect(await locator.isVisible()).toBe(true); }, + async toBeInViewport() { expect(await locator.isVisible()).toBe(true); }, + async toContainText(text: string) { expect(await locator.textContent()).toContain(text); }, +}), { poll: (read: () => Promise) => ({ + async toBe(value: unknown) { expect(await read()).toBe(value); }, + async toEqual(value: unknown) { expect(await read()).toEqual(value); }, +}) }) })); +import { taskDependencyResume } from "./task-dependency-resume.js"; + +interface FixtureOptions { priorRuns?: number; unrelatedActivity?: number; ownedActivity?: number; agentActivity?: number; missingActivity?: boolean; activePrior?: boolean; + closedPanel?: "properties" | "toggle"; missingProperty?: boolean; missingStatusChoice?: boolean } +async function fixture(premature = false, options: FixtureOptions = {}) { + const directory = await mkdtemp(path.join(os.tmpdir(), "dependency-flow-")); + const f = dependencyEvidenceFixture(), s = f.scope; + const blocker = { ...f.created.blocker }, dependent = { ...f.created.dependent }; + const actions: string[] = [], owned: any[] = [], writes: unknown[] = [], reads: string[] = []; + let completed = false, assigned = false, panelOpen = !options.closedPanel; + const reference = () => /review reference is ([a-f0-9]{32})/.exec(blocker.description)?.[1]; + const output = () => JSON.stringify({ reference: reference(), blockerIdentifier: s.blockerIdentifier }); + const runs = () => assigned && (completed || premature) ? [structuredClone(f.finished.dependentRuns[0]!)] : []; + const priorRuns = Array.from({ length: options.priorRuns ?? 0 }, (_, i) => ({ ...structuredClone(f.finished.agentRuns[0]!), + id: `prior-${i}`, nativeIssueId: `prior-task-${i}`, contextSnapshot: { issueId: `prior-task-${i}` }, + startedAt: "2019-12-31T23:00:00Z", status: options.activePrior ? "running" : i % 2 ? "failed" : "succeeded" })); + const activity = (count: number, entityId: string, agentId?: string) => Array.from({ length: count }, (_, i) => ({ + id: `${entityId}-activity-${i}`, companyId: s.companyId, entityType: "issue", entityId, agentId, + actorType: agentId ? "agent" : "user", actorId: agentId ?? s.userId, action: "issue.observed" })); + const taskDocument = () => ({ ...f.finished.document, body: output() }); + const revision = () => ({ ...f.finished.revision, body: output() }); + controls.create.mockImplementation(async (input: any) => { + await input.beforeDispatch?.(); + const row = input.title.includes(" blocker ") ? blocker : dependent; + Object.assign(row, { title: input.title, description: input.prompt }); + actions.push(`create:${row.id}`); + return { issueId: row.id, submittedAtMs: Date.now() }; + }); + const locator = (key: string): any => ({ + key, + filter(value: any) { + if (key === '[data-slot="popover-content"]' && value?.has) { + return locator(options.missingStatusChoice ? "missing-status-choice" : value.has.key); + } + return this; + }, first() { return this; }, last() { return this; }, + locator() { return this; }, + getByRole(_role: string, options?: any) { return locator(options?.name ? String(options.name) : key); }, + async isVisible() { + if (key.includes("data-property-row")) return panelOpen && !options.missingProperty; + if (key === "Toggle side panel") return !panelOpen && options.closedPanel === "toggle"; + if (key.includes("Show properties")) return !panelOpen && options.closedPanel === "properties"; + return true; + }, + async scrollIntoViewIfNeeded() { actions.push(`scroll:${key}`); }, + async fill(value: string) { actions.push(`fill:${key}:${value}`); }, + async click() { + actions.push(`click:${key}`); + if (key === "Toggle side panel" || key.includes("Show properties")) panelOpen = true; + if (key === "missing-status-choice") throw new Error("missing_status_choice"); + if (key === "blocker-choice") dependent.blockedByIssueIds = [blocker.id]; + if (key === "Blocked") dependent.status = "blocked"; + if (key === "agent-choice") { assigned = true; dependent.assigneeAgentId = s.agentId; dependent.assigneeUserId = null; } + if (key === "Done") { + completed = true; blocker.status = "done"; blocker.completedAt = "2020-01-01T00:00:20.000Z"; + dependent.status = "done"; + } + }, + async textContent() { return output(); }, + }); + const page: any = { + locator(selector: string) { return locator(selector); }, + getByRole(_role: string, options?: any) { + if (_role === "button" && options?.exact && ["Blocked", "Done"].includes(options.name)) return locator("missing-status-choice"); + const l = locator(options?.name ? String(options.name) : "buttons"); + l.filter = (options: any) => options?.hasText === blocker.title ? locator("blocker-choice") + : options?.hasText === "Fixture agent" ? locator("agent-choice") : l; + return l; + }, + getByPlaceholder(name: string) { return locator(name); }, + getByTestId(name: string) { return locator(name); }, + async goto(url: string) { actions.push(`goto:${url}`); }, async reload() { actions.push("reload"); }, + }; + const api: any = { origin: "https://qa.example.test", async get(route: string) { + reads.push(route); + if (route === `/api/issues/${s.blockerId}`) return structuredClone(blocker); + if (route === `/api/issues/${s.dependentId}`) return structuredClone(dependent); + if (route === `/api/issues/${s.blockerId}/runs`) return []; + if (route === `/api/issues/${s.dependentId}/runs`) return runs().map(r => ({ runId: r.id })); + if (route.includes("/heartbeat-runs?")) return [...runs(), ...structuredClone(priorRuns)]; + if (route === "/api/heartbeat-runs/run") return runs()[0]; + if (route === `/api/issues/${s.blockerId}/activity`) return completed ? structuredClone(f.reopened.activity) : []; + if (route === `/api/issues/${s.dependentId}/activity`) return activity(options.ownedActivity ?? 0, s.dependentId); + if (route === `/api/companies/${s.companyId}/activity?agentId=${s.agentId}&limit=500`) { + return options.missingActivity ? undefined : activity(options.agentActivity ?? 0, "prior-task", s.agentId); + } + if (route === `/api/companies/${s.companyId}/activity?limit=500`) return activity(options.unrelatedActivity ?? 0, "unrelated-task").slice(0, 500); + if (route === `/api/issues/${s.dependentId}/documents`) return completed ? [taskDocument()] : []; + if (route.endsWith("/documents/dependency-proof")) return taskDocument(); + if (route.endsWith("/revisions/revision")) return revision(); + throw new Error(`unexpected_read:${route}`); + }, async post(...args: unknown[]) { writes.push(args); throw new Error("unexpected_mutation"); }, + async patch(...args: unknown[]) { writes.push(args); throw new Error("unexpected_mutation"); } }; + const context = { page, api, company: { id: s.companyId, issuePrefix: "QA" }, user: { id: s.userId }, environmentId: s.environmentId, + capabilities: ["company", "board-user", "task-write", "agent"], agent: { id: s.agentId, name: "Fixture agent", provider: s.provider, model: s.model, runtimeMode: "native" } } as JourneyContext; + const session = { own: vi.fn(async resource => { owned.push(resource); }) } as unknown as TargetSession; + const evidence = { checks: [] } as unknown as JourneyEvidence; + return { directory, context, session, evidence, actions, owned, writes, reads }; +} +afterEach(() => { vi.useRealTimers(); controls.create.mockReset(); }); +describe("dependency browser flow orchestration", () => { + const evidenceFiles = ["task-dependency-observations.json", "observation-read-failures.json"] as const; + const writerFaults = ["observations", "reads", "both"] as const; + const obstructWriters = async (directory: string, fault: typeof writerFaults[number]) => { + const names = fault === "both" ? [...evidenceFiles] : [evidenceFiles[fault === "observations" ? 0 : 1]]; + await Promise.all(names.map(name => mkdir(path.join(directory, name)))); + return names; + }; + it.each(writerFaults.flatMap(fault => (["unsupported", "failed"] as const).map(outcome => [fault, outcome] as const)))( + "preserves the original %s writer failure outcome %s and attempts both evidence files", async (fault, outcome) => { + const f = await fixture(), blocked = await obstructWriters(f.directory, fault); + const logging = vi.spyOn(console, "error").mockImplementation(() => {}); + const original = Object.freeze(new JourneyError("failed", "dependency_observation_control_failed")); + if (outcome === "unsupported") f.context.agent!.provider = ""; + else vi.spyOn(f.context.api, "get").mockRejectedValue(original); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + if (outcome === "unsupported") await expect(operation).rejects.toMatchObject({ outcome, code: "dependency_execution_identity_unavailable" }); + else await expect(operation).rejects.toBe(original); + for (const name of evidenceFiles.filter(name => !blocked.includes(name))) { + expect(JSON.parse(await readFile(path.join(f.directory, name), "utf8"))).toBeTypeOf("object"); + } + expect(logging).toHaveBeenCalledWith("dependency_evidence_preservation_failed", blocked); + expect(JSON.stringify(logging.mock.calls)).not.toContain(f.directory); + expect(f.actions).toEqual([]); expect(f.owned).toEqual([]); expect(f.writes).toEqual([]); + } finally { logging.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } + }); + it.each(["unsupported", "failed"] as const)("awaits the other evidence file when the read writer throws synchronously after %s", async outcome => { + const f = await fixture(), createReader = observedReader.createObservedReader; + const original = Object.freeze(new JourneyError("failed", "dependency_observation_control_failed")); + const failingWrite = vi.fn(() => { throw new Error("private writer diagnostic"); }); + const reader = vi.spyOn(observedReader, "createObservedReader").mockImplementation((...args) => ({ + ...createReader(...args), preserve: failingWrite, + })); + const logging = vi.spyOn(console, "error").mockImplementation(() => {}); + if (outcome === "unsupported") f.context.agent!.model = ""; + else vi.spyOn(f.context.api, "get").mockRejectedValue(original); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + if (outcome === "unsupported") await expect(operation).rejects.toMatchObject({ outcome, code: "dependency_execution_identity_unavailable" }); + else await expect(operation).rejects.toBe(original); + expect(failingWrite).toHaveBeenCalledOnce(); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", + outcome === "unsupported" ? "prerequisites" : "observe_starting_runs"); + expect(logging).toHaveBeenCalledWith("dependency_evidence_preservation_failed", ["observation-read-failures.json"]); + expect(JSON.stringify(logging.mock.calls)).not.toContain("private writer diagnostic"); + } finally { logging.mockRestore(); reader.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } + }); + it.each(writerFaults)("fails a successful journey when its %s evidence writer fails", async fault => { + const f = await fixture(), blocked = await obstructWriters(f.directory, fault); + const logging = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.useFakeTimers(); vi.setSystemTime(new Date("2020-01-01T00:00:00Z")); + try { + const rejection = expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toMatchObject({ + outcome: "failed", code: "dependency_evidence_preservation_failed", + }); + await vi.advanceTimersByTimeAsync(11_000); await rejection; + expect(f.evidence.checks).toHaveLength(8); expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(logging).toHaveBeenCalledWith("dependency_evidence_preservation_failed", blocked); + for (const name of evidenceFiles.filter(name => !blocked.includes(name))) { + expect(JSON.parse(await readFile(path.join(f.directory, name), "utf8"))).toBeTypeOf("object"); + } + expect(f.writes).toEqual([]); + } finally { logging.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } + }); + it.each((["provider", "model"] as const).flatMap(field => [ + [field, "missing", undefined], [field, "null", null], [field, "empty", ""], [field, "blank", " \t "], + ] as const))("rejects a %s identity that is %s before observation or task work", async (field, _kind, value) => { + const f = await fixture(); + Object.assign(f.context.agent!, { [field]: value }); + const get = vi.spyOn(f.context.api, "get").mockRejectedValue(new Error("preflight_read_attempted")); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toMatchObject({ + outcome: "unsupported", code: "dependency_execution_identity_unavailable", + }); + expect(get).not.toHaveBeenCalled(); expect(controls.create).not.toHaveBeenCalled(); + expect(f.actions).toEqual([]); expect(f.owned).toEqual([]); expect(f.writes).toEqual([]); + expect(f.evidence.checks).toEqual([]); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", "prerequisites"); + expect(JSON.parse(await readFile(path.join(f.directory, "observation-read-failures.json"), "utf8"))).toEqual({ failures: [] }); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each([ + ["empty history", {}], + ["completed earlier campaign runs", { priorRuns: 3 }], + ["more than 100 earlier campaign runs", { priorRuns: 150 }], + ["more than 500 unrelated company events", { unrelatedActivity: 800 }], + ["more than 500 owned task events", { ownedActivity: 600 }], + ["prior agent events", { priorRuns: 2, agentActivity: 300 }], + ] as const)("uses browser actions with %s and never writes output through API observations", async (_name, options) => { + const f = await fixture(false, options); vi.useFakeTimers(); vi.setSystemTime(new Date("2020-01-01T00:00:00Z")); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + await vi.advanceTimersByTimeAsync(11_000); await operation; + expect(f.evidence.checks).toHaveLength(8); expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(f.writes).toEqual([]); + expect(f.actions.indexOf("click:Blocked")).toBeLessThan(f.actions.indexOf("click:agent-choice")); + expect(f.actions.indexOf("click:agent-choice")).toBeLessThan(f.actions.indexOf("click:Done")); + expect(f.owned).toEqual(expect.arrayContaining([ + expect.objectContaining({ kind: "issue", id: "blocker", journeyId: "task-dependency-resume" }), + expect.objectContaining({ kind: "issue", id: "dependent", journeyId: "task-dependency-resume" }), + ])); + const saved = JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8")); + expect(saved.stage).toBe("complete"); expect(saved.scope.prompt).not.toContain(saved.scope.reference); + expect(saved.waiting.at(-1).observedAt - saved.waiting[0].observedAt).toBeGreaterThanOrEqual(10_000); + expect(saved.startingAgentRuns).toHaveLength("priorRuns" in options ? options.priorRuns : 0); + expect(saved.reopened.agentRuns).toHaveLength(saved.startingAgentRuns.length + 1); + expect(f.reads.some(route => route.startsWith("/api/heartbeat-runs/prior-"))).toBe(false); + expect(f.reads).toContain("/api/issues/blocker/activity"); expect(f.reads).toContain("/api/issues/dependent/activity"); + expect(f.reads).not.toContain("/api/companies/company/activity?limit=500"); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each(["properties", "toggle"] as const)("opens a closed properties pane through its %s control before linking", async closedPanel => { + const f = await fixture(false, { closedPanel }); vi.useFakeTimers(); vi.setSystemTime(new Date("2020-01-01T00:00:00Z")); + try { + const operation = taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory }); + await vi.advanceTimersByTimeAsync(11_000); await operation; + expect(f.evidence.checks).toHaveLength(8); expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const opener = f.actions.findIndex(action => action.startsWith("click:") && (action.includes("Show properties") || action === "click:Toggle side panel")); + expect(opener).toBeGreaterThanOrEqual(0); expect(opener).toBeLessThan(f.actions.indexOf("click:blocker-choice")); + expect(f.actions.some(action => action.startsWith("scroll:[data-property-row]:not([inert] *):not([aria-hidden=\"true\"] *)"))).toBe(true); + expect(f.writes).toEqual([]); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each([ + ["missing property", { missingProperty: true }, "link_blocker"], + ["missing matching status glyph", { missingStatusChoice: true }, "set_dependent_blocked"], + ] as const)("fails closed on %s before agent assignment or blocker completion", async (_name, options, stage) => { + const f = await fixture(false, options); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toThrow(); + expect(f.actions).not.toContain("click:agent-choice"); expect(f.actions).not.toContain("click:Done"); expect(f.writes).toEqual([]); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", stage); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it.each([ + ["truncated agent activity", { agentActivity: 500 }, "dependency_activity_coverage_unavailable"], + ["missing agent activity", { missingActivity: true }, "dependency_activity_coverage_unavailable"], + ["active earlier work", { priorRuns: 1, activePrior: true }, "dependency_starting_agent_not_idle"], + ["saturated run history", { priorRuns: 1000 }, "dependency_run_coverage_unavailable"], + ] as const)("rejects %s before completing the blocker", async (_name, options, code) => { + const f = await fixture(false, options); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toThrow(code); + expect(f.actions).not.toContain("click:Done"); expect(f.writes).toEqual([]); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); + it("premature work fails before the browser completes the blocker and preserves its observations", async () => { + const f = await fixture(true); + try { + await expect(taskDependencyResume(f.context, f.session, "journey-123-abcdef01", f.evidence, + { deadlineAt: Date.now() + 60_000, directory: f.directory })).rejects.toThrow("dependency_ran_before_blocker_completed"); + expect(f.actions).not.toContain("click:Done"); expect(f.writes).toEqual([]); + expect(JSON.parse(await readFile(path.join(f.directory, "task-dependency-observations.json"), "utf8"))).toHaveProperty("stage", "observe_blocked_wait"); + expect(JSON.parse(await readFile(path.join(f.directory, "observation-read-failures.json"), "utf8"))).toEqual({ failures: [] }); + } finally { await rm(f.directory, { recursive: true, force: true }); } + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-dependency-resume.ts b/tests/runner-e2e/product-journeys/task-dependency-resume.ts new file mode 100644 index 00000000000..ed13e309878 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-dependency-resume.ts @@ -0,0 +1,243 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { HEARTBEAT_RUN_LIST_MAX_LIMIT } from "../../../packages/shared/src/constants.js"; +import { createTaskThroughUi } from "../user-actions.js"; +import { createObservedReader } from "./observation-read.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; +import { DEPENDENCY_IDLE_MS, dependencyNewAgentRuns, gradeTaskDependencyResume, type DependencyEvidence, type DependencyPoint, type DependencyScope } from "./task-dependency-resume-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "task-dependency-resume" as const; +const taskSnapshot = (task: Row) => ({ id: task.id, companyId: task.companyId, identifier: task.identifier, + title: task.title, description: task.description, status: task.status, assigneeAgentId: task.assigneeAgentId, + assigneeUserId: task.assigneeUserId, createdByUserId: task.createdByUserId, createdByAgentId: task.createdByAgentId, + completedAt: task.completedAt, executionRunId: task.executionRunId, checkoutRunId: task.checkoutRunId, + blockedByIssueIds: task.blockedByIssueIds ?? task.blockedBy?.map((row: Row) => row.id) ?? [] }); +const runSnapshot = (run: Row) => ({ id: run.id, companyId: run.companyId, agentId: run.agentId, + issueId: run.issueId, nativeIssueId: run.nativeIssueId, status: run.status, runtimeMode: run.runtimeMode, + nativePhase: run.nativePhase, invocationSource: run.invocationSource, triggerDetail: run.triggerDetail, startedAt: run.startedAt, + contextSnapshot: { issueId: run.contextSnapshot?.issueId, taskId: run.contextSnapshot?.taskId, + source: run.contextSnapshot?.source, wakeReason: run.contextSnapshot?.wakeReason, + resolvedBlockerIssueId: run.contextSnapshot?.resolvedBlockerIssueId, blockerIssueIds: run.contextSnapshot?.blockerIssueIds, + paperclipEnvironment: { id: run.contextSnapshot?.paperclipEnvironment?.id } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: run.runnerProfileJson?.nativeExecutionInput?.provider?.kind, + model: run.runnerProfileJson?.nativeExecutionInput?.provider?.model } } } }); +const documentSnapshot = (row: Row) => Object.fromEntries(["id", "companyId", "issueId", "key", "body", "latestRevisionId", + "documentId", "createdByAgentId", "createdByUserId", "createdByRunId"].map(key => [key, row[key]])); + +async function properties(context: JourneyContext, label: string) { + const { page } = context; + const row = page.locator('[data-property-row]:not([inert] *):not([aria-hidden="true"] *)') + .filter({ has: page.locator(`[data-property-label="${label}"]`) }).filter({ visible: true }); + const toggle = page.getByRole("button", { name: "Toggle side panel", exact: true, pressed: false }).filter({ visible: true }); + const propertiesButton = page.getByRole("button", { name: /^(Show properties|Properties)$/ }).filter({ visible: true }); + await expect.poll(async () => await row.isVisible() || await toggle.isVisible() || await propertiesButton.isVisible()).toBe(true); + if (!await row.isVisible()) { + if (await toggle.isVisible()) await toggle.click(); + else await propertiesButton.click(); + } + await expect(row).toBeVisible(); + await row.scrollIntoViewIfNeeded(); + await expect(row).toBeInViewport(); + return row; +} +async function setStatus(context: JourneyContext, label: string) { + const { page } = context; + await page.getByTestId("issue-detail-header").filter({ visible: true }) + .getByRole("button", { name: /^Change status \(current:/ }).click(); + await page.locator('[data-slot="popover-content"]').filter({ visible: true }) + .getByRole("button").filter({ has: page.getByRole("img", { name: label, exact: true }) }).click(); +} + +export async function taskDependencyResume(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence, + options: { deadlineAt: number; directory: string }) { + const { api, page, company, user, agent } = context; + const read = createObservedReader(api, options.deadlineAt); + const reference = randomBytes(16).toString("hex"); + let scope: DependencyScope | undefined, stage = "prerequisites", journeyFailed = false; + let latest: DependencyPoint | undefined; + const observations: Partial = { waiting: [] }; + const preserve = async () => { + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + await writeFile(path.join(options.directory, "task-dependency-observations.json"), + JSON.stringify({ schema: "paperclip.task-dependency-observations/v1", stage, ...observations, latest, scope }, null, 2), { mode: 0o600 }); + }; + const url = (identifier: string) => `${api.origin}/${company.issuePrefix}/issues/${identifier}`; + try { + for (const capability of ["company", "board-user", "task-write", "agent"]) { + if (!context.capabilities.includes(capability)) throw new JourneyError("unsupported", `missing_${capability}`); + } + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) throw new JourneyError("unsupported", "native_agent_and_environment_required"); + if ([agent.provider, agent.model].some(value => typeof value !== "string" || !value.trim())) { + throw new JourneyError("unsupported", "dependency_execution_identity_unavailable"); + } + const summaries = async (route: string, limit?: number) => { + const rows = await read.get(route); + if (!Array.isArray(rows) || limit !== undefined && rows.length >= limit) throw new JourneyError("failed", "dependency_run_coverage_unavailable"); + const ids = rows.map(row => row.runId ?? row.id); + if (ids.some(value => typeof value !== "string" || !value) || new Set(ids).size !== ids.length) throw new JourneyError("failed", "dependency_run_identity_invalid"); + return rows.map((row, index) => ({ ...row, id: ids[index] })); + }; + const runDetail = async (runId: string) => { + const run = await read.get(`/api/heartbeat-runs/${runId}`); + if (run.id !== runId) throw new JourneyError("failed", "dependency_run_identity_invalid"); + return runSnapshot(run); + }; + const snapshots = async (route: string) => Promise.all((await summaries(route)).map(row => runDetail(row.id))); + const agentRoute = `/api/companies/${company.id}/heartbeat-runs?agentId=${agent.id}&limit=${HEARTBEAT_RUN_LIST_MAX_LIMIT}`; + stage = "observe_starting_runs"; + const startingRuns = observations.startingAgentRuns = (await summaries(agentRoute, HEARTBEAT_RUN_LIST_MAX_LIMIT)).map(runSnapshot); + if (startingRuns.some(run => run.companyId !== company.id || run.agentId !== agent.id + || !["succeeded", "failed", "cancelled", "timed_out"].includes(run.status))) { + throw new JourneyError("failed", "dependency_starting_agent_not_idle"); + } + const startingIds = new Set(startingRuns.map(run => run.id)); + const agentSnapshots = async () => Promise.all((await summaries(agentRoute, HEARTBEAT_RUN_LIST_MAX_LIMIT)) + .map(row => startingIds.has(row.id) ? runSnapshot(row) : runDetail(row.id))); + stage = "create_blocker"; + const blockerTitle = `Journey QA dependency blocker ${campaignId}`; + const blockerDescription = `Human review gate. The review reference is ${reference}. Keep this task for my review; no agent work is required.`; + const blocker = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: blockerTitle, prompt: blockerDescription, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: blockerTitle, journeyId: id }) }); + await session.own({ kind: "issue", name: blockerTitle, id: blocker.issueId, journeyId: id }); + const savedBlocker = await read.get(`/api/issues/${blocker.issueId}`); + if (typeof savedBlocker.identifier !== "string" || !savedBlocker.identifier) throw new JourneyError("failed", "dependency_blocker_receipt_invalid"); + const dependentTitle = `Journey QA dependency dependent ${campaignId}`, documentKey = "dependency-proof"; + const prompt = `After the blocker ${savedBlocker.identifier} is completed, read its description and save a task document with key ${documentKey}. ` + + `Its body must be JSON with exactly reference (the review reference in the blocker description) and blockerIdentifier ("${savedBlocker.identifier}"). ` + + "Then finish this task. No other tasks, agents or projects are needed."; + stage = "create_dependent"; + const dependent = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: dependentTitle, prompt, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: dependentTitle, journeyId: id }) }); + await session.own({ kind: "issue", name: dependentTitle, id: dependent.issueId, journeyId: id }); + scope = { companyId: company.id, userId: user.id, agentId: agent.id, environmentId: context.environmentId, + provider: agent.provider, model: agent.model, blockerId: blocker.issueId, blockerIdentifier: savedBlocker.identifier, + blockerTitle, blockerDescription, dependentId: dependent.issueId, dependentTitle, prompt, reference, documentKey }; + const observe = async (): Promise => { + const [currentBlocker, currentDependent, blockerRuns, dependentRuns, agentRuns, blockerActivity, dependentActivity, agentActivity] = await Promise.all([ + read.get(`/api/issues/${blocker.issueId}`), read.get(`/api/issues/${dependent.issueId}`), + snapshots(`/api/issues/${blocker.issueId}/runs`), snapshots(`/api/issues/${dependent.issueId}/runs`), + agentSnapshots(), read.get(`/api/issues/${blocker.issueId}/activity`), read.get(`/api/issues/${dependent.issueId}/activity`), + read.get(`/api/companies/${company.id}/activity?agentId=${agent.id}&limit=500`), + ]); + if ([blockerActivity, dependentActivity, agentActivity].some(rows => !Array.isArray(rows)) || agentActivity.length >= 500) { + throw new JourneyError("failed", "dependency_activity_coverage_unavailable"); + } + const newAgentRuns = dependencyNewAgentRuns(agentRuns, startingRuns); + const activity = [...new Map([...blockerActivity, ...dependentActivity, ...agentActivity].map(row => [row.id, row])).values()]; + if (activity.some(row => typeof row.id !== "string" || !row.id || row.companyId !== company.id) + || agentActivity.some(row => row.agentId !== agent.id)) throw new JourneyError("failed", "dependency_activity_identity_invalid"); + const point: DependencyPoint = { observedAt: Date.now(), blocker: taskSnapshot(currentBlocker), dependent: taskSnapshot(currentDependent), + blockerRuns, dependentRuns, agentRuns, activityCoverage: { blocker: true, dependent: true, agent: true }, + activity: activity.filter(row => [blocker.issueId, dependent.issueId].includes(row.entityId) + || newAgentRuns.some(run => run.id === row.runId)).map(row => ({ id: row.id, companyId: row.companyId, actorType: row.actorType, + actorId: row.actorId, runId: row.runId, action: row.action, entityType: row.entityType, entityId: row.entityId, + createdAt: row.createdAt, details: { status: row.details?.status } })) }; + latest = point; + if (dependentRuns.length > 1 || newAgentRuns.length > 1 || blockerRuns.length + || [...dependentRuns, ...newAgentRuns].some(run => ["failed", "cancelled", "timed_out"].includes(run.status))) { + throw new JourneyError("failed", "dependency_unexpected_or_failed_work"); + } + if (currentDependent.status === "done") { + const documents = await read.get(`/api/issues/${dependent.issueId}/documents`); + const candidate = documents.find(row => row.key === documentKey); + if (candidate) { + const document = await read.get(`/api/issues/${dependent.issueId}/documents/${documentKey}`); + point.document = documentSnapshot(document); + point.revision = documentSnapshot(await read.get(`/api/issues/${dependent.issueId}/documents/${documentKey}/revisions/${document.latestRevisionId}`)); + } + } + return point; + }; + observations.created = await observe(); + if ([observations.created.blocker, observations.created.dependent].some(task => task.companyId !== company.id + || task.assigneeUserId !== user.id || task.assigneeAgentId !== null || task.createdByUserId !== user.id + || task.createdByAgentId || task.status !== "todo") || observations.created.blocker.id !== blocker.issueId + || observations.created.dependent.id !== dependent.issueId || observations.created.blockerRuns.length + || observations.created.dependentRuns.length || dependencyNewAgentRuns(observations.created.agentRuns, startingRuns).length) { + throw new JourneyError("failed", "dependency_created_task_binding_mismatch"); + } + stage = "link_blocker"; + const dependentSaved = observations.created.dependent; + await page.goto(url(dependentSaved.identifier ?? dependent.issueId), { waitUntil: "domcontentloaded" }); + const blockedBy = await properties(context, "Blocked by"); + await blockedBy.locator("[data-property-value]").getByRole("button").first().click(); + await page.getByRole("textbox", { name: "Search tasks to add as blockers", exact: true }).fill(blockerTitle); + await page.getByRole("button").filter({ hasText: blockerTitle }).last().click(); + await expect.poll(async () => (taskSnapshot(await read.get(`/api/issues/${dependent.issueId}`)).blockedByIssueIds)).toEqual([blocker.issueId]); + stage = "set_dependent_blocked"; + await setStatus(context, "Blocked"); + await expect.poll(async () => (await read.get(`/api/issues/${dependent.issueId}`)).status).toBe("blocked"); + stage = "assign_blocked_task"; + const assignee = await properties(context, "Assignee"); + await assignee.locator("[data-property-value]").getByRole("button").first().click(); + await page.getByPlaceholder("Search assignees...", { exact: true }).fill(agent.name); + await page.getByRole("button").filter({ hasText: agent.name }).last().click(); + await expect.poll(async () => (await read.get(`/api/issues/${dependent.issueId}`)).assigneeAgentId).toBe(agent.id); + await page.reload(); + stage = "observe_blocked_wait"; + const waiting = observations.waiting!; + waiting.push(await observe()); + if (waiting[0]!.dependentRuns.length || dependencyNewAgentRuns(waiting[0]!.agentRuns, startingRuns).length || waiting[0]!.dependent.status !== "blocked" + || waiting[0]!.blocker.status !== "todo") throw new JourneyError("failed", "dependency_ran_before_blocker_completed"); + const until = waiting[0]!.observedAt + DEPENDENCY_IDLE_MS; + while (Date.now() < until) { + if (Date.now() + 1000 >= options.deadlineAt) throw new JourneyError("failed", "dependency_wait_deadline"); + await new Promise(resolve => setTimeout(resolve, Math.min(1000, until - Date.now()))); + const point = await observe(); waiting.push(point); + if (point.dependentRuns.length || dependencyNewAgentRuns(point.agentRuns, startingRuns).length || point.dependent.status !== "blocked" || point.blocker.status !== "todo") { + throw new JourneyError("failed", "dependency_ran_before_blocker_completed"); + } + } + if (waiting[waiting.length - 1]!.observedAt < until) waiting.push(await observe()); + stage = "complete_blocker"; + await page.goto(url(savedBlocker.identifier), { waitUntil: "domcontentloaded" }); + observations.releaseRequestedAt = Date.now(); + await setStatus(context, "Done"); + await expect.poll(async () => (await read.get(`/api/issues/${blocker.issueId}`)).status).toBe("done"); + observations.released = await observe(); + stage = "await_automatic_resume"; + while (Date.now() < options.deadlineAt) { + const point = await observe(); observations.finished = point; + if (point.dependent.status === "done" && point.dependentRuns.length === 1 && point.dependentRuns[0]?.status === "succeeded" + && point.dependentRuns[0]?.nativePhase === "committed" && point.document && point.revision + && !point.dependent.executionRunId && !point.dependent.checkoutRunId) break; + await new Promise(resolve => setTimeout(resolve, 1000)); + } + if (!observations.finished || observations.finished.dependent.status !== "done" || observations.finished.dependentRuns[0]?.status !== "succeeded" + || observations.finished.dependentRuns[0]?.nativePhase !== "committed" || !observations.finished.document || !observations.finished.revision) { + throw new JourneyError("failed", "dependency_resume_deadline"); + } + stage = "reopen_result"; + await page.goto(url(dependentSaved.identifier ?? dependent.issueId), { waitUntil: "domcontentloaded" }); + await page.reload(); await openDocumentAuthoring(context); + const document = page.locator(`#document-${documentKey}`); + await expect(document).toContainText(reference); + observations.reopened = await observe(); + observations.reopened.resultVisible = await document.isVisible() && (await document.textContent())?.includes(reference) === true; + const checks = gradeTaskDependencyResume({ ...observations, scope } as DependencyEvidence); + evidence.checks.push(...checks); + const failed = checks.find(check => !check.passed); + if (failed) throw new JourneyError("failed", failed.id.replaceAll("-", "_")); + stage = "complete"; + } catch (error) { + journeyFailed = true; + throw error; + } finally { + const writers = [ + { label: "task-dependency-observations.json", write: preserve }, + { label: "observation-read-failures.json", write: () => read.preserve(options.directory) }, + ]; + const results = await Promise.allSettled(writers.map(writer => Promise.resolve().then(writer.write))); + const failedWriters = results.flatMap((result, index) => result.status === "rejected" ? [writers[index]!.label] : []); + if (failedWriters.length) { + console.error("dependency_evidence_preservation_failed", failedWriters); + if (!journeyFailed) throw new JourneyError("failed", "dependency_evidence_preservation_failed"); + } + } +} diff --git a/tests/runner-e2e/product-journeys/task-lifecycle-evidence.test.ts b/tests/runner-e2e/product-journeys/task-lifecycle-evidence.test.ts new file mode 100644 index 00000000000..ab1cfd3176f --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-lifecycle-evidence.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { createIssueLabelSchema } from "../../../packages/shared/src/validators/issue.js"; +import { taskLifecycleEvidence, taskLifecycleLabelName, type LifecycleObservation, type LifecycleTask } from "./task-lifecycle-evidence.js"; +import { parseTargetConfig, preparationPreview } from "./catalog.js"; +import { journeyRequiredChecks } from "./success-contract.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import type { RunnerE2EResult } from "../types.js"; + +function fixture(): Parameters[0] { + const expected = { taskId: "task-1", companyId: "company-1", userId: "user-1", originalTitle: "Original task", + revisedTitle: "Revised task", description: "Exact persisted context", labelName: "Owned label" }; + const initial: LifecycleTask = { id: expected.taskId, companyId: expected.companyId, identifier: "QA-1", title: expected.originalTitle, + description: expected.description, status: "todo", priority: "medium", projectId: "project-1", assigneeUserId: expected.userId, + assigneeAgentId: null, createdByUserId: expected.userId, createdByAgentId: null, labelIds: [], completedAt: null }; + const observe = (task: LifecycleTask, labelVisible: boolean): LifecycleObservation => ({ task: structuredClone(task), runCount: 0, + titleVisible: true, descriptionVisible: true, statusVisible: true, labelVisible }); + const edited = { ...initial, title: expected.revisedTitle, status: "in_progress", labelIds: ["label-1"] }; + return { expected, label: { id: "label-1", name: expected.labelName, companyId: expected.companyId, color: "#0088ff" }, + created: observe(initial, false), edited: observe(edited, true), reloaded: observe(edited, true), + completed: observe({ ...edited, status: "done", completedAt: "2026-01-01T00:00:00.000Z" }, true), + reopened: observe({ ...edited, status: "todo" }, true) }; +} +const stages = ["created", "edited", "reloaded", "completed", "reopened"] as const; + +describe("task lifecycle saved-state oracle", () => { + it("keeps a full generated campaign identity within the production label-name limit", () => { + const campaigns = ["journey-1793720364049-f3055878", "journey-9999999999999-ffffffff"]; + const labels = campaigns.map(taskLifecycleLabelName); + for (const [index, name] of labels.entries()) { + expect(name).toContain(campaigns[index]); + expect(createIssueLabelSchema.shape.name.safeParse(name).success).toBe(true); + } + expect(new Set(labels).size).toBe(campaigns.length); + }); + it("rejects an overlong campaign before it can produce a browser label", () => { + expect(() => taskLifecycleLabelName("c".repeat(32))).toThrow("task_lifecycle_campaign_label_name_invalid"); + }); + it("accepts the complete browser-edit/reload/complete/reopen sequence", () => { + expect(taskLifecycleEvidence(fixture())).toEqual({ created: true, edited: true, reloaded: true, + completedAndReopened: true, uneditedFieldsPreserved: true, noAgentWork: true }); + }); + it("accepts a fresh backlog task and an absent project without changing either contract", () => { + const input = fixture(); input.created.task!.status = "backlog"; + for (const stage of stages) input[stage].task!.projectId = null; + expect(Object.values(taskLifecycleEvidence(input)).every(Boolean)).toBe(true); + }); + it.each(stages)("rejects a missing saved task at %s", stage => { + const input = fixture(); input[stage].task = null; + const result = taskLifecycleEvidence(input); + expect(result.uneditedFieldsPreserved).toBe(false); + expect(Object.values(result).every(Boolean)).toBe(false); + }); + it.each(["id", "companyId", "assigneeUserId", "assigneeAgentId", "createdByUserId", "createdByAgentId", "description"] as const)( + "rejects a wrong owner or saved context field: %s", key => { + const input = fixture(); Object.assign(input.reopened.task!, { [key]: "foreign" }); + expect(taskLifecycleEvidence(input).completedAndReopened).toBe(false); + expect(taskLifecycleEvidence(input).uneditedFieldsPreserved).toBe(false); + }); + it.each(["identifier", "priority", "projectId"] as const)("requires %s to survive every edit", key => { + for (const stage of stages.slice(1)) { + const input = fixture(); Object.assign(input[stage].task!, { [key]: "changed" }); + expect(taskLifecycleEvidence(input).uneditedFieldsPreserved).toBe(false); + } + }); + it.each(["edited", "reloaded", "completed", "reopened"] as const)("rejects a lost title or label at %s", stage => { + for (const patch of [{ title: "Original task" }, { labelIds: [] }, { labelIds: ["foreign"] }, { labelIds: ["label-1", "label-1"] }]) { + const input = fixture(); Object.assign(input[stage].task!, patch); + expect(Object.values(taskLifecycleEvidence(input)).every(Boolean)).toBe(false); + } + }); + it.each(["titleVisible", "descriptionVisible", "statusVisible", "labelVisible"] as const)("requires visible reloaded %s", key => { + for (const stage of ["reloaded", "completed", "reopened"] as const) { + const input = fixture(); input[stage][key] = false; + expect(Object.values(taskLifecycleEvidence(input)).every(Boolean)).toBe(false); + } + }); + it("does not accept a completed-looking UI without saved Done or completion time", () => { + const wrongStatus = fixture(); wrongStatus.completed.task!.status = "in_review"; + expect(taskLifecycleEvidence(wrongStatus).completedAndReopened).toBe(false); + for (const completedAt of [null, "", "not a timestamp"]) { + const input = fixture(); input.completed.task!.completedAt = completedAt; + expect(taskLifecycleEvidence(input).completedAndReopened).toBe(false); + } + }); + it("requires reopened Todo with its completion time cleared", () => { + for (const patch of [{ status: "done" }, { completedAt: "2026-01-01T00:00:00.000Z" }]) { + const input = fixture(); Object.assign(input.reopened.task!, patch); + expect(taskLifecycleEvidence(input).completedAndReopened).toBe(false); + } + }); + it.each(stages)("requires an explicit zero-run observation at %s", stage => { + for (const runCount of [null, 1, -1]) { + const input = fixture(); input[stage].runCount = runCount; + expect(taskLifecycleEvidence(input).noAgentWork).toBe(false); + } + }); + it("rejects stale starting data, a wrong label and an unchanged expected title", () => { + for (const change of [ + (input: ReturnType) => { input.created.task!.labelIds = ["existing"]; }, + (input: ReturnType) => { input.created.task!.status = "done"; }, + (input: ReturnType) => { input.label = null; }, + (input: ReturnType) => { input.label!.companyId = "foreign"; }, + (input: ReturnType) => { input.label!.name = "Other label"; }, + (input: ReturnType) => { input.label!.color = "invalid"; }, + (input: ReturnType) => { input.expected.revisedTitle = input.expected.originalTitle; }, + ]) { + const input = fixture(); change(input); + expect(Object.values(taskLifecycleEvidence(input)).every(Boolean)).toBe(false); + } + }); +}); + +function result(): RunnerE2EResult { + return { schema: "paperclip.runner-e2e.result/v2", suiteId: "product-journeys", suiteDefinitionHash: "test", + executionId: "product-journeys.qa.browser.daytona.task-edit-lifecycle", profileId: "browser", environmentId: "daytona", + caseId: "task-edit-lifecycle", provider: null, model: null, runtimeMode: "none", status: "passed", cleanup: "passed", + attempt: 1, startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", durationMs: 1000, + productJourney: { outcome: "passed", startingState: "recipe", phases: [{ phase: "journey", outcome: "passed" }], + checks: journeyRequiredChecks["task-edit-lifecycle"]!.map(id => ({ id, passed: true })), privacy: "private", resourceDisposition: "released", + deployment: { kind: "cloud", targetId: "qa", origin: "https://qa.example.test", commit: "a".repeat(40), + cloudRelease: "test", runtimeImage: null, configurationHash: "test" } } }; +} + +describe("task lifecycle discovery and report contract", () => { + it("previews a browser-only existing target with no provider allowance", () => { + const config = parseTargetConfig({ version: 1, id: "qa", mode: "existing-cloud", adapterModule: "/private/adapter.mjs", + journeys: ["task-edit-lifecycle"], executionEnvironment: "daytona", baseline: { features: {}, connections: [] }, + limits: { durationMs: 180000, maxRuns: 0, modelBudgetCents: 0 } }); + expect(preparationPreview(config).journeys).toMatchObject([{ id: "task-edit-lifecycle", paid: false }]); + expect(config.limits.maxRuns).toBe(0); + }); + it("rejects every missing required check and unexpected native work", () => { + expect(() => validateRetainedRunnerResult(result())).not.toThrow(); + for (const id of journeyRequiredChecks["task-edit-lifecycle"]!) { + const missing = result(); missing.productJourney!.checks = missing.productJourney!.checks.filter(check => check.id !== id); + expect(() => validateRetainedRunnerResult(missing)).toThrow(); + } + for (const patch of [{ provider: "openai" }, { model: "wrong" }, { runtimeMode: "native" }, { runIds: ["unexpected"] }]) { + expect(() => validateRetainedRunnerResult({ ...result(), ...patch })).toThrow(); + } + const usage = result(); usage.usage = { runs: [{ runId: "unexpected", usage: { inputTokens: 1 } }] }; + expect(() => validateRetainedRunnerResult(usage)).toThrow("result.browserOnly"); + const emptyUsage = result(); emptyUsage.usage = { runs: [] }; + expect(() => validateRetainedRunnerResult(emptyUsage)).not.toThrow(); + }); +}); diff --git a/tests/runner-e2e/product-journeys/task-lifecycle-evidence.ts b/tests/runner-e2e/product-journeys/task-lifecycle-evidence.ts new file mode 100644 index 00000000000..a20b1bab0d4 --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-lifecycle-evidence.ts @@ -0,0 +1,98 @@ +import { createIssueLabelSchema } from "../../../packages/shared/src/validators/issue.js"; +import { JourneyError } from "./types.js"; + +export function taskLifecycleLabelName(campaignId: string): string { + const name = `Journey QA label ${campaignId}`; + if (!createIssueLabelSchema.shape.name.safeParse(name).success) { + throw new JourneyError("unsupported", "task_lifecycle_campaign_label_name_invalid"); + } + return name; +} + +export interface LifecycleTask { + id: string; + companyId: string; + identifier: string; + title: string; + description: string; + status: string; + priority: string; + projectId: string | null; + assigneeUserId: string; + assigneeAgentId: null; + createdByUserId: string; + createdByAgentId: null; + labelIds: string[]; + completedAt: string | null; +} +export interface LifecycleLabel { id: string; companyId: string; name: string; color: string } +export interface LifecycleObservation { + task: LifecycleTask | null; + runCount: number | null; + titleVisible: boolean; + descriptionVisible: boolean; + statusVisible: boolean; + labelVisible: boolean; +} +export interface LifecycleExpectation { + taskId: string; + companyId: string; + userId: string; + originalTitle: string; + revisedTitle: string; + description: string; + labelName: string; +} + +function ownedTask(task: LifecycleTask | null, expected: LifecycleExpectation) { + return Boolean(task && task.id === expected.taskId && task.companyId === expected.companyId && task.identifier + && task.description === expected.description && task.assigneeUserId === expected.userId && task.assigneeAgentId === null + && task.createdByUserId === expected.userId && task.createdByAgentId === null); +} +function singleton(ids: unknown, id: string) { return Array.isArray(ids) && ids.length === 1 && ids[0] === id; } +function sameUneditedFields(task: LifecycleTask | null, initial: LifecycleTask | null) { + return Boolean(task && initial && task.identifier === initial.identifier && task.priority === initial.priority && task.projectId === initial.projectId); +} + +/** Independent saved states accompany visible UI readback. Empty or missing observations cannot pass. */ +export function taskLifecycleEvidence(input: { + expected: LifecycleExpectation; + label: LifecycleLabel | null; + created: LifecycleObservation; + edited: LifecycleObservation; + reloaded: LifecycleObservation; + completed: LifecycleObservation; + reopened: LifecycleObservation; +}) { + const { expected, label, created, edited, reloaded, completed, reopened } = input; + const validExpected = Boolean(expected.taskId && expected.companyId && expected.userId && expected.originalTitle + && expected.revisedTitle && expected.originalTitle !== expected.revisedTitle && expected.description && expected.labelName); + const fresh = validExpected && ownedTask(created.task, expected) && created.task!.title === expected.originalTitle + && ["backlog", "todo"].includes(created.task!.status) && created.task!.completedAt === null + && ["critical", "high", "medium", "low"].includes(created.task!.priority) + && (created.task!.projectId === null || typeof created.task!.projectId === "string" && created.task!.projectId.length > 0) + && Array.isArray(created.task!.labelIds) && created.task!.labelIds.length === 0 + && created.titleVisible && created.descriptionVisible && created.statusVisible; + const labelValid = Boolean(label && label.id && label.companyId === expected.companyId && label.name === expected.labelName + && /^#[a-fA-F0-9]{6}$/.test(label.color)); + const saved = (observation: LifecycleObservation, status: string) => fresh && labelValid && ownedTask(observation.task, expected) + && sameUneditedFields(observation.task, created.task) && observation.task!.title === expected.revisedTitle + && observation.task!.status === status && singleton(observation.task!.labelIds, label!.id); + const shown = (observation: LifecycleObservation) => observation.titleVisible && observation.descriptionVisible + && observation.statusVisible && observation.labelVisible; + const editedMatches = saved(edited, "in_progress") && shown(edited) && edited.task!.completedAt === null; + const reloadedMatches = saved(reloaded, "in_progress") && shown(reloaded) && reloaded.task!.completedAt === null; + const completedMatches = saved(completed, "done") && shown(completed) + && typeof completed.task!.completedAt === "string" && Number.isFinite(Date.parse(completed.task!.completedAt)); + const reopenedMatches = saved(reopened, "todo") && shown(reopened) && reopened.task!.completedAt === null; + const observations = [created, edited, reloaded, completed, reopened]; + return { + created: fresh, + edited: editedMatches, + reloaded: reloadedMatches, + completedAndReopened: completedMatches && reopenedMatches, + uneditedFieldsPreserved: fresh && observations.every(observation => ownedTask(observation.task, expected) + && sameUneditedFields(observation.task, created.task)), + noAgentWork: fresh && observations.every(observation => observation.runCount === 0), + }; +} diff --git a/tests/runner-e2e/product-journeys/task-lifecycle.ts b/tests/runner-e2e/product-journeys/task-lifecycle.ts new file mode 100644 index 00000000000..b2086f890cb --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-lifecycle.ts @@ -0,0 +1,173 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createTaskThroughUi, withBrowserEvent } from "../user-actions.js"; +import { createObservedReader } from "./observation-read.js"; +import { taskLifecycleEvidence, taskLifecycleLabelName, type LifecycleExpectation, type LifecycleLabel, type LifecycleObservation, type LifecycleTask } from "./task-lifecycle-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Stage = "create" | "initial_read" | "rename" | "label" | "in_progress" | "reload" | "complete" | "reopen"; +function taskSnapshot(task: LifecycleTask): LifecycleTask { + return { id: task.id, companyId: task.companyId, identifier: task.identifier, title: task.title, + description: task.description, status: task.status, priority: task.priority, projectId: task.projectId, + assigneeUserId: task.assigneeUserId, assigneeAgentId: task.assigneeAgentId, createdByUserId: task.createdByUserId, + createdByAgentId: task.createdByAgentId, labelIds: task.labelIds, completedAt: task.completedAt }; +} +const emptyObservation = (): LifecycleObservation => ({ task: null, runCount: null, + titleVisible: false, descriptionVisible: false, statusVisible: false, labelVisible: false }); + +async function properties(context: JourneyContext) { + const { page } = context; + // Closed panes keep their rows mounted. Their geometry alone does not make + // an inert or accessibility-hidden panel available for interaction. + const row = page.locator('[data-property-row]:not([inert] *):not([aria-hidden="true"] *)') + .filter({ has: page.locator('[data-property-label="Labels"]') }).filter({ visible: true }); + const toggle = page.getByRole("button", { name: "Toggle side panel", exact: true, pressed: false }).filter({ visible: true }); + const propertiesButton = page.getByRole("button", { name: /^(Show properties|Properties)$/ }).filter({ visible: true }); + await expect.poll(async () => await row.isVisible() || await toggle.isVisible() || await propertiesButton.isVisible()).toBe(true); + if (!await row.isVisible()) { + if (await toggle.isVisible()) await toggle.click(); + else await propertiesButton.click(); + } + await expect(row).toBeVisible(); + await row.scrollIntoViewIfNeeded(); + await expect(row).toBeInViewport(); + return row; +} + +async function setStatus(context: JourneyContext, label: string) { + const { page } = context; + await page.getByTestId("issue-detail-header").filter({ visible: true }) + .getByRole("button", { name: /^Change status \(current:/ }).click(); + // The shipped picker includes both a labeled status glyph and visible text. + // Bind the choice to its exact accessible glyph inside the open popover. + await page.locator('[data-slot="popover-content"]').filter({ visible: true }) + .getByRole("button").filter({ has: page.getByRole("img", { name: label, exact: true }) }).click(); +} + +/** All measured edits use the shipped browser. Independent GETs verify saved state. */ +export async function taskEditLifecycle(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence, + options: { deadlineAt: number; directory: string }) { + const { page, api, company, user } = context; + const read = createObservedReader(api, options.deadlineAt); + const journeyId = "task-edit-lifecycle" as const; + const originalTitle = `Journey QA task lifecycle ${campaignId}`; + const revisedTitle = `Journey QA task lifecycle revised ${campaignId}`; + const description = `Keep this task assigned to me. Saved task context ${campaignId} must survive every edit.`; + const labelName = taskLifecycleLabelName(campaignId); + const observed = { created: emptyObservation(), edited: emptyObservation(), reloaded: emptyObservation(), + completed: emptyObservation(), reopened: emptyObservation() }; + let expected: LifecycleExpectation | undefined; + let label: LifecycleLabel | null = null; + let stage: Stage = "create"; + const check = (id: string, passed: boolean) => { + evidence.checks.push({ id, passed }); + if (!passed) throw new JourneyError("failed", id.replaceAll("-", "_")); + }; + const preserve = async () => { + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + await writeFile(path.join(options.directory, "task-lifecycle-observations.json"), + JSON.stringify({ schema: "paperclip.task-lifecycle-observations/v1", stage, expected, label, ...observed }, null, 2), { mode: 0o600 }); + }; + const grade = () => taskLifecycleEvidence({ expected: expected!, label, ...observed }); + try { + for (const capability of ["company", "board-user", "task-write"]) { + if (!context.capabilities.includes(capability)) throw new JourneyError("unsupported", `missing_${capability}`); + } + if (!campaignId) throw new JourneyError("blocked", "task_lifecycle_campaign_required"); + const created = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: originalTitle, prompt: description, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: originalTitle, journeyId }) }); + await session.own({ kind: "issue", name: originalTitle, id: created.issueId, journeyId }); + const route = `/api/issues/${created.issueId}`; + expected = { taskId: created.issueId, companyId: company.id, userId: user.id, originalTitle, revisedTitle, description, labelName }; + const task = await read.get(route); + await page.goto(`${api.origin}/${company.issuePrefix}/issues/${task.identifier}`); + const capture = async (title: string, statusLabel: string, withLabel: boolean): Promise => { + await expect(page.getByRole("heading", { name: title, exact: true })).toBeVisible(); + await expect(page.getByText(description, { exact: true }).first()).toBeVisible(); + const status = page.getByRole("button", { name: `Change status (current: ${statusLabel})`, exact: true }).filter({ visible: true }).first(); + await expect(status).toBeVisible(); + const labelRow = await properties(context); + if (withLabel) await expect(labelRow.getByText(labelName, { exact: true })).toBeVisible(); + const saved = await read.get(route); + const runs = await read.get(`${route}/runs`); + return { task: structuredClone(taskSnapshot(saved)), runCount: Array.isArray(runs) ? runs.length : null, + titleVisible: await page.getByRole("heading", { name: title, exact: true }).isVisible(), + descriptionVisible: await page.getByText(description, { exact: true }).first().isVisible(), + statusVisible: await status.isVisible(), labelVisible: withLabel && await labelRow.getByText(labelName, { exact: true }).isVisible() }; + }; + stage = "initial_read"; + const initialStatus = task.status === "backlog" ? "Backlog" : "Todo"; + observed.created = await capture(originalTitle, initialStatus, false); + await preserve(); + check("task-lifecycle-browser-created", grade().created); + stage = "rename"; + // Keep the creation receipt stable. Cleanup may accept this one exact, + // predeclared title if the browser save commits but its response is lost. + await session.own({ kind: "issue", name: originalTitle, id: created.issueId, journeyId, taskEdit: { revisedTitle } }); + await page.getByRole("heading", { name: originalTitle, exact: true }).click(); + const titleEditor = page.getByTestId("issue-detail-header").locator("textarea").first(); + await titleEditor.fill(revisedTitle); + await titleEditor.press("Enter"); + await expect.poll(async () => (await read.get(route)).title).toBe(revisedTitle); + stage = "label"; + const labelsRoute = `/api/companies/${company.id}/labels`; + const beforeLabels = await read.get(labelsRoute); + if (beforeLabels.some(row => row.name === labelName)) throw new JourneyError("failed", "task_lifecycle_label_not_fresh"); + const labels = await properties(context); + await labels.locator("[data-property-value]").getByRole("button").first().click(); + await page.getByPlaceholder("New label", { exact: true }).fill(labelName); + await session.own({ kind: "label", name: labelName, journeyId }); + const response = await withBrowserEvent(() => page.waitForResponse(response => response.request().method() === "POST" + && new URL(response.url()).pathname === labelsRoute), + () => page.getByRole("button", { name: "Create label", exact: true }).click()); + if (response.status() !== 201) throw new JourneyError("failed", "task_lifecycle_label_create_failed"); + const createdLabel = await response.json() as LifecycleLabel; + if (typeof createdLabel.id !== "string" || !createdLabel.id || createdLabel.name !== labelName || createdLabel.companyId !== company.id) { + throw new JourneyError("failed", "task_lifecycle_label_receipt_mismatch"); + } + await session.own({ kind: "label", name: labelName, id: createdLabel.id, journeyId }); + label = { id: createdLabel.id, companyId: createdLabel.companyId, name: createdLabel.name, color: createdLabel.color }; + await page.keyboard.press("Escape"); + const savedLabels = await read.get(labelsRoute); + if (savedLabels.filter(row => row.name === labelName).length !== 1 + || !savedLabels.some(row => row.id === label!.id && row.name === labelName && row.companyId === company.id && row.color === label!.color)) { + throw new JourneyError("failed", "task_lifecycle_label_saved_mismatch"); + } + stage = "in_progress"; + await setStatus(context, "In Progress"); + await expect.poll(async () => (await read.get(route)).status).toBe("in_progress"); + await expect.poll(async () => { + const task = await read.get(route); + return Array.isArray(task.labelIds) && task.labelIds.length === 1 && task.labelIds[0] === label!.id; + }).toBe(true); + observed.edited = await capture(revisedTitle, "In Progress", true); + await preserve(); + check("task-lifecycle-visible-edits-persisted", grade().edited); + stage = "reload"; + await page.reload(); + observed.reloaded = await capture(revisedTitle, "In Progress", true); + await preserve(); + check("task-lifecycle-edits-reloaded", grade().reloaded); + stage = "complete"; + await setStatus(context, "Done"); + await expect.poll(async () => (await read.get(route)).status).toBe("done"); + await page.reload(); + observed.completed = await capture(revisedTitle, "Done", true); + await preserve(); + stage = "reopen"; + await setStatus(context, "Todo"); + await expect.poll(async () => (await read.get(route)).status).toBe("todo"); + await page.reload(); + observed.reopened = await capture(revisedTitle, "Todo", true); + await preserve(); + const final = grade(); + check("task-lifecycle-completed-and-reopened", final.completedAndReopened); + check("task-lifecycle-unedited-context-preserved", final.uneditedFieldsPreserved); + check("no-agent-work", final.noAgentWork); + } catch (error) { + if (error instanceof JourneyError) throw error; + throw new JourneyError("failed", `task_lifecycle_${stage}_failed`); + } finally { await preserve(); await read.preserve(options.directory); } +} diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 4c235990f6c..51cacf91541 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -2,8 +2,32 @@ import type { BrowserContext, Page } from "@playwright/test"; import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; -export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "task-document-persistence" | "ask-question" | "cloud-onboarding" - | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse" | "file-input-answer" | "inbox-triage" | "routine-scheduled-run" | "repository-continuation" | "company-access-isolation"; +export type JourneyId = + "task-edit-lifecycle" + | "task-properties" + | "task-create" + | "project-task-context" + | "task-dependency-resume" + | "task-document-persistence" + | "ask-question" + | "chat-continuity" + | "cloud-onboarding" + | "question-answer-resume" + | "task-search-filter" + | "followup-while-running" + | "plan-revise-accept" + | "build-revise" + | "protected-actions" + | "create-skill-studio" + | "stop-redirect" + | "hire-delegate-reuse" + | "agent-settings-persistence" + | "file-input-answer" + | "inbox-triage" + | "routine-scheduled-run" + | "repository-continuation" + | "company-access-isolation" + | "saved-skill-use"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; @@ -113,7 +137,8 @@ export interface TargetSession { /** Publish the observed finalization failure without releasing resources or leases. */ checkpointEvidence?(input?: TargetLifecycleInput): Promise; /** Durable intent before creation; receipt immediately after creation. */ - own(resource: { kind: "issue" | "project" | "agent" | "connection" | "tool-connection" | "skill" | "routine"; name: string; id?: string; journeyId?: JourneyId; + own(resource: { kind: "issue" | "project" | "agent" | "connection" | "tool-connection" | "skill" | "routine" | "label"; name: string; id?: string; journeyId?: JourneyId; + taskEdit?: { revisedTitle: string }; freshChat?: { companyId: string; userId: string; leadAgentId: string; absent: true } }): Promise; } export interface TargetProvider { diff --git a/tests/runner-e2e/report-lock.json b/tests/runner-e2e/report-lock.json index 125e3af0bc9..e65019c8c6c 100644 --- a/tests/runner-e2e/report-lock.json +++ b/tests/runner-e2e/report-lock.json @@ -1,6 +1,6 @@ { - "inputSha256": "e106b3eab2caa8c282bded699a1c217096039641f885c0a4497ab65eaceedf58", - "outputSha256": "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8", - "patchSha256": "0132e268086b9a1e27981a4de76e61a328fe87d9809c7af8ae97f081ea24a0c8", + "inputSha256": "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8", + "outputSha256": "7540075e96f9a93924a1e54775c17346c5760dd5d02902d83fb5228f918996ba", + "patchSha256": "dd4a3451167d3306f941d3e82e6892a6dee57133f0d913b1acf43913df2bcf66", "basis": "Generated with the trusted master lockfile refresh. Review this offline correction and its hashes with the dependency changes." } diff --git a/tests/runner-e2e/report-lock.patch b/tests/runner-e2e/report-lock.patch index de0610deb7e..a342ffa458f 100644 --- a/tests/runner-e2e/report-lock.patch +++ b/tests/runner-e2e/report-lock.patch @@ -1,12 +1,117 @@ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml -index c55bccc..2953e8d 100644 +index 2953e8daef..f586e6f8f2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml -@@ -5039,6 +5039,7 @@ packages: - '@smithy/core@3.35.1': - resolution: {integrity: sha512-i4YPS4B6ts7bjn7UwLnGjiZdprOvHvgGobFZsYK3GIY3E5hIqtj0rReU69BcTpGp+fvtraSNXeG1l+jtJvF55w==} - engines: {node: '>=18.0.0'} -+ deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. +@@ -426,8 +426,8 @@ importers: + specifier: ^24.0.0 + version: 24.13.3 + drizzle-kit: +- specifier: ^0.31.10 +- version: 0.31.10 ++ specifier: ^0.31.11 ++ version: 0.31.11 + tsx: + specifier: ^4.23.15 + version: 4.23.15 +@@ -1015,7 +1015,7 @@ importers: + version: 2.1.0(@grpc/grpc-js@1.14.5)(nice-grpc-common@2.0.4)(nice-grpc@2.1.17) + '@vercel/connect': + specifier: 2.3.3 +- version: 2.3.3(@chat-adapter/slack@4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3))(@modelcontextprotocol/sdk@1.31.0(zod@4.4.3))(ai@7.0.130(zod@4.4.3))(better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)))) ++ version: 2.3.3(@chat-adapter/slack@4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3))(@modelcontextprotocol/sdk@1.31.0(zod@4.4.3))(ai@7.0.130(zod@4.4.3))(better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)))) + acorn: + specifier: 8.18.0 + version: 8.18.0 +@@ -1033,7 +1033,7 @@ importers: + version: 3.0.1(ajv@8.20.0) + better-auth: + specifier: 1.7.2 +- version: 1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))) ++ version: 1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))) + chat: + specifier: 4.39.0 + version: 4.39.0(ai@7.0.130(zod@4.4.3))(zod@4.4.3) +@@ -6667,8 +6667,8 @@ packages: + peerDependencies: + react: ^19.2.8 + +- drizzle-kit@0.31.10: +- resolution: {integrity: sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw==} ++ drizzle-kit@0.31.11: ++ resolution: {integrity: sha512-YCYqxTLIB2OCqf6w9/Vef13baBVbwQIPcbT4Y56AfO6B9ajZhDhD8Jkveg/hJvT/iuMloKD/IYYkyMMNhr7Kqg==} + hasBin: true + + drizzle-orm@0.45.2: +@@ -7066,8 +7066,8 @@ packages: + resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} + engines: {node: '>=10'} + +- get-tsconfig@4.14.0: +- resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} ++ get-tsconfig@4.14.3: ++ resolution: {integrity: sha512-++QEw4DIY7WGoukz+/+A/8dGYPT9l9yIadnmSgZ8Rjr3YVSVDipQSO9CdnJo9ePqFqUUqh+wk9uIaoiAwsiPkA==} + + glob@13.0.6: + resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==} +@@ -10641,7 +10641,7 @@ snapshots: + '@esbuild-kit/esm-loader@2.6.5': + dependencies: + '@esbuild-kit/core-utils': 3.3.2 +- get-tsconfig: 4.14.0 ++ get-tsconfig: 4.14.3 + + '@esbuild/aix-ppc64@0.25.12': + optional: true +@@ -13859,14 +13859,14 @@ snapshots: + dependencies: + execa: 5.1.1 + +- '@vercel/connect@2.3.3(@chat-adapter/slack@4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3))(@modelcontextprotocol/sdk@1.31.0(zod@4.4.3))(ai@7.0.130(zod@4.4.3))(better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))))': ++ '@vercel/connect@2.3.3(@chat-adapter/slack@4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3))(@modelcontextprotocol/sdk@1.31.0(zod@4.4.3))(ai@7.0.130(zod@4.4.3))(better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))))': + dependencies: + '@vercel/oidc': 3.8.9 + optionalDependencies: + '@chat-adapter/slack': 4.39.0(patch_hash=226jj24akljccsfbw7bcxujz4u)(ai@7.0.130(zod@4.4.3))(zod@4.4.3) + '@modelcontextprotocol/sdk': 1.31.0(zod@4.4.3) + ai: 7.0.130(zod@4.4.3) +- better-auth: 1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))) ++ better-auth: 1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))) + + '@vercel/oidc@3.2.0': {} + +@@ -14143,7 +14143,7 @@ snapshots: + + before-after-hook@4.0.0: {} + +- better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))): ++ better-auth@1.7.2(@opentelemetry/api@1.9.1)(drizzle-kit@0.31.11)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)))(pg@8.18.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(jsdom@30.0.1(@noble/hashes@2.4.0))(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))): + dependencies: + '@better-auth/core': 1.7.2(@better-auth/utils@0.4.2)(@better-fetch/fetch@1.3.1)(@opentelemetry/api@1.9.1)(better-call@1.4.0(zod@4.4.3))(jose@6.2.10)(kysely@0.29.5)(nanostores@1.5.3) + '@better-auth/drizzle-adapter': 1.7.2(@better-auth/core@1.7.2(@better-auth/utils@0.4.2)(@better-fetch/fetch@1.3.1)(@opentelemetry/api@1.9.1)(better-call@1.4.0(zod@4.4.3))(jose@6.2.10)(kysely@0.29.5)(nanostores@1.5.3))(@better-auth/utils@0.4.2)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda))) +@@ -14163,7 +14163,7 @@ snapshots: + nanostores: 1.5.3 + zod: 4.4.3 + optionalDependencies: +- drizzle-kit: 0.31.10 ++ drizzle-kit: 0.31.11 + drizzle-orm: 0.45.2(@opentelemetry/api@1.9.1)(kysely@0.29.5)(pg@8.18.0)(postgres@3.4.9(patch_hash=6q4cic6qawywsyeo2spgnjamda)) + pg: 8.18.0 + react: 19.2.8 +@@ -14748,7 +14748,7 @@ snapshots: + react-is: 17.0.2 + tslib: 2.8.1 + +- drizzle-kit@0.31.10: ++ drizzle-kit@0.31.11: + dependencies: + '@drizzle-team/brocli': 0.10.2 + '@esbuild-kit/esm-loader': 2.6.5 +@@ -15156,7 +15156,7 @@ snapshots: + + get-stream@6.0.1: {} + +- get-tsconfig@4.14.0: ++ get-tsconfig@4.14.3: + dependencies: + resolve-pkg-maps: 1.0.0 - '@smithy/credential-provider-imds@4.5.2': - resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 419e37a934d..9906e0d4869 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -434,8 +434,8 @@ export function validateRetainedRunnerResult( if (!evidence) return invalid("result.productJourney"); const required = journeyRequiredChecks[checked.caseId]; if (evidence.outcome !== "passed" || !evidence.deployment || !required || !required.every(id => evidence.checks.some(c => c.id === id && c.passed)) || evidence.checks.some(check => !check.passed) || evidence.phases.some(p => p.outcome !== "passed") || checked.cleanup !== "passed") invalid("result.productJourney"); - if (["inbox-triage", "task-properties", "task-create", "project-task-context", "task-document-persistence", "task-search-filter", "company-access-isolation"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); - if (checked.caseId === "company-access-isolation" && checked.usage != null && Object.entries(checked.usage).some(([key, value]) => key !== "runs" || !Array.isArray(value) || value.length > 0)) invalid("result.browserOnly"); + if (["task-edit-lifecycle", "inbox-triage", "task-properties", "task-create", "project-task-context", "task-document-persistence", "task-search-filter", "company-access-isolation"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); + if (["company-access-isolation", "task-edit-lifecycle"].includes(checked.caseId) && checked.usage != null && Object.entries(checked.usage).some(([key, value]) => key !== "runs" || !Array.isArray(value) || value.length > 0)) invalid("result.browserOnly"); if (Object.hasOwn(agentJourneyRuns, checked.caseId) && (!agentJourneyRunCountValid(checked.caseId, checked.runIds?.length) || !checked.model || !checked.provider || checked.runtimeMode !== "native")) invalid("result.agentJourneyRun"); if (checked.caseId === "cloud-onboarding" && (!checked.runIds?.length || !evidence.operations || !evidence.operations.requestCount || !evidence.operations.traceIds.length || evidence.operations.campaignId !== evidence.campaignId || evidence.operations.attemptId !== evidence.attemptId)) invalid("result.paidJourneyReceipt"); diff --git a/ui/src/components/IssueDocumentsSection.keyless.test.tsx b/ui/src/components/IssueDocumentsSection.keyless.test.tsx new file mode 100644 index 00000000000..d0d2aa6a88d --- /dev/null +++ b/ui/src/components/IssueDocumentsSection.keyless.test.tsx @@ -0,0 +1,161 @@ +// @vitest-environment jsdom + +import { act } from "react"; +import type { ComponentProps, ReactNode } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { Issue, IssueDocument } from "@paperclipai/shared"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { IssueDocumentsSection } from "./IssueDocumentsSection"; +import { queryKeys } from "../lib/queryKeys"; + +const mockIssuesApi = vi.hoisted(() => ({ + listDocuments: vi.fn(), + listDocumentRevisions: vi.fn(), + restoreDocumentRevision: vi.fn(), + upsertDocument: vi.fn(), + lockDocument: vi.fn(), + unlockDocument: vi.fn(), + deleteDocument: vi.fn(), + getDocument: vi.fn(), +})); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +(globalThis as any).IS_REACT_ACT_ENVIRONMENT = true; + +vi.mock("../api/issues", () => ({ issuesApi: mockIssuesApi })); +vi.mock("../api/auth", () => ({ authApi: { getSession: async () => ({ user: { id: "user-1" } }) } })); +vi.mock("@/lib/router", () => ({ useLocation: () => ({ hash: "" }) })); +vi.mock("./IssueDocumentAnnotations", () => ({ + DocumentAnnotationsCountChip: () => null, + IssueDocumentAnnotations: ({ children }: { children: ReactNode }) => <>{children}, +})); +vi.mock("./DocumentMarkdownBody", () => ({ + DocumentMarkdownBody: ({ children }: { children: string }) =>
{children}
, +})); +vi.mock("./MarkdownBody", () => ({ + MarkdownBody: ({ children }: { children: string }) =>
{children}
, +})); +vi.mock("./MarkdownEditor", () => ({ + MarkdownEditor: ({ value }: { value: string }) =>
{value}
, +})); +vi.mock("@/components/ui/button", () => ({ + Button: ({ children, type = "button", ...props }: ComponentProps<"button">) => ( + + ), +})); +vi.mock("@/components/ui/dropdown-menu", () => ({ + DropdownMenu: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuTrigger: ({ children }: { children: ReactNode }) => <>{children}, + DropdownMenuContent: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuItem: ({ children, onClick }: { children: ReactNode; onClick?: () => void }) => ( + + ), + DropdownMenuLabel: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuRadioGroup: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuRadioItem: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuSeparator: () =>
, +})); + +function createIssueDocument(overrides: Partial = {}): IssueDocument { + return { + id: "document-1", + companyId: "company-1", + issueId: "issue-1", + key: "plan", + title: "Plan", + format: "markdown", + body: "", + latestRevisionId: "revision-1", + latestRevisionNumber: 1, + createdByAgentId: null, + createdByUserId: "user-1", + updatedByAgentId: null, + updatedByUserId: "user-1", + lockedAt: null, + lockedByAgentId: null, + lockedByUserId: null, + createdAt: new Date("2026-03-31T12:00:00.000Z"), + updatedAt: new Date("2026-03-31T12:05:00.000Z"), + ...overrides, + }; +} + +function createIssue(): Issue { + return { + id: "issue-1", + companyId: "company-1", + identifier: "DOC-1", + title: "Keyless documents", + documentSummaries: [], + legacyPlanDocument: null, + } as unknown as Issue; +} + +/** A cached row that lost its key, as seen by the task panel before this fix. */ +function keylessDocument(overrides: Partial = {}): IssueDocument { + return { ...createIssueDocument(overrides), key: undefined } as unknown as IssueDocument; +} + +async function flush() { + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)); + }); +} + +describe("IssueDocumentsSection keyless rows", () => { + let container: HTMLDivElement; + let root: Root; + let client: QueryClient; + + beforeEach(() => { + vi.clearAllMocks(); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); + client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + client.clear(); + container.remove(); + }); + + it.each([false, true])("skips a cached row without a key (general only: %s)", async generalDocumentsOnly => { + mockIssuesApi.listDocuments.mockResolvedValue([ + keylessDocument({ id: "keyless", body: "Keyless row content" }), + createIssueDocument({ id: "note", key: "notes", title: "Task note", body: "General document content" }), + ]); + await act(async () => root.render( + + + , + )); + await flush(); + expect(container.textContent).toContain("General document content"); + expect(container.textContent).not.toContain("Keyless row content"); + }); + + it("keeps a keyless mutation response out of the cached document list", async () => { + const document = createIssueDocument({ body: "Lockable plan body" }); + // Hold the post-lock refetch so the test observes the synced cache, not fresh server rows. + mockIssuesApi.listDocuments + .mockResolvedValueOnce([document]) + .mockReturnValue(new Promise(() => {})); + mockIssuesApi.lockDocument.mockResolvedValue(keylessDocument({ lockedAt: new Date("2026-03-31T12:06:00.000Z") })); + await act(async () => root.render( + + + , + )); + await flush(); + await act(async () => container.querySelector('button[title="Lock document"]')?.click()); + await flush(); + expect(mockIssuesApi.lockDocument).toHaveBeenCalledWith("issue-1", "plan"); + const cached = client.getQueryData(queryKeys.issues.documents("issue-1")) ?? []; + expect(cached.map(entry => entry.key)).toEqual(["plan"]); + expect(container.textContent).toContain("Lockable plan body"); + }); +}); diff --git a/ui/src/components/IssueDocumentsSection.tsx b/ui/src/components/IssueDocumentsSection.tsx index 22e80663b05..1c2f53a2f50 100644 --- a/ui/src/components/IssueDocumentsSection.tsx +++ b/ui/src/components/IssueDocumentsSection.tsx @@ -452,6 +452,8 @@ function AccountIssueDocumentsSection({ }, [documentSubject, queryClient]); const syncDocumentCaches = useCallback((document: IssueDocument) => { + // Only a complete document can replace a list row; the refetch fills the gap. + if (typeof document?.key !== "string") return; if (documentSubject.hideSystemDocuments && isSystemIssueDocumentKey(document.key)) return; queryClient.setQueryData( documentSubject.documentsQueryKey, @@ -527,8 +529,10 @@ function AccountIssueDocumentsSection({ }); const sortedDocuments = useMemo(() => { + // A keyless row cannot be rendered or edited; skip it instead of failing the whole page. return (documents ?? []).filter((doc) => - (!generalDocumentsOnly || isGeneralTaskDocumentKey(doc.key)) + typeof doc?.key === "string" + && (!generalDocumentsOnly || isGeneralTaskDocumentKey(doc.key)) && (!documentSubject.hideSystemDocuments || !isSystemIssueDocumentKey(doc.key)), ).sort((a, b) => { if (a.key === "plan" && b.key !== "plan") return -1; diff --git a/ui/src/lib/issue-artifacts.test.ts b/ui/src/lib/issue-artifacts.test.ts index b10c0c7acfd..ff87c35dca7 100644 --- a/ui/src/lib/issue-artifacts.test.ts +++ b/ui/src/lib/issue-artifacts.test.ts @@ -16,6 +16,9 @@ it.each(["plan", " Plan ", "artifact-review-output", "pipeline-case-body", "cont it("allows ordinary general document keys", () => { expect(isGeneralTaskDocumentKey("notes")).toBe(true); }); +it.each([undefined, null])("treats a missing document key (%s) as not general", key => { + expect(isGeneralTaskDocumentKey(key)).toBe(false); +}); function makeAttachment(overrides: Partial & { id: string }): IssueAttachment { return { diff --git a/ui/src/lib/issue-artifacts.ts b/ui/src/lib/issue-artifacts.ts index c4be8194406..624befe1248 100644 --- a/ui/src/lib/issue-artifacts.ts +++ b/ui/src/lib/issue-artifacts.ts @@ -1,8 +1,12 @@ import type { IssueAttachment, IssueDocumentSummary, IssueWorkProduct } from "@paperclipai/shared"; import { getAttachmentArtifactWorkProductMetadata, isArtifactReviewDocumentKey, isSystemIssueDocumentKey } from "@paperclipai/shared"; -/** Documents authored here; Plans and file reviews retain their own workflows. */ -export function isGeneralTaskDocumentKey(key: string): boolean { +/** + * Documents authored here; Plans and file reviews retain their own workflows. + * A cached row without a string key cannot be authored, so it is never general. + */ +export function isGeneralTaskDocumentKey(key: string | null | undefined): boolean { + if (typeof key !== "string") return false; return key.trim().toLowerCase() !== "plan" && !isArtifactReviewDocumentKey(key) && !isSystemIssueDocumentKey(key); }