diff --git a/.github/actions/setup-ci-dependencies/action.yml b/.github/actions/setup-ci-dependencies/action.yml new file mode 100644 index 00000000000..3367001b4b5 --- /dev/null +++ b/.github/actions/setup-ci-dependencies/action.yml @@ -0,0 +1,102 @@ +name: Restore materialized CI dependencies +description: Reuse exact installed dependencies and certified shared build outputs; install on a miss. +inputs: + working-directory: + description: Checkout directory relative to the workspace. + default: '.' + save-cache: + description: Allow a canonical master push to populate the caches. + default: 'false' + restore-cache: + description: Disable materialized restores for a controlled cold-path benchmark. + default: 'true' +outputs: + dependencies-restored: + description: Whether the exact materialized dependency cache was restored. + value: ${{ steps.installed.outputs.cache-hit }} + builds-restored: + description: Whether the exact certified build cache was restored. + value: ${{ steps.builds.outputs.cache-hit }} +runs: + using: composite + steps: + - name: Compute dependency inputs + id: inputs + shell: bash + working-directory: ${{ inputs.working-directory }} + run: node scripts/ci-dependency-cache.mjs + + - name: Restore installed dependencies (read only) + if: inputs.restore-cache == 'true' && steps.inputs.outputs.cacheable == 'true' + id: installed + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: ${{ steps.inputs.outputs.dependency-key }} + path: ${{ steps.inputs.outputs.dependency-paths }} + + - name: Locate pnpm store + if: steps.installed.outputs.cache-hit != 'true' + id: pnpm_store + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" + + - name: Restore pnpm store (read only) + if: steps.installed.outputs.cache-hit != 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ steps.pnpm_store.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles(format('{0}/pnpm-lock.yaml', inputs.working-directory)) }} + restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- + + - name: Install dependencies + if: steps.installed.outputs.cache-hit != 'true' + id: install + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + if pnpm install --frozen-lockfile; then + echo 'frozen=true' >> "$GITHUB_OUTPUT" + else + echo '::notice title=Lockfile::checked-in lockfile is stale for this merge tree; resolving it inline' + pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile + pnpm install --frozen-lockfile + fi + + - name: Restore certified shared build outputs (read only) + if: inputs.restore-cache == 'true' && steps.inputs.outputs.cacheable == 'true' + id: builds + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: ${{ steps.inputs.outputs.build-key }} + path: ${{ steps.inputs.outputs.build-paths }} + + - name: Validate workspace links and shared build outputs + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + pnpm run preflight:workspace-links + node scripts/ensure-plugin-build-deps.mjs + + - name: Save installed dependencies + if: >- + inputs.save-cache == 'true' && steps.inputs.outputs.cacheable == 'true' && github.repository == 'paperclipai/paperclip' && + github.event_name == 'push' && github.ref == 'refs/heads/master' && + steps.installed.outputs.cache-hit != 'true' && steps.install.outputs.frozen == 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: ${{ steps.inputs.outputs.dependency-key }} + path: ${{ steps.inputs.outputs.dependency-paths }} + + - name: Save certified shared build outputs + if: >- + inputs.save-cache == 'true' && steps.inputs.outputs.cacheable == 'true' && github.repository == 'paperclipai/paperclip' && + github.event_name == 'push' && github.ref == 'refs/heads/master' && + steps.builds.outputs.cache-hit != 'true' && + (steps.installed.outputs.cache-hit == 'true' || steps.install.outputs.frozen == 'true') + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: ${{ steps.inputs.outputs.build-key }} + path: ${{ steps.inputs.outputs.build-paths }} diff --git a/.github/scripts/tests/ci-dependency-cache.test.mjs b/.github/scripts/tests/ci-dependency-cache.test.mjs new file mode 100644 index 00000000000..17c1f274061 --- /dev/null +++ b/.github/scripts/tests/ci-dependency-cache.test.mjs @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { cachePlan } from "../../../scripts/ci-dependency-cache.mjs"; + +const entries = { + "package.json": '{"scripts":{"postinstall":"node scripts/link-plugin-dev-sdk.mjs"}}', + "server/package.json": '{"dependencies":{"external":"1.0.0"}}', + "packages/shared/package.json": '{}', + "packages/plugins/sdk/package.json": '{}', + "pnpm-lock.yaml": 'lockfile', + "pnpm-workspace.yaml": 'workspace', + ".npmrc": 'auto-install-peers=false', + "patches/external.patch": 'patch', + "scripts/link-plugin-dev-sdk.mjs": 'postinstall', + "scripts/ci-dependency-cache.mjs": 'cache schema', + "scripts/ensure-plugin-build-deps.mjs": 'build verifier', + "tsconfig.base.json": '{}', + "packages/shared/src/index.ts": 'shared API', + "packages/plugins/sdk/src/index.ts": 'SDK API', + "server/src/server.ts": 'server', +}; +const scope = { node: 'v24.19.0', root: '/work/paperclip', arch: 'x64', libc: '2.39', pnpm: '9.15.4', store: '/store' }; +const plan = (files = entries, runtime = scope) => cachePlan({ + files: Object.keys(files), read: (file) => files[file] === null ? null : Buffer.from(files[file]), scope: runtime, + packageDirs: ['.', 'packages/plugins/sdk', 'packages/shared', 'server'], +}); + +test('dependency identity includes manifests, resolution, patches, installer and runtime boundaries', () => { + const original = plan(); + for (const file of Object.keys(entries).filter((file) => + /package\.json$/.test(file) || /^(pnpm-|\.npmrc|patches\/|scripts\/(link-plugin-dev-sdk|ci-dependency-cache))/.test(file))) { + for (const value of [file.endsWith('package.json') ? '{"changed":true}' : 'changed', null]) { + const changed = plan({ ...entries, [file]: value }); + assert.notEqual(changed.dependencyKey, original.dependencyKey, file); + assert.notEqual(changed.buildKey, original.buildKey, file); + } + } + assert.notEqual(plan({ ...entries, 'packages/new/package.json': '{}' }).dependencyKey, original.dependencyKey); + for (const key of Object.keys(scope)) { + assert.notEqual(plan(entries, { ...scope, [key]: 'different' }).dependencyKey, original.dependencyKey, key); + } +}); + +test('shared and SDK source changes invalidate dependent builds without reinstalling packages', () => { + const original = plan(); + for (const file of ['packages/shared/src/index.ts', 'packages/plugins/sdk/src/index.ts', + 'tsconfig.base.json']) { + const changed = plan({ ...entries, [file]: 'changed' }); + assert.equal(changed.dependencyKey, original.dependencyKey, file); + assert.notEqual(changed.buildKey, original.buildKey, file); + } + assert.deepEqual(plan({ ...entries, 'server/src/server.ts': 'different' }), original); + assert.deepEqual(plan(Object.fromEntries(Object.entries(entries).reverse())), original); + assert.deepEqual(original.dependencyPaths, ['node_modules', 'packages/plugins/sdk/node_modules', + 'packages/shared/node_modules', 'server/node_modules']); + assert.deepEqual(plan({ ...entries, 'tests/fixtures/app/package.json': '{}' }).dependencyPaths, original.dependencyPaths); + for (const directory of ['/outside', '../outside', 'server/../../outside']) { + assert.throws(() => cachePlan({ files: [], read: () => null, scope, packageDirs: [directory] })); + } +}); + +test('unmodelled workspace lifecycle scripts disable materialized reuse', () => { + assert.equal(plan().cacheable, true); + for (const hook of ['preinstall', 'install', 'postinstall', 'prepare']) { + assert.equal(plan({ ...entries, 'server/package.json': JSON.stringify({ scripts: { [hook]: 'node custom.mjs' } }) }).cacheable, false); + } + assert.equal(plan({ ...entries, 'package.json': '{"scripts":{"postinstall":"node other.mjs"}}' }).cacheable, false); + for (const file of ['scripts/ensure-plugin-build-deps.mjs', 'scripts/new-install-helper.mjs', '.pnpmfile.cjs']) { + assert.notEqual(plan({ ...entries, [file]: 'changed' }).dependencyKey, plan().dependencyKey); + } +}); + +test('cache restores are exact and only a frozen canonical master push can publish entries', () => { + const action = readFileSync(new URL('../../actions/setup-ci-dependencies/action.yml', import.meta.url), 'utf8'); + const installed = action.slice(action.indexOf(' - name: Restore installed'), action.indexOf(' - name: Locate pnpm')); + const builds = action.slice(action.indexOf(' - name: Restore certified'), action.indexOf(' - name: Validate workspace')); + for (const restore of [installed, builds]) assert.doesNotMatch(restore, /restore-keys:/); + for (const step of action.split(' - name: Save ').slice(1)) { + assert.match(step, /inputs\.save-cache == 'true'/); + assert.match(step, /github\.repository == 'paperclipai\/paperclip'/); + assert.match(step, /github\.event_name == 'push'/); + assert.match(step, /github\.ref == 'refs\/heads\/master'/); + assert.match(step, /steps\.install\.outputs\.frozen == 'true'/); + } + const workflow = readFileSync(new URL('../../workflows/release-verify.yml', import.meta.url), 'utf8'); + assert.match(workflow, /save-cache: .*inputs\.ref == github\.sha/); +}); diff --git a/.github/workflows/ci-work-benchmark.yml b/.github/workflows/ci-work-benchmark.yml new file mode 100644 index 00000000000..2ff13f8c7cc --- /dev/null +++ b/.github/workflows/ci-work-benchmark.yml @@ -0,0 +1,139 @@ +name: CI work benchmark + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + paths: + - .github/actions/setup-ci-dependencies/** + - .github/workflows/ci-work-benchmark.yml + - scripts/ci-dependency-cache.mjs + - scripts/benchmark-ci-work.mjs + - scripts/run-vitest-stable.mjs + - server/src/__tests__/setup-supertest.ts + - server/src/__tests__/workspace-runtime.test.ts + - server/src/services/workspace-git-snapshot-streaming.test.ts + +permissions: + contents: read + +concurrency: + group: ci-work-benchmark-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + paired: + if: >- + contains(github.event.pull_request.labels.*.name, 'profile-ci') && + (github.event.action != 'labeled' || github.event.label.name == 'profile-ci') + runs-on: ubuntu-latest + timeout-minutes: 25 + defaults: + run: + working-directory: candidate + steps: + - name: Checkout candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.sha }} + path: candidate + fetch-depth: 2 + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 + with: + version: 9.15.4 + run_install: false + + - name: Start cold setup measurement + run: echo "CI_COLD_STARTED=$(date +%s)" >> "$GITHUB_ENV" + + - name: Qualify candidate dependency setup + uses: ./candidate/.github/actions/setup-ci-dependencies + with: + working-directory: candidate + restore-cache: 'false' + + - name: Record cold setup and compute qualification keys + id: qualification + run: | + echo "CI_COLD_SECONDS=$(($(date +%s) - CI_COLD_STARTED))" >> "$GITHUB_ENV" + node scripts/ci-dependency-cache.mjs + + - name: Save PR-scoped qualification dependencies + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: ${{ steps.qualification.outputs.dependency-key }} + path: ${{ steps.qualification.outputs.dependency-paths }} + + - name: Save PR-scoped qualification builds + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: ${{ steps.qualification.outputs.build-key }} + path: ${{ steps.qualification.outputs.build-paths }} + + - name: Remove materialized outputs before restore + run: | + node --input-type=module <<'NODE' + import { execFileSync } from 'node:child_process'; + import { rmSync } from 'node:fs'; + const plan = JSON.parse(execFileSync(process.execPath, ['scripts/ci-dependency-cache.mjs'], { + env: { ...process.env, GITHUB_OUTPUT: '' }, encoding: 'utf8', + })); + for (const directory of [...plan.dependencyPaths, 'packages/shared/dist', 'packages/plugins/sdk/dist']) { + rmSync(directory, { recursive: true, force: true }); + } + NODE + echo "CI_WARM_STARTED=$(date +%s)" >> "$GITHUB_ENV" + + - name: Qualify a real warm restore + id: warm + uses: ./candidate/.github/actions/setup-ci-dependencies + with: + working-directory: candidate + + - name: Verify exact cache hits and record setup comparison + env: + DEPS_RESTORED: ${{ steps.warm.outputs.dependencies-restored }} + BUILDS_RESTORED: ${{ steps.warm.outputs.builds-restored }} + OUTPUT: ${{ runner.temp }}/ci-work-benchmark + run: | + test "$DEPS_RESTORED" = true + test "$BUILDS_RESTORED" = true + mkdir -p "$OUTPUT" + warm_seconds=$(($(date +%s) - CI_WARM_STARTED)) + printf '{"coldSeconds":%s,"warmSeconds":%s}\n' "$CI_COLD_SECONDS" "$warm_seconds" > "$OUTPUT/dependency-setup.json" + echo "Dependency setup: ${CI_COLD_SECONDS}s cold, ${warm_seconds}s with exact materialized restores." >> "$GITHUB_STEP_SUMMARY" + + - name: Prepare unchanged baseline + run: | + base_sha=$(git rev-parse HEAD^1) + git worktree add --detach ../baseline "$base_sha" + cd ../baseline + if ! pnpm install --frozen-lockfile; then + pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile + pnpm install --frozen-lockfile + fi + pnpm run preflight:workspace-links + pnpm --filter @paperclipai/plugin-sdk ensure-build-deps + + - name: Compare complete commands and executed case inventories + env: + BENCHMARK_OUTPUT: ${{ runner.temp }}/ci-work-benchmark + run: | + node scripts/benchmark-ci-work.mjs ../baseline "$BENCHMARK_OUTPUT" + cat "$BENCHMARK_OUTPUT/summary.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Retain timings, inventories and failure logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: ci-work-benchmark + path: ${{ runner.temp }}/ci-work-benchmark + retention-days: 14 diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 3a4db1c85ac..e225da5cebd 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -186,13 +186,19 @@ jobs: # Share the checked-in lockfile key with master. PR merge refs must not # save full copies of the store or evict the post-merge build caches. + - name: Restore materialized dependencies and shared builds + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') != '' + uses: ./.github/actions/setup-ci-dependencies + - name: Locate pnpm store + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' id: pnpm_store run: | echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} @@ -200,6 +206,7 @@ jobs: restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- - name: Install dependencies + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' # Manifest-changing and stacked PRs can hold a pnpm-lock.yaml that is # stale for the merge tree. Resolve it inline instead of waiting on a # policy-job artifact: that dependency put the policy job's queue and @@ -484,13 +491,19 @@ jobs: # Share the checked-in lockfile key with master. PR merge refs must not # save full copies of the store or evict the post-merge build caches. + - name: Restore materialized dependencies and shared builds + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') != '' + uses: ./.github/actions/setup-ci-dependencies + - name: Locate pnpm store + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' id: pnpm_store run: | echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} @@ -498,6 +511,7 @@ jobs: restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- - name: Install dependencies + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' # Manifest-changing and stacked PRs can hold a pnpm-lock.yaml that is # stale for the merge tree. Resolve it inline instead of waiting on a # policy-job artifact: that dependency put the policy job's queue and @@ -684,13 +698,19 @@ jobs: # Share the checked-in lockfile key with master. PR merge refs must not # save full copies of the store or evict the post-merge build caches. + - name: Restore materialized dependencies and shared builds + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') != '' + uses: ./.github/actions/setup-ci-dependencies + - name: Locate pnpm store + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' id: pnpm_store run: | echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} @@ -698,6 +718,7 @@ jobs: restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- - name: Install dependencies + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' # Manifest-changing and stacked PRs can hold a pnpm-lock.yaml that is # stale for the merge tree. Resolve it inline instead of waiting on a # policy-job artifact: that dependency put the policy job's queue and @@ -885,13 +906,19 @@ jobs: # Share the checked-in lockfile key with master. PR merge refs must not # save full copies of the store or evict the post-merge build caches. + - name: Restore materialized dependencies and shared builds + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') != '' + uses: ./.github/actions/setup-ci-dependencies + - name: Locate pnpm store + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' id: pnpm_store run: | echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} @@ -899,6 +926,7 @@ jobs: restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- - name: Install dependencies + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' # Manifest-changing and stacked PRs can hold a pnpm-lock.yaml that is # stale for the merge tree. Resolve it inline instead of waiting on a # policy-job artifact: that dependency put the policy job's queue and @@ -1160,13 +1188,19 @@ jobs: # Share the checked-in lockfile key with master. PR merge refs must not # save full copies of the store or evict the post-merge build caches. + - name: Restore materialized dependencies and shared builds + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') != '' + uses: ./.github/actions/setup-ci-dependencies + - name: Locate pnpm store + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' id: pnpm_store run: | echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} @@ -1174,6 +1208,7 @@ jobs: restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- - name: Install dependencies + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' # Manifest-changing and stacked PRs can hold a pnpm-lock.yaml that is # stale for the merge tree. Resolve it inline instead of waiting on a # policy-job artifact: that dependency put the policy job's queue and @@ -1239,13 +1274,19 @@ jobs: # Share the checked-in lockfile key with master. PR merge refs must not # save full copies of the store or evict the post-merge build caches. + - name: Restore materialized dependencies and shared builds + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') != '' + uses: ./.github/actions/setup-ci-dependencies + - name: Locate pnpm store + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' id: pnpm_store run: | echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} @@ -1253,6 +1294,7 @@ jobs: restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- - name: Install dependencies + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' # Manifest-changing and stacked PRs can hold a pnpm-lock.yaml that is # stale for the merge tree. Resolve it inline instead of waiting on a # policy-job artifact: that dependency put the policy job's queue and @@ -1486,13 +1528,19 @@ jobs: # Share the checked-in lockfile key with master. PR merge refs must not # save full copies of the store or evict the post-merge build caches. + - name: Restore materialized dependencies and shared builds + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') != '' + uses: ./.github/actions/setup-ci-dependencies + - name: Locate pnpm store + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' id: pnpm_store run: | echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} @@ -1500,6 +1548,7 @@ jobs: restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- - name: Install dependencies + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' # Manifest-changing and stacked PRs can hold a pnpm-lock.yaml that is # stale for the merge tree. Resolve it inline instead of waiting on a # policy-job artifact: that dependency put the policy job's queue and diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index a4632dfb442..cf333aaa618 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -52,7 +52,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 - cache: pnpm + package-manager-cache: false # rust-cache hashes its absolute cache paths into the entry's version, # and GitHub only serves an entry whose key and version both match. The @@ -102,7 +102,14 @@ jobs: - name: Validate release package manifest run: node ./scripts/release-package-map.mjs check + - name: Restore materialized dependencies and shared builds + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') != '' + uses: ./.github/actions/setup-ci-dependencies + with: + save-cache: ${{ github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha }} + - name: Install dependencies + if: hashFiles('.github/actions/setup-ci-dependencies/action.yml') == '' run: pnpm install --no-frozen-lockfile - name: Typecheck diff --git a/doc/ci-profiling.md b/doc/ci-profiling.md index 9d836537bd8..fe8472a7577 100644 --- a/doc/ci-profiling.md +++ b/doc/ci-profiling.md @@ -154,3 +154,44 @@ References: [Honeycomb trace fields](https://docs.honeycomb.io/get-started/best- [Events API](https://docs.honeycomb.io/api/events/create-events), [large trace display limit](https://docs.honeycomb.io/troubleshoot/common-issues/data-in-honeycomb/#your-very-large-trace-displays-only-32000-spans), [GitHub workflow_run](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_run). + +## Qualify reductions in CI work + +For changes to the stable test launcher, dependency setup, or the large Git and +readiness fixtures, `profile-ci` also runs **CI work benchmark**. It uses one +GitHub-hosted runner without secrets or AWS access. It compares the unchanged +PR base and candidate sequentially. It records complete command times, test +inventories, revisions, and failure logs in the `ci-work-benchmark` artifact. +The serialized shard must retain the same cases. The fixture comparison retains +the database contract and splits large Git scanning from bounded staging. +These command timings exclude queue time and setup; use the full Honeycomb +waterfall to assess the complete CI critical path. + +The benchmark also exercises cold dependency setup and a real materialized +cache restore after removing the installed directories. Its two qualification +entries are scoped to the PR merge ref and a separate checkout path. They cannot +populate master's caches. Normal PR lanes restore only; the exact canonical +master push in release verification is the normal cache writer. + +Materialized dependency keys include every tracked package manifest, lockfile, +workspace configuration, patches, repository scripts, Node version/ABI, platform, +architecture, libc, pnpm version, store location, and physical checkout path. +There are no prefix restores. An input change takes the normal install path. +A stale lockfile uses the existing resolution fallback but cannot publish an +entry. An additional workspace install/prepare hook disables materialized reuse +until its inputs are modeled. Shared/SDK build outputs have a separate key that also includes both +packages' source/configuration and the build verifier. The existing verifier +checks source and output fingerprints and rebuilds incomplete or changed output. +Tests and their results are never cached or omitted by this mechanism. + +Serialized shards share one Vitest controller with `forks`, `isolate`, one +worker, and file parallelism disabled. Each file gets a separate process and +test home/configuration. The isolation regression runs files in both orders and +checks process IDs, environment, globals, fake timers, and filesystem state. + +The readiness regression uses a real one-second PostgreSQL idle timeout and +verifies that the backend connection changes while the parent write can commit. +Its configured readiness window stays long. The Git regression keeps all four +filename lanes above 32 MiB, while the staging case transfers and deletes 4,000 +files whose filename list exceeds 2 MiB. It still checks every staged file, +unusual names, snapshot ownership, cleanup, and bounded command lengths. diff --git a/package.json b/package.json index 32ed9601455..93ef1dee218 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", + "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/__tests__/vitest-worker-isolation.test.mjs b/scripts/__tests__/vitest-worker-isolation.test.mjs new file mode 100644 index 00000000000..bd70f733df4 --- /dev/null +++ b/scripts/__tests__/vitest-worker-isolation.test.mjs @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const root = fileURLToPath(new URL('../../', import.meta.url)); +const vitest = path.join(root, 'node_modules/vitest/vitest.mjs'); +const api = fileURLToPath(new URL('../../node_modules/vitest/dist/index.js', import.meta.url)); +const setup = path.join(root, 'server/src/__tests__/setup-supertest.ts'); + +test('one controller retains fresh fork, globals, environment, timers and home per serialized file', () => { + const directory = realpathSync(mkdtempSync(path.join(os.tmpdir(), 'pc-fork-proof-'))); + try { + const record = path.join(directory, 'workers.ndjson'); + const config = path.join(directory, 'vitest.config.mjs'); + const files = ['first.test.ts', 'second.test.ts']; + for (const file of files) writeFileSync(path.join(directory, file), ` + import { expect, it, vi } from ${JSON.stringify(api)}; + import { appendFileSync, existsSync, writeFileSync } from 'node:fs'; + it('starts clean and can contaminate only its own worker', () => { + expect(globalThis.serializedLeak).toBeUndefined(); + expect(process.env.PAPERCLIP_SERIALIZED_LEAK).toBeUndefined(); + expect(vi.isFakeTimers()).toBe(false); + expect(existsSync(process.env.PAPERCLIP_CONFIG)).toBe(false); + appendFileSync(${JSON.stringify(record)}, JSON.stringify({ + name: ${JSON.stringify(file)}, pid: process.pid, home: process.env.PAPERCLIP_HOME, + config: process.env.PAPERCLIP_CONFIG, instance: process.env.PAPERCLIP_INSTANCE_ID, + }) + '\\n'); + writeFileSync(process.env.PAPERCLIP_CONFIG, '{}'); + globalThis.serializedLeak = true; + process.env.PAPERCLIP_SERIALIZED_LEAK = 'leaked'; + vi.useFakeTimers(); + }); + `); + for (const order of [files, [...files].reverse()]) { + rmSync(record, { force: true }); + writeFileSync(config, `import Sequencer from './sequencer.mjs'; export default ${JSON.stringify({ + root: directory, + test: { + include: order, pool: 'forks', isolate: true, fileParallelism: false, + maxWorkers: 1, setupFiles: [setup], + sequence: { sequencer: '__SEQUENCER__' }, + }, + }).replace('"__SEQUENCER__"', 'Sequencer')};`); + writeFileSync(path.join(directory, 'sequencer.mjs'), `export default class { + shard(tests) { return tests; } + sort(tests) { return tests.sort((a, b) => ${JSON.stringify(order)}.indexOf(a.moduleId.split('/').pop()) - ${JSON.stringify(order)}.indexOf(b.moduleId.split('/').pop())); } + }`); + const result = spawnSync(process.execPath, [vitest, 'run', '--config', config], { + cwd: root, encoding: 'utf8', timeout: 60_000, + env: { ...process.env, PAPERCLIP_HOME: path.join(directory, 'controller-home'), + PAPERCLIP_CONFIG: path.join(directory, 'controller-config.json'), PAPERCLIP_INSTANCE_ID: 'controller' }, + }); + assert.equal(result.status, 0, `${result.error ?? ''}\n${result.stdout}\n${result.stderr}`); + const workers = readFileSync(record, 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + assert.equal(workers.length, 2); + assert.deepEqual(workers.map((worker) => worker.name), order); + for (const key of ['pid', 'home', 'config', 'instance']) assert.equal(new Set(workers.map((worker) => worker[key])).size, 2, key); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/scripts/benchmark-ci-work.mjs b/scripts/benchmark-ci-work.mjs new file mode 100644 index 00000000000..7fa6dec7661 --- /dev/null +++ b/scripts/benchmark-ci-work.mjs @@ -0,0 +1,76 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const candidate = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const baseline = path.resolve(process.argv[2] ?? ""); +assert.ok(process.argv[2] && baseline !== candidate, 'Pass a separately installed unchanged baseline checkout'); +const output = path.resolve(process.argv[3] ?? path.join(candidate, '.ci-benchmark')); +mkdirSync(output, { recursive: true }); +const results = []; +const commands = [ + ['serialized', process.execPath, ['scripts/run-vitest-stable.mjs', '--mode', 'serialized', '--shard-index', '0', '--shard-count', '9']], + ['fixtures', 'pnpm', ['exec', 'vitest', 'run', '--project', '@paperclipai/server', + 'server/src/services/workspace-git-snapshot-streaming.test.ts', 'server/src/__tests__/workspace-runtime.test.ts', + '-t', 'streams.*all four real Git|commits fixed-port reservations']], +]; +for (const [variant, cwd] of [['baseline', baseline], ['candidate', candidate]]) { + for (const [name, command, args] of commands) { + const profile = path.join(output, `${variant}-${name}`); + mkdirSync(profile, { recursive: true }); + const started = performance.now(); + const result = spawnSync(command, args, { + cwd, encoding: 'utf8', timeout: 15 * 60_000, maxBuffer: 64 * 1024 * 1024, + env: { ...process.env, GITHUB_WORKSPACE: cwd, NODE_ENV: 'test', PAPERCLIP_CI_PROFILE: 'true', + PAPERCLIP_CI_PROFILE_DIR: profile, PAPERCLIP_CI_PROFILE_JOB: `benchmark-${variant}-${name}` }, + }); + const wallMs = performance.now() - started; + writeFileSync(path.join(output, `${variant}-${name}.log`), `${result.stdout ?? ''}\n${result.stderr ?? ''}`); + const records = readdirSync(profile).filter((file) => file.endsWith('.ndjson')) + .flatMap((file) => readFileSync(path.join(profile, file), 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line))); + results.push({ variant, name, wallMs, status: result.status, + sha: execFileSync('git', ['rev-parse', 'HEAD'], { cwd, encoding: 'utf8' }).trim(), + tests: records.filter((record) => record.kind === 'test' && record.status !== 'skipped') + .map((record) => [record.file, record.name, record.status]).sort(), + files: records.filter((record) => record.kind === 'file').map((record) => record.file).sort(), + }); + writeFileSync(path.join(output, 'results.json'), JSON.stringify({ node: process.version, results }, null, 2)); + assert.equal(result.status, 0, `${variant}/${name} failed: ${result.error ?? ''}; see ${output}`); + console.log(`${variant}/${name}: ${(wallMs / 1000).toFixed(2)}s`); + } +} +const rows = []; +for (const [name] of commands) { + const before = results.find((result) => result.variant === 'baseline' && result.name === name); + const after = results.find((result) => result.variant === 'candidate' && result.name === name); + assert.ok(before.tests.length > 0, `${name}: missing test evidence`); + assert.deepEqual(after.files, before.files, `${name}: file inventory changed`); + if (name === 'serialized') assert.deepEqual(after.tests, before.tests, 'serialized case inventory changed'); + else { + const contracts = (tests) => { + const found = new Set(); + for (const [file, title, status] of tests) { + assert.equal(status, 'passed'); + if (file.endsWith('/workspace-runtime.test.ts') && title.includes('commits fixed-port reservations')) found.add('database'); + if (file.endsWith('/workspace-git-snapshot-streaming.test.ts')) { + if (title.includes('above 32 MiB')) found.add('large Git lanes'); + if (/stag(?:es|ing)/.test(title)) found.add('bounded staging'); + } + } + return [...found].sort(); + }; + for (const result of [before, after]) assert.deepEqual(contracts(result.tests), + ['bounded staging', 'database', 'large Git lanes'], 'fixture contracts must all execute'); + } + rows.push(`| ${name} | ${(before.wallMs / 1000).toFixed(2)}s | ${(after.wallMs / 1000).toFixed(2)}s | ${((1 - after.wallMs / before.wallMs) * 100).toFixed(1)}% | ${after.tests.length} |`); +} +const summary = ['| Work | Baseline | Candidate | Reduction | Executed cases |', + '|---|---:|---:|---:|---:|', ...rows, + '', 'Paired sequential runs on one host with Node ' + process.version + '. Setup, cache transfer, queue time and the full CI critical path are outside these command timings.', +].join('\n'); +writeFileSync(path.join(output, 'summary.md'), summary + '\n'); +console.log(summary); +const total = (variant) => results.filter((result) => result.variant === variant).reduce((sum, result) => sum + result.wallMs, 0); +assert.ok(total('candidate') < total('baseline'), 'No combined speedup measured; inspect the paired evidence before claiming an improvement'); diff --git a/scripts/ci-dependency-cache.mjs b/scripts/ci-dependency-cache.mjs new file mode 100644 index 00000000000..b447c01d362 --- /dev/null +++ b/scripts/ci-dependency-cache.mjs @@ -0,0 +1,83 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { appendFileSync, existsSync, readFileSync, realpathSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { readPluginsUnder } from "./link-plugin-dev-sdk.mjs"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +export function cachePlan({ files, read, scope, packageDirs }) { + const dependencyInputs = files.filter((file) => + /(^|\/)package\.json$/.test(file) + || ["pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc", ".pnpmfile.cjs", "pnpmfile.cjs"].includes(file) + || file.startsWith("scripts/") + || file.startsWith("patches/")); + const digest = (inputs, salt) => { + const hash = createHash("sha256").update(JSON.stringify(salt)); + for (const file of [...inputs].sort()) { + const bytes = read(file); + hash.update(JSON.stringify([file, bytes === null ? null : bytes.length])); + if (bytes !== null) hash.update(bytes); + } + return hash.digest("hex"); + }; + const dependencyKey = `ci-dependencies-v1-${digest(dependencyInputs, scope)}`; + const cacheable = packageDirs.every((directory) => { + const manifest = read(path.posix.join(directory, 'package.json')); + if (manifest === null) return false; + const scripts = JSON.parse(manifest).scripts ?? {}; + return ['preinstall', 'install', 'postinstall', 'prepare'].every((hook) => + scripts[hook] === undefined || (directory === '.' && hook === 'postinstall' + && scripts[hook] === 'node scripts/link-plugin-dev-sdk.mjs')); + }); + const buildInputs = files.filter((file) => + file.startsWith("packages/shared/") || file.startsWith("packages/plugins/sdk/") + || /^tsconfig[^/]*\.json$/.test(file) + || file === "scripts/ensure-plugin-build-deps.mjs"); + return { + cacheable, + dependencyKey, + buildKey: `ci-build-dependencies-v1-${digest(buildInputs, dependencyKey)}`, + dependencyPaths: [...new Set(packageDirs.map((directory) => { + if (path.isAbsolute(directory) || directory.split('/').includes('..')) throw new Error('Package directory escapes the checkout'); + return path.posix.join(directory, 'node_modules'); + }))].sort(), + }; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const files = execFileSync("git", ["ls-files", "-z"], { cwd: root }).toString().split("\0").filter(Boolean); + const command = (args) => execFileSync("pnpm", args, { cwd: root, encoding: "utf8" }).trim(); + const workspaces = JSON.parse(command(['-r', 'list', '--depth=-1', '--json'])); + const packageDirs = ['.', + ...workspaces.map((workspace) => path.relative(root, workspace.path).split(path.sep).join('/') || '.'), + ...readPluginsUnder(path.join(root, 'packages/plugins/sandbox-providers')).map((directory) => path.relative(root, directory).split(path.sep).join('/')), + 'packages/plugins/examples/plugin-orchestration-smoke-example', + ]; + const plan = cachePlan({ + files, + read: (file) => existsSync(path.join(root, file)) ? readFileSync(path.join(root, file)) : null, + packageDirs, + scope: { + root: realpathSync(root), node: process.version, abi: process.versions.modules, + platform: process.platform, arch: process.arch, + libc: process.report.getReport().header.glibcVersionRuntime ?? null, + pnpm: command(["--version"]), store: command(["store", "path", "--silent"]), + }, + }); + const output = [ + `cacheable=${plan.cacheable}`, + `dependency-key=${plan.dependencyKey}`, + `build-key=${plan.buildKey}`, + "dependency-paths< path.join(root, file)), + "PAPERCLIP_CACHE_PATHS", + "build-paths< 0) { runVitest( [ "--project", "@paperclipai/server", - routeTest.repoPath, + ...shardTests.map((test) => test.repoPath), "--pool=forks", "--isolate", + ...serializedServerVitestArgs, ], - routeTest.repoPath, + `serialized shard ${shardIndex + 1}/${shardCount}`, ); } } diff --git a/server/src/__tests__/setup-supertest.ts b/server/src/__tests__/setup-supertest.ts index 82472092bf3..be189a6abe0 100644 --- a/server/src/__tests__/setup-supertest.ts +++ b/server/src/__tests__/setup-supertest.ts @@ -29,6 +29,8 @@ process.env.PAPERCLIP_TEST_HOST_HOME ??= process.env.PAPERCLIP_HOME?.trim() || p // Receipt-spool recovery must never scan a developer's instance during tests. const { home: paperclipTestHome, cleanup: cleanupPaperclipTestHome } = createIsolatedTestHome(); process.env.PAPERCLIP_HOME = paperclipTestHome; +process.env.PAPERCLIP_CONFIG = path.join(paperclipTestHome, "config.json"); +process.env.PAPERCLIP_INSTANCE_ID = `vt-${process.pid}`; // Setup-file afterAll hooks run before suites drain asynchronous heartbeats. // Cleanup at worker exit, after all suite teardown, avoids deleting live state. process.once("exit", cleanupPaperclipTestHome); diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index 3e193c74a16..1969d8a4bf7 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -7326,8 +7326,10 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => } }, 20_000); - it("commits fixed-port reservations before readiness windows longer than 60 seconds", async () => { + it("commits fixed-port reservations across the configured database idle timeout", async () => { const fixture = await createRuntimeFixture(); + const readinessDb = createDb(tempDb!.connectionString, { idleTimeoutSeconds: 1, maxConnections: 1 }); + const [initialConnection] = await readinessDb.$client<{ pid: number }[]>`select pg_backend_pid() as pid`; const cleanupRuntimeHome = await createRuntimeHome(); const workspace = fixture.workspaces[0]!; const basePort = await findFreePort(); @@ -7353,7 +7355,7 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => try { startPromise = startRuntimeServicesForWorkspaceControl({ - db, + db: readinessDb, actor: fixture.actor, issue: null, workspace: fixture.realizedWorkspace(workspace), @@ -7377,16 +7379,18 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => parentWrite.then(() => "committed" as const), new Promise<"locked">((resolve) => setTimeout(() => resolve("locked"), 2_000)), ]); + expect(parentWriteOutcome).toBe("committed"); const readinessWaitStartedAt = Date.now(); - await new Promise((resolve) => setTimeout(resolve, 60_250)); - expect(Date.now() - readinessWaitStartedAt).toBeGreaterThanOrEqual(60_000); + await new Promise((resolve) => setTimeout(resolve, 1_250)); + expect(Date.now() - readinessWaitStartedAt).toBeGreaterThanOrEqual(1_000); + const [reconnected] = await readinessDb.$client<{ pid: number }[]>`select pg_backend_pid() as pid`; + expect(reconnected!.pid).not.toBe(initialConnection!.pid); await fs.writeFile(releaseMarkerPath, "release"); const started = (await startPromise)[0]!; await parentWrite; - expect(parentWriteOutcome).toBe("committed"); expect(started).toMatchObject({ port: basePort, url: `http://127.0.0.1:${basePort}`, @@ -7413,6 +7417,7 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => }).catch(() => undefined); await cleanupRuntimeHome(); await fixture.cleanup(); + await readinessDb.$client.end({ timeout: 1 }); } }, 80_000); diff --git a/server/src/services/workspace-git-snapshot-streaming.test.ts b/server/src/services/workspace-git-snapshot-streaming.test.ts index fa6a8247cb6..6dbfe303ed3 100644 --- a/server/src/services/workspace-git-snapshot-streaming.test.ts +++ b/server/src/services/workspace-git-snapshot-streaming.test.ts @@ -53,7 +53,10 @@ function localClient(commands: string[]): SandboxManagedRuntimeClient { // macOS adds per-file process and filesystem overhead to the real deletion lane. const realGitTimeoutMs = process.platform === "darwin" ? 300_000 : 180_000; -it("streams and stages all four real Git filename lanes above 32 MiB through the shared scheduler", async () => { +it.each([ + { label: "above 32 MiB", fileCount: 40_000, expectedBytes: 34_988_890, minimumBytes: 32 * 1024 * 1024, stageFiles: false }, + { label: "with staging above argument limits", fileCount: 4_000, expectedBytes: 3_494_890, minimumBytes: 2 * 1024 * 1024, stageFiles: true }, +])("streams all four real Git filename lanes $label through the shared scheduler", async ({ fileCount, expectedBytes, minimumBytes, stageFiles }) => { const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-stream-git-"))); directories.push(root); const repo = path.join(root, "repo"); @@ -68,7 +71,7 @@ it("streams and stages all four real Git filename lanes above 32 MiB through the await fs.mkdir(parent, { recursive: true }); const fileName = (index: number) => `${"asset-".repeat(36)}${index}.js`; const writeFiles = async (content: string) => { - for (let start = 0; start < 40_000; start += 100) { + for (let start = 0; start < fileCount; start += 100) { await Promise.all(Array.from({ length: 100 }, (_, offset) => fs.writeFile(path.join(parent, fileName(start + offset)), content))); } }; @@ -95,9 +98,9 @@ it("streams and stages all four real Git filename lanes above 32 MiB through the count++; bytes += Buffer.byteLength(relative) + 1; } - expect(count).toBe(40_000); - expect(bytes).toBe(34_988_890); - expect(bytes).toBeGreaterThan(32 * 1024 * 1024); + expect(count).toBe(fileCount); + expect(bytes).toBe(expectedBytes); + expect(bytes).toBeGreaterThan(minimumBytes); }; // Standalone callers must use the same streaming semantics. setExpensiveWorkspaceGitExecutor(null); @@ -107,7 +110,6 @@ it("streams and stages all four real Git filename lanes above 32 MiB through the bindScheduler(); const snapshot = await scan(); checkLargeLane(snapshot.overlayPaths); - await fs.writeFile(path.join(parent, "late.js"), "must not stage"); const commands: string[] = []; const remote = path.join(root, "remote"); const stage = async (gitSnapshot: GitWorkspaceSnapshot) => { @@ -118,19 +120,22 @@ it("streams and stages all four real Git filename lanes above 32 MiB through the runtimes.push(runtime); return runtime; }; - const runtime = await stage(snapshot); const remoteParent = path.join(remote, path.relative(repo, parent)); - let stagedCount = 0; - for await (const entry of await fs.opendir(remoteParent)) { - expect(await fs.readFile(path.join(remoteParent, entry.name), "utf8")).toBe("base"); - stagedCount++; + if (stageFiles) { + await fs.writeFile(path.join(parent, "late.js"), "must not stage"); + const runtime = await stage(snapshot); + let stagedCount = 0; + for await (const entry of await fs.opendir(remoteParent)) { + expect(await fs.readFile(path.join(remoteParent, entry.name), "utf8")).toBe("base"); + stagedCount++; + } + expect(stagedCount).toBe(fileCount); + for (const name of unusual) expect(await fs.readFile(path.join(remote, name), "utf8")).toBe(name); + const manifestFile = isPathManifest(snapshot.overlayPaths) ? snapshot.overlayPaths.filePath : ""; + await runtime.cleanupWorkspaceSnapshot(); + await expect(fs.stat(manifestFile)).rejects.toMatchObject({ code: "ENOENT" }); + await fs.rm(path.join(parent, "late.js")); } - expect(stagedCount).toBe(40_000); - for (const name of unusual) expect(await fs.readFile(path.join(remote, name), "utf8")).toBe(name); - const manifestFile = isPathManifest(snapshot.overlayPaths) ? snapshot.overlayPaths.filePath : ""; - await runtime.cleanupWorkspaceSnapshot(); - await expect(fs.stat(manifestFile)).rejects.toMatchObject({ code: "ENOENT" }); - await fs.rm(path.join(parent, "late.js")); await fs.writeFile(path.join(repo, ".gitignore"), "*.js\n"); const ignored = await scan(); @@ -146,10 +151,13 @@ it("streams and stages all four real Git filename lanes above 32 MiB through the await fs.rm(parent, { recursive: true }); const deleted = await scan(); checkLargeLane(deleted.deletedPaths); - await stage(deleted); - let remaining = 0; - for await (const _entry of await fs.opendir(remoteParent)) remaining++; - expect(remaining).toBe(0); + if (stageFiles) { + await stage(deleted); + let remaining = 0; + for await (const _entry of await fs.opendir(remoteParent)) remaining++; + expect(remaining).toBe(0); + expect(commands.length).toBeGreaterThan(0); + } expect(commands.every((command) => command.length < 16 * 1024)).toBe(true); expect(scheduler.snapshot()).toMatchObject({ activeCount: 0, queuedCount: 0, inFlightCount: 0, cacheBytes: 0 }); }, realGitTimeoutMs);