From 066c8244b5efba84335f400f2f191cc5a6850560 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 15 Nov 2026 05:57:44 -0600 Subject: [PATCH] perf(release): build canary daemons during source verification and keep their dependency cache (#21554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Every master push runs a canary that publishes npm packages and reaches staging > - Only one canary runs at a time, so a canary's duration sets how often merged work ships > - A canary takes about 70 min. Two parts of it are avoidable waiting: the Runner daemons build only after the source proof, and darwin-x64 often rebuilds every dependency crate > - This pull request starts the daemon builds with the canary and keeps their dependency cache across Runner changes > - The benefit is about 15-25 min less per canary, with no change to what must pass before publication ## Linked Issues or Issue Description **What happened?** In successful canaries on 2026-11-13 the critical path was: `Reuse exact-source verification` 11-14 min, then `Release daemon (darwin-x64)` 10-28 min (9 runs: 10, 16, 18, 20, 20, 21, 24, 24, 28), then `publish_canary` 27-37 min. The daemon builds waited for the source proof, although they only build and upload artifacts. The darwin-x64 build was 16.4 min with a cache miss and 3.9 min with a hit. The native cache key included a hash of `packages/paperclip-runner/src/**`, `protocol/**` and `*.json`, but the cache holds dependency crates only (`cache-workspace-crates: false`). So every Runner change caused a full dependency rebuild. **Expected behavior** Daemons build in parallel with the source wait. The dependency cache changes only when dependencies, Cargo configuration, host or target change. Refs #21399, #16028 ## What Changed - `release.yml`: `release_runner_source` no longer needs `verify_canary`. On a canonical master push it starts at once. `publish_canary` still requires `verify_canary.result == 'success'`, so nothing publishes from unverified source. - `release-verify.yml`: the native Rust cache `shared-key` drops the Runner source hash. It keeps the Cargo manifest and configuration, host and target. - `release-verify-workflow.test.mjs`: tests that a canonical master push selects the source without the proof, that a fork, other ref or other event does not, that `publish_canary` still needs the proof, and that the cache key contains no source globs. - `doc/RELEASING.md`: describes both behaviors. ## Verification - `node --test scripts/__tests__/release-verify-workflow.test.mjs`: 73 of 73 pass. - Red: the same tests against master's `release.yml` and `release-verify.yml` fail 2 tests (the source selection and the cache key). - After merge: the next canary shows `Release daemon (*)` starting at the same time as `Reuse exact-source verification`, and darwin-x64 restores its cache. ## Risks - Low. If verification fails, that commit's daemon builds are wasted macOS minutes. Verification failures on master are now rare. - The first canary after merge saves a new cache entry, so it is not faster for darwin-x64. Later canaries are. - Stable, beta and nightly still select their source through their own gates, unchanged. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, in Claude Code with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .github/workflows/release-verify.yml | 2 +- .github/workflows/release.yml | 4 ++-- doc/RELEASING.md | 8 ++++++++ .../release-verify-workflow.test.mjs | 20 +++++++++++++++---- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index e69c8c9b86f..77d34faa0d3 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -693,7 +693,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 with: workspaces: ${{ steps.native_rust_workspace.outputs.path }} -> target - shared-key: release-native-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }}-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }} + shared-key: release-native-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }}-${{ matrix.host }}-${{ matrix.target }} cache-workspace-crates: false cache-bin: false cache-on-failure: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f64604741c7..940cfd8c29f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -345,10 +345,10 @@ jobs: run: node scripts/cloud-source-verification.mjs "$SOURCE_SHA" release_runner_source: - needs: [verify_canary, select_nightly, select_beta, preflight_stable] + needs: [select_nightly, select_beta, preflight_stable] if: >- !cancelled() && ( - (github.event_name == 'push' && needs.verify_canary.result == 'success') || + (github.event_name == 'push' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master') || (needs.select_nightly.result == 'success' && needs.select_nightly.outputs.proceed == 'true') || needs.select_beta.result == 'success' || needs.preflight_stable.result == 'success') runs-on: ubuntu-latest diff --git a/doc/RELEASING.md b/doc/RELEASING.md index ac2b1a02405..68230fcb677 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -129,6 +129,14 @@ and the next push publishes. If a canary is missing after a quiet period, look at the failed `Reuse exact-source verification` job. It names the readiness run and lists each failed job as a lost runner or a failure. +A canary builds its Runner daemons (`release_runner_binaries`) while it waits +for the source proof. Those jobs only build and upload artifacts. +`publish_canary` still requires `verify_canary`, so nothing is published from +unverified source. If verification fails, the daemon builds for that commit are +discarded. The native daemon build cache holds dependency crates only. Its key +covers the Cargo manifest, Cargo configuration, host and target, but not the +Runner's own source, so a Runner change does not discard the dependencies. + After publication, the onboarding smoke checks out the same source SHA and installs its test dependencies from the publisher's resolved lockfile with `--frozen-lockfile`. The same-run artifact is captured before tracked files are diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 1eebb910c7e..5b4d4aaa1fd 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -934,7 +934,18 @@ test("release daemon production uses exact-source native hosts and no publisher const release = readWorkflow("release.yml"); const source = release.split(" release_runner_source:\n")[1].split(/\n [a-z_]+:\n/)[0]; const binaries = release.split(" release_runner_binaries:\n")[1].split(/\n [a-z_]+:\n/)[0]; - assert.match(source, /needs: \[verify_canary, select_nightly, select_beta, preflight_stable\]/); + assert.match(source, /needs: \[select_nightly, select_beta, preflight_stable\]/); + const sourceCondition = source.match(/^ if: >-\n((?: {6}.+\n)+)/m)?.[1].replace(/\s+/g, " ").trim(); + assert.ok(sourceCondition, "release_runner_source needs a condition"); + const pushGithub = { event_name: "push", repository: "paperclipai/paperclip", repository_id: "1170821064", ref: "refs/heads/master" }; + const skippedLanes = { select_nightly: { result: "skipped" }, select_beta: { result: "skipped" }, preflight_stable: { result: "skipped" } }; + const selectsSource = (github) => runInNewContext(sourceCondition, { cancelled: () => false, github, needs: skippedLanes }); + assert.equal(selectsSource(pushGithub), true, "a canary builds its daemons without waiting for the source proof"); + for (const change of [{ repository: "someone/paperclip" }, { repository_id: "1" }, { ref: "refs/heads/feature" }, { event_name: "schedule" }]) { + assert.equal(selectsSource({ ...pushGithub, ...change }), false, JSON.stringify(change)); + } + const publishCanary = release.split(" publish_canary:\n")[1].split(/\n [a-z_]+:\n/)[0]; + assert.match(publishCanary, /needs\.verify_canary\.result == 'success'/, "publication still waits for the source proof"); assert.match(source, /!cancelled\(\)/); assert.match(binaries, /needs: release_runner_source/); assert.match(binaries, /ref: \$\{\{ needs\.release_runner_source\.outputs\.sha \}\}/); @@ -942,7 +953,7 @@ test("release daemon production uses exact-source native hosts and no publisher const condition = binaries.match(/^ if: (.+)$/m)?.[1].replace(/^\$\{\{\s*|\s*\}\}$/g, ""); assert.match(condition, /^!cancelled\(\) && needs\.release_runner_source\.result == 'success' && needs\.release_runner_source\.outputs\.required == 'true'$/, "Native compilation must bypass deliberately skipped channel ancestors and require successful exact-source selection"); - const laneJobs = ["verify_canary", "select_nightly", "select_beta", "preflight_stable"]; + const laneJobs = ["select_nightly", "select_beta", "preflight_stable"]; for (const activeLane of laneJobs) { for (const [result, required, cancelled, expected] of [ ["success", "true", false, true], ["failure", "true", false, false], @@ -952,7 +963,7 @@ test("release daemon production uses exact-source native hosts and no publisher ]) { const needs = Object.fromEntries(laneJobs.map(job => [job, { result: job === activeLane ? "success" : "skipped" }])); needs.release_runner_source = { result, outputs: { required } }; - assert.equal(Object.values(needs).filter(job => job.result === "skipped").length, result === "skipped" ? 4 : 3); + assert.equal(Object.values(needs).filter(job => job.result === "skipped").length, result === "skipped" ? 3 : 2); assert.equal(runInNewContext(condition, { cancelled: () => cancelled, needs }), expected, `${activeLane}: source=${result}, required=${required}, cancelled=${cancelled}`); } @@ -987,7 +998,8 @@ test('native dependency caches are target-bound and unmerged master callers cann assert.match(cache, /workspaces: \$\{\{ steps.native_rust_workspace.outputs.path \}\} -> target/); const key = cache.match(/^\s*shared-key: (.+)$/m)?.[1]; assert.ok(key?.startsWith("release-native-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', ")); - assert.ok(key?.endsWith("-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }}")); + assert.ok(key?.endsWith("-${{ matrix.host }}-${{ matrix.target }}")); + assert.doesNotMatch(key, /protocol\/\*\*|src\/\*\*/, "only dependency crates are cached, so runner source must not invalidate them"); for (const option of ['cache-workspace-crates', 'cache-bin', 'cache-on-failure']) { assert.match(cache, new RegExp(option + ': false')); }