From 058c55bf8f9beb74b2d07c0854172e1d4454ded1 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Mon, 14 Sep 2026 23:57:12 -0700 Subject: [PATCH] fix(ci): overlap preview migrator publication waits (#13454) ## Thinking Path > - Paperclip manages AI agents and their work. > - Cloud deploys an application image with a migrator from the same source commit. > - The migrator consists of the DB package and its matching shared package. > - npm can accept a package several minutes before readers can see it. > - The publisher waits for shared visibility before it submits the DB package. > - This change submits both validated packages before polling their visibility. > - Their availability delays can then overlap while the final gate still requires both packages. ## Linked Issues or Issue Description Related: #13334. No duplicate open migrator-publication fix was found. **What happened?** The cloud migrator publisher waits up to ten minutes for the shared package before submitting the DB package. The two registry delays therefore accumulate. A shared visibility timeout can prevent the DB package from being submitted at all. **Expected behavior** Submit both valid packages, then wait for both to pass the existing exact-source metadata checks. Report which package remains unavailable. **Steps to reproduce** Return 404 for shared metadata after npm accepts the shared archive. Observe that the old publisher never submits the DB archive until shared becomes visible. The new regression test requires both submissions before the first visibility sleep. **Paperclip version** Base commit 08adcc70d5ec45b7ced9619a3dc10c1d1bec397d. GitHub Actions cloud-migrator publication. ## What Changed - Validate both package archives before either publication starts. - Submit missing packages before polling their visibility. - Report each visible package and name packages missing at timeout. - Cover delayed visibility, partial reuse, and invalid package pairs. - Document the publication order. ## Verification - Focused preview tests: 19 passed. - Release registry tests: 132 passed. - `pnpm -r typecheck`: passed. - `pnpm build`: passed. - All 33 latest-head GitHub checks are green or intentionally skipped. Greptile is 5/5 with no unresolved findings. - The local full Vitest run found three failures in unchanged company-skills cache tests. A standalone filesystem test reproduces this macOS host rejecting a read-only directory rename with EACCES. Those tests pass in Linux CI. The local full run continues. ## Risks The DB package can become visible before its shared dependency. The publisher still requires both to pass before succeeding. This removes avoidable serialization; it does not eliminate npm propagation delays or prove archive downloadability. Those remain separate parts of the migrator availability work. ## Model Used OpenAI GPT-6 through Codex, with reasoning, repository tools, and code execution. The exact serving model ID and context window are not exposed by this environment. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass (focused release tests; the full-suite macOS limitation is documented above) - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- doc/preview-release-artifacts.md | 6 ++++ scripts/preview-artifacts.mjs | 27 +++++++++++---- scripts/preview-artifacts.test.mjs | 53 ++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/doc/preview-release-artifacts.md b/doc/preview-release-artifacts.md index d7a5fe5279..3ffec98887 100644 --- a/doc/preview-release-artifacts.md +++ b/doc/preview-release-artifacts.md @@ -37,6 +37,12 @@ or existing package identity mismatches fail the workflow. Retries reuse matchin published artifacts, including a shared package published before a DB publish failure. Allow npm's visibility polling to finish before retrying. +The publisher validates both package archives first, then submits each missing +package without waiting for the other to become visible. One visibility poll +checks both accepted packages, so their registry propagation delays overlap. +The publisher succeeds only after both packages pass the identity and +distribution-pin checks. A visibility timeout names the package still missing. + The final `stack-deploy-result` artifact contains `result.json` with contract version 1, request ID, SHA, stage `build`, and status `ready`. It expires after 30 days. This confirms artifact availability; it does not certify a tenant deploy. diff --git a/scripts/preview-artifacts.mjs b/scripts/preview-artifacts.mjs index 6a6eff6fb2..3eeb68996e 100644 --- a/scripts/preview-artifacts.mjs +++ b/scripts/preview-artifacts.mjs @@ -165,22 +165,35 @@ export function packPreview(source, output, sha, { exec = execFileSync } = {}) { } export async function publishPreview(dir, sha, { fetchImpl = fetch, exec = execFileSync, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) { - for (const short of ["shared", "db"]) { + // Validate the entire pair before publishing either immutable package. + const packages = ["shared", "db"].map((short) => { const name = `@paperclipai/${short}`; const file = path.resolve(dir, `${short}.tgz`); const bytes = readFileSync(file); assertMetadata(tarManifest(bytes), name, sha); + return { name, file, bytes }; + }); + const pending = new Set(); + for (const { name, file, bytes } of packages) { if (await packageExists(name, sha, fetchImpl)) { console.log(`Reusing ${name}@${versionFor(sha)}`); continue; } console.log(`Publishing ${name}@${versionFor(sha)} (${createHash("sha256").update(bytes).digest("hex").slice(0, 12)})`); // No package checkout, lifecycle scripts, npmrc, or branch code runs here. exec("npm", ["publish", file, "--tag", "preview", "--access", "public", "--ignore-scripts", "--provenance", "--registry", "https://registry.npmjs.org"], { stdio: "inherit" }); - let published = false; - for (let attempt = 0; attempt < 60; attempt++) { - if (await packageExists(name, sha, fetchImpl)) { published = true; break; } - await sleep(10_000); - } - if (!published) throw new Error("npm accepted the preview but it is not yet visible. Retry reuses published packages."); + pending.add(name); } + // npm accepts a package without resolving its dependencies. Submit both + // packages before waiting so their registry propagation can overlap. + for (let attempt = 0; pending.size && attempt < 60; attempt++) { + const checks = await Promise.all([...pending].map(async (name) => ({ name, visible: await packageExists(name, sha, fetchImpl) }))); + for (const { name, visible } of checks) { + if (visible) { + pending.delete(name); + console.log(`Visible ${name}@${versionFor(sha)}`); + } + } + if (pending.size) await sleep(10_000); + } + if (pending.size) throw new Error(`npm accepted the preview but it is not yet visible: ${[...pending].join(", ")}. Retry reuses published packages.`); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/scripts/preview-artifacts.test.mjs b/scripts/preview-artifacts.test.mjs index ebdd9a6122..711423533f 100644 --- a/scripts/preview-artifacts.test.mjs +++ b/scripts/preview-artifacts.test.mjs @@ -97,6 +97,59 @@ test("publishing reuses existing previews and never executes package lifecycle h } finally { rmSync(dir, { recursive: true, force: true }); } }); +test("publishing submits both packages before waiting for either to propagate", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "preview-publish-overlap-")); + const submitted = []; + let polls = 0; + try { + for (const short of ["shared", "db"]) writeFileSync(path.join(dir, `${short}.tgz`), pack(manifest(`@paperclipai/${short}`))); + await publishPreview(dir, sha, { + exec: (_command, args) => submitted.push(path.basename(args[1], ".tgz")), + fetchImpl: async (url) => { + const name = decodeURIComponent(new URL(url).pathname.split("/")[1]); + // Both packages become visible after the first shared visibility wait. + return submitted.length === 2 && polls > 0 + ? json({ ...manifest(name), dist: { integrity: "test-integrity", tarball: "https://registry.npmjs.org/package.tgz" } }) + : json({}, 404); + }, + sleep: async () => { assert.deepEqual(submitted, ["shared", "db"]); polls++; }, + }); + assert.deepEqual(submitted, ["shared", "db"]); + assert.equal(polls, 1); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("a visibility timeout identifies the missing package after both were submitted", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "preview-publish-timeout-")); + const submitted = []; + try { + for (const short of ["shared", "db"]) writeFileSync(path.join(dir, `${short}.tgz`), pack(manifest(`@paperclipai/${short}`))); + await assert.rejects(publishPreview(dir, sha, { + exec: (_command, args) => submitted.push(path.basename(args[1], ".tgz")), + fetchImpl: async (url) => { + const name = decodeURIComponent(new URL(url).pathname.split("/")[1]); + return name === "@paperclipai/db" && submitted.includes("db") + ? json({ ...manifest(name), dist: { integrity: "test-integrity", tarball: "https://registry.npmjs.org/package.tgz" } }) + : json({}, 404); + }, + sleep: async () => {}, + }), /not yet visible: @paperclipai\/shared\./); + assert.deepEqual(submitted, ["shared", "db"]); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("invalid DB package metadata prevents publication of either package", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "preview-publish-invalid-")); + try { + writeFileSync(path.join(dir, "shared.tgz"), pack(manifest("@paperclipai/shared"))); + writeFileSync(path.join(dir, "db.tgz"), pack({ ...manifest("@paperclipai/db"), gitHead: "b".repeat(40) })); + await assert.rejects(publishPreview(dir, sha, { + exec: () => assert.fail("Invalid package pairs must not be published"), + fetchImpl: async () => assert.fail("Validate the pair before registry requests"), + }), /identity or dependency pin mismatch/); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + test("preview workflow separates branch compilation from trusted publishing", () => { const workflow = readFileSync(new URL("../.github/workflows/release.yml", import.meta.url), "utf8"); const builder = workflow.split(" package_preview:")[1].split(" publish_preview:")[0];