diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0f234d2553c..7bd904c5316 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -30,6 +30,75 @@ updates: update-types: - "version-update:semver-major" + # Sandbox providers (packages/plugins/sandbox-providers/*). pnpm-workspace.yaml + # keeps them out of the root workspace on purpose, so the "/" entry above never + # sees them. Each provider is a standalone published package with its own + # version ranges. + # + # Each lockfile has exactly one entry, so every pull request changes a + # manifest and the lockfile that installs it together, and CI can pass. + # + # scripts/typecheck-sandbox-providers is a small pnpm workspace that lists every + # provider without its own lockfile, and holds one lockfile for them. CI + # installs that lockfile frozen. Dependabot updates this workspace as one unit: + # one pull request per dependency changes every provider that uses the + # dependency, and the shared lockfile, together. So this entry needs no + # `groups`. One entry per provider would open a separate pull request for each + # provider for the same package, and each one would fail CI because the shared + # lockfile would stay old. + # + # Dependabot does not expand a workspace glob that starts with ../, so that + # workspace lists each provider by its full path. A test checks the list. + # + # Providers are published libraries, so Dependabot's default versioning + # strategy keeps a range when a new version fits it. It widens the range when a + # release falls outside it, for example zod 3 to zod 4. + - package-ecosystem: npm + directory: "/scripts/typecheck-sandbox-providers" + schedule: + interval: weekly + day: monday + time: "06:00" + open-pull-requests-limit: 10 + labels: + - "dependencies" + ignore: + # Same reason as the "/" entry. + - dependency-name: "@types/node" + update-types: + - "version-update:semver-major" + + # Daytona is the only provider with its own lockfile. The release build and the + # Daytona end-to-end job install it frozen, so the shared workspace above leaves + # Daytona out, and scripts/typecheck-test-programs.mjs installs Daytona from + # this lockfile (LOCKED_PROVIDERS). This entry is the only one that changes + # Daytona's package.json, and it changes this lockfile in the same pull request. + - package-ecosystem: npm + directory: "/packages/plugins/sandbox-providers/daytona" + schedule: + interval: weekly + day: monday + time: "06:00" + open-pull-requests-limit: 5 + labels: + - "dependencies" + ignore: + # Same reason as the "/" entry. + - dependency-name: "@types/node" + update-types: + - "version-update:semver-major" + + # No entry for packages/paperclip-runner/scripts/pi-distribution or + # kimi-distribution. Every direct dependency there is a pinned provider + # release, and the pins feed each provider's ACPX commandDigest. The Runner + # checks compare the Pi lockfile with PI_DISTRIBUTION_PINS, and the Kimi build + # compares the SHA-256 of its package-lock.json with a pin. So a Dependabot + # change to either lockfile fails CI (for example #18998), and a real upgrade + # also starts saved sessions fresh. Change these by hand (see + # rules/acpx-profile-identity.mdc). pi-acp@0.0.33 needs zod ^3.25.0 as a + # direct dependency, so the Pi zod pin (3.25.76) cannot move to zod 4 until + # pi-acp allows it. GitHub security alerts still cover both lockfiles. + - package-ecosystem: github-actions directory: "/" schedule: diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index da29540ff9c..29108db1513 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1368,6 +1368,19 @@ Do not add a lockfile inside a provider. exists, which changes what a release resolves. Daytona already has one; the release build resolves it in place when it is stale. +Daytona is the one exception: it commits its own lockfile, so the workspace +leaves it out and the script installs it from that lockfile (`LOCKED_PROVIDERS`). +The same rule applies to it: do not refresh or commit that lockfile. The script +resolves a stale one in place for the run, and the lockfile refresh workflow +commits the new one after merge. + +Dependabot updates the providers through this workspace (see +`.github/dependabot.yml`), and Daytona through a second entry. Dependabot does +not expand a workspace glob that starts with `../`, so +`scripts/typecheck-sandbox-providers/pnpm-workspace.yaml` lists each provider by +its full path. When you add a provider, add it there and to `SANDBOX_PROVIDERS` +in `scripts/typecheck-test-programs.mjs`. A test checks that the two lists match. + #### Sandbox provider tests `pnpm test:run` does not reach the sandbox providers, because they are outside the diff --git a/packages/plugins/sandbox-providers/daytona/.npmrc b/packages/plugins/sandbox-providers/daytona/.npmrc new file mode 100644 index 00000000000..694168ff403 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/.npmrc @@ -0,0 +1,5 @@ +# pnpm reads this file for installs in this directory. Without it, Dependabot +# uses the root .npmrc (auto-install-peers=false). That rewrites this lockfile, +# which was made with pnpm's default (true), and frozen installs fail. +# npm and pnpm never publish .npmrc, so this file does not ship. +auto-install-peers=true diff --git a/scripts/typecheck-sandbox-providers/.npmrc b/scripts/typecheck-sandbox-providers/.npmrc new file mode 100644 index 00000000000..020baa78cfa --- /dev/null +++ b/scripts/typecheck-sandbox-providers/.npmrc @@ -0,0 +1,4 @@ +# pnpm reads this file for installs in this workspace. Without it, Dependabot +# uses the root .npmrc (auto-install-peers=false). That rewrites this lockfile, +# which was made with pnpm's default (true), and the frozen install in CI fails. +auto-install-peers=true diff --git a/scripts/typecheck-sandbox-providers/package.json b/scripts/typecheck-sandbox-providers/package.json index c26c721beb8..9f62b83b09c 100644 --- a/scripts/typecheck-sandbox-providers/package.json +++ b/scripts/typecheck-sandbox-providers/package.json @@ -2,6 +2,7 @@ "name": "@paperclipai/typecheck-sandbox-providers", "private": true, "description": "Installs sandbox provider dependencies for scripts/typecheck-test-programs.mjs. See doc/DEVELOPING.md.", + "packageManager": "pnpm@9.15.4", "engines": { "node": ">=24.11.0" } diff --git a/scripts/typecheck-sandbox-providers/pnpm-workspace.yaml b/scripts/typecheck-sandbox-providers/pnpm-workspace.yaml index 8a7199ba45e..1e32ae3d8ac 100644 --- a/scripts/typecheck-sandbox-providers/pnpm-workspace.yaml +++ b/scripts/typecheck-sandbox-providers/pnpm-workspace.yaml @@ -2,6 +2,21 @@ # commit a lockfile: scripts/build-standalone-public-packages.mjs uses a frozen # install when one exists. This workspace pins their dependencies in one lockfile # for scripts/typecheck-test-programs.mjs, without changing how they are released. +# +# List each provider by its full path. Do not use a glob such as +# ../../packages/plugins/sandbox-providers/*. Dependabot updates this workspace +# (see .github/dependabot.yml), and it does not expand a glob that starts with +# ../, so it would not see the providers. Daytona is not listed: it has its own +# lockfile and its own Dependabot entry, and scripts/typecheck-test-programs.mjs +# installs it from that lockfile (LOCKED_PROVIDERS). A test checks that this list +# matches SANDBOX_PROVIDERS without LOCKED_PROVIDERS. packages: - - ../../packages/plugins/sandbox-providers/* + - ../../packages/plugins/sandbox-providers/boxd + - ../../packages/plugins/sandbox-providers/cloudflare - ../../packages/plugins/sandbox-providers/cloudflare/bridge-template + - ../../packages/plugins/sandbox-providers/createos + - ../../packages/plugins/sandbox-providers/e2b + - ../../packages/plugins/sandbox-providers/exe-dev + - ../../packages/plugins/sandbox-providers/kubernetes + - ../../packages/plugins/sandbox-providers/modal + - ../../packages/plugins/sandbox-providers/novita diff --git a/scripts/typecheck-test-programs.mjs b/scripts/typecheck-test-programs.mjs index c073bac93c1..3d5dcc26e2d 100644 --- a/scripts/typecheck-test-programs.mjs +++ b/scripts/typecheck-test-programs.mjs @@ -6,7 +6,8 @@ // in PROGRAMS reads them without emitting. Sandbox providers are outside the // workspace, so the root install has none of their dependencies. They install from // typecheck-sandbox-providers/pnpm-lock.yaml here, because their SDK types are part -// of what a test checks. +// of what a test checks. A provider with its own lockfile (LOCKED_PROVIDERS) installs +// from that lockfile instead, so each lockfile has one owner and one Dependabot entry. // // The script then fails if a tracked test file is in no program. tsc decides that, // not a glob: `include` silently drops foo.test.tsx when foo.test.ts exists. @@ -37,6 +38,10 @@ export const SANDBOX_PROVIDERS = [ "packages/plugins/sandbox-providers/novita", ]; +// Providers that commit their own pnpm-lock.yaml. The release build and the Daytona +// end-to-end job install it frozen, so the shared workspace leaves them out. +export const LOCKED_PROVIDERS = ["packages/plugins/sandbox-providers/daytona"]; + export const PROGRAMS = [ "packages/db/tsconfig.test.json", "packages/kv-demo-mcp-server/tsconfig.test.json", @@ -60,10 +65,12 @@ const PREPARE_STEPS = [ ]; // Replacing a provider's standalone node_modules would otherwise ask for confirmation. const INSTALL_ARGS = ["install", "--frozen-lockfile", "--ignore-scripts", "--prefer-offline", "--config.confirmModulesPurge=false"]; +const LOCKED_INSTALL_ARGS = ["install", "--ignore-workspace", ...INSTALL_ARGS.slice(1)]; // Contributors do not commit provider lockfiles: the lockfile refresh workflow regenerates them after // merge, as it does the root pnpm-lock.yaml. Until then a changed manifest leaves the lockfile stale, // so resolve it in place, like the root install in CI. export const RESOLVE_ARGS = ["install", "--resolution-only", "--ignore-scripts", "--no-frozen-lockfile"]; +const LOCKED_RESOLVE_ARGS = ["install", "--ignore-workspace", ...RESOLVE_ARGS.slice(1)]; export function findUncheckedTests({ trackedFiles, checkedFiles, checkedElsewhere = CHECKED_ELSEWHERE }) { const checked = new Set(checkedFiles); @@ -168,21 +175,27 @@ export function typecheckTestPrograms({ const status = pnpm(args, { cwd: root, env }); if (status !== 0) return status; } - const workspace = path.join(root, PROVIDER_WORKSPACE); - let installed = pnpm(INSTALL_ARGS, { cwd: workspace, env }); - if (installed !== 0) { - log( - `typecheck:test-programs: ${PROVIDER_WORKSPACE}/pnpm-lock.yaml does not match the provider manifests. ` + - `Resolving it in place. Do not commit it: the lockfile refresh workflow updates it after merge.`, - ); - installed = pnpm(RESOLVE_ARGS, { cwd: workspace, env }); - if (installed === 0) installed = pnpm(INSTALL_ARGS, { cwd: workspace, env }); + const install = (directory, frozenArgs, resolveArgs) => { + const cwd = path.join(root, directory); + let status = pnpm(frozenArgs, { cwd, env }); + if (status !== 0) { + log( + `typecheck:test-programs: ${directory}/pnpm-lock.yaml does not match its manifests. ` + + `Resolving it in place. Do not commit it: the lockfile refresh workflow updates it after merge.`, + ); + status = pnpm(resolveArgs, { cwd, env }); + if (status === 0) status = pnpm(frozenArgs, { cwd, env }); + } + if (status !== 0) logError(`typecheck:test-programs: could not install the sandbox providers in ${directory}`); + return status; + }; + const installed = install(PROVIDER_WORKSPACE, INSTALL_ARGS, RESOLVE_ARGS); + if (installed !== 0) return installed; + for (const provider of LOCKED_PROVIDERS) { + const status = install(provider, LOCKED_INSTALL_ARGS, LOCKED_RESOLVE_ARGS); + if (status !== 0) return status; } - if (installed !== 0) { - logError(`typecheck:test-programs: could not install the sandbox providers in ${PROVIDER_WORKSPACE}`); - return installed; - } - // The install removes the SDK link from each provider's node_modules. + // The installs remove the SDK link from each provider's node_modules. linkPlugins(); let exitCode = 0; diff --git a/scripts/typecheck-test-programs.test.mjs b/scripts/typecheck-test-programs.test.mjs index 9e82a0a929e..3fb020597fa 100644 --- a/scripts/typecheck-test-programs.test.mjs +++ b/scripts/typecheck-test-programs.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { execFileSync, spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; @@ -8,6 +8,7 @@ import { fileURLToPath } from "node:url"; import { checkCoverage, findUncheckedTests, + LOCKED_PROVIDERS, PROGRAMS, PROVIDER_WORKSPACE, RESOLVE_ARGS, @@ -63,6 +64,22 @@ test("every program exists, and the provider list matches the tracked provider p assert.deepEqual(manifests.map((manifest) => path.dirname(manifest)).sort(), [...SANDBOX_PROVIDERS].sort()); }); +test("the provider workspace lists every provider without its own lockfile by full path, so Dependabot sees them", () => { + const workspace = readFileSync(path.join(root, PROVIDER_WORKSPACE, "pnpm-workspace.yaml"), "utf8"); + const listed = [...workspace.matchAll(/^\s+-\s+(\S+)\s*$/gm)].map((match) => match[1]); + assert.ok(listed.every((entry) => !entry.includes("*")), "Dependabot does not expand a glob that starts with ../"); + const providers = listed.map((entry) => path.posix.normalize(path.posix.join(PROVIDER_WORKSPACE, entry))); + const unlocked = SANDBOX_PROVIDERS.filter((provider) => !LOCKED_PROVIDERS.includes(provider)); + assert.deepEqual(providers.sort(), unlocked.sort()); +}); + +test("each provider with its own lockfile is a provider and has that lockfile", () => { + for (const provider of LOCKED_PROVIDERS) { + assert.ok(SANDBOX_PROVIDERS.includes(provider), provider); + assert.ok(existsSync(path.join(root, provider, "pnpm-lock.yaml")), provider); + } +}); + test("no provider commits a lockfile, except the one Daytona already has", () => { const tracked = execFileSync("git", ["ls-files", "packages/plugins/sandbox-providers"], { cwd: root, encoding: "utf8" }); assert.deepEqual( @@ -105,26 +122,65 @@ test("providers install frozen without scripts, then relink the SDK", () => { assert.equal(calls.linked, 1); }); -test("a stale provider lockfile is resolved in place, then installed frozen", () => { - const workspace = path.join("/repo", PROVIDER_WORKSPACE); - let frozenAttempts = 0; - const { options, calls } = harness({ - pnpm: (args, { cwd }) => { - calls.pnpm.push({ args, cwd }); - if (cwd !== workspace || !args.includes("--frozen-lockfile")) return 0; - frozenAttempts += 1; - return frozenAttempts === 1 ? 1 : 0; - }, - }); +test("a locked provider installs frozen from its own lockfile, outside any workspace", () => { + const { options, calls } = harness(); assert.equal(typecheckTestPrograms(options), 0); - assert.deepEqual( - calls.pnpm.filter(({ cwd }) => cwd === workspace).map(({ args }) => args.includes("--frozen-lockfile") ? "frozen" : args), - ["frozen", RESOLVE_ARGS, "frozen"], - ); + for (const provider of LOCKED_PROVIDERS) { + const install = calls.pnpm.find(({ cwd }) => cwd === path.join("/repo", provider)); + assert.ok(install.args.includes("--ignore-workspace")); + assert.ok(install.args.includes("--frozen-lockfile")); + assert.ok(install.args.includes("--ignore-scripts")); + } +}); + +// Each directory's first frozen install fails; resolution and the second frozen install pass. +const staleOnce = (directory, calls) => { + const cwd = path.join("/repo", directory); + let frozenAttempts = 0; + return (args, options) => { + calls.pnpm.push({ args, cwd: options.cwd }); + if (options.cwd !== cwd || !args.includes("--frozen-lockfile")) return 0; + frozenAttempts += 1; + return frozenAttempts === 1 ? 1 : 0; + }; +}; +const installSteps = (calls, directory) => + calls.pnpm + .filter(({ cwd }) => cwd === path.join("/repo", directory)) + .map(({ args }) => (args.includes("--frozen-lockfile") ? "frozen" : args)); + +test("a stale provider lockfile is resolved in place, then installed frozen", () => { + const { options, calls } = harness(); + options.pnpm = staleOnce(PROVIDER_WORKSPACE, calls); + assert.equal(typecheckTestPrograms(options), 0); + assert.deepEqual(installSteps(calls, PROVIDER_WORKSPACE), ["frozen", RESOLVE_ARGS, "frozen"]); assert.ok(RESOLVE_ARGS.includes("--resolution-only") && RESOLVE_ARGS.includes("--ignore-scripts")); assert.equal(calls.tsc.length, 2); }); +test("a stale locked-provider lockfile is resolved in place, outside any workspace", () => { + const [provider] = LOCKED_PROVIDERS; + const { options, calls } = harness(); + options.pnpm = staleOnce(provider, calls); + assert.equal(typecheckTestPrograms(options), 0); + const [, resolve] = installSteps(calls, provider); + assert.deepEqual(installSteps(calls, provider), ["frozen", resolve, "frozen"]); + for (const flag of ["--ignore-workspace", "--resolution-only", "--ignore-scripts", "--no-frozen-lockfile"]) { + assert.ok(resolve.includes(flag), flag); + } + assert.equal(calls.tsc.length, 2); +}); + +test("a locked provider install that still fails stops before any program runs", () => { + const [provider] = LOCKED_PROVIDERS; + const { options, calls } = harness({ + pnpm: (args, { cwd }) => (cwd === path.join("/repo", provider) ? 1 : 0), + }); + assert.equal(typecheckTestPrograms(options), 1); + assert.equal(calls.tsc.length, 0); + assert.ok(calls.errors[0].includes(provider), calls.errors[0]); +}); + test("a provider install that still fails stops before any program runs", () => { const { options, calls } = harness({ pnpm: (args, { cwd }) => (cwd === path.join("/repo", PROVIDER_WORKSPACE) ? 1 : 0),