diff --git a/.github/cloud-migrator-deploy/cloudfront-read-statement.json b/.github/cloud-migrator-deploy/cloudfront-read-statement.json new file mode 100644 index 0000000000..cf36a18d2f --- /dev/null +++ b/.github/cloud-migrator-deploy/cloudfront-read-statement.json @@ -0,0 +1,14 @@ +{ + "Sid": "AllowCloudFrontReadCloudMigrators", + "Effect": "Allow", + "Principal": { + "Service": "cloudfront.amazonaws.com" + }, + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::paperclipai-runner-e2e-history-078455283791-us-east-1/cloud-migrators/v1/*", + "Condition": { + "StringEquals": { + "AWS:SourceArn": "arn:aws:cloudfront::078455283791:distribution/E3GTU28BBO2SFR" + } + } +} diff --git a/.github/cloud-migrator-deploy/trust-policy.json b/.github/cloud-migrator-deploy/trust-policy.json new file mode 100644 index 0000000000..db8d148aa3 --- /dev/null +++ b/.github/cloud-migrator-deploy/trust-policy.json @@ -0,0 +1,18 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::078455283791:oidc-provider/token.actions.githubusercontent.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + "token.actions.githubusercontent.com:sub": "repo:paperclipai/paperclip:ref:refs/heads/master" + } + } + } + ] +} diff --git a/.github/cloud-migrator-deploy/upload-policy.json b/.github/cloud-migrator-deploy/upload-policy.json new file mode 100644 index 0000000000..478ba9d36d --- /dev/null +++ b/.github/cloud-migrator-deploy/upload-policy.json @@ -0,0 +1,25 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::paperclipai-runner-e2e-history-078455283791-us-east-1/cloud-migrators/v1/*", + "Condition": { + "StringEquals": { + "s3:if-none-match": "*" + } + } + }, + { + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::paperclipai-runner-e2e-history-078455283791-us-east-1", + "Condition": { + "StringLike": { + "s3:prefix": "cloud-migrators/v1/*" + } + } + } + ] +} diff --git a/.github/workflows/cloud-migrator-artifacts.yml b/.github/workflows/cloud-migrator-artifacts.yml new file mode 100644 index 0000000000..f4da1b25b1 --- /dev/null +++ b/.github/workflows/cloud-migrator-artifacts.yml @@ -0,0 +1,90 @@ +name: Cloud migrator artifacts +run-name: Cloud migrator artifacts ${{ github.sha }} + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: {} +concurrency: + group: cloud-migrator-artifacts-${{ github.sha }} + cancel-in-progress: false + +jobs: + build: + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - name: Install migrator build dependencies + run: pnpm install --ignore-scripts --no-frozen-lockfile --filter @paperclipai/db... --filter @paperclipai/shared... + - name: Build exact-source packages and dependency lockfile + env: + SOURCE_SHA: ${{ github.sha }} + run: | + node scripts/preview-artifacts.mjs pack . migrator-artifacts "$SOURCE_SHA" + node scripts/cloud-migrator-artifacts.mjs build migrator-artifacts "$SOURCE_SHA" + node scripts/cloud-migrator-artifacts.mjs verify-install migrator-artifacts "$SOURCE_SHA" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: cloud-migrator-bundle + path: | + migrator-artifacts/db.tgz + migrator-artifacts/shared.tgz + migrator-artifacts/package-lock.json + migrator-artifacts/manifest.json + if-no-files-found: error + retention-days: 3 + + publish: + needs: build + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + id-token: write + attestations: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: scripts + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: cloud-migrator-bundle + path: migrator-artifacts + - name: Validate the complete bundle before attestation + env: + SOURCE_SHA: ${{ github.sha }} + run: node scripts/cloud-migrator-artifacts.mjs validate migrator-artifacts "$SOURCE_SHA" + - name: Attest the manifest and every content hash it pins + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: migrator-artifacts/manifest.json + - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 + with: + role-to-assume: arn:aws:iam::078455283791:role/paperclip-cloud-migrator-github + aws-region: us-east-1 + role-duration-seconds: 900 + - name: Publish immutable migrator and verify public downloads + env: + SOURCE_SHA: ${{ github.sha }} + run: node scripts/cloud-migrator-artifacts.mjs publish migrator-artifacts "$SOURCE_SHA" diff --git a/doc/preview-release-artifacts.md b/doc/preview-release-artifacts.md index 3ffec98887..4157882ef9 100644 --- a/doc/preview-release-artifacts.md +++ b/doc/preview-release-artifacts.md @@ -122,3 +122,62 @@ node scripts/preview-artifacts.mjs pack /path/to/source /path/to/output FULL_SHA This executes source build scripts. Keep output outside the repository and use an environment without publishing or cloud-admin credentials. + +## Direct cloud migrator artifacts + +`cloud-migrator-artifacts.yml` builds the DB and shared preview packages on each +canonical `master` push. A manual run also requires `master` and uses its exact +commit. This workflow runs on GitHub-hosted runners. It has no PR trigger. + +The build resolves a complete npm lockfile from the two local archives. It then +pins their download URLs to immutable, content-addressed objects. The cloud +migration runner can use `npm ci` with this lockfile before either new package +version is available on npm. Existing external dependencies still come from npm +and carry SHA-512 integrity pins. Package lifecycle scripts remain disabled. +Before upload, the build job smoke-installs the real archives and their complete +external and bundled dependency graph with an empty npm cache. It imports both +installed packages. This check uses local archive URLs because public objects +do not exist yet; all versions and integrity pins remain unchanged. + +Artifacts use the existing runner-history S3 bucket and CloudFront distribution, +under the separate `cloud-migrators/v1/` prefix. The manifest at +`https://d1p6rlowie26tp.cloudfront.net/cloud-migrators/v1//manifest.json` +records the full source SHA, exact preview version, and the size, URL, and SHA-512 +hash of each archive and the lockfile. Blob URLs include the content hash. +The publisher validates the complete bundle before any write, writes all blobs +before the manifest, and verifies downloads through the public endpoint. +A retry reuses a complete existing manifest after verification. The publisher +also creates a GitHub/Sigstore build-provenance attestation for the manifest +before upload. This independently binds all package and lockfile content hashes +to the canonical workflow, master ref, repository identity, and source commit. +Cloud must verify this signature and its certificate claims before accepting +the executable archives; hashes served by the artifact store alone are not +sufficient provenance. + +The build job has no AWS credential. The publish job downloads only the four +fixed files, validates them, and uploads them without executing their code. +The dedicated `paperclip-cloud-migrator-github` OIDC role trusts only +`repo:paperclipai/paperclip:ref:refs/heads/master`. Its policy permits prefix +listing and conditional `PutObject` calls in this one prefix. It permits no +object deletion or overwrite. PRs, including allowlisted PRs, cannot assume it. + +The deploy policies are checked in under `.github/cloud-migrator-deploy/`: + +- `trust-policy.json`: the role trust policy. +- `upload-policy.json`: the role's inline permission policy. +- `cloudfront-read-statement.json`: append this statement to the existing bucket + policy, preserving its other statements and public-access blocks. + +There is no lifecycle expiry on this prefix. Keep referenced artifacts for +rollback; deleting them can prevent a fresh migrator install for an old release. +This producer rollout is additive. npm preview publication and the current +cloud readiness gate remain active until the cloud consumer supports the new +manifest. A later cutover must preserve source verification, image identity, +migration compatibility, and the cloud runner's integrity checks. + +Local verification: + +```sh +node --test scripts/cloud-migrator-artifacts.test.mjs +node scripts/cloud-migrator-artifacts.mjs verify +``` diff --git a/package.json b/package.json index 9df886d7b7..8d1bdd7a0d 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/select-cloud-cache.test.mjs", + "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/select-cloud-cache.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/cloud-migrator-artifacts.mjs b/scripts/cloud-migrator-artifacts.mjs new file mode 100644 index 0000000000..6c097a584d --- /dev/null +++ b/scripts/cloud-migrator-artifacts.mjs @@ -0,0 +1,208 @@ +#!/usr/bin/env node +// The build job has no publish credential. The publisher only validates and +// uploads fixed data files; it never installs or executes package code. +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { copyFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { assertMetadata, tarManifest, versionFor } from "./preview-artifacts.mjs"; + +export const artifactBase = "https://d1p6rlowie26tp.cloudfront.net/cloud-migrators/v1"; +export const artifactBucket = "paperclipai-runner-e2e-history-078455283791-us-east-1"; +const prefix = "cloud-migrators/v1/"; +const names = ["db", "shared"]; +const maximumBytes = 32 * 1024 * 1024; +const integrityFor = (bytes) => `sha512-${createHash("sha512").update(bytes).digest("base64")}`; + +export function descriptor(bytes, extension) { + const hash = createHash("sha512").update(bytes).digest("hex"); + return { url: `${artifactBase}/blobs/${hash}.${extension}`, integrity: integrityFor(bytes), size: bytes.length }; +} + +function assertDescriptor(pin, extension) { + if (!pin || typeof pin.integrity !== "string" || !/^sha512-[A-Za-z0-9+/]{86}==$/.test(pin.integrity) || + !Number.isSafeInteger(pin.size) || pin.size <= 0 || pin.size > maximumBytes) throw new Error("Invalid artifact integrity or size."); + const digest = Buffer.from(pin.integrity.slice(7), "base64"); + if (digest.toString("base64") !== pin.integrity.slice(7) || pin.url !== `${artifactBase}/blobs/${digest.toString("hex")}.${extension}`) { + throw new Error("Artifact URL does not match its content hash and trusted origin."); + } +} + +export function assertManifest(manifest, sha) { + if (manifest?.version !== 1 || manifest.sourceSha !== sha || manifest.packageVersion !== versionFor(sha)) throw new Error("Artifact source identity mismatch."); + for (const name of names) assertDescriptor(manifest.packages?.[name], "tgz"); + assertDescriptor(manifest.lockfile, "json"); +} + +export function assertLockfile(lock, manifest) { + const version = manifest.packageVersion; + if (lock?.lockfileVersion !== 3 || !lock.packages || Array.isArray(lock.packages) || + JSON.stringify(lock.packages[""]?.dependencies) !== JSON.stringify({ "@paperclipai/db": version })) throw new Error("Invalid migrator lockfile root."); + for (const name of names) { + const pin = lock.packages[`node_modules/@paperclipai/${name}`]; + const expected = manifest.packages[name]; + if (pin?.version !== version || pin.integrity !== expected.integrity || pin.resolved !== expected.url || pin.link || pin.inBundle) throw new Error("Migrator lockfile package pin mismatch."); + } + if (lock.packages["node_modules/@paperclipai/db"].dependencies?.["@paperclipai/shared"] !== version) throw new Error("Migrator shared dependency mismatch."); + for (const [key, entry] of Object.entries(lock.packages)) { + if (key === "") continue; + if (!entry || typeof entry !== "object" || entry.link) throw new Error("Invalid migrator lockfile entry."); + if (/(?:^|\/)node_modules\/@paperclipai\/[^/]+$/.test(key) && !names.some((name) => key === `node_modules/@paperclipai/${name}`)) throw new Error("Unexpected internal migrator dependency."); + if (entry.inBundle === true) { + if (!key.startsWith("node_modules/@paperclipai/db/node_modules/")) throw new Error("Unexpected bundled dependency."); + continue; + } + if (!/^sha512-[A-Za-z0-9+/]{86}==$/.test(entry.integrity ?? "")) throw new Error("Migrator dependency has no strong integrity pin."); + if (names.some((name) => key === `node_modules/@paperclipai/${name}`)) continue; + const url = new URL(entry.resolved); + if (url.origin !== "https://registry.npmjs.org" || url.username || url.password || url.search || url.hash) throw new Error("Migrator dependency must resolve to npm."); + } +} + +export function buildBundle(directory, sha, { exec = execFileSync } = {}) { + versionFor(sha); + directory = path.resolve(directory); + const packages = {}; + for (const name of names) { + const bytes = readFileSync(path.join(directory, `${name}.tgz`)); + assertMetadata(tarManifest(bytes), `@paperclipai/${name}`, sha); + packages[name] = descriptor(bytes, "tgz"); + } + const scratch = mkdtempSync(path.join(os.tmpdir(), "cloud-migrator-lock-")); + try { + for (const name of names) copyFileSync(path.join(directory, `${name}.tgz`), path.join(scratch, `${name}.tgz`)); + const root = { name: "paperclip-migrator-install-root", version: "0.0.0", private: true, + dependencies: { "@paperclipai/db": "file:db.tgz", "@paperclipai/shared": "file:shared.tgz" } }; + writeFileSync(path.join(scratch, "package.json"), JSON.stringify(root)); + exec("npm", ["install", "--package-lock-only", "--ignore-scripts", "--no-audit", "--no-fund", "--registry=https://registry.npmjs.org"], { cwd: scratch, stdio: "inherit", timeout: 180_000 }); + const lock = JSON.parse(readFileSync(path.join(scratch, "package-lock.json"), "utf8")); + // Both new packages are local during resolution. npm ci subsequently uses + // these immutable URLs, without looking up the new npm versions. + lock.packages[""].dependencies = { "@paperclipai/db": versionFor(sha) }; + for (const name of names) lock.packages[`node_modules/@paperclipai/${name}`].resolved = packages[name].url; + const lockBytes = Buffer.from(JSON.stringify(lock) + "\n"); + const manifest = { version: 1, sourceSha: sha, packageVersion: versionFor(sha), packages, lockfile: descriptor(lockBytes, "json") }; + assertManifest(manifest, sha); + assertLockfile(lock, manifest); + writeFileSync(path.join(directory, "package-lock.json"), lockBytes); + writeFileSync(path.join(directory, "manifest.json"), JSON.stringify(manifest) + "\n"); + return manifest; + } finally { rmSync(scratch, { recursive: true, force: true }); } +} + +function verifyBytes(bytes, pin) { + if (bytes.length !== pin.size || integrityFor(bytes) !== pin.integrity) throw new Error("Artifact bytes do not match their immutable pin."); +} + +export function validateBundle(directory, sha) { + const manifest = JSON.parse(readFileSync(path.join(directory, "manifest.json"), "utf8")); + assertManifest(manifest, sha); + for (const name of names) { + const bytes = readFileSync(path.join(directory, `${name}.tgz`)); + verifyBytes(bytes, manifest.packages[name]); + assertMetadata(tarManifest(bytes), `@paperclipai/${name}`, sha); + } + const bytes = readFileSync(path.join(directory, "package-lock.json")); + verifyBytes(bytes, manifest.lockfile); + assertLockfile(JSON.parse(bytes), manifest); + return manifest; +} + +/** Exercise the real dependency graph before publishing, with no new npm versions. */ +export function verifyInstall(directory, sha, { exec = execFileSync } = {}) { + const manifest = validateBundle(directory, sha); + const scratch = mkdtempSync(path.join(os.tmpdir(), "cloud-migrator-install-")); + try { + const lock = JSON.parse(readFileSync(path.join(directory, "package-lock.json"), "utf8")); + for (const name of names) { + copyFileSync(path.join(directory, `${name}.tgz`), path.join(scratch, `${name}.tgz`)); + // The public objects do not exist yet. Only transport changes for this + // smoke install; exact versions, integrity, root and transitive pins stay. + lock.packages[`node_modules/@paperclipai/${name}`].resolved = `file:${name}.tgz`; + } + writeFileSync(path.join(scratch, "package.json"), JSON.stringify({ name: "paperclip-migrator-install-root", version: "0.0.0", private: true, + dependencies: { "@paperclipai/db": manifest.packageVersion } })); + writeFileSync(path.join(scratch, "package-lock.json"), JSON.stringify(lock)); + exec("npm", ["ci", "--ignore-scripts", "--no-audit", "--no-fund", "--update-notifier=false", "--cache", path.join(scratch, "empty-cache"), + "--registry=https://registry.npmjs.org"], { cwd: scratch, stdio: "inherit", timeout: 180_000 }); + for (const name of names) assertMetadata(JSON.parse(readFileSync(path.join(scratch, "node_modules", "@paperclipai", name, "package.json"), "utf8")), `@paperclipai/${name}`, sha); + exec(process.execPath, ["--input-type=module", "--eval", "await import('@paperclipai/db'); await import('@paperclipai/shared');"], { cwd: scratch, stdio: "inherit", timeout: 30_000 }); + } finally { rmSync(scratch, { recursive: true, force: true }); } +} + +async function download(url, fetchImpl) { + const response = await fetchImpl(url, { redirect: "error", signal: AbortSignal.timeout(60_000) }); + if (!response.ok) throw new Error(`Artifact download failed: HTTP ${response.status}`, { cause: { status: response.status } }); + const reader = response.body.getReader(); + const chunks = []; + let size = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + size += value.length; + if (size > maximumBytes) throw new Error("Artifact exceeds size limit."); + chunks.push(value); + } + } finally { await reader.cancel(); } + return Buffer.concat(chunks); +} + +export async function verifyPublished(sha, fetchImpl = fetch) { + versionFor(sha); + const manifest = JSON.parse(await download(`${artifactBase}/${sha}/manifest.json`, fetchImpl)); + assertManifest(manifest, sha); + await Promise.all(names.map(async (name) => { + const bytes = await download(manifest.packages[name].url, fetchImpl); + verifyBytes(bytes, manifest.packages[name]); + assertMetadata(tarManifest(bytes), `@paperclipai/${name}`, sha); + })); + const lock = await download(manifest.lockfile.url, fetchImpl); + verifyBytes(lock, manifest.lockfile); + assertLockfile(JSON.parse(lock), manifest); + return manifest; +} + +export async function publishBundle(directory, sha, { exec = execFileSync, fetchImpl = fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) } = {}) { + const manifest = validateBundle(directory, sha); + const key = `${prefix}${sha}/manifest.json`; + const verifyVisible = async () => { + for (let attempt = 0; ; attempt++) { + try { return await verifyPublished(sha, fetchImpl); } + catch (error) { + // A consumer may have cached a missing-object response just before + // publication. Wait through the CDN error TTL, never through bad bytes. + if (attempt >= 6 || ![403, 404].includes(error.cause?.status)) throw error; + await sleep(2_000); + } + } + }; + const aws = (args) => exec("aws", ["s3api", ...args, "--bucket", artifactBucket, "--region", "us-east-1"], { encoding: "utf8", maxBuffer: 1024 * 1024 }); + // Exact prefix listing distinguishes missing objects from permission errors. + const exists = (objectKey) => JSON.parse(aws(["list-objects-v2", "--prefix", objectKey, "--max-keys", "1"])).Contents?.some((object) => object.Key === objectKey); + if (exists(key)) return verifyVisible(); + const upload = (file, objectKey, contentType) => { + if (exists(objectKey)) return; + aws(["put-object", "--key", objectKey, "--body", path.resolve(directory, file), "--content-type", contentType, + "--cache-control", "public,max-age=31536000,immutable", "--if-none-match", "*"]); + }; + for (const name of names) upload(`${name}.tgz`, prefix + manifest.packages[name].url.slice(`${artifactBase}/`.length), "application/gzip"); + upload("package-lock.json", prefix + manifest.lockfile.url.slice(`${artifactBase}/`.length), "application/json"); + // Publish the commit marker last; readers can never observe a partial bundle. + upload("manifest.json", key, "application/json"); + return verifyVisible(); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [command, directory, sha] = process.argv.slice(2); + try { + if (command === "build") buildBundle(directory, sha); + else if (command === "validate") validateBundle(directory, sha); + else if (command === "verify-install") verifyInstall(directory, sha); + else if (command === "publish") await publishBundle(directory, sha); + else if (command === "verify") await verifyPublished(directory); + else throw new Error("Expected build, validate, verify-install, publish, or verify."); + } catch (error) { console.error(error.message); process.exitCode = 1; } +} diff --git a/scripts/cloud-migrator-artifacts.test.mjs b/scripts/cloud-migrator-artifacts.test.mjs new file mode 100644 index 0000000000..61374f3206 --- /dev/null +++ b/scripts/cloud-migrator-artifacts.test.mjs @@ -0,0 +1,144 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { createServer } from "node:http"; +import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { gzipSync } from "node:zlib"; +import { artifactBase, assertManifest, assertLockfile, buildBundle, descriptor, validateBundle, verifyPublished, publishBundle } from "./cloud-migrator-artifacts.mjs"; +import { previewManifest, versionFor } from "./preview-artifacts.mjs"; + +const sha = "a".repeat(40); +function fixture(t) { + const dir = mkdtempSync(path.join(os.tmpdir(), "migrator-artifact-test-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + for (const name of ["db", "shared"]) { + const bytes = Buffer.from(JSON.stringify(previewManifest({ name: `@paperclipai/${name}`, dependencies: {} }, sha))); + const header = Buffer.alloc(512); + header.write("package/package.json"); header.write(bytes.length.toString(8).padStart(11, "0"), 124, 11); header[156] = 48; + // A real tar header, so npm can install this fixture as well as inspect it. + header.fill(32, 148, 156); + const sum = header.reduce((a, b) => a + b, 0); + header.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, 8); + const padded = Buffer.alloc(Math.ceil(bytes.length / 512) * 512); bytes.copy(padded); + writeFileSync(path.join(dir, `${name}.tgz`), gzipSync(Buffer.concat([header, padded, Buffer.alloc(1024)]))); + } + const manifest = buildBundle(dir, sha, { exec: (cmd, args, options) => { + assert.equal(cmd, "npm"); assert.ok(args.includes("--ignore-scripts")); + const localRoot = JSON.parse(readFileSync(path.join(options.cwd, "package.json"))); + assert.deepEqual(localRoot.dependencies, { "@paperclipai/db": "file:db.tgz", "@paperclipai/shared": "file:shared.tgz" }); + const packages = { "": localRoot }; + for (const name of ["db", "shared"]) packages[`node_modules/@paperclipai/${name}`] = { + version: versionFor(sha), integrity: descriptor(readFileSync(path.join(dir, `${name}.tgz`)), "tgz").integrity, + resolved: `file:${name}.tgz`, ...(name === "db" ? { dependencies: { "@paperclipai/shared": versionFor(sha) } } : {}), + }; + writeFileSync(path.join(options.cwd, "package-lock.json"), JSON.stringify({ lockfileVersion: 3, packages })); + } }); + const files = new Map([[`${artifactBase}/${sha}/manifest.json`, readFileSync(path.join(dir, "manifest.json"))]]); + for (const name of ["db", "shared"]) files.set(manifest.packages[name].url, readFileSync(path.join(dir, `${name}.tgz`))); + files.set(manifest.lockfile.url, readFileSync(path.join(dir, "package-lock.json"))); + const fetchImpl = async (url, options) => { + assert.equal(options.redirect, "error"); assert.ok(options.signal); + assert.ok(files.has(url), `unexpected download: ${url}`); + return new Response(files.get(url)); + }; + return { dir, manifest, files, fetchImpl }; +} + +test("bundle pins the exact source pair and complete lockfile without new npm lookups", async (t) => { + const { dir, manifest, fetchImpl } = fixture(t); + assert.deepEqual(validateBundle(dir, sha), manifest); + assert.deepEqual(await verifyPublished(sha, fetchImpl), manifest); +}); + +test("source identity, content hashes, size, and origin fail closed", async (t) => { + const { dir, manifest, files, fetchImpl } = fixture(t); + for (const mutate of [ + (m) => { m.sourceSha = "b".repeat(40); }, + (m) => { m.packages.db.url = "https://evil.invalid/db.tgz"; }, + (m) => { m.packages.shared.size = 0; }, + (m) => { m.lockfile.integrity = "sha1-weak"; }, + ]) { + const bad = structuredClone(manifest); mutate(bad); assert.throws(() => assertManifest(bad, sha)); + } + files.set(manifest.packages.db.url, Buffer.from("corrupt")); + await assert.rejects(verifyPublished(sha, fetchImpl), /immutable pin/); + writeFileSync(path.join(dir, "db.tgz"), "corrupt"); + await assert.rejects(publishBundle(dir, sha, { exec: () => assert.fail("no upload before pair validation") })); +}); + +test("lockfile rejects mutable, foreign, linked, and mismatched dependencies", (t) => { + const { dir, manifest } = fixture(t); + const lock = JSON.parse(readFileSync(path.join(dir, "package-lock.json"))); + for (const mutate of [ + (l) => { l.packages[""].dependencies["@paperclipai/db"] = "latest"; }, + (l) => { l.packages["node_modules/@paperclipai/shared"].version = "0.0.0"; }, + (l) => { l.packages["node_modules/@paperclipai/db"].link = true; }, + (l) => { l.packages["node_modules/evil"] = { inBundle: true }; }, + (l) => { l.packages["node_modules/evil"] = { integrity: manifest.packages.db.integrity, resolved: "https://evil.invalid/pkg.tgz" }; }, + (l) => { l.packages["node_modules/evil"] = { integrity: "sha1-weak", resolved: "https://registry.npmjs.org/pkg.tgz" }; }, + (l) => { l.packages["node_modules/a/node_modules/@paperclipai/shared"] = l.packages["node_modules/@paperclipai/shared"]; }, + ]) { + const bad = structuredClone(lock); mutate(bad); assert.throws(() => assertLockfile(bad, manifest)); + } +}); + +test("publisher writes blobs first, marker last, and never overwrites existing objects", async (t) => { + const { dir, fetchImpl } = fixture(t); + const objects = new Set(); const uploads = []; + const exec = (cmd, args) => { + assert.equal(cmd, "aws"); + const arg = (key) => args[args.indexOf(key) + 1]; + if (args[1] === "list-objects-v2") return JSON.stringify({ Contents: objects.has(arg("--prefix")) ? [{ Key: arg("--prefix") }] : [] }); + assert.equal(args[1], "put-object"); assert.equal(arg("--if-none-match"), "*"); + objects.add(arg("--key")); uploads.push(arg("--key")); return "{}"; + }; + await publishBundle(dir, sha, { exec, fetchImpl }); + assert.equal(uploads.length, 4); assert.equal(uploads.at(-1), `cloud-migrators/v1/${sha}/manifest.json`); + await publishBundle(dir, sha, { exec, fetchImpl }); assert.equal(uploads.length, 4); +}); + +test("download failures and oversized objects never count as available", async (t) => { + fixture(t); + for (const status of [403, 404, 500]) await assert.rejects(verifyPublished(sha, async () => new Response(null, { status })), /download failed/); + await assert.rejects(verifyPublished(sha, async () => new Response(Buffer.alloc(32 * 1024 * 1024 + 1))), /size limit/); +}); + +test("real npm ci installs the new pair from pinned archives with an empty cache", async (t) => { + const { dir } = fixture(t); + // Real npm resolution uses local archives; neither package version exists on npm. + const manifest = buildBundle(dir, sha); + const lock = JSON.parse(readFileSync(path.join(dir, "package-lock.json"))); + const requests = []; + const server = createServer((req, res) => { + requests.push(req.url); + if (!["/db.tgz", "/shared.tgz"].includes(req.url)) { res.writeHead(500); res.end(); return; } + res.end(readFileSync(path.join(dir, req.url.slice(1)))); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + const base = `http://127.0.0.1:${server.address().port}`; + for (const name of ["db", "shared"]) lock.packages[`node_modules/@paperclipai/${name}`].resolved = `${base}/${name}.tgz`; + writeFileSync(path.join(dir, "package.json"), JSON.stringify({ name: "paperclip-migrator-install-root", version: "0.0.0", private: true, dependencies: { "@paperclipai/db": versionFor(sha) } })); + writeFileSync(path.join(dir, "package-lock.json"), JSON.stringify(lock)); + await promisify(execFile)("npm", ["ci", "--update-notifier=false", "--ignore-scripts", "--no-audit", "--no-fund", "--registry", base, "--cache", path.join(dir, "empty-cache")], { cwd: dir, timeout: 60_000 }); + assert.deepEqual(requests.sort(), ["/db.tgz", "/shared.tgz"]); + for (const name of ["db", "shared"]) assert.equal(JSON.parse(readFileSync(path.join(dir, `node_modules/@paperclipai/${name}/package.json`))).version, manifest.packageVersion); +}); + +test("AWS trust is master-only and publication policy cannot overwrite objects", () => { + const read = (name) => JSON.parse(readFileSync(new URL(`../.github/cloud-migrator-deploy/${name}.json`, import.meta.url))); + assert.equal(read("trust-policy").Statement[0].Condition.StringEquals["token.actions.githubusercontent.com:sub"], "repo:paperclipai/paperclip:ref:refs/heads/master"); + const policy = read("upload-policy").Statement; + assert.deepEqual(policy.map((s) => s.Action), ["s3:PutObject", "s3:ListBucket"]); + assert.equal(policy[0].Condition.StringEquals["s3:if-none-match"], "*"); + const workflow = readFileSync(new URL("../.github/workflows/cloud-migrator-artifacts.yml", import.meta.url), "utf8"); + assert.ok(!workflow.includes("pull_request") && !workflow.includes("self-hosted") && !workflow.includes("runs-on/fleet=")); + assert.equal((workflow.match(/id-token: write/g) ?? []).length, 1); + assert.equal((workflow.match(/attestations: write/g) ?? []).length, 1); + assert.ok(workflow.indexOf(" validate migrator-artifacts") < workflow.indexOf("uses: actions/attest@")); + assert.ok(workflow.indexOf("uses: actions/attest@") < workflow.indexOf(" publish migrator-artifacts")); + assert.ok(workflow.indexOf(" verify-install migrator-artifacts") < workflow.indexOf("actions/upload-artifact@"), "the real dependency smoke must pass before artifact upload"); +});