Reconcile work-folder foundation with current master

Keep current task continuation, chat UI, provider security, and multi-architecture build behavior. Regenerate the work-folder schema after the published migration sequence and make it replay-safe for preview databases. Verify preserved file/trash/checkpoint references and company constraints on PostgreSQL.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-11 14:12:19 -05:00
commit f03b8e7097
1325 files changed
+1488090 -52241

No files matched your search

@@ -0,0 +1,50 @@
// Run before building, and again inside the protected deployment job on reruns.
module.exports = async function authorizeStorybookDeploy({ github, context }) {
const fail = (message) => { throw new Error(message); };
if (context.repo.owner !== "paperclipai" || context.repo.repo !== "paperclip") {
fail("Storybook publishing is restricted to paperclipai/paperclip.");
}
if (context.eventName !== "workflow_dispatch" || !context.ref.startsWith("refs/heads/")) {
fail("Storybook publishing requires a manual run from a repository branch.");
}
// The selected branch must never be able to add itself to the allowlist.
const { data: repository } = await github.rest.repos.get(context.repo);
const { data: file } = await github.rest.repos.getContent({
...context.repo,
path: ".github/CODEOWNERS",
ref: repository.default_branch,
});
if (file.encoding !== "base64" || typeof file.content !== "string") {
fail("Cannot read the default branch CODEOWNERS file.");
}
const owners = new Set();
for (const line of Buffer.from(file.content, "base64").toString("utf8").split(/\r?\n/)) {
const fields = line.split("#", 1)[0].trim().split(/\s+/);
for (const owner of fields.slice(1)) {
// Individual GitHub accounts only. Teams/email entries do not grant access.
if (/^@[a-z\d](?:[a-z\d-]*[a-z\d])?$/i.test(owner)) {
owners.add(owner.slice(1).toLowerCase());
}
}
}
if (owners.size === 0) fail("CODEOWNERS has no individual GitHub accounts.");
for (const actor of [context.actor, process.env.GITHUB_TRIGGERING_ACTOR]) {
if (!actor || !owners.has(actor.toLowerCase())) {
fail(`Only default-branch CODEOWNERS may publish Storybook (${actor || "missing actor"}).`);
}
}
// A branch can edit its workflow. Require a GitHub-enforced CODEOWNER review
// as well, so editing this check cannot grant an outsider deployment access.
const { data: environment } = await github.rest.repos.getEnvironment({
...context.repo,
environment_name: "storybook-deploy",
});
const reviewers = environment.protection_rules
?.find((rule) => rule.type === "required_reviewers")?.reviewers;
if (environment.can_admins_bypass !== false || !reviewers?.length ||
reviewers.some(({ type, reviewer }) => type !== "User" || !owners.has(reviewer.login.toLowerCase()))) {
fail("storybook-deploy must require CODEOWNER reviewers and disable administrator bypass.");
}
};
+46
View File
@@ -0,0 +1,46 @@
const { execFileSync } = require('node:child_process');
const fs = require('node:fs');
const path = require('node:path');
const { storybookDestination, branchIndex } = require('./storybook-destination.cjs');
const destination = storybookDestination({
branch: process.env.SOURCE_BRANCH, sha: process.env.SOURCE_SHA,
runId: process.env.GITHUB_RUN_ID, runAttempt: process.env.GITHUB_RUN_ATTEMPT,
bucket: process.env.STORYBOOK_S3_BUCKET, baseUrl: process.env.STORYBOOK_PUBLIC_BASE_URL,
});
const source = path.resolve('storybook-static');
// Treat the artifact as public files, never as executable publisher code.
function validateTree(dir) {
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
if (entry.isSymbolicLink() || entry.name.startsWith('.') || (!entry.isDirectory() && !entry.isFile())) {
throw new Error(`Unsupported artifact entry: ${path.join(dir, entry.name)}`);
}
if (entry.isDirectory()) validateTree(path.join(dir, entry.name));
}
}
validateTree(source);
for (const name of ['index.html', 'iframe.html', 'index.json']) {
if (!fs.statSync(path.join(source, name)).isFile() || fs.statSync(path.join(source, name)).size === 0) {
throw new Error(`Missing Storybook output: ${name}`);
}
}
fs.writeFileSync(path.join(source, 'deployment.json'), JSON.stringify(destination, null, 2) + '\n');
const aws = (args) => execFileSync('aws', args, { stdio: 'inherit' });
// Complete a unique build before changing the branch's entry point. No deletion
// permissions, shared root writes or mixed-version branch assets are needed.
aws(['s3', 'cp', source, `s3://${destination.bucket}/${destination.buildPrefix}/`,
'--recursive', '--no-follow-symlinks', '--only-show-errors',
'--cache-control', 'public,max-age=31536000,immutable']);
const indexFile = path.join(process.env.RUNNER_TEMP, 'storybook-branch-index.html');
fs.writeFileSync(indexFile, branchIndex(destination.buildUrl));
aws(['s3', 'cp', indexFile, `s3://${destination.bucket}/${destination.prefix}/index.html`,
'--content-type', 'text/html; charset=utf-8', '--cache-control', 'no-cache,max-age=0,must-revalidate', '--only-show-errors']);
aws(['s3', 'cp', indexFile, `s3://${destination.bucket}/${destination.bookmarkPrefix}/index.html`,
'--content-type', 'text/html; charset=utf-8', '--cache-control', 'no-cache,max-age=0,must-revalidate', '--only-show-errors']);
const report = `[Branch Storybook](${destination.url})\n\n[This build](${destination.buildUrl})\n\nCommit: \`${destination.sha}\`\n`;
const reportPath = path.join(process.env.RUNNER_TEMP, 'storybook-deployment.md');
fs.writeFileSync(reportPath, report);
if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT,
`url=${destination.url}\nbuild_url=${destination.buildUrl}\nreport_path=${reportPath}\n`);
if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, report);
console.log(JSON.stringify(destination));
+47
View File
@@ -0,0 +1,47 @@
const { createHash } = require('node:crypto');
function storybookDestination({ branch, sha, runId, runAttempt, bucket, baseUrl }) {
if (typeof branch !== 'string' || !branch || /[\x00-\x20\x7f]/.test(branch)) {
throw new Error('A non-empty repository branch name is required.');
}
if (!/^[a-f0-9]{40}$/.test(sha)) throw new Error('A full source commit SHA is required.');
if (![runId, runAttempt].every((value) => /^[1-9]\d*$/.test(String(value)))) {
throw new Error('A valid workflow run and attempt are required.');
}
if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket)) throw new Error('Invalid Storybook S3 bucket.');
const base = new URL(baseUrl);
if (base.protocol !== 'https:' || base.username || base.password || base.search || base.hash || base.pathname !== '/') {
throw new Error('Storybook base URL must be a credential-free HTTPS origin.');
}
const label = branch.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0,60) || 'branch';
const digest = createHash('sha256').update(branch).digest('hex').slice(0,16);
const branchKey = `${label}-${digest}`;
const prefix = `storybook/branches/${branchKey}`;
const buildPrefix = `${prefix}/builds/${runId}-${runAttempt}`;
// Use one reversible path segment: slashes and special characters become
// ~HH UTF-8 bytes, so feature/foo and feature-foo never share a bookmark.
let bookmarkKey = [...Buffer.from(branch)].map((byte) =>
/[A-Za-z0-9_-]/.test(String.fromCharCode(byte))
? String.fromCharCode(byte) : `~${byte.toString(16).toUpperCase().padStart(2, '0')}`).join('');
// Reserve the existing hashed directories, including all immutable builds.
bookmarkKey = bookmarkKey.replace(/-([a-f0-9]{16})$/, '~2D$1');
// Keep arbitrarily long ref names within S3's object-key limit. ~long cannot
// occur in the reversible encoding, whose escapes contain only hex digits.
if (bookmarkKey.length > 900) bookmarkKey = `${bookmarkKey.slice(0, 800)}~long-${digest}`;
const bookmarkPrefix = `storybook/branches/${bookmarkKey}`;
return {
branch, sha, bucket, branchKey, prefix, buildPrefix, bookmarkPrefix,
url: `${base.origin}/${bookmarkPrefix}/`,
legacyUrl: `${base.origin}/${prefix}/index.html`,
buildUrl: `${base.origin}/${buildPrefix}/index.html`,
};
}
function branchIndex(buildUrl) {
// The target is generated from a validated origin and ASCII path segments.
const target = JSON.stringify(buildUrl).replace(/</g, '\\u003c');
return `<!doctype html><html lang="en"><meta charset="utf-8"><title>Storybook preview</title>
<script>const target = new URL(${target}); target.search = location.search; target.hash = location.hash; location.replace(target.href);</script>
<noscript><a href="${buildUrl}">Open this branch's Storybook</a></noscript></html>\n`;
}
module.exports = { storybookDestination, branchIndex };
@@ -0,0 +1,91 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { waitForCloudArtifacts } from "../../../scripts/cloud-readiness.mjs";
import { previewManifest } from "../../../scripts/preview-artifacts.mjs";
const sha = "a".repeat(40);
const digest = `sha256:${"b".repeat(64)}`;
const json = (body, status = 200) => new Response(JSON.stringify(body), { status });
function registry({ missing = new Set(), failure, wrongImage = false, wrongPackage = false } = {}) {
return async (url) => {
if (failure) return json({}, failure);
if (url.startsWith("https://registry.npmjs.org/")) {
const name = decodeURIComponent(new URL(url).pathname.split("/")[1]);
if (missing.has(name.split("/")[1])) return json({}, 404);
const pkg = previewManifest({ name, version: "0.0.0" }, sha);
return json({ ...pkg, ...(wrongPackage ? { gitHead: "c".repeat(40) } : {}), dist: { integrity: "sha512-fixture", tarball: "https://registry.npmjs.org/fixture.tgz" } });
}
if (url.includes("/token?")) return json({ token: "fixture" });
if (url.includes("/manifests/")) return missing.has("image") ? json({}, 404) : json({ config: { digest } });
if (url.includes("/blobs/")) return json({ config: { Labels: { "org.opencontainers.image.revision": wrongImage ? "c".repeat(40) : sha } } });
throw new Error(`Unexpected request: ${url}`);
};
}
test("readiness requires the image and both exact-source packages on the successful poll", async () => {
const missing = new Set(["image", "shared", "db"]);
let clock = 0;
const states = [];
const result = await waitForCloudArtifacts(sha, {
fetchImpl: registry({ missing }), now: () => clock, intervalMs: 10, timeoutMs: 100, log: (message) => states.push(message),
sleep: async (ms) => {
clock += ms;
if (clock === 10) missing.delete("image");
if (clock === 20) missing.delete("shared");
if (clock === 30) { missing.delete("db"); missing.add("image"); }
if (clock === 40) missing.delete("image");
},
});
assert.equal(clock, 40, "an artifact disappearing before the final poll must prevent readiness");
assert.deepEqual(result, { version: 1, sha, packageVersion: `0.0.0-preview.g${sha}` });
assert.match(states.at(-1), /Cloud artifacts available/);
});
test("missing artifacts time out with a precise inventory and bounded sleep", async () => {
let clock = 0;
const sleeps = [];
await assert.rejects(waitForCloudArtifacts(sha, {
fetchImpl: registry({ missing: new Set(["db"]) }), now: () => clock, timeoutMs: 25, intervalMs: 20, log: () => {},
sleep: async (ms) => { sleeps.push(ms); clock += ms; },
}), /timed out.*missing: db/);
assert.deepEqual(sleeps, [20, 5]);
});
for (const fixture of [{ failure: 403 }, { failure: 503 }, { wrongImage: true }, { wrongPackage: true }]) {
test(`registry errors and identity mismatches fail without waiting: ${JSON.stringify(fixture)}`, async () => {
await assert.rejects(waitForCloudArtifacts(sha, {
fetchImpl: registry(fixture), sleep: async () => assert.fail("must not retry an invalid artifact or upstream error"), log: () => {},
}));
});
}
test("invalid source and timing configuration are rejected before registry access", async () => {
const fetchImpl = async () => assert.fail("invalid inputs must not reach a registry");
await assert.rejects(waitForCloudArtifacts("master", { fetchImpl }), /full immutable commit SHA/);
for (const options of [{ timeoutMs: 0 }, { intervalMs: -1 }, { timeoutMs: Infinity }]) {
await assert.rejects(waitForCloudArtifacts(sha, { ...options, fetchImpl }), /positive finite/);
}
});
test("the versioned readiness job requires successful source, image and artifact jobs", () => {
const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8");
assert.match(workflow, /push:\s*\n\s*branches: \[master\]/);
assert.match(workflow, /group: cloud-readiness-\$\{\{ github.sha \}\}/);
assert.match(workflow, /uses: \.\/\.github\/workflows\/release-verify.yml\s+with:\s+ref: \$\{\{ github.sha \}\}/);
assert.match(workflow, /uses: \.\/\.github\/workflows\/docker-cloud.yml/);
const ready = workflow.split(" ready:")[1];
assert.match(ready, /name: Cloud deployable v1/);
assert.match(ready, /needs: \[verify, image, artifacts\]/);
assert.match(ready, /if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'/);
assert.doesNotMatch(ready, /^\s*(?:if:.*always\(|continue-on-error:)/m);
assert.doesNotMatch(workflow, /secrets: inherit|id-token: write|actions: write|checks: write|uses: .*@v\d\b/);
const cloud = readFileSync(new URL("../../workflows/docker-cloud.yml", import.meta.url), "utf8");
assert.doesNotMatch(cloud, /^ push:/m, "the master image must build only once");
const migrator = readFileSync(new URL("../../workflows/cloud-artifacts.yml", import.meta.url), "utf8");
assert.match(migrator, /push:\s*\n\s*branches: \[master\]/);
assert.match(migrator, /SOURCE_SHA: \$\{\{ github.sha \}\}/);
assert.match(migrator, /gh workflow run release.yml .*--ref master/);
assert.match(migrator, /--field channel=cloud-migrator/);
assert.match(migrator, /--field source_ref="\$SOURCE_SHA"/);
});
@@ -0,0 +1,35 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { runInNewContext } from "node:vm";
const workflow = readFileSync(new URL("../../workflows/docker-cloud.yml", import.meta.url), "utf8");
// Exercise the workflow's actual boolean expression. Its string comparisons and
// boolean operators have the same results in JS for these canonical contexts.
const expression = workflow.match(/^ runs-on: \$\{\{ (.+) \}\}$/m)?.[1];
assert.ok(expression, "cloud routing must remain an explicit job expression");
const timeoutExpression = workflow.match(/^ timeout-minutes: \$\{\{ (.+) \}\}$/m)?.[1];
assert.ok(timeoutExpression, "AWS jobs must finish before the Fleet instance lifetime");
const fleet = "runs-on/fleet=paperclip-cloud-build-x64/env=public-ci";
const base = { repository: "paperclipai/paperclip", repository_id: "1170821064", ref: "refs/heads/master", event_name: "push" };
for (const { name, github = {}, enabled = "true", expected = "ubuntu-latest" } of [
{ name: "canonical master push", expected: fleet },
{ name: "manual master build", github: { event_name: "workflow_dispatch" }, expected: fleet },
{ name: "disabled switch", enabled: "false" },
{ name: "missing switch", enabled: "" },
{ name: "invalid switch", enabled: "yes" },
{ name: "fork", github: { repository: "someone/paperclip", repository_id: "123" } },
{ name: "wrong repository identity", github: { repository_id: "123" } },
{ name: "pull request", github: { event_name: "pull_request", ref: "refs/pull/123/merge" } },
{ name: "privileged PR event", github: { event_name: "pull_request_target" } },
{ name: "release tag", github: { ref: "refs/tags/v2026.911.0" } },
{ name: "branch push", github: { ref: "refs/heads/feature" } },
{ name: "manual branch build", github: { event_name: "workflow_dispatch", ref: "refs/heads/feature" } },
{ name: "workflow completion event", github: { event_name: "workflow_run" } },
]) {
test(`cloud runner routing: ${name}`, () => {
const context = { github: { ...base, ...github }, vars: { AWS_CLOUD_BUILDS_ENABLED: enabled } };
assert.equal(runInNewContext(expression, context), expected);
assert.equal(runInNewContext(timeoutExpression, context), expected === fleet ? 40 : 60);
});
}
@@ -0,0 +1,65 @@
import test from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { spawnSync } from "node:child_process";
const workflow = readFileSync(new URL("../../workflows/docker-cloud.yml", import.meta.url), "utf8");
const step = workflow.split(" - name: Free runner disk")[1].split(" - name: Login to GitHub Container Registry")[0];
const script = step.split(" run: |\n")[1].split("\n").map((line) => line.replace(/^ {10}/, "")).join("\n");
const threshold = 64 * 1024 * 1024;
for (const { name, dockerFree, workspaceFree, dfStatus = "0", infoStatus = "0", cleanup } of [
{ name: "ample free space", dockerFree: threshold + 1, workspaceFree: threshold + 1, cleanup: false },
{ name: "exactly the headroom threshold", dockerFree: threshold, workspaceFree: threshold, cleanup: false },
{ name: "Docker filesystem below threshold", dockerFree: threshold - 1, workspaceFree: threshold + 1, cleanup: true },
{ name: "workspace filesystem below threshold", dockerFree: threshold + 1, workspaceFree: threshold - 1, cleanup: true },
{ name: "invalid Docker measurement", dockerFree: "unknown", workspaceFree: threshold + 1, cleanup: true },
{ name: "invalid workspace measurement", dockerFree: threshold + 1, workspaceFree: "unknown", cleanup: true },
{ name: "failed df command", dockerFree: threshold + 1, workspaceFree: threshold + 1, dfStatus: "1", cleanup: true },
{ name: "failed Docker inspection", dockerFree: threshold + 1, workspaceFree: threshold + 1, infoStatus: "1", cleanup: true },
]) {
test(`cloud disk cleanup: ${name}`, () => {
const dir = mkdtempSync(path.join(tmpdir(), "cloud-disk-test-"));
const log = path.join(dir, "commands.log");
// Every mutating command is a recording fixture; no real SDKs, caches,
// images, or directories are deleted when the workflow shell executes.
const fixture = `#!/bin/bash
printf '%s %s\\n' "\${0##*/}" "$*" >> "$COMMAND_LOG"
case "\${0##*/}" in
df)
printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\\n'
if [ "$1" = '-Pk' ]; then
printf '/dev/docker 200000000 1 %s 1%% /docker\\n' "$DOCKER_FREE"
printf '/dev/workspace 200000000 1 %s 1%% /workspace\\n' "$WORKSPACE_FREE"
exit "$DF_STATUS"
fi
;;
docker)
if [ "$1" = 'info' ]; then
printf '/docker-data\\n'
exit "$INFO_STATUS"
fi
;;
esac
`;
try {
for (const command of ["df", "docker", "pnpm", "sudo"]) {
writeFileSync(path.join(dir, command), fixture, { mode: 0o755 });
}
const result = spawnSync("bash", ["-c", script], {
encoding: "utf8",
env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, GITHUB_WORKSPACE: "/workspace", COMMAND_LOG: log,
DOCKER_FREE: String(dockerFree), WORKSPACE_FREE: String(workspaceFree), DF_STATUS: dfStatus, INFO_STATUS: infoStatus },
});
assert.equal(result.status, 0, result.stderr);
const commands = readFileSync(log, "utf8");
if (infoStatus === "0") assert.match(commands, /df -Pk \/docker-data \/workspace/);
assert.equal(commands.includes("pnpm store prune"), cleanup);
assert.equal(commands.includes("sudo rm -rf /usr/share/dotnet"), cleanup);
assert.equal(commands.includes("docker system prune -af"), cleanup);
assert.equal(result.stdout.includes("skipping cleanup"), !cleanup);
} finally { rmSync(dir, { recursive: true, force: true }); }
});
}
@@ -6,6 +6,7 @@ const workflows = [
'.github/workflows/refresh-lockfile.yml',
'.github/workflows/pr-trusted.yml',
'.github/workflows/docker.yml',
'.github/workflows/docker-cloud.yml',
];
test('lockfile repair workflows resolve dependencies instead of updating metadata only', async () => {
@@ -0,0 +1,37 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
const workflow = readFileSync(new URL("../../workflows/release-verify.yml", import.meta.url), "utf8");
const runner = workflow.split(" verify_paperclip_runner:")[1].split(" build:")[0];
test("Runner dependency caching selects the package's pinned compiler before computing its key", () => {
const select = runner.indexOf(" - name: Select the pinned Runner Rust toolchain");
const cache = runner.indexOf(" - name: Cache Runner Rust dependencies");
assert.ok(select >= 0 && cache > select);
const setup = runner.slice(select, cache);
assert.match(setup, /working-directory: packages\/paperclip-runner/);
assert.match(setup, /rustup show active-toolchain/);
assert.match(setup, /echo "RUSTUP_TOOLCHAIN=\$toolchain" >> "\$GITHUB_ENV"/);
assert.match(runner, /uses: Swatinem\/rust-cache@[0-9a-f]{40} # v[0-9.]+/);
assert.match(runner, /workspaces: packages\/paperclip-runner\/runner -> target/);
assert.match(runner, /shared-key: release-runner-v1/);
});
test("the shared cache excludes workspace artifacts and only restores or saves the exact master-push source", () => {
assert.match(runner, /cache-workspace-crates: false/);
assert.match(runner, /cache-bin: false/);
const saveIf = runner.match(/^\s*save-if: (.+)$/m)?.[1];
assert.equal(saveIf, "${{ github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha }}");
const cacheStep = runner.split(" - name: Cache Runner Rust dependencies")[1].split(" - name: Install dependencies")[0];
assert.equal(cacheStep.match(/^\s*if: (.+)$/m)?.[1], saveIf);
assert.doesNotMatch(runner, /cache-on-failure: true|cache-all-crates: true/);
});
test("cache hits cannot bypass Runner verification", () => {
const verify = runner.split(" - name: Verify Paperclip Runner")[1];
assert.match(verify, /run: pnpm --filter @paperclipai\/paperclip-runner check:all/);
assert.doesNotMatch(verify, /if:|continue-on-error:/);
assert.ok(runner.indexOf("Cache Runner Rust dependencies") < runner.indexOf(" - name: Verify Paperclip Runner\n"));
assert.doesNotMatch(runner, /id-token: write|packages: write|secrets: inherit/);
});
+26
View File
@@ -0,0 +1,26 @@
const { branchIndex } = require('./storybook-destination.cjs');
async function verifyStorybook({ branchUrl, buildUrl, sha, fetch = globalThis.fetch,
sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), attempts = 6 }) {
for (let attempt = 1; attempt <= attempts; attempt++) {
try {
const [metadata, index] = await Promise.all([
fetch(new URL('deployment.json', buildUrl), { signal: AbortSignal.timeout(15000) }),
fetch(branchUrl, { signal: AbortSignal.timeout(15000) }),
]);
if (!metadata.ok || !index.ok) throw new Error(`Public deployment returned HTTP ${metadata.status}/${index.status}.`);
if ((await metadata.json()).sha !== sha) throw new Error('Public build has the wrong source commit.');
if ((await index.text()) !== branchIndex(buildUrl)) throw new Error('Public branch URL does not point to this build.');
return;
} catch (error) {
if (attempt === attempts) throw error;
await sleep(10000);
}
}
}
module.exports = { verifyStorybook };
if (require.main === module) {
verifyStorybook({ branchUrl: process.env.BRANCH_URL, buildUrl: process.env.BUILD_URL,
sha: process.env.SOURCE_SHA }).catch((error) => { console.error(error); process.exitCode = 1; });
}
@@ -0,0 +1,10 @@
{
"Sid": "AllowCloudFrontReadStorybook",
"Effect": "Allow",
"Principal": {"Service": "cloudfront.amazonaws.com"},
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::paperclipai-runner-e2e-history-078455283791-us-east-1/storybook/branches/*",
"Condition": {"StringEquals": {
"AWS:SourceArn": "arn:aws:cloudfront::078455283791:distribution/E3GTU28BBO2SFR"
}}
}
@@ -0,0 +1,12 @@
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {"Federated": "arn:aws:iam::078455283791:oidc-provider/token.actions.githubusercontent.com"},
"Action": "sts:AssumeRoleWithWebIdentity",
"Condition": {"StringEquals": {
"token.actions.githubusercontent.com:aud": "sts.amazonaws.com",
"token.actions.githubusercontent.com:sub": "repo:paperclipai/paperclip:environment:storybook-deploy"
}}
}]
}
@@ -0,0 +1,8 @@
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:PutObject", "s3:GetObject", "s3:AbortMultipartUpload"],
"Resource": "arn:aws:s3:::paperclipai-runner-e2e-history-078455283791-us-east-1/storybook/branches/*"
}]
}
+34
View File
@@ -0,0 +1,34 @@
name: Cloud artifacts
on:
push:
branches: [master]
workflow_dispatch:
permissions: {}
jobs:
dispatch_migrator:
name: Start exact-source cloud migrator publication
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
actions: write
steps:
# This separate workflow starts at merge, outside the full npm release's
# concurrency group. Publication stays in release.yml so npm recognizes
# the established trusted-publisher identity and npm-canary environment.
# No source checkout or package code runs with the dispatch credential.
- name: Dispatch the migrator-only release
env:
GH_TOKEN: ${{ github.token }}
SOURCE_SHA: ${{ github.sha }}
run: |
set -euo pipefail
request_id="$(cat /proc/sys/kernel/random/uuid)"
gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref master \
--field channel=cloud-migrator \
--field source_ref="$SOURCE_SHA" \
--field request_id="$request_id"
echo "Started Cloud migrator $SOURCE_SHA in release.yml (request $request_id)." >> "$GITHUB_STEP_SUMMARY"
+93
View File
@@ -0,0 +1,93 @@
name: Cloud readiness
run-name: Cloud readiness ${{ github.sha }}
on:
push:
branches: [master]
workflow_dispatch:
permissions: {}
# Source verification must start outside the full npm release's queue.
concurrency:
group: cloud-readiness-${{ github.sha }}
cancel-in-progress: false
jobs:
image:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
permissions:
contents: read
packages: write
uses: ./.github/workflows/docker-cloud.yml
verify:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
permissions:
contents: read
uses: ./.github/workflows/release-verify.yml
with:
ref: ${{ github.sha }}
artifacts:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
name: Wait for exact-source cloud artifacts
runs-on: ubuntu-latest
timeout-minutes: 35
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Wait for verified image and exact-source migrator
env:
SOURCE_SHA: ${{ github.sha }}
run: node scripts/cloud-readiness.mjs "$SOURCE_SHA"
source_verified:
# npm canary publication reuses this exact-source verification proof.
# Keep it independent of image/migrator availability, and fail closed when
# any source check fails, is cancelled, or is skipped.
name: Cloud source verified v1
needs: [verify]
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Check the source verification consumer
run: node --test scripts/cloud-source-verification.test.mjs
- name: Record source verification
env:
SOURCE_SHA: ${{ github.sha }}
run: |
echo "Cloud source verified v1: $SOURCE_SHA" >> "$GITHUB_STEP_SUMMARY"
ready:
# Versioned consumer contract. Never add always() or continue-on-error:
# failed, cancelled, or skipped prerequisites must not report readiness.
name: Cloud deployable v1
needs: [verify, image, artifacts]
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Record cloud readiness
env:
SOURCE_SHA: ${{ github.sha }}
run: |
echo "Cloud deployable v1: $SOURCE_SHA" >> "$GITHUB_STEP_SUMMARY"
echo "Source verification passed; the full-SHA image and exact-source migrator are available." >> "$GITHUB_STEP_SUMMARY"
echo "Deployment tooling must still resolve and pin the image and migrator and validate migration compatibility." >> "$GITHUB_STEP_SUMMARY"
+300
View File
@@ -0,0 +1,300 @@
name: Docker cloud
on:
workflow_dispatch:
workflow_call:
permissions: {}
# Independent SHAs can build immediately on separate runners.
# Repeated requests for the same source serialize without cancelling a build.
# No mutable canary channel is promoted here; docker.yml owns that operation.
concurrency:
group: docker-cloud-${{ github.sha }}
cancel-in-progress: false
jobs:
build-and-push-cloud:
# Only canonical master builds can consume the release Fleet. The runner
# group must also allow this workflow only at refs/heads/master.
# Keep an operator switch for a full-run retry on GitHub-hosted runners.
runs-on: ${{ vars.AWS_CLOUD_BUILDS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-cloud-build-x64/env=public-ci' || 'ubuntu-latest' }}
# Fleet instances expire after 45 minutes, including bootstrap and cleanup.
timeout-minutes: ${{ vars.AWS_CLOUD_BUILDS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 40 || 60 }}
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# Full history and tags so `git describe` below can compute the
# release version to stamp into the image.
fetch-depth: 0
# `.git` is dockerignored, so a running image cannot derive its own
# version and otherwise reports the source package.json placeholder in
# analytics and the debug panel. Compute it here from the pristine
# checkout (real CalVer drift from the nearest release tag) and pass it
# into the build. Empty when no release tag is reachable — the server
# then keeps its existing fallbacks.
- name: Compute build version
id: build-version
run: |
set -euo pipefail
case "${GITHUB_REF}" in
refs/tags/nightly/v*)
# Lane tags carry the exact published version; stamp it verbatim
# instead of describing drift from the nearest stable tag.
version="${GITHUB_REF#refs/tags/nightly/v}"
;;
refs/tags/beta/v*)
version="${GITHUB_REF#refs/tags/beta/v}"
;;
*)
version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)"
;;
esac
echo "version=${version}" >> "$GITHUB_OUTPUT"
echo "Stamping build version: ${version:-<none>}"
# ISO week stamp for the Dockerfile's tool layer: the layer caches
# across commits and re-pulls the @latest CLI tools when the week rolls
# over, instead of on every build.
- name: Compute tool cache epoch
id: tools-epoch
run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
# Each SHA exports its own cache. Import recent first-parent caches so
# a late older build cannot overwrite a newer build's cache manifest.
# The legacy ref keeps the first builds warm during the transition.
- name: Select cloud cache ancestry
id: cloud-cache
env:
CACHE_IMAGE: ghcr.io/${{ github.repository }}
run: |
set -euo pipefail
{
echo 'sources<<CACHE_SOURCES'
for commit in $(git rev-list --first-parent --max-count=10 HEAD); do
echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud-$commit"
done
echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud"
echo 'CACHE_SOURCES'
} >> "$GITHUB_OUTPUT"
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
run_install: false
# No dependency cache here: this workflow publishes release images, and
# restoring a shared Actions cache into the build inputs would let a
# poisoned cache entry reach the published artifact.
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Refresh lockfile for Docker build context
run: |
set -euo pipefail
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
changed="$(git status --porcelain)"
if [ -z "$changed" ]; then
echo "Lockfile already matches package metadata."
exit 0
fi
if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then
echo "Unexpected files changed during lockfile refresh:"
echo "$changed"
exit 1
fi
echo "Using refreshed pnpm-lock.yaml in the Docker build context."
- name: Free runner disk
run: |
set -euo pipefail
echo "Disk before cleanup:"
df -h
# A measured hosted cloud build started with 86 GB available.
# Keep ample headroom for BuildKit and image verification, but
# avoid minutes deleting SDKs when neither filesystem needs space.
minimum_free_kib=$((64 * 1024 * 1024))
if docker_root="$(docker info --format '{{.DockerRootDir}}')" \
&& available_kib="$(df -Pk "$docker_root" "$GITHUB_WORKSPACE" | awk 'NR > 1 { rows++; if ($4 !~ /^[0-9]+$/) invalid = 1; if (min == "" || $4 < min) min = $4 } END { if (invalid || rows != 2) exit 1; print min }')" \
&& [[ "$available_kib" =~ ^[0-9]+$ ]] \
&& (( available_kib >= minimum_free_kib )); then
echo "At least 64 GiB is available for Docker and the workspace; skipping cleanup."
exit 0
fi
pnpm store prune || true
sudo apt-get clean || true
sudo rm -rf \
/usr/share/dotnet \
/usr/share/swift \
/usr/local/lib/android \
/usr/local/share/boost \
/usr/local/share/powershell \
/opt/ghc \
/opt/hostedtoolcache/CodeQL \
/opt/hostedtoolcache/PyPy \
/opt/hostedtoolcache/Ruby || true
docker system prune -af || true
echo "Disk after cleanup:"
df -h
- name: Login to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
# Deployment tooling reads these labels from the registry to verify an
# image's schema expectations against a migrator before deploying it,
# without pulling the image. The server refuses to start when the
# database is missing bundled migrations, so orchestrators need a cheap
# way to check image/migrator compatibility up front.
- name: Compute schema migration labels
id: schema
run: |
set -euo pipefail
last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
echo "last=${last}" >> "$GITHUB_OUTPUT"
echo "count=${count}" >> "$GITHUB_OUTPUT"
# Published under the same lane tag set as the self-hosted image, with a
# `-cloud` suffix (nightly-cloud, latest-cloud, <version>-cloud,
# sha-<short>-cloud). `:canary-cloud` follows the same retag-step
# ownership rule as `:canary` above.
- name: Docker meta (cloud)
id: meta-cloud
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
with:
images: ghcr.io/${{ github.repository }}
flavor: |
suffix=-cloud,onlatest=true
tags: |
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=sha
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Build and push (cloud)
id: build-cloud
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
target: cloud
# Space-separated sandbox-provider directory names to build into
# the variant; add here when managed deployments need another.
# CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages the
# variant installs from server/package.json's declared version;
# add another name there when a managed tenant needs it.
build-args: |
USER_UID=1001
USER_GID=1001
CLOUD_BUNDLED_PLUGINS=daytona
CLOUD_BUNDLED_SERVER_DEPS=@sentry/node
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
# amd64 only, unlike the self-hosted image above: the cloud variant
# is consumed exclusively by managed-deployment hosts, which run
# amd64. The QEMU-emulated arm64 half dominated this job's wall
# clock, and dropping it roughly halves time-to-deployable-image.
platforms: linux/amd64
push: true
# Same-SHA builds serialize above; different SHAs never share a
# writable cache ref. Registry layers are content-addressed and
# shared even when cache manifests have separate tags.
cache-from: ${{ steps.cloud-cache.outputs.sources }}
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud-${{ github.sha }},mode=max
tags: ${{ steps.meta-cloud.outputs.tags }}
labels: ${{ steps.meta-cloud.outputs.labels }}
# The cloud target installs @sentry/node at the version
# server/package.json declares, into a directory the server's own
# module resolution walks. Verify the image this job just pushed, not
# a local build, so a build-cache or layer-ordering regression is
# caught before any tenant runs the image.
- name: Verify the pushed image resolves the declared Sentry version
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
run: |
set -euo pipefail
expected="$(node -e "process.stdout.write(require('./server/package.json').peerDependencies['@sentry/node'])")"
test -n "$expected"
installed="$(docker run --rm --pull always \
-v "$PWD/scripts/assert-cloud-image-sentry.mjs:/app/server/.ci-sentry-probe.mjs:ro" \
--entrypoint node "$IMAGE" /app/server/.ci-sentry-probe.mjs)"
echo "Declared optional peer version: $expected"
echo "Installed in the pushed image: $installed"
if [ "$installed" != "$expected" ]; then
echo "ERROR: the pushed image resolves @sentry/node@$installed, expected @sentry/node@$expected" >&2
exit 1
fi
echo "The pushed image resolves the declared @sentry/node version."
# Managed hosts run node as 1001:1001. Bake that identity into the image
# so usermod does not walk the mounted home on every container start.
# Check before the entrypoint can repair a wrongly built identity.
- name: Verify cloud runtime user
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
run: |
set -euo pipefail
docker run --rm --entrypoint sh "$IMAGE" -ec '
test "$(id -u node)" = 1001
test "$(id -g node)" = 1001
test "$USER_UID" = 1001
test "$USER_GID" = 1001
'
docker run --rm -e USER_UID=1001 -e USER_GID=1001 "$IMAGE" sh -ec '
test "$(id -u)" = 1001
test "$(id -g)" = 1001
test -w "$PAPERCLIP_HOME"
'
# Verify the independently published cloud image without waiting for
# the self-hosted manifest job. The Sentry check already pulled it.
- name: Verify cloud PID 1 reaps orphaned processes
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
run: docker run --rm -i "$IMAGE" sh -s < scripts/assert-orphan-reaping.sh
# Cloud's commit resolver and preview-artifact planner use the full SHA.
# Publish that address only after checking this build's exact digest.
# Retagging reuses the registry manifest and does not rebuild the image.
- name: Publish verified full-SHA cloud tag
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
FULL_SHA_TAG: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-cloud
run: |
set -euo pipefail
revision="$(docker image inspect "$IMAGE" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')"
platform="$(docker image inspect "$IMAGE" --format '{{ .Os }}/{{ .Architecture }}')"
test "$revision" = "$GITHUB_SHA"
test "$platform" = linux/amd64
docker buildx imagetools create --prefer-index=false --tag "$FULL_SHA_TAG" "$IMAGE"
+37
View File
@@ -0,0 +1,37 @@
name: Docker Runner check
on:
pull_request:
paths:
- .github/workflows/docker-runner-check.yml
- Dockerfile
- .dockerignore
- packages/paperclip-runner/rust-toolchain.toml
- packages/paperclip-runner/runner/**
- packages/paperclip-runner/protocol/**
permissions: {}
concurrency:
group: docker-runner-check-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
runner:
name: Compile isolated native Runner
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
# Compile the real target with the real .dockerignore. This catches new
# Cargo or embedded protocol inputs that the isolated COPY set omits.
# No registry credentials, cache imports/exports, or image publication.
- name: Compile the Runner from its isolated Docker context
run: docker buildx build --target runner-build --progress plain .
+223 -217
View File
@@ -19,13 +19,20 @@ on:
type: string
default: ""
permissions:
contents: read
packages: write
# Least privilege: nothing at the workflow level; each job declares exactly
# the token scopes it uses (checkout needs contents:read, GHCR pushes need
# packages:write).
permissions: {}
# Serialise builds per ref without killing an in-flight one: a newer push
# supersedes only the pending slot, so the image build that is already
# running always finishes and publishes.
# running always finishes and publishes. Canary TAG refs each get their
# own group on purpose: their builds run in parallel so every published
# canary gets its sha images regardless of merge cadence. The mutable
# `:canary` channel tags are NOT written by the build matrix (which
# would race across parallel runs) — each canary-tag run retags the
# channel afterwards, only if it still matches the npm `canary`
# dist-tag, so the channel moves monotonically and always mirrors npm.
concurrency:
group: docker-${{ github.ref }}
cancel-in-progress: false
@@ -64,9 +71,51 @@ jobs:
if-no-files-found: error
retention-days: 30
# Multi-arch by native runner, not QEMU.
#
# This was one job building linux/amd64,linux/arm64 together on an x86
# runner. The arm64 half is emulated there, and it did not merely run slow:
# it wedged, every time, in `RUN pnpm --filter @paperclipai/server build`,
# emitting nothing for 38-45 minutes until `timeout-minutes: 60` killed the
# job. Verified across three consecutive runs on 2026-09-04; the amd64 half
# reached `production 5/5` minutes earlier in every one.
#
# A timed-out job is reported as *cancelled*, not failed, so the run read
# "cancelled" and the production image simply stopped publishing without
# anything going red in an obvious way.
#
# It also starved the queue. A run that burns the full hour holds the
# top-level concurrency slot for that hour, and `cancel-in-progress: false`
# keeps exactly one pending slot — so with merges arriving faster than one
# an hour, most runs were superseded before they ever started a job. Five of
# ten master commits sampled that day never produced an image at all.
#
# Each platform now builds on a runner of its own architecture and pushes by
# digest; `merge` assembles the manifest list. arm64 is kept rather than
# dropped (the cloud variant below dropped it and is amd64-only) because
# this is the self-hosted image, and ARM hosts consume it.
build-and-push:
runs-on: ubuntu-latest
strategy:
# Independent legs: one architecture failing should still publish
# nothing, but it must not also hide the other's logs behind a
# cancellation.
fail-fast: false
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
arch: amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
arch: arm64
runs-on: ${{ matrix.runner }}
# Native builds land well inside this; it is a backstop, not a budget.
# (The interim fix while this PR landed raised the single QEMU job's cap
# to 120 minutes; native per-arch legs make that headroom unnecessary.)
timeout-minutes: 60
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v7
@@ -188,16 +237,18 @@ jobs:
echo "last=${last}" >> "$GITHUB_OUTPUT"
echo "count=${count}" >> "$GITHUB_OUTPUT"
# Lane tag mapping: master pushes publish `:canary`, nightly/v* tags
# publish `:nightly`, and only stable v* tags move `:latest` and the
# versioned tags. `:sha-<short>` is published on every build.
# Lane tag mapping: nightly/v* tags publish `:nightly`, and only
# stable v* tags move `:latest` and the versioned tags.
# `:sha-<short>` is published on every build. `:canary` is
# deliberately absent here — the channel tag is moved by the
# dist-tag-checked retag step below, never by the build matrix,
# so parallel canary builds cannot race it backwards.
- name: Docker meta
id: meta
uses: docker/metadata-action@v6
with:
images: ghcr.io/${{ github.repository }}
tags: |
type=raw,value=canary,enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
@@ -207,8 +258,8 @@ jobs:
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Build and push
- name: Build and push by digest
id: build
uses: docker/build-push-action@v7
with:
context: .
@@ -220,25 +271,123 @@ jobs:
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
platforms: linux/amd64,linux/arm64
push: true
platforms: ${{ matrix.platform }}
# By digest, not by tag: two runners cannot each push the same tag
# and end up with a manifest list. Each leg publishes an untagged
# image and `merge` names them together.
outputs: type=image,name=ghcr.io/${{ github.repository }},push-by-digest=true,name-canonical=true,push=true
# Registry-backed BuildKit cache instead of type=gha: the Actions
# cache is capped at 10GB per repo, and two multi-arch mode=max jobs
# evict each other, so most builds ran effectively cold. The cache
# ref lives in ghcr next to the image and is written only by this
# workflow (docker.yml runs on master/tag pushes, never on PRs).
cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max
tags: ${{ steps.meta.outputs.tags }}
#
# Per-arch refs now the legs are separate runners: a shared ref would
# have each leg overwrite the other's cache on every build.
cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }}
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }},mode=max
labels: ${{ steps.meta.outputs.labels }}
# The digest is the only thing `merge` needs from this job. Carried as an
# empty file named for it, which is the upstream pattern — the name is
# the payload, so several legs can upload without colliding on content.
- name: Export digest
run: |
set -euo pipefail
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
test -n "$digest"
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: digests-production-${{ matrix.arch }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
# Names the per-architecture digests as one manifest list under the real
# tags. Nothing is publicly tagged until this runs, so a half-published
# multi-arch image is not a state anything can pull.
merge-and-push:
needs: build-and-push
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: read
packages: write
steps:
# Checked out for `packages/db` (schema labels) and the orphan-reaping
# script the verification step pipes in.
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- name: Download digests
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
path: /tmp/digests
pattern: digests-production-*
merge-multiple: true
- name: Login to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
# Repeated from the build job rather than passed between them: job
# outputs would have to survive a matrix, and this is two `ls` calls.
- name: Compute schema migration labels
id: schema
run: |
set -euo pipefail
last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
echo "last=${last}" >> "$GITHUB_OUTPUT"
echo "count=${count}" >> "$GITHUB_OUTPUT"
# Same lane mapping as the build job; this is where it is actually
# applied, since the legs push untagged. `:canary` is deliberately
# absent, exactly as in the build job's mapping: the channel tag is
# moved only by the dist-tag-checked promote_canary_channel job below,
# so parallel canary-tag builds can never race the channel backwards.
- name: Docker meta
id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
with:
images: ghcr.io/${{ github.repository }}
tags: |
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=sha
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Create manifest list and push
working-directory: /tmp/digests
run: |
set -euo pipefail
docker buildx imagetools create \
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf 'ghcr.io/${{ github.repository }}@sha256:%s ' *)
# PID 1 must be an init that reaps adopted orphans. With node there, the
# orphans agent runs leave behind are never wait()ed and pin as zombies
# until the cgroup pid limit is exhausted and every fork() in the
# container fails. Run against the pushed image rather than a local
# build: the step above is multi-arch with `push: true`, so nothing is
# loaded into the runner's daemon. The cloud variant is FROM production
# and inherits the same ENTRYPOINT, so checking this image covers both.
# container fails. Run against the pushed manifest rather than a local
# build: the legs push by digest, so nothing is loaded into this
# runner's daemon. The independent cloud workflow checks its own image.
- name: Verify PID 1 reaps orphaned processes
env:
# Through the environment, not interpolated into the script body, so
@@ -251,113 +400,47 @@ jobs:
echo "Verifying orphan reaping in $image"
docker run --rm -i --pull always "$image" sh -s < scripts/assert-orphan-reaping.sh
# The cloud variant carries built bundled plugins for managed deployments
# (see the `cloud` stage in the Dockerfile). It runs as its own job with no
# `needs:` on the stock publish above, so the two builds run in parallel and
# a failure or slow build in one never gates, delays, or skips the other.
# Both jobs share only the single top-level concurrency slot. Each job is a
# separate runner, so this one carries its own copy of the prep steps
# (checkout through schema labels) — the accepted cost of that isolation.
# Master cloud builds start independently in docker-cloud.yml. Tag builds
# and manual Docker dispatches call the same implementation, preserving the
# release tags and the canary promotion dependency below.
build-and-push-cloud:
if: github.event_name != 'push' || github.ref != 'refs/heads/master'
uses: ./.github/workflows/docker-cloud.yml
permissions:
contents: read
packages: write
# Moves the mutable `:canary` / `:canary-cloud` channel tags. Kept OUT
# of the build jobs and serialized in its own lane, and — the load-
# bearing property — CONVERGENT rather than self-interested: a
# promotion does not promote "its own" canary, it retags the channel
# to whatever the npm `canary` dist-tag names at execution time,
# provided that version's sha images are published. GitHub's shared
# concurrency lane keeps one running and one pending promotion and
# REPLACES the pending slot with the latest enqueued — an older build
# finishing late can therefore evict the newest canary's pending
# promotion. With convergent promotion that eviction is harmless:
# whichever promotion survives resolves the current dist-tag fresh
# and lands the channel there (the current canary's images always
# exist by the time any later promotion runs, because per-tag build
# groups mean canary builds are never superseded and each run's
# promotion is gated on its own completed pushes). Every interleaving
# converges the Docker channel onto the npm channel.
promote_canary_channel:
if: startsWith(github.ref, 'refs/tags/canary/v')
# merge-and-push, not build-and-push: the per-arch legs push untagged
# digests, and the production `sha-*` tags this promotion retags only
# exist once the manifest merge has named them.
needs: [merge-and-push, build-and-push-cloud]
runs-on: ubuntu-latest
timeout-minutes: 60
timeout-minutes: 10
permissions:
contents: read
packages: write
concurrency:
group: docker-canary-channel-promotion
cancel-in-progress: false
steps:
- name: Checkout
uses: actions/checkout@v7
with:
# Full history and tags so `git describe` below can compute the
# release version to stamp into the image.
fetch-depth: 0
# `.git` is dockerignored, so a running image cannot derive its own
# version and otherwise reports the source package.json placeholder in
# analytics and the debug panel. Compute it here from the pristine
# checkout (real CalVer drift from the nearest release tag) and pass it
# into the build. Empty when no release tag is reachable — the server
# then keeps its existing fallbacks.
- name: Compute build version
id: build-version
run: |
set -euo pipefail
case "${GITHUB_REF}" in
refs/tags/nightly/v*)
# Lane tags carry the exact published version; stamp it verbatim
# instead of describing drift from the nearest stable tag.
version="${GITHUB_REF#refs/tags/nightly/v}"
;;
refs/tags/beta/v*)
version="${GITHUB_REF#refs/tags/beta/v}"
;;
*)
version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)"
;;
esac
echo "version=${version}" >> "$GITHUB_OUTPUT"
echo "Stamping build version: ${version:-<none>}"
# ISO week stamp for the Dockerfile's tool layer: the layer caches
# across commits and re-pulls the @latest CLI tools when the week rolls
# over, instead of on every build.
- name: Compute tool cache epoch
id: tools-epoch
run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
- name: Setup pnpm
uses: pnpm/action-setup@v6
with:
version: 9.15.4
run_install: false
# No dependency cache here: this workflow publishes release images, and
# restoring a shared Actions cache into the build inputs would let a
# poisoned cache entry reach the published artifact.
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: 24
- name: Refresh lockfile for Docker build context
run: |
set -euo pipefail
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
changed="$(git status --porcelain)"
if [ -z "$changed" ]; then
echo "Lockfile already matches package metadata."
exit 0
fi
if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then
echo "Unexpected files changed during lockfile refresh:"
echo "$changed"
exit 1
fi
echo "Using refreshed pnpm-lock.yaml in the Docker build context."
- name: Free runner disk
run: |
set -euo pipefail
echo "Disk before cleanup:"
df -h
pnpm store prune || true
sudo apt-get clean || true
sudo rm -rf \
/usr/share/dotnet \
/usr/share/swift \
/usr/local/lib/android \
/usr/local/share/boost \
/usr/local/share/powershell \
/opt/ghc \
/opt/hostedtoolcache/CodeQL \
/opt/hostedtoolcache/PyPy \
/opt/hostedtoolcache/Ruby || true
docker system prune -af || true
echo "Disk after cleanup:"
df -h
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
with:
@@ -365,99 +448,22 @@ jobs:
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
# Deployment tooling reads these labels from the registry to verify an
# image's schema expectations against a migrator before deploying it,
# without pulling the image. The server refuses to start when the
# database is missing bundled migrations, so orchestrators need a cheap
# way to check image/migrator compatibility up front.
- name: Compute schema migration labels
id: schema
run: |
set -euo pipefail
last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
echo "last=${last}" >> "$GITHUB_OUTPUT"
echo "count=${count}" >> "$GITHUB_OUTPUT"
# Published under the same lane tag set as the self-hosted image, with a
# `-cloud` suffix (canary-cloud, nightly-cloud, latest-cloud,
# <version>-cloud, sha-<short>-cloud).
- name: Docker meta (cloud)
id: meta-cloud
uses: docker/metadata-action@v6
with:
images: ghcr.io/${{ github.repository }}
flavor: |
suffix=-cloud,onlatest=true
tags: |
type=raw,value=canary,enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=sha
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Build and push (cloud)
uses: docker/build-push-action@v7
with:
context: .
target: cloud
# Space-separated sandbox-provider directory names to build into
# the variant; add here when managed deployments need another.
# CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages the
# variant installs from server/package.json's declared version;
# add another name there when a managed tenant needs it.
build-args: |
CLOUD_BUNDLED_PLUGINS=daytona
CLOUD_BUNDLED_SERVER_DEPS=@sentry/node
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
# amd64 only, unlike the self-hosted image above: the cloud variant
# is consumed exclusively by managed-deployment hosts, which run
# amd64. The QEMU-emulated arm64 half dominated this job's wall
# clock, and dropping it roughly halves time-to-deployable-image.
platforms: linux/amd64
push: true
# Registry-backed BuildKit cache, separate ref from the self-hosted
# job so the two parallel builds never clobber each other's cache
# manifest (see the rationale on the job above).
cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud,mode=max
tags: ${{ steps.meta-cloud.outputs.tags }}
labels: ${{ steps.meta-cloud.outputs.labels }}
# The cloud target installs @sentry/node at the version
# server/package.json declares, into a directory the server's own
# module resolution walks. Verify the image this job just pushed, not
# a local build, so a build-cache or layer-ordering regression is
# caught before any tenant runs the image.
- name: Verify the pushed image resolves the declared Sentry version
- name: Converge the channel tags onto the current npm canary
env:
IMAGE_TAGS: ${{ steps.meta-cloud.outputs.tags }}
IMAGE: ghcr.io/${{ github.repository }}
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
image="$(printf '%s\n' "$IMAGE_TAGS" | head -n 1)"
test -n "$image"
expected="$(node -e "process.stdout.write(require('./server/package.json').peerDependencies['@sentry/node'])")"
test -n "$expected"
installed="$(docker run --rm --pull always \
-v "$PWD/scripts/assert-cloud-image-sentry.mjs:/app/server/.ci-sentry-probe.mjs:ro" \
--entrypoint node "$image" /app/server/.ci-sentry-probe.mjs)"
echo "Declared optional peer version: $expected"
echo "Installed in the pushed image: $installed"
if [ "$installed" != "$expected" ]; then
echo "ERROR: the pushed image resolves @sentry/node@$installed, expected @sentry/node@$expected" >&2
exit 1
current="$(curl -fsS "https://registry.npmjs.org/-/package/@paperclipai%2Fdb/dist-tags" | jq -er .canary)"
sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/$(printf 'canary/v%s' "$current" | jq -sRr @uri)" --jq .sha 2>/dev/null || true)"
if [ -z "$sha" ]; then
echo "canary/v${current} does not resolve yet; a later promotion converges the channel"
exit 0
fi
echo "The pushed image resolves the declared @sentry/node version."
short="$(printf '%s' "$sha" | cut -c1-7)"
if ! docker buildx imagetools inspect "$IMAGE:sha-${short}-cloud" >/dev/null 2>&1; then
echo "images for ${current} (sha-${short}) not published yet; its own promotion converges the channel"
exit 0
fi
docker buildx imagetools create -t "$IMAGE:canary" "$IMAGE:sha-${short}"
docker buildx imagetools create -t "$IMAGE:canary-cloud" "$IMAGE:sha-${short}-cloud"
echo "channel tags moved to canary ${current} (sha-${short})"
+50 -4
View File
@@ -335,7 +335,7 @@ jobs:
run: node --test ./scripts/__tests__/e2e-shard.test.mjs
- name: Test release verify workflow wiring
run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs
run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/cloud-source-verification.test.mjs
- name: Test standalone package build concurrency
run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs
@@ -553,7 +553,7 @@ jobs:
# Preserve the legacy required-check name while the underlying work runs in parallel.
name: verify
if: ${{ always() }}
needs: [gate, policy, typecheck_release_registry, general_tests, build, docker_context_integrity]
needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity]
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 5
@@ -564,6 +564,7 @@ jobs:
POLICY_RESULT: ${{ needs.policy.result }}
TYPECHECK_RELEASE_REGISTRY_RESULT: ${{ needs.typecheck_release_registry.result }}
GENERAL_TESTS_RESULT: ${{ needs.general_tests.result }}
RUNNER_VERIFICATION_RESULT: ${{ needs.verify_paperclip_runner.result }}
BUILD_RESULT: ${{ needs.build.result }}
DOCKER_CONTEXT_INTEGRITY_RESULT: ${{ needs.docker_context_integrity.result }}
run: |
@@ -572,12 +573,14 @@ jobs:
true)
test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "success"
test "$GENERAL_TESTS_RESULT" = "success"
test "$RUNNER_VERIFICATION_RESULT" = "success"
test "$BUILD_RESULT" = "success"
test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "success"
;;
false)
test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped"
test "$GENERAL_TESTS_RESULT" = "skipped"
test "$RUNNER_VERIFICATION_RESULT" = "skipped"
test "$BUILD_RESULT" = "skipped"
test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "skipped"
;;
@@ -587,8 +590,8 @@ jobs:
;;
esac
build:
name: Build
verify_paperclip_runner:
name: Verify Paperclip Runner
needs: [gate, policy]
if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
@@ -633,6 +636,49 @@ jobs:
- name: Verify Paperclip Runner
run: pnpm --filter @paperclipai/paperclip-runner check:all
build:
name: Build
needs: [gate, policy]
if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Setup Node.js for pnpm bootstrap
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
env:
NPM_CONFIG_AUDIT: "false"
NPM_CONFIG_FUND: "false"
NPM_CONFIG_UPDATE_NOTIFIER: "false"
with:
version: 9.15.4
- name: Restore regenerated PR lockfile (if policy uploaded one)
if: needs.policy.outputs.lockfile_regenerated == '1'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: pr-lockfile
path: .
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Build Runner Evalbook viewer
run: pnpm --filter @paperclipai/paperclip-runner build:issue-thread
+102 -8
View File
@@ -58,16 +58,59 @@ jobs:
fail-fast: false
matrix:
include:
- group: general-server
group_label: server (1/3)
# Split the long chat file by collected test locations, and balance
# the remaining server files across ten runners. Normal PR/local
# invocations retain their complete general-server group.
- group: general-server-without-chat
group_label: server (1/10)
shard_index: 0
shard_count: 10
- group: general-server-without-chat
group_label: server (2/10)
shard_index: 1
shard_count: 10
- group: general-server-without-chat
group_label: server (3/10)
shard_index: 2
shard_count: 10
- group: general-server-without-chat
group_label: server (4/10)
shard_index: 3
shard_count: 10
- group: general-server-without-chat
group_label: server (5/10)
shard_index: 4
shard_count: 10
- group: general-server-without-chat
group_label: server (6/10)
shard_index: 5
shard_count: 10
- group: general-server-without-chat
group_label: server (7/10)
shard_index: 6
shard_count: 10
- group: general-server-without-chat
group_label: server (8/10)
shard_index: 7
shard_count: 10
- group: general-server-without-chat
group_label: server (9/10)
shard_index: 8
shard_count: 10
- group: general-server-without-chat
group_label: server (10/10)
shard_index: 9
shard_count: 10
- group: general-chat
group_label: chat (1/3)
shard_index: 0
shard_count: 3
- group: general-server
group_label: server (2/3)
- group: general-chat
group_label: chat (2/3)
shard_index: 1
shard_count: 3
- group: general-server
group_label: server (3/3)
- group: general-chat
group_label: chat (3/3)
shard_index: 2
shard_count: 3
# Keep parity with pr.yml: workspaces-a is split with Vitest's
@@ -191,8 +234,8 @@ jobs:
- name: Run deterministic Runner workflow scorer tests
run: pnpm test:runner-workflow-evals
build:
name: Build
verify_paperclip_runner:
name: Verify Paperclip Runner
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
@@ -215,11 +258,62 @@ jobs:
node-version: 24
cache: pnpm
- name: Select the pinned Runner Rust toolchain
working-directory: packages/paperclip-runner
run: |
set -euo pipefail
rustup show
toolchain="$(rustup show active-toolchain | awk '{print $1}')"
echo "RUSTUP_TOOLCHAIN=$toolchain" >> "$GITHUB_ENV"
- name: Cache Runner Rust dependencies
# Restore and save only within trusted master-push verification. GitHub
# isolates branch/PR caches from master; other callers compile afresh.
if: ${{ github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha }}
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
workspaces: packages/paperclip-runner/runner -> target
shared-key: release-runner-v1
# Rebuild workspace code and rerun every check. Cache only compiled
# dependencies; never restore installed executables from cargo/bin.
cache-workspace-crates: false
cache-bin: false
# The step guard also restricts restores. Save only after a successful
# master-push verification of that push's exact commit.
save-if: ${{ github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha }}
- name: Install dependencies
run: pnpm install --no-frozen-lockfile
- name: Verify Paperclip Runner
run: pnpm --filter @paperclipai/paperclip-runner check:all
build:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ inputs.ref }}
persist-credentials: false
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Install dependencies
run: pnpm install --no-frozen-lockfile
- name: Build
run: pnpm build
+278 -6
View File
@@ -1,4 +1,5 @@
name: Release
run-name: ${{ inputs.channel == 'preview' && format('Stack deploy {0} build', inputs.request_id) || inputs.channel == 'cloud-migrator' && format('Cloud migrator {0}', inputs.source_ref) || 'Release' }}
on:
push:
@@ -17,12 +18,22 @@ on:
- stable
- beta
- nightly
- preview
- cloud-migrator
default: stable
source_ref:
description: (stable) Commit SHA, branch, or tag to publish as stable
description: Stable source ref, or full immutable SHA for a preview or cloud migrator build
required: true
type: string
default: master
request_id:
description: (preview/cloud-migrator) Correlation UUID
type: string
default: ""
preview_migrator:
description: (preview) Publish isolated shared and database packages if missing
type: boolean
default: false
stable_date:
description: Enter a UTC date in YYYY-MM-DD format, for example 2026-03-18. Do not enter a version string. The workflow will resolve that date to a stable version such as 2026.318.0, then 2026.318.1 for the next same-day stable.
required: false
@@ -46,7 +57,7 @@ on:
default: false
concurrency:
group: release-${{ github.event_name }}-${{ github.ref }}
group: ${{ (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && format('{0}-{1}', inputs.channel, inputs.source_ref) || format('release-{0}-{1}', github.event_name, github.ref) }}
cancel-in-progress: false
env:
@@ -64,11 +75,247 @@ env:
NPM_PUBLISH_VERIFY_DELAY_SECONDS: "10"
jobs:
plan_preview:
name: Check preview artifacts
if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && !inputs.dry_run
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
image: ${{ steps.plan.outputs.image }}
packages: ${{ steps.plan.outputs.packages }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Validate immutable source and inspect existing artifacts
id: plan
env:
SOURCE_SHA: ${{ inputs.source_ref }}
REQUEST_ID: ${{ inputs.request_id }}
PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }}
PLAN_COMMAND: ${{ inputs.channel == 'cloud-migrator' && 'plan-migrator' || 'plan' }}
run: node scripts/preview-artifacts.mjs "$PLAN_COMMAND" "$SOURCE_SHA" "$REQUEST_ID" "$PREVIEW_MIGRATOR"
package_preview:
name: Build preview migrator
needs: plan_preview
if: needs.plan_preview.outputs.packages == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
path: trusted
persist-credentials: false
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ inputs.source_ref }}
path: source
persist-credentials: false
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
run_install: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Install build dependencies without lifecycle scripts
working-directory: source
run: pnpm install --ignore-scripts --no-frozen-lockfile
- name: Build and pack exact-source preview packages
env:
SOURCE_SHA: ${{ inputs.source_ref }}
run: node trusted/scripts/preview-artifacts.mjs pack source packages "$SOURCE_SHA"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: preview-packages
overwrite: true
path: packages/*.tgz
if-no-files-found: error
retention-days: 7
publish_preview:
name: Publish preview migrator
needs: [plan_preview, package_preview]
if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.packages == 'true' && needs.package_preview.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 30
# A manual preview and a merge-triggered migrator may compile in parallel.
# Serialize only publication so they cannot race an immutable npm version,
# without making the migrator wait for a preview's separate image build.
concurrency:
group: preview-package-publish-${{ inputs.source_ref }}
cancel-in-progress: false
# Reuse release.yml's established npm trusted-publisher identity. This job
# publishes only isolated preview versions; it cannot advance lane tags.
environment: npm-canary
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Install npm with trusted publishing support
run: npm install --global npm@11.18.0 --ignore-scripts
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: preview-packages
path: preview-packages
- name: Publish immutable preview packages without running package code
env:
SOURCE_SHA: ${{ inputs.source_ref }}
run: node scripts/preview-artifacts.mjs publish preview-packages "$SOURCE_SHA"
image_preview:
name: Build preview cloud image
needs: plan_preview
if: needs.plan_preview.outputs.image == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ inputs.source_ref }}
persist-credentials: false
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
run_install: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Prepare locked image context
env:
SOURCE_SHA: ${{ inputs.source_ref }}
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
pnpm install --resolution-only --ignore-scripts --ignore-pnpmfile --no-frozen-lockfile
echo "TOOLS_EPOCH=$(date -u +%G-W%V)" >> "$GITHUB_ENV"
- uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- name: Build the immutable cloud image without registry credentials
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
target: cloud
platforms: linux/amd64
push: false
provenance: false # Docker archives cannot carry registry attestations.
outputs: type=docker,dest=${{ runner.temp }}/preview-image.tar
tags: ghcr.io/paperclipai/paperclip:sha-${{ inputs.source_ref }}-cloud
build-args: |
CLOUD_BUNDLED_PLUGINS=daytona
CLOUD_BUNDLED_SERVER_DEPS=@sentry/node
PAPERCLIP_BUILD_COMMIT=${{ inputs.source_ref }}
PAPERCLIP_BUILD_VERSION=0.0.0-preview.g${{ inputs.source_ref }}
CLI_TOOLS_CACHE_EPOCH=${{ env.TOOLS_EPOCH }}
labels: |
org.opencontainers.image.revision=${{ inputs.source_ref }}
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: preview-image
overwrite: true
path: ${{ runner.temp }}/preview-image.tar
compression-level: 0
if-no-files-found: error
retention-days: 1
publish_image_preview:
name: Publish preview cloud image
needs: [plan_preview, image_preview]
if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.image == 'true' && needs.image_preview.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 30
# This existing environment has an external master-only branch policy.
environment: npm-canary
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: preview-image
path: preview-image
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Verify image identity and publish without executing image code
env:
SOURCE_SHA: ${{ inputs.source_ref }}
run: node scripts/preview-artifacts.mjs publish-image preview-image/preview-image.tar "$SOURCE_SHA"
result_preview:
name: Verify preview artifacts
needs: [plan_preview, image_preview, publish_image_preview, package_preview, publish_preview]
if: >-
always() && inputs.channel == 'preview' && needs.plan_preview.result == 'success' &&
(needs.publish_image_preview.result == 'success' || needs.plan_preview.outputs.image == 'false') &&
(needs.publish_preview.result == 'success' || needs.plan_preview.outputs.packages == 'false')
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Confirm exact artifacts are visible
env:
SOURCE_SHA: ${{ inputs.source_ref }}
REQUEST_ID: ${{ inputs.request_id }}
PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }}
run: node scripts/preview-artifacts.mjs result "$SOURCE_SHA" "$REQUEST_ID"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: stack-deploy-result
overwrite: true
path: stack-deploy-result/result.json
if-no-files-found: error
retention-days: 30
verify_canary:
if: github.event_name == 'push'
uses: ./.github/workflows/release-verify.yml
with:
ref: ${{ github.sha }}
name: Reuse exact-source verification
if: github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
timeout-minutes: 50
permissions:
contents: read
actions: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Require successful source checks for this exact master push
env:
GITHUB_TOKEN: ${{ github.token }}
SOURCE_SHA: ${{ github.sha }}
run: node scripts/cloud-source-verification.mjs "$SOURCE_SHA"
publish_canary:
if: github.event_name == 'push'
@@ -81,6 +328,8 @@ jobs:
permissions:
contents: write
id-token: write
# For the explicit docker.yml dispatch below.
actions: write
steps:
- name: Checkout repository
@@ -139,6 +388,29 @@ jobs:
git push origin "refs/tags/${tag}"
echo "version=${tag#canary/v}" >> "$GITHUB_OUTPUT"
# Canary images previously relied on the master-push docker.yml run,
# whose single pending concurrency slot gets superseded by every
# newer push — on a busy day no canary image publishes at all (five
# consecutive canaries shipped npm packages with no cloud image on
# 2026-09-06, starving downstream managed deploys for ~18 hours).
# Tag pushes made with GITHUB_TOKEN do not fire docker.yml's
# triggers, so dispatch the image build at the canary tag
# explicitly, exactly like the nightly and beta lanes: the run keys
# its concurrency off the tag ref, so no master push can supersede
# it, and docker.yml's `type=sha` mapping publishes the
# sha-<short> and sha-<short>-cloud images either way.
- name: Build Docker images for the canary tag
env:
GH_TOKEN: ${{ github.token }}
run: |
{
echo "## Canary published"
echo ""
echo "- Published canary: \`${{ steps.canary_tag.outputs.version }}\`"
echo "- Docker build dispatched at \`canary/v${{ steps.canary_tag.outputs.version }}\`"
} >> "$GITHUB_STEP_SUMMARY"
gh workflow run docker.yml --ref "refs/tags/canary/v${{ steps.canary_tag.outputs.version }}" --repo "$GITHUB_REPOSITORY"
# The package is already public when this gate runs. A red result leaves the
# immutable canary in npm, but makes the release workflow visibly fail before
# anyone mistakes an installable package for an onboardable one.
+3 -1
View File
@@ -12,7 +12,9 @@ on:
type: string
concurrency:
group: runner-chaos-evals-${{ inputs.ref || github.ref }}
# Reusable calls inherit the caller's workflow name. Cloud readiness and
# Release verify the same SHA independently and must not cancel each other.
group: runner-chaos-evals-${{ github.workflow }}-${{ inputs.ref || github.ref }}
cancel-in-progress: true
jobs:
+169
View File
@@ -0,0 +1,169 @@
name: Storybook Deploy
on:
workflow_dispatch:
inputs:
branch:
description: "Repository branch to publish (empty uses the selected workflow branch)"
type: string
default: ""
# Also exposed by Storybook Visual, which is already available on master.
workflow_call:
inputs:
branch:
type: string
default: ""
permissions:
contents: read
jobs:
authorize:
name: Authorize Storybook publisher
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
sha: ${{ steps.source.outputs.sha }}
branch: ${{ steps.source.outputs.branch }}
branch_key: ${{ steps.source.outputs.branch_key }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Require CODEOWNER initiator and rerunner
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
script: |
const authorize = require('./.github/scripts/authorize-storybook-deploy.cjs');
await authorize({ github, context });
- name: Pin requested repository branch
id: source
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
SOURCE_BRANCH: ${{ inputs.branch }}
STORYBOOK_S3_BUCKET: ${{ vars.STORYBOOK_S3_BUCKET }}
STORYBOOK_PUBLIC_BASE_URL: ${{ vars.STORYBOOK_PUBLIC_BASE_URL }}
with:
script: |
const branch = process.env.SOURCE_BRANCH || context.ref.slice('refs/heads/'.length);
const { data } = await github.rest.git.getRef({ ...context.repo, ref: `heads/${branch}` });
if (data.ref !== `refs/heads/${branch}` || data.object.type !== 'commit') {
throw new Error('Select an existing branch in this repository.');
}
// With no source override, preserve the exact dispatched commit.
const sha = process.env.SOURCE_BRANCH ? data.object.sha : context.sha;
const { storybookDestination } = require('./.github/scripts/storybook-destination.cjs');
const destination = storybookDestination({ branch, sha,
runId: context.runId, runAttempt: process.env.GITHUB_RUN_ATTEMPT,
bucket: process.env.STORYBOOK_S3_BUCKET, baseUrl: process.env.STORYBOOK_PUBLIC_BASE_URL });
core.setOutput('sha', sha);
core.setOutput('branch_key', destination.branchKey);
core.setOutput('branch', branch);
build:
name: Build selected branch Storybook
permissions: {}
needs: authorize
runs-on: ubuntu-latest
timeout-minutes: 25
outputs:
artifact_name: ${{ steps.artifact.outputs.name }}
env:
STORYBOOK_DISABLE_TELEMETRY: "1"
steps:
- name: Download public source without repository credentials
env:
SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
run: |
[[ "$SOURCE_SHA" =~ ^[a-f0-9]{40}$ ]]
curl --fail --silent --show-error --location --retry 3 \
"https://codeload.github.com/paperclipai/paperclip/tar.gz/$SOURCE_SHA" \
--output "$RUNNER_TEMP/source.tar.gz"
tar -xzf "$RUNNER_TEMP/source.tar.gz" --strip-components=1
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
package-manager-cache: false
- run: pnpm install --frozen-lockfile --ignore-scripts
- run: pnpm build-storybook
- name: Record source and validate output
id: artifact
env:
SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
SOURCE_BRANCH: ${{ needs.authorize.outputs.branch }}
run: |
test -s ui/storybook-static/index.html
test -s ui/storybook-static/iframe.html
test -s ui/storybook-static/index.json
jq -n --arg sha "$SOURCE_SHA" --arg branch "$SOURCE_BRANCH" \
'{sha: $sha, branch: $branch}' > ui/storybook-static/deployment.json
echo "name=storybook-deploy-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: ${{ steps.artifact.outputs.name }}
path: ui/storybook-static
if-no-files-found: error
retention-days: 7
deploy:
name: Publish branch Storybook to S3
needs: [authorize, build]
runs-on: ubuntu-latest
timeout-minutes: 15
concurrency:
group: storybook-deploy-${{ needs.authorize.outputs.branch_key }}
cancel-in-progress: false
permissions:
contents: read
id-token: write
environment:
name: storybook-deploy
url: ${{ steps.deployment.outputs.url }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
sparse-checkout: .github/scripts
- name: Recheck CODEOWNER access before publishing
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
script: |
const authorize = require('./.github/scripts/authorize-storybook-deploy.cjs');
await authorize({ github, context });
- name: Download the successful build artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: ${{ needs.build.outputs.artifact_name }}
path: storybook-static
- name: Assume the Storybook-only uploader role
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
with:
role-to-assume: ${{ vars.STORYBOOK_AWS_ROLE_ARN }}
aws-region: ${{ vars.STORYBOOK_AWS_REGION }}
role-duration-seconds: 900
- name: Publish this branch preview
id: deployment
env:
SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
SOURCE_BRANCH: ${{ needs.authorize.outputs.branch }}
STORYBOOK_S3_BUCKET: ${{ vars.STORYBOOK_S3_BUCKET }}
STORYBOOK_PUBLIC_BASE_URL: ${{ vars.STORYBOOK_PUBLIC_BASE_URL }}
run: node .github/scripts/publish-storybook.cjs
- name: Verify public build and stable branch URL
env:
BUILD_URL: ${{ steps.deployment.outputs.build_url }}
BRANCH_URL: ${{ steps.deployment.outputs.url }}
SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
run: node .github/scripts/verify-storybook.cjs
- name: Upload deployment links
if: ${{ !cancelled() && steps.deployment.outcome == 'success' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: storybook-deployment-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ steps.deployment.outputs.report_path }}
if-no-files-found: error
retention-days: 30
+22 -2
View File
@@ -3,6 +3,16 @@ name: Storybook Visual
on:
workflow_dispatch:
inputs:
branch:
description: "Repository branch to publish (deployment only; empty uses the selected workflow branch)"
required: false
type: string
default: ""
deploy_preview:
description: "Publish this branch to S3/CloudFront instead of running visual tests (CODEOWNERS only)"
required: false
type: boolean
default: false
update_snapshots:
description: "Generate updated snapshots and a baseline review bundle"
required: false
@@ -18,7 +28,7 @@ on:
- labeled
concurrency:
group: storybook-visual-${{ github.event.pull_request.number || github.ref }}
group: storybook-visual-${{ inputs.deploy_preview && github.run_id || 'visual' }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
@@ -28,7 +38,7 @@ jobs:
visual:
name: Storybook visual regression
if: >-
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'workflow_dispatch' && !inputs.deploy_preview) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'storybook-visual'))
runs-on: ubuntu-latest
timeout-minutes: 35
@@ -100,3 +110,13 @@ jobs:
path: tests/storybook-visual/baseline-review/snapshots.tgz
retention-days: 30
if-no-files-found: error
preview:
name: Deploy selected branch Storybook
if: github.event_name == 'workflow_dispatch' && inputs.deploy_preview
permissions:
contents: read
id-token: write
uses: ./.github/workflows/storybook-deploy.yml
with:
branch: ${{ inputs.branch }}