Merge master and preserve question and search journeys

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 14:47:32 -05:00
commit 04f34f278e
113 files changed
+6319 -466

No files matched your search

+22
View File
@@ -0,0 +1,22 @@
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
// Read trees only. The routing job fetches Git objects into a bare repository;
// no pull request code, hooks, filters or package scripts run here.
export function mergeDiff(merge, prHead, cwd = process.cwd()) {
assert.match(merge ?? "", /^[a-f0-9]{40}$/);
assert.match(prHead ?? "", /^[a-f0-9]{40}$/);
const git = (...args) => execFileSync("git", args, {
cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
}).trim();
const header = git("--no-replace-objects", "cat-file", "-p", merge).split("\n\n", 1)[0];
const parents = [...header.matchAll(/^parent ([a-f0-9]{40})$/gm)].map((match) => match[1]);
assert.equal(parents.length, 2, "Expected a GitHub pull request merge commit");
assert.equal(parents[1], prHead, "Merge commit does not contain the expected PR head");
const base = parents[0];
const changed = execFileSync("git", ["diff", "--no-ext-diff", "--no-textconv", "--no-renames",
"--name-only", "-z", base, merge, "--"], {
cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
}).split("\0").filter(Boolean).sort();
return { diff: { source: "git-merge-parent", base, head: merge, prHead }, changed };
}
+10
View File
@@ -0,0 +1,10 @@
// Audited prose inputs, shared by routing, Vitest selection and task hashing.
// Do not broaden this to all Markdown: docs/api and bundled skills are runtime inputs.
export function documentationInput(file) {
if (file === "doc/DEVELOPING.md" || file === "doc/DATABASE.md") return "guidance";
if (/^doc\/plans\/(?:[^/]+\/)*[^/]+\.md$/.test(file)
&& !file.split("/").includes("..")) return "historical";
return null;
}
export const productInputs = (files) => files.filter((file) => !documentationInput(file));
+18 -6
View File
@@ -2,6 +2,8 @@ import { appendFileSync, readFileSync, realpathSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { matchesGlob } from "node:path";
import { fileURLToPath } from "node:url";
import { documentationInput, productInputs } from "./ci-inputs.mjs";
import { mergeDiff } from "./ci-diff.mjs";
const releaseNote = /^releases\/(beta\/)?v[^/]+\.md$/;
const validPath = (file) => typeof file === "string" && file.length > 0
@@ -52,28 +54,33 @@ export function planTasks({ config, lanes, paths, mode = "enabled", forceReason
const changed = [...new Set(paths)];
if (!changed.length || changed.some((file) => !validPath(file))) forceReason ??= "Missing or invalid diff";
const matches = (file, globs) => globs.some((glob) => matchesGlob(file, glob));
const globalChanges = changed.filter((file) => matches(file, config.globalDependencies));
const sourceChanges = productInputs(changed);
const globalChanges = sourceChanges.filter((file) => matches(file, config.globalDependencies));
if (globalChanges.length) forceReason ??= `Global CI input changed: ${globalChanges[0]}`;
const unknown = changed.filter((file) => !releaseNote.test(file)
const unknown = sourceChanges.filter((file) => !releaseNote.test(file)
&& !matches(file, config.globalDependencies) && ![...inputs.values()].some((globs) => matches(file, globs)));
if (unknown.length) forceReason ??= `Unclassified path: ${unknown[0]}`;
if (mode === "full") forceReason ??= "Full CI requested";
const reasons = Object.fromEntries([...inputs].map(([task, globs]) => [task,
forceReason ? [forceReason] : [...new Set([...globalChanges, ...changed.filter((file) => matches(file, globs))])],
forceReason ? [forceReason] : [...new Set([...globalChanges, ...sourceChanges.filter((file) => matches(file, globs))])],
]));
// This uncached lane owns the repository-wide CLI guidance check.
if (reasons["ci:typecheck"]) reasons["ci:typecheck"].push(...changed.filter((file) => documentationInput(file) === "guidance"));
const jobs = Object.fromEntries(Object.entries(lanes).map(([job, entries]) => {
const affected = entries.filter((entry) => reasons[entry.task].length > 0);
const selected = mode === "shadow" ? entries : affected;
return [job, { run: selected.length > 0, matrix: { include: selected.length ? selected : entries },
selected: selected.length, proposed: affected.length, total: entries.length }];
}));
return { version: 1, mode, forceReason, changed, reasons, jobs };
return { version: 1, mode, forceReason, changed, documentation: changed.filter(documentationInput), reasons, jobs };
}
export function planSummary(plan) {
const rows = Object.entries(plan.jobs).map(([job, lane]) =>
`| ${job} | ${lane.proposed}/${lane.total} | ${lane.selected}/${lane.total} |`);
return ["### Incremental CI", "", `Mode: **${plan.mode}**${plan.forceReason ? ` — ${plan.forceReason}` : ""}`,
...(plan.diff ? [`Diff: \`${plan.diff.base}\` → \`${plan.diff.head}\` (complete Git tree comparison)`, ""] : []),
`Documentation inputs: ${(plan.documentation ?? []).join(", ") || "none"}`,
"", "| Suite | Affected jobs | Scheduled jobs |", "| --- | ---: | ---: |", ...rows,
"", "<details><summary>Selection reasons</summary>", "", "```json", JSON.stringify(plan.reasons, null, 2),
"```", "</details>", ""].join("\n");
@@ -83,9 +90,13 @@ if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.me
try {
const config = JSON.parse(readFileSync(process.env.CI_TASK_CONFIG ?? "turbo.json", "utf8"));
const lanes = JSON.parse(readFileSync(process.env.CI_TASK_MATRIX ?? ".github/ci-task-matrix.json", "utf8"));
let paths = [], forceReason = process.env.CI_FORCE_REASON || null;
let paths = [], diff, forceReason = process.env.CI_FORCE_REASON || null;
try {
if (process.env.CI_COMPARISON) {
if (process.env.CI_DIFF_REPOSITORY) {
const result = mergeDiff(process.env.EVENT_MERGE_SHA, process.env.EVENT_HEAD_SHA, process.env.CI_DIFF_REPOSITORY);
paths = result.changed;
diff = result.diff;
} else if (process.env.CI_COMPARISON) {
paths = comparisonPaths(JSON.parse(readFileSync(process.env.CI_COMPARISON, "utf8")));
} else {
const [base, head = "HEAD"] = process.argv.slice(2);
@@ -95,6 +106,7 @@ if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.me
}
} catch (error) { forceReason ??= `Diff unavailable: ${error.message}`; }
const plan = planTasks({ config, lanes, paths, mode: process.env.CI_INCREMENTAL_MODE || "enabled", forceReason });
if (diff) plan.diff = diff;
if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `plan=${JSON.stringify(plan)}\n`);
else console.log(JSON.stringify(plan, null, 2));
if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, planSummary(plan));
+85 -6
View File
@@ -5,6 +5,7 @@ import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { comparisonPaths, planTasks } from "../ci-plan.mjs";
import { mergeDiff } from "../ci-diff.mjs";
const root = new URL("../../../", import.meta.url);
const config = JSON.parse(readFileSync(new URL("turbo.json", root), "utf8"));
@@ -18,7 +19,8 @@ test("trusted workflow fallbacks preserve every declared partition and command",
const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8");
for (const [job, entries] of Object.entries(lanes)) {
const block = workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0];
assert.ok(block.includes(`fromJSON(needs.gate.outputs.plan).jobs.${job}.run`), job);
const normalized = block.replaceAll("(needs.test_selection.outputs.plan || needs.gate.outputs.plan)", "needs.gate.outputs.plan");
assert.ok(normalized.includes(`fromJSON(needs.gate.outputs.plan).jobs.${job}.run`), job);
if (!block.includes(" matrix:")) { assert.equal(entries.length, 1); continue; }
const fallback = block.match(/matrix:.*\|\| '(.*)'\) \}\}/)?.[1];
assert.ok(fallback, `${job}: missing full fallback`);
@@ -115,7 +117,7 @@ test("cache-affecting definition changes force execution across successive PR pu
}
}
const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8");
assert.equal((workflow.match(/force:.*fromJSON\(needs.gate.outputs.plan\).forceReason != null/g) ?? []).length, 3);
assert.equal((workflow.match(/force:.*\.forceReason != null/g) ?? []).length, 3);
const action = readFileSync(new URL(".github/actions/run-ci-task/action.yml", root), "utf8");
assert.match(action, /CI_FORCE_TASKS: \$\{\{ inputs.force \}\}/);
assert.equal((action.match(/if:.*inputs.force != 'true'/g) ?? []).length, 2);
@@ -169,6 +171,9 @@ test("required aggregates reject omitted, failed and cancelled selected work", (
env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(resultPlan), CI_RESULTS: JSON.stringify(results) } });
assert.equal(run().status, 0);
for (const name of names) {
// Optional analysis can fail or be skipped; the unchanged task plan
// still has to pass. Selected-plan cases are exercised separately.
if (name === "test_selection") continue;
const previous = results[name].result;
for (const bad of previous === "success" ? ["failure", "cancelled", "skipped"] : ["success"]) {
results[name].result = bad;
@@ -185,6 +190,30 @@ test("the real routing step uses pinned trusted definitions and defaults to full
const shell = block.split(" run: |\n")[1].split("\n").map((line) => line.slice(10)).join("\n");
const directory = mkdtempSync(path.join(tmpdir(), "ci-routing-plan-"));
try {
const repository = path.join(directory, "source");
mkdirSync(repository);
const gitBin = execFileSync("which", ["git"], { encoding: "utf8" }).trim();
const git = (...args) => execFileSync(gitBin, args, { cwd: repository, encoding: "utf8" }).trim();
git("init", "-q", "-b", "main"); git("config", "user.name", "CI"); git("config", "user.email", "ci@example.invalid");
writeFileSync(path.join(repository, "base"), "base"); git("add", "."); git("commit", "-qm", "base");
git("checkout", "-qb", "pr");
mkdirSync(path.join(repository, "ui/src/components"), { recursive: true });
writeFileSync(path.join(repository, "ui/src/components/Sidebar.tsx"), "change");
git("add", "."); git("commit", "-qm", "PR");
const prHead = git("rev-parse", "HEAD");
git("checkout", "-q", "main");
writeFileSync(path.join(repository, "unrelated-master"), "already merged");
git("add", "."); git("commit", "-qm", "advance base");
const base = git("rev-parse", "HEAD");
git("merge", "--no-ff", "pr", "-m", "tested merge");
const merge = git("rev-parse", "HEAD");
writeFileSync(path.join(directory, "git"), `#!/usr/bin/env node
const {spawnSync}=require('node:child_process');
if(process.env.TEST_GIT_FAILURE==='true')process.exit(1);
const args=process.argv.slice(2).map(a=>a==='https://github.com/paperclipai/paperclip.git'?process.env.TEST_GIT_REPOSITORY:a);
const r=spawnSync(${JSON.stringify(gitBin)},args,{stdio:'inherit'});process.exit(r.status??1);
`);
chmodSync(path.join(directory, "git"), 0o755);
const gh = path.join(directory, "gh");
writeFileSync(gh, `#!/usr/bin/env node
const fs = require('node:fs'), path = require('node:path');
@@ -195,27 +224,77 @@ else if (endpoint?.includes('/contents/')) {
const [file, query] = endpoint.split('/contents/')[1].split('?');
if (query !== 'ref=' + 'a'.repeat(40)) process.exit(2);
process.stdout.write(fs.readFileSync(path.join(process.env.TEST_TRUSTED_ROOT, file)));
} else if (endpoint?.includes('/compare/')) console.log(JSON.stringify({ files: [{filename:'ui/src/components/Sidebar.tsx'}] }));
}
else process.exit(3);
`);
chmodSync(gh, 0o755);
for (const overrides of [{}, { TEST_FETCH_FAILURE: "true" }, { PROFILE_CI: "true" }, { STACK_JSON: "{}" }, { FULL_CI: "false" }]) {
for (const overrides of [{}, { TEST_FETCH_FAILURE: "true" }, { TEST_GIT_FAILURE: "true" }, { PROFILE_CI: "true" }, { STACK_JSON: "{}" }, { FULL_CI: "false" }]) {
const output = path.join(directory, "output");
writeFileSync(output, "");
const result = spawnSync("bash", ["-c", shell], { encoding: "utf8", env: {
...process.env, PATH: `${directory}:${process.env.PATH}`, TEST_TRUSTED_ROOT: root.pathname,
RUNNER_TEMP: directory, FULL_CI: "true", PROFILE_CI: "false", FORCE_FULL_CI: "false",
CI_INCREMENTAL_MODE: "enabled", STACK_JSON: "null", EVENT_BASE_SHA: "b".repeat(40), EVENT_MERGE_SHA: "c".repeat(40),
CI_INCREMENTAL_MODE: "enabled", STACK_JSON: "null", EVENT_BASE_SHA: "b".repeat(40), EVENT_HEAD_SHA: prHead, EVENT_MERGE_SHA: merge,
TEST_GIT_REPOSITORY: repository,
GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: "", ...overrides,
} });
assert.equal(result.status, 0, result.stderr);
const value = readFileSync(output, "utf8").trim();
if (overrides.TEST_FETCH_FAILURE || overrides.FULL_CI === "false") assert.equal(value, "plan=");
if (overrides.TEST_FETCH_FAILURE || overrides.TEST_GIT_FAILURE || overrides.FULL_CI === "false") assert.equal(value, "plan=");
else {
assert.notEqual(value, "plan=", `${JSON.stringify(overrides)}: ${result.stderr}`);
const resultPlan = JSON.parse(value.slice("plan=".length));
assert.equal(resultPlan.jobs.general_tests.selected, overrides.PROFILE_CI || overrides.STACK_JSON ? 19 : 2);
assert.equal(resultPlan.diff.base, base);
assert.deepEqual(resultPlan.changed, ["ui/src/components/Sidebar.tsx"]);
}
}
} finally { rmSync(directory, { recursive: true, force: true }); }
});
test("audited prose does not widen source selection, but guidance still runs its uncached guard", () => {
const source = "ui/src/components/Sidebar.tsx";
const sourcePlan = plan([source]);
for (const doc of ["doc/DEVELOPING.md", "doc/DATABASE.md", "doc/plans/2026-10-30-example.md"]) {
const mixed = plan([source, doc]);
assert.equal(mixed.forceReason, null);
assert.deepEqual(mixed.jobs, sourcePlan.jobs);
assert.deepEqual(mixed.documentation, [doc]);
}
const guidance = plan(["doc/DEVELOPING.md"]);
assert.deepEqual([...selectedTasks(guidance)], ["ci:typecheck"]);
assert.equal(config.tasks["//#ci:typecheck"].cache, false);
assert.deepEqual([...selectedTasks(plan(["doc/plans/2026-10-30-example.md"]))], []);
for (const file of ["docs/api/issues.md", "doc/unknown.md", "doc/plans/tool.sh", "skills/paperclip/SKILL.md",
"server/src/onboarding-assets/default/AGENTS.md", "tests/fixtures/example.md"]) {
assert.ok(selectedTasks(plan([source, file])).has("ci:server"), file);
}
});
test("complete merge trees handle more than 300 files, base advancement, renames and deletions", (t) => {
const directory = mkdtempSync(path.join(tmpdir(), "ci-merge-diff-"));
t.after(() => rmSync(directory, { recursive: true, force: true }));
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
const write = (file, value) => { mkdirSync(path.dirname(path.join(directory, file)), { recursive: true }); writeFileSync(path.join(directory, file), value); };
const commit = () => { git("add", "."); git("commit", "-qm", "fixture"); return git("rev-parse", "HEAD"); };
git("init", "-q", "-b", "main"); git("config", "user.name", "CI"); git("config", "user.email", "ci@example.invalid");
write("ui/src/deleted.ts", "old"); write("ui/src/renamed.ts", "rename"); const staleBase = commit();
git("checkout", "-qb", "pr");
for (let index = 0; index < 350; index++) write(`ui/src/component-${index}.ts`, "new");
git("rm", "ui/src/deleted.ts");
mkdirSync(path.join(directory, "doc/plans"), { recursive: true });
git("mv", "ui/src/renamed.ts", "doc/plans/2026-10-30-moved.md");
const prHead = commit();
git("checkout", "-q", "main"); write(".github/workflows/unrelated.yml", "already merged CI change"); const base = commit();
git("merge", "--no-ff", "pr", "-m", "tested merge"); const head = git("rev-parse", "HEAD");
const result = mergeDiff(head, prHead, directory);
assert.equal(result.diff.base, base); assert.notEqual(result.diff.base, staleBase);
assert.equal(result.changed.length, 353);
assert.ok(result.changed.includes("ui/src/deleted.ts")); assert.ok(result.changed.includes("ui/src/renamed.ts"));
assert.ok(!result.changed.includes(".github/workflows/unrelated.yml"));
assert.equal(plan(result.changed).forceReason, null);
assert.throws(() => mergeDiff(head, base, directory), /expected PR head/);
assert.throws(() => mergeDiff(prHead, prHead, directory), /merge commit/);
assert.throws(() => mergeDiff("f".repeat(40), prHead, directory));
});
@@ -7,6 +7,8 @@ on:
- turbo.json
- .github/ci-task-matrix.json
- .github/scripts/ci-plan.mjs
- .github/scripts/ci-diff.mjs
- .github/scripts/ci-inputs.mjs
- .github/scripts/tests/ci-plan.test.mjs
- .github/actions/run-ci-task/**
- .github/workflows/ci-routing.yml
@@ -22,6 +24,50 @@ concurrency:
cancel-in-progress: true
jobs:
qualify_test_selection:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: 24
package-manager-cache: false
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86
with:
version: 9.15.4
- uses: ./.github/actions/setup-ci-dependencies
- name: Prove conservative graphs, exact partitions, manifests and failure fallbacks
run: node --test scripts/ci/test-selection*.test.mjs scripts/ci/vitest-impact-report.integration.test.mjs
- name: Qualify selection against the real Paperclip workspace
env:
NODE_ENV: test
VITEST: 'true'
run: |
node --input-type=module <<'NODE'
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { planTasks } from './.github/scripts/ci-plan.mjs';
import { createTestSelection } from './scripts/ci/select-tests.mjs';
const config = JSON.parse(readFileSync('turbo.json', 'utf8'));
const lanes = JSON.parse(readFileSync('.github/ci-task-matrix.json', 'utf8'));
const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
for (const [file, project, related] of [
['ui/src/pages/Inbox.tsx', '@paperclipai/ui', 'ui/src/pages/Inbox.test.tsx'],
['server/src/middleware/auth.ts', '@paperclipai/server', 'server/src/__tests__/agent-auth-middleware.test.ts'],
]) {
const selected = await createTestSelection(planTasks({ config, lanes, paths: [file, "doc/DEVELOPING.md", "doc/plans/2026-10-30-ci-proof.md"] }), head);
const graph = selected.graphs.find(graph => graph.project === project);
assert.ok(graph.related.includes(related));
assert.ok(graph.alwaysRun.length > 0, 'Filesystem contracts must remain covered');
assert.ok(graph.selected.length < graph.inventory.length, 'The real workspace must support narrowing');
console.log(file, JSON.stringify(selected.plan.testSelection.stats, null, 2));
}
NODE
qualify:
runs-on: ubuntu-latest
timeout-minutes: 10
+9 -6
View File
@@ -350,24 +350,27 @@ jobs:
FORCE_FULL_CI: ${{ contains(github.event.pull_request.labels.*.name, 'full-ci') }}
CI_INCREMENTAL_MODE: ${{ vars.CI_INCREMENTAL_MODE || 'enabled' }}
STACK_JSON: ${{ toJSON(github.event.pull_request.stack) }}
EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }}
EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
EVENT_MERGE_SHA: ${{ github.sha }}
run: |
set -euo pipefail
fallback() { echo 'plan=' >> "$GITHUB_OUTPUT"; exit 0; }
[[ "$FULL_CI" == 'true' ]] || fallback
[[ "$EVENT_BASE_SHA" =~ ^[0-9a-f]{40}$ && "$EVENT_MERGE_SHA" =~ ^[0-9a-f]{40}$ ]] || fallback
[[ "$EVENT_HEAD_SHA" =~ ^[0-9a-f]{40}$ && "$EVENT_MERGE_SHA" =~ ^[0-9a-f]{40}$ ]] || fallback
trusted="$(gh api /repos/paperclipai/paperclip/git/ref/heads/master --jq .object.sha)" || fallback
[[ "$trusted" =~ ^[0-9a-f]{40}$ ]] || fallback
directory="$(mktemp -d)"
trap 'rm -rf "$directory"' EXIT
for file in .github/scripts/ci-plan.mjs .github/ci-task-matrix.json turbo.json; do
for file in .github/scripts/ci-plan.mjs .github/scripts/ci-inputs.mjs .github/scripts/ci-diff.mjs .github/ci-task-matrix.json turbo.json; do
gh api -H 'Accept: application/vnd.github.raw+json' \
"/repos/paperclipai/paperclip/contents/$file?ref=$trusted" \
> "$directory/${file##*/}" || fallback
done
gh api "/repos/paperclipai/paperclip/compare/$EVENT_BASE_SHA...$EVENT_MERGE_SHA" \
> "$directory/comparison.json" || fallback
# Read the exact tested merge and both parents. The event's base SHA can
# lag behind its actual first parent. A bare tree diff has no API file cap.
git init --bare "$directory/repository.git" >/dev/null || fallback
git -C "$directory/repository.git" fetch --quiet --no-tags --filter=blob:none --depth=2 \
https://github.com/paperclipai/paperclip.git "$EVENT_MERGE_SHA" || fallback
if [[ "$PROFILE_CI" == 'true' || "$FORCE_FULL_CI" == 'true' ]]; then
export CI_FORCE_REASON='Full suite requested by label'
elif [[ "$STACK_JSON" != 'null' && -n "$STACK_JSON" ]]; then
@@ -375,5 +378,5 @@ jobs:
fi
export CI_TASK_CONFIG="$directory/turbo.json"
export CI_TASK_MATRIX="$directory/ci-task-matrix.json"
export CI_COMPARISON="$directory/comparison.json"
export CI_DIFF_REPOSITORY="$directory/repository.git"
node "$directory/ci-plan.mjs" || fallback
+105 -10
View File
@@ -24,6 +24,82 @@ jobs:
gate:
uses: paperclipai/paperclip/.github/workflows/ci-routing.yml@master
test_selection:
name: Select Vitest files
needs: [gate]
if: >-
needs.gate.outputs.full_ci == 'true' && needs.gate.outputs.plan != '' &&
fromJSON(needs.gate.outputs.plan).mode == 'enabled' &&
fromJSON(needs.gate.outputs.plan).forceReason == null &&
(vars.CI_VITEST_SELECTION == '' || vars.CI_VITEST_SELECTION == 'enabled')
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 10
# An unavailable optimization must not suppress the original test matrix.
continue-on-error: true
outputs:
plan: ${{ steps.publish.outputs.plan }}
env:
CI_PLAN: ${{ needs.gate.outputs.plan }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
fetch-depth: 0
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: 24
package-manager-cache: false
- name: Check source-diff eligibility
id: eligible
run: |
if [[ -f scripts/ci/select-tests.mjs ]]; then
node scripts/ci/select-tests.mjs --eligible
else
echo 'enabled=false' >> "$GITHUB_OUTPUT"
fi
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86
if: steps.eligible.outputs.enabled == 'true'
with:
version: 9.15.4
- name: Restore dependencies for graph analysis
if: steps.eligible.outputs.enabled == 'true'
uses: ./.github/actions/setup-ci-dependencies
- name: Select files and rebalance test lanes
id: select
if: steps.eligible.outputs.enabled == 'true'
timeout-minutes: 5
run: node scripts/ci/select-tests.mjs
- name: Upload validated selection and audit
id: upload_selection
if: steps.select.outputs.selected == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: vitest-selection-${{ github.run_attempt }}
path: |
.ci-test-selection/manifest.json
.ci-test-selection/audit.json
include-hidden-files: true
if-no-files-found: error
retention-days: 7
# Publish only after upload succeeds. Missing/failed/timed-out selectors
# produce no plan, and dependent jobs use the original gate's matrix.
- name: Publish the executable plan
id: publish
if: steps.eligible.outputs.enabled == 'true'
env:
ARTIFACT_ID: ${{ steps.upload_selection.outputs.artifact-id }}
run: |
node --input-type=module <<'NODE'
import assert from 'node:assert/strict';
import { appendFileSync, readFileSync } from 'node:fs';
const plan = JSON.parse(readFileSync('.ci-test-selection/plan.json', 'utf8'));
if (plan.testSelection) {
assert.match(process.env.ARTIFACT_ID ?? '', /^[0-9]+$/);
plan.testSelection.artifactId = process.env.ARTIFACT_ID;
}
appendFileSync(process.env.GITHUB_OUTPUT, `plan=${JSON.stringify(plan)}\n`);
NODE
policy:
needs: [gate]
runs-on: ${{ needs.gate.outputs.runner }}
@@ -358,15 +434,17 @@ jobs:
general_tests:
name: General tests (${{ matrix.group_label }})
needs: [gate]
if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.general_tests.run) }}
needs: [gate, test_selection]
if: ${{ always() && !cancelled() && needs.gate.result == 'success' && needs.gate.outputs.full_ci == 'true' && ((needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.general_tests.run) }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.general_tests.matrix) || '{"include":[{"group":"general-server-without-chat","group_label":"server (1/12)","shard_index":0,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (2/12)","shard_index":1,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (3/12)","shard_index":2,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (4/12)","shard_index":3,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (5/12)","shard_index":4,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (6/12)","shard_index":5,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (7/12)","shard_index":6,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (8/12)","shard_index":7,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (9/12)","shard_index":8,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (10/12)","shard_index":9,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (11/12)","shard_index":10,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (12/12)","shard_index":11,"shard_count":12,"task":"ci:server"},{"group":"general-chat","group_label":"chat (1/3)","shard_index":0,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (2/3)","shard_index":1,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (3/3)","shard_index":2,"shard_count":3,"task":"ci:chat"},{"group":"general-workspaces-a","group_label":"workspaces-a (1/2)","shard_index":0,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-a","group_label":"workspaces-a (2/2)","shard_index":1,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-b","group_label":"workspaces-b","task":"ci:workspaces-b"},{"group":"general-workspaces-c","group_label":"workspaces-c","task":"ci:db"}]}') }}
matrix: ${{ fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan) != '' && toJSON(fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.general_tests.matrix) || '{"include":[{"group":"general-server-without-chat","group_label":"server (1/12)","shard_index":0,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (2/12)","shard_index":1,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (3/12)","shard_index":2,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (4/12)","shard_index":3,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (5/12)","shard_index":4,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (6/12)","shard_index":5,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (7/12)","shard_index":6,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (8/12)","shard_index":7,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (9/12)","shard_index":8,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (10/12)","shard_index":9,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (11/12)","shard_index":10,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (12/12)","shard_index":11,"shard_count":12,"task":"ci:server"},{"group":"general-chat","group_label":"chat (1/3)","shard_index":0,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (2/3)","shard_index":1,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (3/3)","shard_index":2,"shard_count":3,"task":"ci:chat"},{"group":"general-workspaces-a","group_label":"workspaces-a (1/2)","shard_index":0,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-a","group_label":"workspaces-a (2/2)","shard_index":1,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-b","group_label":"workspaces-b","task":"ci:workspaces-b"},{"group":"general-workspaces-c","group_label":"workspaces-c","task":"ci:db"}]}') }}
env:
PAPERCLIP_CI_TEST_SELECTION: ${{ matrix.selection && '.ci-test-selection/manifest.json' || '' }}
PAPERCLIP_CI_SELECTION_HASH: ${{ matrix.selection && fromJSON(needs.test_selection.outputs.plan).testSelection.hash || '' }}
PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }}
PAPERCLIP_CI_PROFILE_JOB: General tests (${{ matrix.group_label }})
@@ -376,6 +454,13 @@ jobs:
with:
persist-credentials: false
- name: Download selected test files
if: matrix.selection
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
with:
artifact-ids: ${{ fromJSON(needs.test_selection.outputs.plan).testSelection.artifactId }}
path: .ci-test-selection
- name: Initialize CI profile
continue-on-error: true
if: env.PAPERCLIP_CI_PROFILE == 'true'
@@ -438,7 +523,7 @@ jobs:
task: ${{ matrix.task }}
args: ${{ matrix.shard_count && format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) || '[]' }}
partition: ${{ strategy.job-index || 0 }}
force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }}
force: ${{ needs.gate.outputs.profile_ci == 'true' || (needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).mode != 'enabled' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).forceReason != null }}
- name: Run grouped general test suites
if: hashFiles('turbo.json') == ''
@@ -521,14 +606,14 @@ jobs:
verify:
name: verify
if: ${{ always() }}
needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run]
needs: [gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run]
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 5
steps:
- name: Validate every planned suite result
env:
FULL_CI: ${{ needs.gate.outputs.full_ci }}
CI_PLAN: ${{ needs.gate.outputs.plan }}
CI_PLAN: ${{ needs.test_selection.outputs.plan || needs.gate.outputs.plan }}
CI_RESULTS: ${{ toJSON(needs) }}
run: |
node --input-type=module <<'NODE'
@@ -539,6 +624,7 @@ jobs:
assert.ok(['true', 'false'].includes(process.env.FULL_CI), 'Valid scope required');
const plan = process.env.CI_PLAN ? JSON.parse(process.env.CI_PLAN) : null;
if (plan) assert.equal(plan.version, 1, 'Supported task plan required');
if (plan?.testSelection) assert.equal(results.test_selection.result, 'success', 'Published selection must succeed');
for (const job of ["typecheck_release_registry","general_tests","verify_paperclip_runner","build","docker_context_integrity","verify_serialized_server","canary_dry_run"]) {
if (plan) assert.equal(typeof plan.jobs[job]?.run, 'boolean', 'Missing suite decision: ' + job);
const selected = process.env.FULL_CI === 'true' && (!plan || plan.jobs[job].run);
@@ -1021,15 +1107,17 @@ jobs:
verify_serialized_server:
name: Verify serialized server suites (${{ matrix.shard_label }})
needs: [gate]
if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.run) }}
needs: [gate, test_selection]
if: ${{ always() && !cancelled() && needs.gate.result == 'success' && needs.gate.outputs.full_ci == 'true' && ((needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.verify_serialized_server.run) }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.matrix) || '{"include":[{"shard_index":0,"shard_count":9,"shard_label":"1/9","task":"ci:serialized"},{"shard_index":1,"shard_count":9,"shard_label":"2/9","task":"ci:serialized"},{"shard_index":2,"shard_count":9,"shard_label":"3/9","task":"ci:serialized"},{"shard_index":3,"shard_count":9,"shard_label":"4/9","task":"ci:serialized"},{"shard_index":4,"shard_count":9,"shard_label":"5/9","task":"ci:serialized"},{"shard_index":5,"shard_count":9,"shard_label":"6/9","task":"ci:serialized"},{"shard_index":6,"shard_count":9,"shard_label":"7/9","task":"ci:serialized"},{"shard_index":7,"shard_count":9,"shard_label":"8/9","task":"ci:serialized"},{"shard_index":8,"shard_count":9,"shard_label":"9/9","task":"ci:serialized"}]}') }}
matrix: ${{ fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan) != '' && toJSON(fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.verify_serialized_server.matrix) || '{"include":[{"shard_index":0,"shard_count":9,"shard_label":"1/9","task":"ci:serialized"},{"shard_index":1,"shard_count":9,"shard_label":"2/9","task":"ci:serialized"},{"shard_index":2,"shard_count":9,"shard_label":"3/9","task":"ci:serialized"},{"shard_index":3,"shard_count":9,"shard_label":"4/9","task":"ci:serialized"},{"shard_index":4,"shard_count":9,"shard_label":"5/9","task":"ci:serialized"},{"shard_index":5,"shard_count":9,"shard_label":"6/9","task":"ci:serialized"},{"shard_index":6,"shard_count":9,"shard_label":"7/9","task":"ci:serialized"},{"shard_index":7,"shard_count":9,"shard_label":"8/9","task":"ci:serialized"},{"shard_index":8,"shard_count":9,"shard_label":"9/9","task":"ci:serialized"}]}') }}
env:
PAPERCLIP_CI_TEST_SELECTION: ${{ matrix.selection && '.ci-test-selection/manifest.json' || '' }}
PAPERCLIP_CI_SELECTION_HASH: ${{ matrix.selection && fromJSON(needs.test_selection.outputs.plan).testSelection.hash || '' }}
PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }}
PAPERCLIP_CI_PROFILE_JOB: Verify serialized server suites (${{ matrix.shard_label }})
@@ -1039,6 +1127,13 @@ jobs:
with:
persist-credentials: false
- name: Download selected test files
if: matrix.selection
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
with:
artifact-ids: ${{ fromJSON(needs.test_selection.outputs.plan).testSelection.artifactId }}
path: .ci-test-selection
- name: Initialize CI profile
continue-on-error: true
if: env.PAPERCLIP_CI_PROFILE == 'true'
@@ -1101,7 +1196,7 @@ jobs:
task: ci:serialized
args: ${{ format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) }}
partition: ${{ strategy.job-index || 0 }}
force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }}
force: ${{ needs.gate.outputs.profile_ci == 'true' || (needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).mode != 'enabled' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).forceReason != null }}
- name: Run serialized server test shard
if: hashFiles('turbo.json') == ''
+1
View File
@@ -99,6 +99,7 @@ test-results/
# Opt-in CI timing artifacts
.ci-profile/
.ci-test-selection/
# Content-addressed CI task results
.turbo/
+16
View File
@@ -71,6 +71,22 @@
"packages/paperclip-runner/src/**",
"packages/paperclip-runner/runner/**"
]
},
{
"path": "rules/acpx-profile-identity.mdc",
"description": "ACPX provider identity model: provider digests attest upstream releases, the pack contract digest covers Runner source and the shared ACPX layer, and profile versions change only for deliberate resets.",
"scope": [
"packages/paperclip-runner/acpx-profiles.json",
"packages/paperclip-runner/cursor-contract.json",
"packages/paperclip-runner/cursor-distributions.json",
"packages/paperclip-runner/antigravity-distributions.json",
"packages/paperclip-runner/scripts/acpx-release-attestations.mjs",
"packages/paperclip-runner/scripts/generate-acpx-profiles.mjs",
"packages/paperclip-runner/scripts/contracts/**",
"packages/paperclip-runner/src/drivers/acpx/**",
"packages/paperclip-runner/src/providers/**",
"packages/paperclip-runner/protocol/**"
]
}
]
}
+4
View File
@@ -41,6 +41,10 @@ For `server/src/services/chat-channels.ts` and everything under
For `server/src/services/heartbeat.ts`, `server/src/services/heartbeat/`,
heartbeat-prefixed sibling services, and heartbeat regression tests, read
[`rules/heartbeat-execution-boundaries.mdc`](rules/heartbeat-execution-boundaries.mdc).
For ACPX provider profiles, their release attestations and pins, the provider-pack
contract registry, and ACPX driver and provider code under
`packages/paperclip-runner/`, read
[`rules/acpx-profile-identity.mdc`](rules/acpx-profile-identity.mdc).
See [`rules/README.md`](rules/README.md) for rule authoring and Greptile setup.
## 3. Repo Map
+63
View File
@@ -722,6 +722,60 @@ connection, before enabling the new server and UI.
- The repo keeps only the newest 5 snapshots. `generate` runs `prune:snapshots` afterwards to delete older ones. Drizzle only reads the newest snapshot, and each snapshot is a full copy of the schema (over 1 MB each). Older snapshots are still in git history.
- `packages/db/src/migration-snapshot-drift.test.ts` is the enforcement backstop. It repeats the diff that `generate` performs and fails when the newest snapshot no longer matches `packages/db/src/schema/`.
## Managed adapter settings
Cloud-managed instances keep adapter selection preferences in the private
`instance_settings` singleton `adapter-settings:v1`. Its `general` JSON contains
`version: 1`, a monotonic `revision`, and `disabledTypes`. It is separate from
the public `default` settings row and never appears in the instance settings API.
Each update locks this row in a transaction. The update and company activity
records commit together. Discovery and new-agent admission read the database
at their asynchronous operation boundaries; no polling or mutable settings cache
is used. Disabled adapters remain available to existing agents.
`PAPERCLIP_ADAPTERS` and `PAPERCLIP_ADAPTERS_FILE` declare the effective operator
adapter list. The declaration is parsed once for a process configuration and
validated at startup. It does not rewrite stored preferences. A conflicting
adapter toggle returns 403. Removing the declaration restores saved choices.
Operator file edits require a restart. Self-hosted instances continue to use
`adapter-settings.json` for saved preferences.
The server-only adapter settings service exposes `inspectStored()` and
`seedIfAbsent({ disabledTypes }, transaction?)` for migration tooling. Seeding
uses the unique singleton key, returns `{ seeded, current }`, and never replaces
an existing record. The caller can use the same transaction for its import
receipt. Before admission, `initializeAdapterSettings` checks the durable row.
When absent, it imports a valid legacy settings file once, retaining unknown
adapter types, or seeds empty preferences if no file exists. It rejects files
larger than 64 KiB, malformed input, and read failures. An existing durable row
wins over every stale file. Ordinary Cloud reads do not read a legacy file.
The focused PostgreSQL suite can use an owned disposable PostgreSQL database via
`PAPERCLIP_ADAPTER_SETTINGS_TEST_DATABASE_URL` when embedded PostgreSQL is
unavailable. It applies migrations and inserts test data, so never point this
variable at an existing application database.
## Cloud configuration migration receipt
The private `instance_settings` singleton `cloud-configuration-migration:v1`
records atomic import completion, semantic source digests, imported revisions,
and metadata-only external adapter blockers. It composes the adapter and
startup-config owners without replacing an existing authoritative row. Receipt
locking and the import transaction handle concurrent attempts and retries; no
local completion marker participates in conflict resolution.
Qualification records current configuration revisions and selected image package
evidence before normal plugin reconciliation. Image inspection does not hold
admin write locks; configuration and registration snapshots are rechecked in a
short transaction before the report is saved. A historical successful report
never grants current eligibility. External metadata blockers survive loss of the
legacy home and require a deliberate revision-checked recheck of a present valid
original source file. Package mismatches remain recorded after maintenance
reconciliation. See the [bounded migration and qualification procedure](cloud-configuration-migration.md)
for the stopped-writer prerequisite, commands, retry semantics, integrity limits,
and the explicit exclusion of secret bytes, whole-server zero downtime, and
volume removal.
## Cloud runtime identity singleton
The private `instance_settings` row whose singleton key is
@@ -1490,3 +1544,12 @@ It uses disposable embedded PostgreSQL by default. A disposable local database
named `review_wait_benchmark...` can be supplied through
`PAPERCLIP_TEST_DATABASE_URL` to compare another PostgreSQL major version.
DataRow bytes exclude response metadata and transport overhead.
## Cloud runtime configuration singleton
The private `instance_settings` key `cloud-runtime-config:v1` holds a strictly
validated, versioned nonsecret configuration and revision. It is separate from
the ordinary instance settings API. Seed inserts preserve an existing winner;
replacements lock the row and reject stale revisions. Database/transaction
handles permit atomic migration receipts. Connection URLs, API keys and unknown
fields are excluded. See [field ownership and startup order](cloud-runtime-configuration.md).
+81 -5
View File
@@ -171,10 +171,12 @@ does not include the lower PRs' code.
For ordinary full-scope PRs, the routing job further selects affected suites
using `turbo.json` and `.github/ci-task-matrix.json`. It downloads the planner
and both definitions from one canonical master revision, without executing PR
code. The comparison covers the tested merge commit. Both rename paths count;
failed, empty, malformed, or potentially truncated comparisons select the full
suite. Unclassified files also select full CI. Stack verification retains the
and its helpers and definitions from one canonical master revision, without executing PR
code. It fetches the exact tested merge commit and its parents into a temporary
bare Git repository, validates the PR head, and compares the merge tree against
its first parent. The event's base SHA can be stale. Git tree comparisons have
no GitHub compare-API 300-file cap. Renames count as deletion plus addition;
failed, empty, or malformed comparisons select the full suite. Unclassified files also select full CI. Stack verification retains the
existing scope policy and runs all suites whenever that policy requires them.
Changes to global inputs, including the task graph, package scripts, workflows,
and execution helpers, force full execution and disable result-cache restore and
@@ -193,6 +195,69 @@ canary dry run.
The uncached typecheck job also runs the repository-wide CLI guidance safety
guard. It still scans UI changes when unrelated server shards are omitted.
The shared policy in `.github/scripts/ci-inputs.mjs` exempts only audited prose:
`doc/DEVELOPING.md`, `doc/DATABASE.md`, and Markdown under `doc/plans/`.
The first two still select the uncached typecheck/guidance lane; historical plans
need only the always-running policy checks. These inputs do not widen product
test selection or enter product task cache hashes. All other docs, bundled
skills, fixtures and unknown paths keep their existing conservative coverage.
A source change plus exempt prose remains graph eligible. When adding a reader
of exempt prose, update this policy as well as the task inputs.
For ordinary source PRs (including the audited prose inputs above), a separate `Select Vitest files` job refines the
general and serialized test lanes before their runners are allocated. It runs
on the gate-approved test fleet, loads the real Vitest projects, and follows
imports across workspace packages. Only projects with their own source changes
are narrowed; unchanged consumer projects retain their complete coarse lane
coverage without paying for another graph traversal. Filesystem, subprocess, network, CommonJS,
dynamic-loader, unresolved, and unapproved external-package consumers are
retained conservatively, including indirect consumers through local helpers.
The selector explicitly tracks setup/configuration dependencies as well as test
imports. The small pure-dependency allowlist lives in
`scripts/ci/vitest-selection-graph.mjs`; new external packages remain opaque
until reviewed. This is deliberately broader than `vitest related` alone.
For conservative retention, explicit complete local mock factories cut the
mocked implementation's I/O edges. Partial mocks, spies, automocks, mock bypasses,
and dynamic or escaped mocking APIs do not. Edits to a mocked module still use
Vitest's unmodified related-file graph. Non-isolated projects run in full.
Literal Vite globs use Vite's expanded dependency edges, including newly added
members. Consumers of generated or untracked modules remain in the conservative
set because the import graph does not describe their build inputs.
The server's temporary-home and Supertest setup has a reviewed, content-hashed
exception. It still follows local helper imports. Any change to the reviewed
contents disables the exception until reviewed again; setup dependency changes
invalidate the whole project. Unreviewed setup I/O uses the full task plan.
Selected files are repartitioned within their existing owners using the current
per-runner budgets and server timing weights. Small selections allocate fewer
jobs; empty lanes allocate none. Serialized files remain serialized, and chat's
line shards, native Runner lanes, Rust, builds, typechecking and E2E retain their
existing selection/execution rules. The job summary reports files and jobs
selected per lane, and the `vitest-selection` artifact records the manifest and
dependency audit. Selected jobs execute exact Vitest test specifications, so
similarly named `.test.ts` and `.test.tsx` files cannot overlap between shards.
Missing/deleted/renamed source files, non-exempt non-source inputs, unrecognized graph inputs,
setup/config changes, failed dependency installs, graph errors and timeouts use
the complete original task plan. Source mutations during dependency setup also
disable narrowing. A published manifest must match the checked-out
merge SHA and parent diff, its plan hash and the expected shard/project ownership. Missing or
invalid manifests fail selected jobs and the required aggregate; they cannot
turn reduced matrices green. Failed-job retries download the original artifact
ID saved with the plan, even when GitHub increments the run attempt. Turbo hashes the selected file assignment through
`PAPERCLIP_CI_SELECTION_HASH`, so a passing subset cannot satisfy a full run or a
different selection.
Set the repository variable `CI_VITEST_SELECTION=off` to disable file selection
while keeping the existing task-level optimization. Empty or `enabled` enables
it; other values disable it. `full-ci`, profiling, and task-level `full`/`shadow`
modes also bypass file selection. Qualification uses real Vitest graphs and real
Turbo cache entries, including a filesystem contract omitted by the raw graph:
```sh
node --test scripts/ci/test-selection*.test.mjs scripts/ci/cache.integration.test.mjs
```
General and serialized test suites and the standalone Rust suite use Turbo's
content-addressed cache. An execution adapter maps selected lanes to the existing
package scripts and their declared inputs. It selects tracked and non-ignored
@@ -1488,7 +1553,9 @@ Paperclip also persists an empty `OPENAI_API_KEY` override for those agents so a
If the `codex` CLI is not installed or not on `PATH`, `codex_local` agent runs fail at execution time with a clear adapter error. Quota polling uses a short-lived `codex app-server` subprocess: when `codex` cannot be spawned, that provider reports `ok: false` in aggregated quota results and the API server keeps running (it must not exit on a missing binary).
Local adapters require their corresponding CLI/session setup on the machine running Paperclip. External adapters are installed through the adapter/plugin flow and should not require hardcoded imports in `server/` or `ui/`.
Local adapters require their corresponding CLI/session setup on the machine running Paperclip. Self-hosted instances can install external adapters through the adapter/plugin flow without hardcoded imports in `server/` or `ui/`.
Cloud-managed instances use only adapter implementations supplied by the release image. A tenant server token or the presence of `PAPERCLIP_MANAGED_CONFIG` enforces this boundary. Startup and UI parser requests do not read external adapter registrations or packages. Install, reinstall, reload, remove, and override operations return `403` with `adapter_install_platform_managed`, including for instance admins. Legacy `adapter-plugins.json` files stay untouched and do not change the built-in registry.
### Filesystem access boundary (2026-10-20)
@@ -3815,3 +3882,12 @@ in `_test` for `PAPERCLIP_MCP_SLEEP_TEST_DATABASE_URL`,
`PAPERCLIP_DOT_RUNNER_TEST_DATABASE_URL`. The interaction-service regression suite
also accepts `PAPERCLIP_INTERACTIONS_TEST_DATABASE_URL`. The suites migrate and write to those
URLs; the sleep suite truncates company data between tests.
## Cloud startup configuration
Cloud stores nonsecret startup choices in a private PostgreSQL singleton.
Bootstrap connection credentials stay in the supplied environment. A bounded
one-time config-file seed preserves existing choices, then later starts use the
database without loading home `.env` files or repairing worktree config. See
[Cloud runtime configuration](cloud-runtime-configuration.md) for field ownership,
secret handling, initialization order and migration interfaces.
+148
View File
@@ -0,0 +1,148 @@
# Cloud configuration migration and package qualification
This procedure covers saved adapter preferences, the allowlisted nonsecret
startup configuration, and selected bundled plugin packages. The result is a
point-in-time report. It does not authorize a deployment, guarantee zero
downtime, establish general Railway admission, or prove that a volume can be
removed. Secret and key bytes, database credentials, artifacts, logs, local
storage, workspaces, and plugin state outside PostgreSQL need their own storage
and replacement checks. This includes the decision signing key and local
encryption keys. Their byte authority is unchanged by this procedure.
Runtime environment and secret equality, plus an actual HTTP handoff between
application processes, remain separate rollout prerequisites. The package digest
does not establish those conditions.
## Migration prerequisite
Stop every legacy file writer before the first import. This includes old
application replicas and operator commands that can write configuration or
adapter settings. One maintenance window is sufficient. The importer cannot
make an old file writer participate in its database transaction. Keep those
writers stopped after import; later file edits do not update database authority.
Keep the normal database backup and recovery procedure for the instance. Use the
new release with the same managed environment and database connection. The
server runs migration before constructing configuration-dependent services.
For an explicit source-checkout invocation, use:
```sh
pnpm exec tsx server/src/qualify-cloud-configuration.ts --migrate
```
The command uses `DATABASE_URL` and the normal Cloud environment. It does not
accept credentials as arguments. It writes the receipt to PostgreSQL and prints
a JSON report; no report file or local completion marker is required. Exit code
0 means the requested configuration/package check passed, 2 means the report has
blockers, and 1 means the operation failed. A fresh installation can report a
missing plugin registration before normal boot installation completes. Run the
check again after that installation settles.
## Database authority and retry behavior
The private `instance_settings` singleton `cloud-configuration-migration:v1`
contains the versioned receipt. The transaction locks that row and composes the
adapter and startup-config owners' atomic seed methods. Completion and both
seeds commit together. A failed write or crash before commit leaves no completed
receipt. A retry uses committed database state and never replaces a later
settings revision. Earlier independently seeded owner records remain authoritative.
The receipt records source kind, a digest of the nonsecret semantic candidate,
the imported revision, and bounded legacy external-registration evidence.
An absent startup configuration remains an authoritative `config: null`; it is
distinct from an uninitialized database record. No raw secret values are copied.
Once the receipt exists, ordinary boots do not reread legacy source files.
A replacement home can be empty or absent without changing saved authority.
All legacy readers have a 64 KiB limit. They require regular files, reject
symlinks and FIFOs, and use bounded reads. External registration inspection reads
metadata only. It never loads a registered module or follows a registered package
path. A nonempty, invalid, or unreadable registration file creates a durable
blocker. Removing the old home does not clear that blocker.
To resolve an external-registration blocker, stop its writer, deliberately
remove the unsupported registration metadata at the original source location,
and leave a present valid empty JSON array file. Read the current receipt revision
from the report, then run:
```sh
pnpm exec tsx server/src/qualify-cloud-configuration.ts --recheck-external-revision 3
```
Use the actual current revision in place of `3`. A changed revision, a different
source location, or a missing/invalid source file rejects the recheck. Reusing
the same container path with an empty home is insufficient. This is an explicit
operator resolution; qualification never clears the blocker automatically.
## Current packages and overlap comparison
Every qualification reads current settings revisions and plugin registrations.
It resolves selected packages from this image with the same bundled package
identity rules as normal loading. It verifies package bytes before importing a
trusted image manifest, then compares that manifest with the approved database
registration. Missing packages, missing evidence, incompatible versions,
unselected registrations, and integrity failures prevent qualification. Legacy
package paths and npm discovery cannot supply missing image bytes.
Image inspection happens outside database write locks. A short transaction then
locks and rereads the current configuration and registration records. A change
during inspection produces a blocker and requires a new check. The report gives
`adapterRevision`, `configRevision`, `configurationDigest`, and
`packageEvidenceDigest`. A historical completed receipt is evidence, not current
authorization. Call qualification again for a current result.
The server records this report before ordinary bundled-plugin reconciliation.
It retains `lastPackageMismatch` after later successful checks, so reconciliation
cannot erase the evidence seen before an upgrade. The report does not block
normal maintenance upgrades or replace capability approval. After an upgrade
settles, its current packages can qualify normally.
For a same-package comparison, capture the current release's
`packageEvidenceDigest` and pass it explicitly on the replacement:
```sh
pnpm exec tsx server/src/qualify-cloud-configuration.ts --expected-package-evidence sha256:REPLACE_WITH_THE_64_HEX_DIGEST
```
`configurationPackagesEligible` describes current database/configuration/package
compatibility. `samePackageOverlapEligible` is true only when an explicit expected
digest also matches. There is no permanent first-release baseline that would
prevent future maintenance releases from establishing their own evidence.
The package evidence digest covers selected packages, not container image bytes.
Distribution packages reuse and reverify their catalog digest. Builtin packages
have a bounded inventory of physical file bytes, modes, and logical symlink
targets, including standalone pnpm dependency trees. With the selected catalog
root, inventory may follow links only into that provider, the catalog SDK,
`packages/shared`, and the current image's root `node_modules`. Reached pnpm
package units include their sibling dependency links; unrelated root packages
are not scanned. Canonical directory/file deduplication bounds cycles, and
logical root labels preserve the digest when identical image bytes relocate.
Dangling or escaping links, special files, more than 50,000 entries, depth above
32, metadata above 128 KiB, or more than 256 MiB per package fail verification.
The measured cached Cloud Daytona closure has about 22,253 entries and 130 MB;
that artifact predates this change and is not a complete current release proof.
Inventory work is synchronous and linear in selected package bytes. It runs only during startup before the application is constructed
or in the explicit operator process, with no idle polling. The report includes
`packageInspectionDurationMs`. Packages must remain immutable throughout a check.
## Verification scope
The focused PostgreSQL tests use two independent clients and separate home/image
roots. They import legacy settings, change saved revisions, remove the old roots,
reconstruct from the database, and start a real plugin worker from the replacement
image fixture. They also cover concurrent migration, commit failure and retry,
database failure, stale sources, durable external blockers, package mismatch,
maintenance reconciliation, and changes during manifest inspection.
The bundled-plugin registration suite also has an actual prebuilt Daytona case.
Build its standalone package with the normal SDK linker before running that case;
it is skipped when the provider build is absent. It qualifies the real linked
SDK/shared package graph and uses the production loader and worker manager for
credential-free initialize/health calls. The runtime-path tests also invoke the
production server bundler to verify its preserved module locations.
Set `PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL` only to an owned disposable
PostgreSQL database when embedded PostgreSQL is unavailable. The suite applies
migrations and writes/deletes test records. A complete release image, hosted
replacement, overlapping production processes, secret-byte migration, and volume
removal are not established by these local fixtures.
+118
View File
@@ -0,0 +1,118 @@
# Cloud runtime configuration
Cloud-managed servers store nonsecret startup and application choices in the
private `instance_settings` singleton `cloud-runtime-config:v1`. The row contains
`version: 1`, an increasing `revision`, and a strict allowlist of configuration
fields. It is separate from the normal instance settings row and is not exposed
by the instance settings API.
The canonical managed-instance predicate applies: a nonblank Cloud tenant token
or the presence of `PAPERCLIP_MANAGED_CONFIG`, including a blank value, selects
this configuration path. Invalid managed feature documents still retain their
existing validation at application startup.
## Startup and authority
1. The hosting environment supplies `DATABASE_URL` and, if needed,
`DATABASE_MIGRATION_URL`. Cloud requires a PostgreSQL URL and never falls back
to embedded PostgreSQL. Credentials, trust keys, provider configuration and
deployment wiring needed before the database connection remain supplied by
the environment or the existing secret system. This phase parses only the
database connection record. Runtime mode, host binding and authentication
checks wait until the saved snapshot has loaded.
2. After schema migration and database connection, startup reads the private
runtime configuration row. An existing invalid row or a database error stops
startup. Neither condition selects a legacy file or an old process snapshot.
3. If no row exists, startup imports the legacy config once. It uses
`PAPERCLIP_CONFIG` when supplied, otherwise the instance home config path.
It does not search working-directory ancestors. A truly absent file saves
an authoritative `config: null` marker. This retains the existing dynamic
environment/image defaults, including authentication and secret strictness.
The marker never enables a later file fallback. A present malformed, linked, nonregular, or oversized file
stops initialization. Reads and stored snapshots are limited to 64 KiB.
4. The seed uses an insert that does not replace an existing row. Concurrent
initializers read the committed winner. Later starts do not inspect the
legacy file, even when it is absent, corrupt, or changed.
5. Startup publishes a detached configuration snapshot, restores the signed
Cloud runtime identity, then initializes telemetry, secret-provider defaults,
authentication and application services. A saved telemetry opt-out therefore
takes effect before a telemetry client can be created.
Existing operator environment overlays keep precedence over saved values.
Reading an overlaid value does not save it back to the database. Credentials and
connection URLs are never copied into this JSONB row.
Cloud does not load instance or working-directory `.env` files and does not run
legacy worktree configuration repair. The hosting system must supply those
bootstrap variables directly. In particular, a database connection kept only
in `.env` cannot bootstrap this path. Self-hosted configuration and dotenv
behavior remain unchanged.
## Field ownership
The table lists every field consumed from the existing config file and the
remaining known schema fields. Environment-only `Config` fields, such as
heartbeat maintenance flags and feedback-export credentials, retain their
existing environment sources and are not frozen into the saved snapshot.
| Config file fields | Cloud owner |
| --- | --- |
| `server.deploymentMode`, `exposure`, `bind`, `customBindHost`, `host`, `port`, `allowedHostnames`, `serveUi` | Saved runtime configuration, with existing environment overlays. |
| `auth.baseUrlMode`, `publicBaseUrl`, `disableSignUp` | Saved runtime configuration, with environment overlays and the signed Cloud identity restoration. Saved URLs cannot contain userinfo, query parameters, or fragments. |
| `database.backup.enabled`, `intervalMinutes`, `retentionDays`, `dir` | Saved runtime configuration, with existing backup environment overlays. |
| `database.mode`, `connectionString` | Cloud bootstrap requires the environment `DATABASE_URL`. The legacy URL is never saved. |
| `database.embeddedPostgresDataDir`, `embeddedPostgresPort` | Local embedded-database settings. Cloud does not use embedded PostgreSQL or these paths. |
| `storage.provider`, `localDisk.baseDir`, `s3.bucket`, `s3.region`, `s3.endpoint`, `s3.prefix`, `s3.forcePathStyle` | Saved runtime configuration, with existing environment overlays. Saved endpoints cannot contain userinfo, query parameters, or fragments. |
| `secrets.provider`, `strictMode`, `localEncrypted.keyFilePath` | Saved nonsecret provider choice and locator, with existing environment overlays. No key bytes are imported or saved. |
| `telemetry.enabled` | Saved choice, passed to the existing telemetry initialization and opt-out policy before client creation. |
| `llm.provider` | Saved provider metadata. |
| `llm.apiKey` | Never saved. The existing OpenAI model-discovery consumer uses `OPENAI_API_KEY` first. A legacy OpenAI key requires that environment credential before import, or startup stops. Claude key metadata has no server consumer. |
| `logging.mode`, `logDir`, `updates.checkEnabled` | Nonsecret legacy/CLI metadata is preserved. Server logging continues to use its existing environment configuration. |
| `$meta` | CLI file-write metadata is not application authority. The compatibility view synthesizes valid metadata. |
| Unknown extension keys | No server config consumer exists. They are excluded from the strict saved allowlist, including any unknown credential fields. |
A saved path is a locator. This change does not make local storage objects,
backup bytes, logging files, or secret-key bytes durable. Their storage owners
and existing environment/secret mechanisms retain responsibility for those
bytes. Restart tests prove configuration restoration under a different home;
they do not prove durable file, secret, or backup storage.
The decision signing service still uses `PAPERCLIP_DECISION_SIGNING_SECRET` when
supplied and otherwise reads or creates its instance key file before the database
connection. The local encrypted secret provider still uses its environment key
or existing key-file mechanism. Those byte-storage fallbacks are outside this
configuration migration. Qualification must account for them separately; this
PR does not establish file-free secret storage on Cloud.
## Internal update and migration interface
`cloudRuntimeConfigService(db)` provides `inspectStored(handle?)`,
`seedIfAbsent(config, handle?)`, and `replace(config, expectedRevision, handle?)`.
Each method accepts a database or transaction handle. A migration can compose a
config seed, related settings and a receipt in one transaction.
Replacement validates the same allowlist, locks the singleton row, checks the
expected revision, and increases it. A stale revision fails. There is no new
public configuration endpoint or settings UI. Startup consumers use a snapshot;
a saved replacement takes effect on the next initialization/restart.
`readLegacyCloudRuntimeConfig(env?)` returns the validated nonsecret candidate
without writing it. `initializeCloudRuntimeConfig` performs the standalone
one-time seed and publishes its committed snapshot. Integrated migration
receipts and deployed Cloud qualification are separate consumers of this
interface.
## Verification
`server/src/__tests__/cloud-runtime-config.test.ts` exercises real PostgreSQL
clients, concurrent writes, transaction rollback, changed homes, a fresh Node
process, environment precedence, secret exclusion, invalid data and database
failure. It uses embedded PostgreSQL by default. A dedicated disposable database
can be selected with `PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL` when the host
cannot start embedded PostgreSQL. That database must be reserved for this test.
The test applies the current schema before use.
The startup orchestration controls in
`server-startup-feedback-export.test.ts` hold configuration initialization and
verify that telemetry, authentication and app construction wait for it. These
local controls do not claim a live Cloud rollout.
+36
View File
@@ -34,6 +34,42 @@ Current limitations to keep in mind:
In practice, that means the current implementation is a good fit for local development and self-hosted persistent deployments, but not yet for multi-instance cloud plugin distribution.
### Managed image packages
Managed Cloud provisions only plugins selected from the current release image
catalog. A Cloud tenant token or managed configuration keeps this image-only
policy active. Without an explicit selected package list, Cloud selects no
packages and cannot fall back to local installs. An explicit operator-managed
list also keeps this policy active outside Cloud. The database registration's
plugin key, package name, version and
approved manifest identify that package. Its legacy `packagePath` remains for
local/self-hosted compatibility; Cloud never uses that path to locate code.
Workers, plugin migrations and UI files resolve through the same current-image
package resolver. Cloud does not install npm dependencies, build packages, use
the development UI proxy, or fall back to a server home directory.
Prebuilt workers use the current release's baked CLI `tsx` loader for the SDK's
workspace TypeScript dependencies. The loader is located from the owning server
package in both regular and bundled server layouts. Missing loaders and loader
links outside the release's CLI or root `node_modules` fail activation. This
loads image-supplied code without installing dependencies or building packages.
Equivalent packages at different image roots reuse the existing registration,
configuration, approval status and company scopes without rewriting its stored
path. Concurrent first registration of the same package retains one plugin ID.
Conflicting package identities fail explicitly. Normal maintenance boot retains
the existing bundled manifest/version reconciliation and approval gates.
`createBundledPluginPackageResolver(...).inspect(registration, verifiedManifest)`
is a read-only qualification hook. Resolve the current image path, load its
manifest with the validated plugin loader, and inspect the registration before
normal boot reconciliation. Missing manifest evidence and incompatible packages
produce explicit blockers. Distribution packages retain their catalog digest
verification; built-in packages trust the immutable release image and must bump
their version when their code changes. Qualification supports identical package
sets only. This package-resolution boundary does not coordinate workers,
background jobs, or deployment overlap between servers.
## 1. Scope
This spec covers:
@@ -74,6 +74,9 @@ API: `present` — representative operation IDs:
- `PATCH /api/instance/settings`.
- `GET /api/instance/settings/experimental`.
- `PATCH /api/instance/settings/experimental`.
- `GET /api/adapters`.
- `GET /api/adapters/{type}`.
- `PATCH /api/adapters/{type}`.
Agent access: `restricted` — Board/admin instance settings; CLI configure/env; local
configuration files.
@@ -88,9 +91,38 @@ Verification: `not-run` — assert the documented agent denial with an owned tes
verify any listed agent request/result path separately. A delegated operation needs its
own policy and access review.
Cloud runtime configuration uses a separate private PostgreSQL singleton and
adds no public read or mutation endpoint. Existing instance settings APIs do not
expose it. Bootstrap database credentials and trust keys remain in the supplied
environment; the strict saved allowlist excludes API keys and connection URLs.
Internal configuration updates require a matching revision and transaction lock.
[Database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) cover
seed/restart, credential exclusion, conflicts and rollback. These are local
controls, not agent authorization or deployed Cloud qualification.
Gaps: Settings APIs cover parts of configuration; pre-start database/binding/storage
configuration is local operator tooling.
Adapter toggles require instance-admin authority and remain blocked when
`instance.adapters` is hidden. Ordinary board members can read the catalog;
agent keys cannot toggle adapters. Managed Cloud preferences are stored in a
private database singleton. The operator adapter declaration sets the effective
list and rejects conflicting toggles. [PostgreSQL regression coverage](../../server/src/services/adapter-settings-postgres.test.ts)
exercises persistence, concurrent writers, admission freshness, activity audit,
and failed writes; [authorization tests](../../server/src/__tests__/adapter-routes-authz.test.ts)
cover admin and hidden-setting gates. These are local tests, not deployed proof.
Cloud configuration migration and package qualification are operator commands,
not agent-accessible REST capabilities. They require the managed environment and
database operator access. The [command](../../server/src/qualify-cloud-configuration.ts)
composes transaction-safe owner seeds and writes a private receipt. Every
qualification rereads current revisions and image package evidence; a completed
historical receipt grants no deployment or execution authority. [PostgreSQL coverage](../../server/src/__tests__/cloud-configuration-migration-postgres.test.ts)
uses independent clients, lost-home reconstruction, a real replacement fixture
worker, rollback/retry, and durable blockers. [Reader coverage](../../server/src/__tests__/cloud-configuration-readers.test.ts)
checks bounded sources and pnpm-style link integrity. These are local fixtures;
see the [scope and operator procedure](../../doc/cloud-configuration-migration.md).
Cloud stack Admins may update experimental settings through the trusted Cloud
actor without receiving other instance-admin powers. This uses the current
stack role on each request; company membership alone does not grant it.
+8 -1
View File
@@ -55,7 +55,14 @@ Policy: Company model discovery requires company access; account-backed discover
also enforces its account/pool and target-agent rules. Instance adapter inventory
and installation retain board/admin gates. Delegated managed
AI-account login operations are enumerated under `model-accounts`; they do not
grant raw host administration or execution-session reset.
grant raw host administration or execution-session reset. On Cloud-managed
instances, external adapter install, reinstall, reload, removal and override
operations return `403` with `adapter_install_platform_managed`, including for
instance admins. Built-in metadata remains available to board users. Startup
and UI parser reads do not consume saved external registrations or package code.
[Local fixture controls](../../server/src/adapters/cloud-external-adapters.test.ts)
and [route controls](../../server/src/__tests__/adapter-routes-authz.test.ts)
cover this boundary; they do not establish live agent or provider qualification.
Evidence: `source-only` — [implementation](../../server/src/routes/agents.ts#L6608) at
the reviewed commit above.
+30
View File
@@ -63,8 +63,38 @@ unfinished application write must still prevent sleep after its client leaves.
### `settings-backup`
Cloud startup reads nonsecret application choices from a private, versioned
PostgreSQL singleton after a bounded one-time legacy config seed. A fresh
process and different home restore the saved choices without consulting a stale
`.env` file. Invalid saved data or an unavailable database prevents startup;
saved telemetry opt-out applies before client creation. The
[database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) and
[startup-order controls](../../server/src/__tests__/server-startup-feedback-export.test.ts)
cover these local boundaries. See [field ownership](../../doc/cloud-runtime-configuration.md).
These controls do not qualify a deployed Cloud rollout or durable file storage.
Automated: [experimental settings](../../ui/src/pages/InstanceExperimentalSettings.test.tsx) and [database backup routes](../../server/src/__tests__/instance-database-backups-routes.test.ts) cover their UI/API contracts.
Cloud migration: in one maintenance window, stop legacy file writers and run the
[configuration/package qualification command](../../doc/cloud-configuration-migration.md).
Inspect its database receipt, update a saved preference, then repeat against a
second owned application/image root with the first home unavailable. Use an
explicit package evidence digest for same-package comparison. Confirm stale
qualification and missing legacy files cannot clear blockers. This recipe covers
configuration/packages and does not qualify whole-server zero downtime, secret
bytes, or volume removal. [Local PostgreSQL evidence path](../../server/src/__tests__/cloud-configuration-migration-postgres.test.ts)
includes a real fixture worker; it does not establish a hosted replacement.
Managed adapters: change a visible adapter preference as an instance admin,
then use a second application process against the same disposable database to
confirm catalog and new-agent selection reflect it. Verify saved agents remain
usable. Restart with an operator adapter declaration, confirm its effective
list, and reject conflicting toggles. Remove the declaration and confirm saved
preferences return. [Database tests](../../server/src/services/adapter-settings-postgres.test.ts)
cover local persistence, concurrent writers, audit rollback, and failure responses;
[local compatibility tests](../../server/src/services/adapter-settings-local.test.ts)
cover the legacy file. No deployed walkthrough is claimed.
Manual: Change a harmless experimental setting, reload, and confirm the affected route honors it. Back up a disposable instance, inspect the reported file and metadata, and restore using the documented procedure into a second isolated target. Verify a known record after restoration.
Cloud staging: On a managed `*.staging.paperclip.app` instance, verify experiments
+9
View File
@@ -170,6 +170,15 @@ controls do not establish live provider acceptance.
### `adapter-setup`
Cloud-managed instances load adapters from the release image. Saved external
registrations cannot replace built-ins or supply UI parser code. Install,
reinstall, reload, remove, and override operations remain platform-managed.
The [fixture loader controls](../../server/src/adapters/cloud-external-adapters.test.ts)
exercise real packages, import side effects, both managed-instance signals and
self-hosted compatibility. The [route authorization controls](../../server/src/__tests__/adapter-routes-authz.test.ts)
check managed denials before package or registration access. These are local
controls, not live Cloud provider qualification.
Automated: [adapter validation routes](../../server/src/__tests__/agent-adapter-validation-routes.test.ts) and [model refresh](../../server/src/__tests__/adapter-model-refresh-routes.test.ts) cover server behavior.
Claude native setup uses the existing Anthropic API-key or subscription connection
+6
View File
@@ -27,6 +27,12 @@ checks only assignment to the signed-in user and persistence after reload. Its
source is reusable across self-hosted and Cloud; Cloud live qualification is
separate. It does not cover other property operations or agent authorization.
The [Product task-search-filter journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md)
checks `tasks/organize` with owned matching and nonmatching controls, exact task
identities, saved task content, and search/status persistence after reload and
detail/back navigation. It is model-free and reusable across deployment targets.
Live Cloud qualification is separate from the journey's assertion unit tests.
## How to get to it (user POV)
Cold collection loads use a lightweight index and batched hierarchy context,
+1
View File
@@ -101,6 +101,7 @@
"agentmail": "^0.5.31",
"cross-env": "^10.1.0",
"esbuild": "^0.28.2",
"oxc-parser": "0.127.0",
"typescript": "^7.0.2",
"vitest": "^5.0.3",
"yaml": "^2.8.1",
+7
View File
@@ -552,6 +552,13 @@ pnpm --filter @paperclipai/paperclip-runner verify
The verification command requires a stable Rust toolchain with `cargo` on
`PATH`, in addition to Node.js 24.11+ and pnpm 9+.
Rust development builds use `debug = "line-tables-only"` to reduce build storage
while retaining file names and line numbers in backtraces. The default test
profile inherits this setting. Incremental compilation remains enabled, and
release profiles keep their existing settings. For full debugger support,
including local variables and types, set `CARGO_PROFILE_DEV_DEBUG=2` and
`CARGO_PROFILE_TEST_DEBUG=2` for the build or test command.
Minimal Debian/Ubuntu hosts without root access can extract the required
Playwright browser libraries into a user-owned cache and run the same acceptance
sequence with:
@@ -73,17 +73,30 @@ real transport; it does not qualify a hosted OpenCode or ACPX deployment.
## Profile identity
Every ACPX provider's `commandDigest` attests only its own upstream release.
For Copilot (profile 92), Grok (20) and Pi (37) it binds the upstream package
versions, the platform closure pins and, for Grok, the launcher bytes.
`scripts/acpx-release-attestations.mjs` computes these digests and
`generate:acpx-profiles` writes them. Runner source changes do not change them.
Every ACPX harness binds the shared ACPX layer the same way: saved inputs must
match the current ACPX version, and the provider pack's `contractDigest` binds
the Runner's ACPX patch with the other Runner contracts.
Saved inputs with any profile version from 1 through the current one decode, so
they reach recovery. Launch and recovery admission require the exact current
profile and digest, and refuse the rest. A profile identity does not establish
hosted harness qualification.
Each provider declares how in `ACPX_RELEASE_ATTESTATIONS`
(`scripts/acpx-release-attestations.mjs`):
| Kind | Providers | The digest is |
| --- | --- | --- |
| `installed-command` | Claude, Codex | the upstream package's launch command, hashed at installation |
| `launcher` | Grok | the SHA-256 of the launcher shipped verbatim |
| `closure` | Copilot, Pi | upstream versions and the complete platform closure pins |
| `declaration` | Cursor, Antigravity | a checked-in release declaration, as canonical JSON |
`generate:acpx-profiles` writes the computed digests and `check:acpx-profiles`
refuses a stale one, so a digest changes only with its upstream release. Every
ACPX harness binds the shared ACPX layer the same way: saved inputs must match
the current ACPX version, and the provider pack's `contractDigest` binds the
Runner's ACPX patch with the other Runner contracts. Runner source changes
change no provider digest.
A profile version changes only to reset every saved session for that provider
on purpose; an upstream release keeps it, and a new provider starts at 1. Saved
inputs with any version from 1 through the current one decode, so they reach
recovery. Launch and recovery admission require the exact current profile and
digest, and refuse the rest. A profile identity does not establish hosted
harness qualification. [`rules/acpx-profile-identity.mdc`](../../../rules/acpx-profile-identity.mdc)
summarizes these practices for contributors.
## External controller checkpoints
@@ -2,6 +2,9 @@
members = ["crates/runner-core"]
resolver = "2"
[profile.dev]
debug = "line-tables-only"
[workspace.package]
version = "0.0.0"
edition = "2021"
@@ -3,13 +3,34 @@ import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { pathToFileURL } from "node:url";
// Copilot, Grok and Pi profile identities attest upstream releases, like Claude,
// Codex, Cursor and Antigravity. They bind upstream versions and the bytes a
// repackager copies verbatim: complete platform closures and Grok's launcher.
// Never bind Paperclip source, the shared ACPX layer, absolute paths, the
// provider-pack manifest digest or install layout. The profile's acpxVersion and
// the pack contract digest bind ACPX and Runner source for every harness alike.
export const ATTESTED_ACPX_AGENTS = Object.freeze(["copilot", "grok", "pi"]);
/**
* How each ACPX provider's `commandDigest` attests its upstream release. Every
* profile in acpx-profiles.json declares one kind here:
*
* - `installed-command`: installation hashes the upstream package's launch
* command and compares it with the digest (Claude, Codex).
* - `launcher`: the SHA-256 of a launcher shipped verbatim (Grok).
* - `closure`: upstream versions and the complete platform closure pins.
* - `declaration`: a checked-in release declaration, hashed as canonical JSON.
*
* A digest changes only when the upstream release changes. It never binds
* Paperclip source, the shared ACPX layer, absolute paths, the provider-pack
* manifest digest or install layout: the profile's acpxVersion and the pack
* contract digest bind ACPX and Runner source for every provider alike.
* See rules/acpx-profile-identity.mdc.
*/
export const ACPX_RELEASE_ATTESTATIONS = Object.freeze({
claude: { kind: "installed-command" },
codex: { kind: "installed-command" },
grok: { kind: "launcher", path: "src/providers/grok/launcher.cjs" },
copilot: { kind: "closure", pins: { closure: ["copilot-closure-pins.ts", "COPILOT_CLOSURE_SHA256"] } },
pi: { kind: "closure", pins: {
nodeVersion: ["pi-node-pins.ts", "PI_NODE_VERSION"],
closure: ["pi-closure-pins.ts", "PI_DISTRIBUTION_CLOSURE_SHA256"],
} },
cursor: { kind: "declaration", path: "cursor-contract.json", digestField: "commandDigest" },
antigravity: { kind: "declaration", path: "antigravity-distributions.json" },
});
const sha256 = bytes => createHash("sha256").update(bytes).digest("hex");
const canonicalJson = value => value && typeof value === "object"
@@ -17,28 +38,42 @@ const canonicalJson = value => value && typeof value === "object"
: `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
: JSON.stringify(value);
/** The release attestation for one provider, computed from a runner package root. */
/**
* The release attestation for one provider, computed from a runner package
* root, or null when installation verifies the digest instead.
*/
export async function acpxReleaseAttestation(agent, packageRoot) {
const manifest = JSON.parse(readFileSync(resolve(packageRoot, "acpx-profiles.json"), "utf8"));
const profile = manifest.profiles[agent];
if (!ATTESTED_ACPX_AGENTS.includes(agent) || !profile) throw new Error(`No upstream release attestation for ${agent}`);
// Grok keeps its physical byte-digest contract: admission hashes the launcher.
if (agent === "grok") {
return { commandDigest: `sha256:${sha256(readFileSync(resolve(packageRoot, "src/providers/grok/launcher.cjs")))}` };
const attestation = Object.hasOwn(ACPX_RELEASE_ATTESTATIONS, agent) ? ACPX_RELEASE_ATTESTATIONS[agent] : undefined;
if (!profile || !attestation) {
throw new Error(`${agent} needs a release attestation: declare how its commandDigest attests the upstream release `
+ "in ACPX_RELEASE_ATTESTATIONS (scripts/acpx-release-attestations.mjs). See rules/acpx-profile-identity.mdc.");
}
switch (attestation.kind) {
case "installed-command":
return null;
case "launcher":
return { commandDigest: `sha256:${sha256(readFileSync(resolve(packageRoot, attestation.path)))}` };
case "declaration": {
const { [attestation.digestField]: _digest, ...declaration } = JSON.parse(readFileSync(resolve(packageRoot, attestation.path), "utf8"));
return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
}
case "closure": {
const declaration = {
schema: "paperclip.acpx-release-attestation.v1", agent,
agentProfileVersion: profile.agentProfileVersion,
agentServerPackage: profile.agentServerPackage, agentServerVersion: profile.agentServerVersion,
agentRuntimePackage: profile.agentRuntimePackage, agentRuntimeVersion: profile.agentRuntimeVersion,
};
for (const [field, [file, name]] of Object.entries(attestation.pins)) {
// Key each import on content so a process never reuses pins from edited bytes.
const path = resolve(packageRoot, "src/drivers/acpx", file);
declaration[field] = (await import(`${pathToFileURL(path).href}?sha256=${sha256(readFileSync(path))}`))[name];
}
return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
}
default:
throw new Error(`${agent} declares an unknown release attestation kind: ${attestation.kind}`);
}
// Key each import on content so a process never reuses pins from edited bytes.
const pins = async file => {
const path = resolve(packageRoot, "src/drivers/acpx", file);
return import(`${pathToFileURL(path).href}?sha256=${sha256(readFileSync(path))}`);
};
const declaration = {
schema: "paperclip.acpx-release-attestation.v1", agent,
agentProfileVersion: profile.agentProfileVersion,
agentServerPackage: profile.agentServerPackage, agentServerVersion: profile.agentServerVersion,
agentRuntimePackage: profile.agentRuntimePackage, agentRuntimeVersion: profile.agentRuntimeVersion,
...agent === "copilot"
? { closure: (await pins("copilot-closure-pins.ts")).COPILOT_CLOSURE_SHA256 }
: { nodeVersion: (await pins("pi-node-pins.ts")).PI_NODE_VERSION, closure: (await pins("pi-closure-pins.ts")).PI_DISTRIBUTION_CLOSURE_SHA256 },
};
return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
}
@@ -5,10 +5,12 @@ import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import test from "node:test";
import { acpxReleaseAttestation, ATTESTED_ACPX_AGENTS } from "./acpx-release-attestations.mjs";
import { ACPX_RELEASE_ATTESTATIONS, acpxReleaseAttestation } from "./acpx-release-attestations.mjs";
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const manifest = JSON.parse(readFileSync(join(packageRoot, "acpx-profiles.json"), "utf8"));
// Providers whose digest the generator computes; installation verifies the rest.
const COMPUTED = Object.keys(manifest.profiles).filter(agent => ACPX_RELEASE_ATTESTATIONS[agent]?.kind !== "installed-command");
const packageFiles = [
"acpx-profiles.json", "package.json", "cursor-distributions.json", "cursor-contract.json", "antigravity-distributions.json",
"scripts/acpx-release-attestations.mjs", "scripts/generate-acpx-profiles.mjs",
@@ -33,15 +35,27 @@ function relocatedPackage(t) {
const generate = (root, ...args) => spawnSync(process.execPath, [join(root, "scripts/generate-acpx-profiles.mjs"), ...args],
{ cwd: root, env: { PATH: process.env.PATH }, encoding: "utf8", timeout: 20_000 });
test("the manifest carries each provider's current release attestation", async () => {
for (const agent of ATTESTED_ACPX_AGENTS) {
test("every profile declares how its digest attests the upstream release", async () => {
assert.deepEqual(Object.keys(ACPX_RELEASE_ATTESTATIONS).sort(), Object.keys(manifest.profiles).sort());
assert.deepEqual(COMPUTED.sort(), ["antigravity", "copilot", "cursor", "grok", "pi"]);
for (const agent of COMPUTED) {
assert.equal(manifest.profiles[agent].commandDigest, (await acpxReleaseAttestation(agent, packageRoot)).commandDigest, agent);
}
});
test("a new provider is told to declare its release attestation", async t => {
const relocated = relocatedPackage(t);
const path = join(relocated, "acpx-profiles.json"); const value = JSON.parse(readFileSync(path, "utf8"));
value.profiles.newagent = { ...value.profiles.codex };
writeFileSync(path, JSON.stringify(value));
const result = generate(relocated, "--check");
assert.notEqual(result.status, 0);
assert.match(result.stderr, /newagent needs a release attestation: declare how its commandDigest attests the upstream release/u);
});
test("a relocated copy of the provider inputs computes the same attestations", async t => {
const relocated = relocatedPackage(t);
for (const agent of ATTESTED_ACPX_AGENTS) {
for (const agent of COMPUTED) {
const here = await acpxReleaseAttestation(agent, packageRoot);
const there = await acpxReleaseAttestation(agent, relocated);
assert.equal(there.commandDigest, here.commandDigest, agent);
@@ -76,24 +90,52 @@ for (const [name, agents, edit] of [
const path = join(root, "src/providers/grok/launcher.cjs");
writeFileSync(path, `${readFileSync(path, "utf8")}\n// changed\n`);
}],
["the Cursor release declaration", ["cursor"], root => {
const path = join(root, "cursor-contract.json"); const value = JSON.parse(readFileSync(path, "utf8"));
value.nativePatch = "changed"; writeFileSync(path, JSON.stringify(value));
}],
["the Antigravity distribution declaration", ["antigravity"], root => {
const path = join(root, "antigravity-distributions.json"); const value = JSON.parse(readFileSync(path, "utf8"));
value.version = "9.9.9"; writeFileSync(path, JSON.stringify(value));
}],
]) test(`changing ${name} changes only the affected attestations`, async t => {
const relocated = relocatedPackage(t);
const before = Object.fromEntries(await Promise.all(ATTESTED_ACPX_AGENTS.map(async agent => [agent, (await acpxReleaseAttestation(agent, relocated)).commandDigest])));
const before = Object.fromEntries(await Promise.all(COMPUTED.map(async agent => [agent, (await acpxReleaseAttestation(agent, relocated)).commandDigest])));
edit(relocated);
for (const agent of ATTESTED_ACPX_AGENTS) {
for (const agent of COMPUTED) {
const after = (await acpxReleaseAttestation(agent, relocated)).commandDigest;
if (agents.includes(agent)) assert.notEqual(after, before[agent], agent);
else assert.equal(after, before[agent], agent);
}
});
test("the generator refreshes a release declaration's own digest with the profile", async t => {
const relocated = relocatedPackage(t);
const path = join(relocated, "cursor-contract.json"); const contract = JSON.parse(readFileSync(path, "utf8"));
contract.modes = [...contract.modes, "review"];
writeFileSync(path, `${JSON.stringify(contract, null, 2)}\n`);
const stale = generate(relocated, "--check");
assert.notEqual(stale.status, 0);
assert.match(stale.stderr, /cursor commandDigest no longer matches its upstream release attestation/u);
assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), contract, "check mode writes nothing");
const refreshed = generate(relocated);
assert.equal(refreshed.status, 0, refreshed.stderr);
const { commandDigest } = await acpxReleaseAttestation("cursor", relocated);
assert.notEqual(commandDigest, manifest.profiles.cursor.commandDigest);
assert.equal(JSON.parse(readFileSync(path, "utf8")).commandDigest, commandDigest);
assert.equal(JSON.parse(readFileSync(join(relocated, "acpx-profiles.json"), "utf8")).profiles.cursor.commandDigest, commandDigest);
assert.match(readFileSync(join(relocated, "src/drivers/acpx/generated-profiles.ts"), "utf8"), new RegExp(commandDigest));
assert.match(readFileSync(join(relocated, "runner/crates/runner-core/src/generated_acpx_profiles.rs"), "utf8"), new RegExp(commandDigest));
assert.equal(generate(relocated, "--check").status, 0);
});
test("the generator refuses a stale attestation in check mode and refreshes it otherwise", async t => {
const relocated = relocatedPackage(t);
const launcher = join(relocated, "src/providers/grok/launcher.cjs");
writeFileSync(launcher, `${readFileSync(launcher, "utf8")}\n// changed\n`);
const stale = generate(relocated, "--check");
assert.notEqual(stale.status, 0);
assert.match(stale.stderr, /grok release attestation is stale/u);
assert.match(stale.stderr, /grok commandDigest no longer matches its upstream release attestation/u);
const refreshed = generate(relocated);
assert.equal(refreshed.status, 0, refreshed.stderr);
const written = JSON.parse(readFileSync(join(relocated, "acpx-profiles.json"), "utf8"));
@@ -1,30 +1,36 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { acpxReleaseAttestation, ATTESTED_ACPX_AGENTS } from "./acpx-release-attestations.mjs";
import { ACPX_RELEASE_ATTESTATIONS, acpxReleaseAttestation } from "./acpx-release-attestations.mjs";
const root = new URL("../", import.meta.url);
const readJson = async path => JSON.parse(await readFile(new URL(path, root), "utf8"));
const manifest = await readJson("acpx-profiles.json");
const distributions = await readJson("cursor-distributions.json");
const antigravityDistributions = await readJson("antigravity-distributions.json");
const contract = await readJson("cursor-contract.json");
const pkg = await readJson("package.json");
assert.equal(manifest.schema, "paperclip.acpx-profiles.v1");
assert.equal(manifest.acpxVersion, pkg.dependencies.acpx);
// Copilot, Grok and Pi digests attest upstream releases; they change only when
// an upstream version, a platform closure or the Grok launcher changes.
// Every profile's commandDigest attests its upstream release, so it changes only
// when that release changes. Installation verifies installed-command digests.
const check = process.argv.includes("--check");
let attestationsChanged = false;
for (const agent of ATTESTED_ACPX_AGENTS) {
const { commandDigest } = await acpxReleaseAttestation(agent, fileURLToPath(root));
if (manifest.profiles[agent].commandDigest === commandDigest) continue;
assert.ok(!check, `${agent} release attestation is stale in acpx-profiles.json; run generate:acpx-profiles`);
manifest.profiles[agent].commandDigest = commandDigest;
attestationsChanged = true;
const writes = new Map();
for (const agent of Object.keys(manifest.profiles)) {
const attestation = await acpxReleaseAttestation(agent, fileURLToPath(root));
if (!attestation) continue;
// A release declaration may carry its own copy of the digest; keep it in step.
const { path, digestField } = ACPX_RELEASE_ATTESTATIONS[agent];
const declaration = digestField ? await readJson(path) : null;
if (manifest.profiles[agent].commandDigest === attestation.commandDigest
&& (!declaration || declaration[digestField] === attestation.commandDigest)) continue;
assert.ok(!check, `${agent} commandDigest no longer matches its upstream release attestation. `
+ "Run `pnpm --filter @paperclipai/paperclip-runner generate:acpx-profiles`; an upstream change keeps the profile version.");
manifest.profiles[agent].commandDigest = attestation.commandDigest;
writes.set("acpx-profiles.json", manifest);
if (declaration) writes.set(path, { ...declaration, [digestField]: attestation.commandDigest });
}
if (attestationsChanged) await writeFile(new URL("acpx-profiles.json", root), `${JSON.stringify(manifest, null, 2)}\n`);
for (const [path, value] of writes) await writeFile(new URL(path, root), `${JSON.stringify(value, null, 2)}\n`);
const contract = await readJson("cursor-contract.json");
const entries = Object.entries(manifest.profiles);
for (const [agent, profile] of entries) {
assert.match(agent, /^[a-z]+$/);
@@ -38,21 +44,13 @@ for (const [agent, profile] of entries) {
if (name && Object.hasOwn(pkg.dependencies, name)) assert.equal(version, pkg.dependencies[name], `${agent}: installed dependency drift`);
}
}
// The Cursor contract is an immutable release attestation, not another runtime
// lookup table. Refuse generation if a release declaration disagrees with it.
// Release declarations must agree with the profiles they attest.
assert.equal(manifest.profiles.cursor.agentServerVersion, distributions.version);
assert.equal(manifest.profiles.cursor.agentServerVersion, contract.vendor);
assert.equal(manifest.profiles.cursor.agentProfileVersion, contract.profileVersion);
assert.equal(manifest.profiles.cursor.commandDigest, contract.commandDigest);
assert.equal(contract.commandDigest, manifest.profiles.cursor.commandDigest, "cursor-contract.json carries a stale commandDigest");
assert.equal(distributions.patchVersion, contract.nativePatch);
const canonicalJson = value => value && typeof value === "object"
? Array.isArray(value) ? `[${value.map(canonicalJson).join(",")}]`
: `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
: JSON.stringify(value);
assert.equal(manifest.profiles.antigravity.agentServerVersion, antigravityDistributions.version);
assert.equal(manifest.profiles.antigravity.commandDigest, `sha256:${createHash("sha256").update(canonicalJson(antigravityDistributions)).digest("hex")}`, "Antigravity distribution identity is stale");
const { commandDigest, ...attestation } = contract;
assert.equal(commandDigest, `sha256:${createHash("sha256").update(canonicalJson(attestation)).digest("hex")}`);
for (const distribution of Object.values(distributions.platforms)) assert.match(distribution.closureSha256, /^[a-f0-9]{64}$/);
const quote = JSON.stringify;
@@ -2,6 +2,23 @@ import { describe, expect, it } from "vitest";
import { QUALIFIED_ACPX_PROFILE_DATA } from "./generated-profiles.js";
import { isSupportedAcpxProfileVersion } from "./profile-compatibility.js";
// A profile version changes only to reset saved sessions on purpose. An upstream
// release changes the commandDigest and keeps the version; a new provider starts
// at 1. Update this table, with the reason, only for a deliberate reset.
const PROFILE_VERSIONS: Readonly<Record<string, number>> = {
antigravity: 1, claude: 1, codex: 1, copilot: 92, cursor: 17, grok: 20, pi: 37,
};
describe("ACPX profile versions", () => {
it("change only for a deliberate session reset", () => {
for (const [agent, profile] of Object.entries(QUALIFIED_ACPX_PROFILE_DATA)) {
expect(profile.agentProfileVersion, `${agent}: a profile version bump resets every saved ${agent} session. `
+ "An upstream release changes only the commandDigest. See rules/acpx-profile-identity.mdc.")
.toBe(PROFILE_VERSIONS[agent] ?? 1);
}
});
});
describe("historical ACPX profile decoding", () => {
it("decodes every revision from 1 through the current one so saved runs reach recovery", () => {
for (const [agent, current] of Object.entries(QUALIFIED_ACPX_PROFILE_DATA)) {
@@ -1,6 +1,6 @@
{
"name": "@paperclipai/plugin-daytona",
"version": "0.1.1",
"version": "0.1.9",
"description": "Daytona sandbox provider plugin for Paperclip environments",
"license": "MIT",
"homepage": "https://github.com/paperclipai/paperclip",
@@ -1,6 +1,6 @@
{
"name": "@paperclipai/plugin-kubernetes",
"version": "0.1.0",
"version": "0.1.0-alpha.1",
"description": "Kubernetes sandbox provider plugin for Paperclip environments",
"license": "MIT",
"homepage": "https://github.com/paperclipai/paperclip",
+3
View File
@@ -68,6 +68,9 @@ importers:
esbuild:
specifier: ^0.28.2
version: 0.28.2
oxc-parser:
specifier: 0.127.0
version: 0.127.0
turbo:
specifier: 2.11.10
version: 2.11.10
+72
View File
@@ -0,0 +1,72 @@
---
description: Give every ACPX provider the same identity model - its commandDigest attests the upstream release, the pack contract digest covers Runner source and the shared ACPX layer, and profile versions change only for deliberate resets.
globs: "packages/paperclip-runner/acpx-profiles.json,packages/paperclip-runner/cursor-contract.json,packages/paperclip-runner/cursor-distributions.json,packages/paperclip-runner/antigravity-distributions.json,packages/paperclip-runner/scripts/acpx-release-attestations.mjs,packages/paperclip-runner/scripts/generate-acpx-profiles.mjs,packages/paperclip-runner/scripts/contracts/**,packages/paperclip-runner/src/drivers/acpx/**,packages/paperclip-runner/src/providers/**,packages/paperclip-runner/protocol/**"
alwaysApply: false
---
# ACPX profile identity
## Scope and purpose
This rule covers ACPX provider profiles, their release declarations and pins,
the profile generator, the provider-pack contract registry, and the ACPX
driver and provider code. Every ACPX harness (Claude, Codex, Copilot, Cursor,
Grok, Pi, Antigravity and any new one) uses the same identity model, so a
change behaves the same way for all of them.
## The identity model
A provider profile in `acpx-profiles.json` has a version and a `commandDigest`.
Three things decide whether saved work and installed packs stay compatible:
| What changed | What carries it | Effect |
| --- | --- | --- |
| The provider's upstream release: package versions, platform closures, a launcher shipped verbatim | The profile's `commandDigest` | Saved sessions for that provider start fresh |
| The shared ACPX layer | The profile's `acpxVersion` and the Runner's ACPX patch in the pack `contractDigest` | The same for every provider |
| A runner/pack contract: schemas, generated catalogs, Rust shapes and env keys | The provider pack's `contractDigest` | Older image packs are not reused |
Ordinary Runner source changes, refactors and comments change none of these.
## Required behavior
- **Attest the upstream release.** Declare each provider's attestation in
`ACPX_RELEASE_ATTESTATIONS` (`scripts/acpx-release-attestations.mjs`), using
the kind that fits: `installed-command`, `launcher`, `closure` or
`declaration`. The digest binds only what a repackager copies verbatim from
the upstream release, so the same release gives the same digest wherever the
pack is built or installed.
- **Keep Paperclip source and the ACPX layer out of provider digests.**
Runner/pack compatibility belongs to the `contractDigest`. When Runner code
and a pack must agree on a new field or behavior, declare it in a schema
under `protocol/` and generate the TypeScript, so the contract registry
covers it.
- **Regenerate instead of hand-editing.** After an upstream bump or a pin
change, run `pnpm --filter @paperclipai/paperclip-runner generate:acpx-profiles`.
`check:acpx-profiles` runs in the build and refuses a stale digest.
- **Keep the profile version.** An upstream release changes the digest, not the
version. A new provider starts at version 1. Bump a version only to reset
every saved session for that provider on purpose, such as a change in saved
session semantics that exact admission cannot detect. Record the reason in
the PR and in the version table in `profile-compatibility.test.ts`.
- **Keep earlier versions decodable.** Saved inputs with any version from 1
through the current one decode, so recovery can refuse them. Launch and
recovery admission require the exact current profile and digest.
## Adding a provider
1. Add its profile at version 1 to `acpx-profiles.json`.
2. Add its attestation to `ACPX_RELEASE_ATTESTATIONS`, with its pins in
erasable TypeScript or JSON that the generator can read.
3. Run `generate:acpx-profiles`, then `node --test scripts/acpx-release-attestations.test.mjs`.
## Review criteria
- Does every provider digest change only with its upstream release?
- Is a new runner/pack contract declared where the `contractDigest` reads it?
- Does a version bump state which saved sessions it resets, and why?
- Does the change treat every ACPX harness the same way?
The canonical reference is
[`packages/paperclip-runner/docs/durable-recovery.md`](../packages/paperclip-runner/docs/durable-recovery.md#profile-identity).
The pack contract digest is described in
[`docker/daytona-runner/README.md`](../docker/daytona-runner/README.md).
+6 -2
View File
@@ -249,7 +249,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
]) {
assert.match(
jobs.get(jobId),
/^ {4}if: \$\{\{ needs\.gate\.outputs\.full_ci == 'true' && /m,
/^ {4}if: \$\{\{ (?:always\(\) && !cancelled\(\) && needs\.gate\.result == 'success' && )?needs\.gate\.outputs\.full_ci == 'true' && /m,
`${jobId} must run only when the gate selects full CI`,
);
}
@@ -263,10 +263,14 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
const verify = jobs.get("verify");
assert.match(
verify,
/^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m,
/^ {4}needs: \[gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m,
);
assert.equal(runAggregate(verify, true).status, 0);
assert.equal(runAggregate(verify, false).status, 0);
for (const result of ["failure", "cancelled", "skipped"]) {
assert.equal(runAggregate(verify, true, { test_selection: result }).status, 0,
"an optional selector cannot block the successful original matrix");
}
for (const job of ["gate", "policy", "typecheck_release_registry", "general_tests", "verify_paperclip_runner",
"build", "docker_context_integrity", "verify_serialized_server", "canary_dry_run"]) {
for (const result of ["failure", "cancelled", "skipped"]) {
+62 -1
View File
@@ -10,6 +10,63 @@ import { executionPlan, sourceFiles } from "./execution-plan.mjs";
const repositoryRoot = new URL("../../", import.meta.url);
const turbo = process.env.PAPERCLIP_CI_TEST_TURBO ?? new URL("node_modules/.bin/turbo", repositoryRoot).pathname;
test("real Turbo keeps full runs and different selected test sets in separate cache entries", () => {
const directory = mkdtempSync(path.join(tmpdir(), "ci-selection-cache-"));
const write = (file, value) => writeFileSync(path.join(directory, file), value);
const json = (file, value) => write(file, JSON.stringify(value));
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
try {
git("init", "-q");
for (const folder of ["scripts/ci", "server/src", "packages/fixture", "bin", ".ci-test-selection"]) mkdirSync(path.join(directory, folder), { recursive: true });
for (const file of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) {
copyFileSync(new URL(`scripts/ci/${file}`, repositoryRoot), path.join(directory, "scripts/ci", file));
}
mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
write(".gitignore", ".turbo/\n.ci-test-selection/\nresult.txt\ncalls.txt\n");
json("package.json", { name: "selection-cache-proof", private: true, packageManager: "npm@10.0.0", workspaces: ["packages/*"], scripts: { "test:run:general": "node check.cjs" } });
json("packages/fixture/package.json", { name: "fixture", private: true });
json("package-lock.json", { name: "selection-cache-proof", lockfileVersion: 3, packages: { "": { name: "selection-cache-proof" } } });
json("turbo.json", { globalDependencies: ["doc/**", "docs/**"], globalEnv: ["CI_TASK_CACHE_SCOPE", "PAPERCLIP_*"], tasks: {
"//#ci:server": { inputs: ["check.cjs", "server/**"], outputs: ["result.txt"], cache: true },
} });
mkdirSync(path.join(directory, "doc/plans"), { recursive: true });
mkdirSync(path.join(directory, "docs/api"), { recursive: true });
write("doc/DEVELOPING.md", "before"); write("doc/plans/example.md", "before");
write("docs/api/issues.md", "before");
write("server/src/a.test.ts", "// a"); write("server/src/b.test.ts", "// b");
write("check.cjs", `const fs=require('node:fs');fs.appendFileSync('calls.txt','call\\n');const file=process.env.PAPERCLIP_CI_TEST_SELECTION;fs.writeFileSync('result.txt',file?JSON.parse(fs.readFileSync(file)).lanes['general-server-without-chat'].shards[0]['@paperclipai/server'].join(','):'full');`);
write("bin/pnpm", `#!${process.execPath}\nconst {spawnSync}=require('node:child_process');const args=process.argv.slice(2);if(args[0]==='--version')console.log('9.15.4');else{const r=spawnSync(${JSON.stringify(turbo)},args.slice(2),{stdio:'inherit'});process.exit(r.status??1);}`);
chmodSync(path.join(directory, "bin/pnpm"), 0o755);
git("add", ".");
git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qm", "fixture");
const head = git("rev-parse", "HEAD");
const calls = () => readFileSync(path.join(directory, "calls.txt"), "utf8").trim().split("\n").length;
const run = (testFile) => {
const selection = path.join(directory, ".ci-test-selection/manifest.json");
json(".ci-test-selection/manifest.json", { version: 1, head, lanes: {
"general-server-without-chat": { shards: [{ "@paperclipai/server": [testFile ?? "server/src/a.test.ts"] }] },
} });
const result = spawnSync(process.execPath, [path.join(directory, "scripts/ci/run-task.mjs")], {
cwd: directory, encoding: "utf8", env: { ...process.env, PATH: `${directory}/bin:${process.env.PATH}`,
CI_TASK: "ci:server", CI_TASK_ARGS: "[]", CI_FORCE_TASKS: "false", CI_TASK_DRY_RUN: "false",
PAPERCLIP_CI_TEST_SELECTION: testFile ? selection : "", PAPERCLIP_CI_SELECTION_HASH: "",
TURBO_TOKEN: "", TURBO_TEAM: "", TURBO_TELEMETRY_DISABLED: "1" },
});
assert.equal(result.status, 0, result.stdout + result.stderr);
assert.equal(readFileSync(path.join(directory, "result.txt"), "utf8"), testFile ?? "full");
};
run(); run(); assert.equal(calls(), 1);
write("doc/DEVELOPING.md", "after"); write("doc/plans/example.md", "after");
run(); assert.equal(calls(), 1, "audited prose does not invalidate product test results");
run("server/src/a.test.ts"); run("server/src/a.test.ts"); assert.equal(calls(), 2);
run("server/src/b.test.ts"); assert.equal(calls(), 3);
run(); assert.equal(calls(), 3, "the original full result is reusable without accepting a subset result");
write("docs/api/issues.md", "after");
run(); assert.equal(calls(), 4, "runtime documentation must invalidate results");
} finally { rmSync(directory, { recursive: true, force: true }); }
});
test("every cached lane maps to an existing package script with complete inputs", () => {
const config = JSON.parse(readFileSync(new URL("turbo.json", repositoryRoot), "utf8"));
for (const [name, definition] of Object.entries(config.tasks).filter(([, task]) => task.cache)) {
@@ -35,6 +92,8 @@ test("real Turbo executes existing suite scripts and invalidates source, argumen
const json = (name, value) => write(name, JSON.stringify(value));
try {
execFileSync("git", ["init", "-q"], { cwd: directory });
mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
write(".gitignore", ".turbo/\ngenerated/\nresult.txt\ncalls.txt\n");
mkdirSync(path.join(directory, "packages/fixture"), { recursive: true });
json("packages/fixture/package.json", { name: "fixture", private: true });
@@ -83,9 +142,11 @@ test("the task launcher bypasses cache reads and writes when execution is forced
mkdirSync(path.join(directory, "scripts/ci"), { recursive: true });
mkdirSync(path.join(directory, "bin"));
mkdirSync(path.join(directory, "packages/fixture"), { recursive: true });
for (const name of ["run-task.mjs", "execution-plan.mjs"]) {
for (const name of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) {
copyFileSync(new URL(`scripts/ci/${name}`, repositoryRoot), path.join(directory, "scripts/ci", name));
}
mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
write(".gitignore", ".turbo/\ncalls.txt\nresult.txt\nbin/\n");
json("packages/fixture/package.json", { name: "fixture", private: true });
json("package.json", { name: "forced-cache-proof", private: true, packageManager: "npm@10.0.0",
+2 -1
View File
@@ -1,5 +1,6 @@
import { execFileSync } from "node:child_process";
import { matchesGlob } from "node:path";
import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
export function sourceFiles(cwd = process.cwd()) {
return [...new Set(execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z"],
@@ -30,7 +31,7 @@ export function executionPlan(config, task, args, files = sourceFiles()) {
target = "@paperclipai/paperclip-runner#check:runner";
commandArgs = args;
} else throw new Error(`Missing task command: ${task}`);
const select = (globs) => files.filter((file) => globs.some((glob) => matchesGlob(file, glob))).map(literalGlob);
const select = (globs) => productInputs(files).filter((file) => globs.some((glob) => matchesGlob(file, glob))).map(literalGlob);
const selectedInputs = select(definition.inputs);
const inputs = target.startsWith("//#") ? selectedInputs
: selectedInputs.map((input) => `$TURBO_ROOT$/${input}`);
+29
View File
@@ -0,0 +1,29 @@
import assert from "node:assert/strict";
import path from "node:path";
import { createVitest, parseCLI } from "vitest/node";
// Positional Vitest CLI filters are substrings: foo.test.ts also selects
// foo.test.tsx. Run the exact discovered specifications assigned to this shard.
const { filter, options } = parseCLI(["vitest", "run", ...process.argv.slice(2)]);
assert.ok(filter.length > 0, "Selected execution requires explicit test files");
const expected = new Set(filter.map((file) => path.resolve(file)));
assert.equal(expected.size, filter.length, "Duplicate selected file");
process.env.TEST = "true";
process.env.VITEST = "true";
process.env.NODE_ENV ??= "test";
// Match the CLI's additive --exclude semantics without replacing project rules.
options.cliExclude = options.exclude;
delete options.exclude;
const ctx = await createVitest({ ...options, watch: false });
try {
await ctx.standalone();
const specifications = (await ctx.globTestSpecifications()).filter((spec) => expected.has(spec.moduleId));
assert.deepEqual(specifications.map((spec) => spec.moduleId).sort(), [...expected].sort(),
"Selected files must match exactly one discovered specification in this project");
await ctx.runTestSpecifications(specifications);
} catch (error) {
console.error(error);
process.exitCode = 1;
} finally {
await ctx.exit();
}
+8 -1
View File
@@ -3,6 +3,8 @@ import { lstatSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from
import { createHash } from "node:crypto";
import { matchesGlob } from "node:path";
import { executionPlan, sourceFiles } from "./execution-plan.mjs";
import { loadSelection, selectionHash } from "./test-selection-manifest.mjs";
import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
const task = process.env.CI_TASK;
if (!/^ci:[a-z-]+$/.test(task ?? "")) throw new Error("Invalid CI task");
@@ -15,13 +17,18 @@ const version = (command, argv) => {
const config = JSON.parse(readFileSync("turbo.json", "utf8"));
const files = sourceFiles();
const execution = executionPlan(config, task, args, files);
// Validate before Turbo can return a cache hit. The hash separates selected
// executions from full executions and from every different selected file set.
if (process.env.PAPERCLIP_CI_TEST_SELECTION) {
process.env.PAPERCLIP_CI_SELECTION_HASH = selectionHash(loadSelection());
}
mkdirSync(".turbo/task-configs", { recursive: true });
const runtimeConfig = `.turbo/task-configs/${task.replace(":", "-")}.json`;
writeFileSync(runtimeConfig, JSON.stringify(execution.config));
const classified = [...config.globalDependencies, ...Object.values(config.tasks).flatMap((entry) => entry.inputs)];
const unknownInputs = createHash("sha256");
let uncacheable = false;
for (const file of files) {
for (const file of productInputs(files)) {
if (classified.some((glob) => matchesGlob(file, glob))) continue;
unknownInputs.update(JSON.stringify(file));
try {
+124
View File
@@ -0,0 +1,124 @@
import { execFileSync } from "node:child_process";
import { appendFileSync, lstatSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import path from "node:path";
import assert from "node:assert/strict";
import { refineTestPlan, selectionDurations } from "./test-selection-plan.mjs";
import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
import { mergeDiff } from "../../.github/scripts/ci-diff.mjs";
export function selectionEligibility(plan) {
if (plan?.version !== 1 || plan.mode !== "enabled" || plan.forceReason) return "Full/shadow/forced task plan";
if (!Array.isArray(plan.changed) || !plan.changed.length) return "Incomplete comparison";
const changed = productInputs(plan.changed);
if (!changed.length) return "Documentation-only diff";
for (const file of changed) {
if (!/^(?:ui\/src|server\/src|cli\/src|packages\/.+\/src)\/.+\.[cm]?[jt]sx?$/.test(file)
|| /(?:config|setup)|\.d\.[cm]?ts$/i.test(file) || file.split("/").includes("..")) return `Broad input: ${file}`;
try {
if (!lstatSync(file).isFile()) return `Non-regular source: ${file}`;
} catch { return `Deleted or missing source: ${file}`; }
}
return null;
}
function runnerInventory(group) {
const args = group === "serialized" ? ["--mode", "serialized", "--shard-index", "0", "--shard-count", "1"]
: ["--mode", "general", "--group", group,
...(group === "general-server-without-chat" ? ["--shard-index", "0", "--shard-count", "1"] : [])];
const env = { ...process.env, GITHUB_WORKFLOW: "PR" };
delete env.PAPERCLIP_CI_TEST_SELECTION;
return JSON.parse(execFileSync(process.execPath, ["scripts/run-vitest-stable.mjs", ...args, "--dry-run"],
{ env, encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }));
}
export async function createTestSelection(plan, head) {
const reason = selectionEligibility(plan);
if (reason) throw new Error(reason);
const groups = new Set();
for (const job of ["general_tests", "verify_serialized_server"]) {
if (plan.jobs[job].run) for (const entry of plan.jobs[job].matrix.include) groups.add(entry.group ?? "serialized");
}
groups.delete("general-chat"); // Keep its line sharding and DB ownership intact.
const owners = {}, projects = new Set();
for (const group of groups) {
const inventory = runnerInventory(group);
if (group === "serialized" || group === "general-server-without-chat") {
owners[group] = { "@paperclipai/server": group === "serialized"
? inventory.selectedSerializedSuites : inventory.selectedGeneralServerSuites };
projects.add("@paperclipai/server");
} else {
owners[group] = Object.fromEntries(inventory.workspaceProjects.map((project) => [project, null]));
for (const project of inventory.workspaceProjects) projects.add(project);
}
}
const { selectProjectTests } = await import("./vitest-selection-graph.mjs");
const graphs = [];
const changed = productInputs(plan.changed);
for (const project of projects) {
const graph = await selectProjectTests({ project, changed, onlyOwnChanges: true });
graphs.push(graph);
console.log(`${project}: ${graph.selected.length}/${graph.inventory.length} files, ${graph.alwaysRun.length} retained conservatively, ${graph.seconds.toFixed(1)}s`);
for (const owner of Object.values(owners)) if (owner[project] === null) owner[project] = graph.inventory;
}
const covered = new Set(graphs.flatMap((graph) => graph.coveredChanges));
for (const file of changed) assert.ok(covered.has(file), `Change outside the resolved test graphs: ${file}`);
const server = graphs.find((graph) => graph.project === "@paperclipai/server");
if (server) {
const general = runnerInventory("general-server-without-chat");
const native = runnerInventory("general-server-native-runner");
assert.deepEqual([...general.selectedGeneralServerSuites, ...general.selectedSerializedSuites,
...native.selectedGeneralServerSuites, "server/src/__tests__/chat-channels.integration.test.ts"].sort(),
server.inventory, "Server inventory must be completely covered by existing lane ownership");
}
return { ...refineTestPlan({ plan, head, owners, graphs, durations: selectionDurations() }), graphs };
}
async function main() {
const plan = JSON.parse(process.env.CI_PLAN || "null");
const output = (key, value) => {
if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${value}\n`);
};
if (process.argv.includes("--eligible")) {
const reason = selectionEligibility(plan);
console.log(reason ?? "Source diff is eligible for graph selection");
output("enabled", !reason);
return;
}
const directory = ".ci-test-selection";
mkdirSync(directory, { recursive: true });
const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).trim();
let result;
try {
const head = git("rev-parse", "HEAD");
assert.equal(head, process.env.GITHUB_SHA, "Selection must describe the tested merge commit");
assert.ok(plan.diff, "Missing trusted merge diff");
const local = mergeDiff(head, plan.diff.prHead);
assert.deepEqual(local.diff, plan.diff, "Selection must use the trusted merge parents");
assert.deepEqual(local.changed, [...new Set(plan.changed)].sort(), "Local diff disagrees with the trusted routing diff");
assert.equal(git("status", "--porcelain", "--untracked-files=normal"), "", "Source inputs changed during dependency setup");
process.env.NODE_ENV = "test";
process.env.VITEST = "true";
result = await createTestSelection(plan, head);
writeFileSync(`${directory}/manifest.json`, JSON.stringify(result.manifest));
writeFileSync(`${directory}/audit.json`, JSON.stringify(result.graphs, null, 2));
output("selected", "true");
} catch (error) {
console.warn(`Using the complete task plan: ${error.message}`);
result = { plan: { ...plan, testSelectionFallback: error.message } };
output("selected", "false");
}
writeFileSync(`${directory}/plan.json`, JSON.stringify(result.plan));
if (process.env.GITHUB_STEP_SUMMARY) {
const stats = result.plan.testSelection?.stats;
appendFileSync(process.env.GITHUB_STEP_SUMMARY, stats
? ["### Vitest test selection", "", "| Lane | Test files | Jobs |", "| --- | ---: | ---: |",
...stats.map((s) => `| ${s.group} | ${s.selected}/${s.total} | ${s.jobs}/${s.beforeJobs} |`), "",
"Filesystem/process/opaque consumers are retained. Chat and native Runner lanes keep their existing ownership.", ""].join("\n")
: `### Vitest test selection\n\nFull task plan: ${result.plan.testSelectionFallback}\n`);
}
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main().catch((error) => { console.error(error); process.exitCode = 1; });
}
+66
View File
@@ -0,0 +1,66 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { execFileSync } from "node:child_process";
import { readFileSync, statSync } from "node:fs";
export const selectableGroups = ["general-server-without-chat", "general-workspaces-a",
"general-workspaces-b", "general-workspaces-c", "serialized"];
export function selectionHash(manifest) {
return createHash("sha256").update(JSON.stringify({ version: manifest.version, lanes: manifest.lanes })).digest("hex");
}
export function validateSelection(manifest, { head, hash } = {}) {
assert.equal(manifest.version, 1, "Unsupported test selection");
assert.match(manifest.head, /^[a-f0-9]{40}$/);
if (head) assert.equal(manifest.head, head, "Selection belongs to another checkout");
assert.ok(manifest.lanes && typeof manifest.lanes === "object" && !Array.isArray(manifest.lanes));
const seen = new Set();
for (const [group, lane] of Object.entries(manifest.lanes)) {
assert.ok(selectableGroups.includes(group), `Unknown selection group: ${group}`);
assert.ok(Array.isArray(lane.shards) && lane.shards.length > 0 && lane.shards.length <= 12);
for (const shard of lane.shards) {
assert.ok(shard && typeof shard === "object" && !Array.isArray(shard));
let count = 0;
for (const [project, files] of Object.entries(shard)) {
assert.ok(project && Array.isArray(files) && files.length > 0);
for (const file of files) {
assert.equal(typeof file, "string");
assert.ok(/^(ui|server|cli|packages)\//.test(file) && !/[\0\r\n\\:*?\[\]{}]/.test(file)
&& !file.split("/").some((part) => ["", ".", ".."].includes(part)), `Invalid selected path: ${file}`);
const key = `${project}:${file}`;
assert.ok(!seen.has(key), `Test assigned twice: ${key}`);
seen.add(key);
count++;
}
}
assert.ok(count > 0, "Empty selection shard");
}
}
if (hash) assert.equal(selectionHash(manifest), hash, "Selection artifact does not match its plan");
return manifest;
}
export function loadSelection(file = process.env.PAPERCLIP_CI_TEST_SELECTION) {
const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
const manifest = validateSelection(JSON.parse(readFileSync(file, "utf8")), {
head, hash: process.env.PAPERCLIP_CI_SELECTION_HASH,
});
for (const lane of Object.values(manifest.lanes)) {
for (const shard of lane.shards) for (const files of Object.values(shard)) {
for (const file of files) assert.ok(statSync(file).isFile(), `Selected test missing: ${file}`);
}
}
return manifest;
}
export function selectedShard(manifest, options, projects) {
const group = options.mode === "serialized" ? "serialized" : options.group;
const lane = manifest.lanes[group];
assert.ok(lane, `Missing selection for ${group}`);
assert.equal(lane.shards.length, options.shardCount ?? 1, "Selection shard count mismatch");
const shard = lane.shards[options.shardIndex ?? 0];
assert.ok(shard, "Missing selection shard");
for (const project of Object.keys(shard)) assert.ok(projects.includes(project), `Wrong project for ${group}: ${project}`);
return shard;
}
+77
View File
@@ -0,0 +1,77 @@
import assert from "node:assert/strict";
import { loadShardDurations, partitionGeneralServerSuites } from "../general-server-shard.mjs";
import { selectionHash, validateSelection } from "./test-selection-manifest.mjs";
// Preserve the existing per-job budget. Small selections use fewer runners;
// large selections retain the existing parallelism and serialized ownership.
export function partitionSelection(inventory, selected, maxShards, durations = {}) {
const totalWeight = partitionGeneralServerSuites(inventory, 1, durations)[0].totalWeight;
const selectedWeight = partitionGeneralServerSuites(selected, 1, durations)[0].totalWeight;
const budget = totalWeight / maxShards || 1;
const count = Math.min(maxShards, selected.length, Math.max(1, Math.ceil(selectedWeight / budget)));
return selected.length ? partitionGeneralServerSuites(selected, count, durations).map((shard) => shard.files) : [];
}
export function refineTestPlan({ plan, head, owners, graphs, durations = {} }) {
const result = structuredClone(plan);
const manifest = { version: 1, head, lanes: {} };
const stats = [];
const byProject = new Map(graphs.map((graph) => [graph.project, graph]));
for (const job of ["general_tests", "verify_serialized_server"]) {
if (!plan.jobs[job].run) continue;
const entries = plan.jobs[job].matrix.include;
const groups = [...new Set(entries.map((entry) => entry.group ?? "serialized"))];
const matrix = [];
for (const group of groups) {
const original = entries.filter((entry) => (entry.group ?? "serialized") === group);
if (group === "general-chat") { matrix.push(...original); continue; }
const owned = owners[group];
assert.ok(owned, `Missing inventory owner: ${group}`);
const projectByFile = new Map();
const selected = [];
for (const [project, files] of Object.entries(owned)) {
const graph = byProject.get(project);
assert.ok(graph, `Missing project graph: ${project}`);
const inventory = new Set(graph.inventory);
const affected = new Set(graph.selected);
for (const file of files) {
assert.ok(inventory.has(file), `Runner/Vitest inventory disagreement: ${file}`);
assert.ok(!projectByFile.has(file), `Ambiguous owner: ${file}`);
projectByFile.set(file, project);
if (affected.has(file)) selected.push(file);
}
}
const partitions = partitionSelection([...projectByFile.keys()], selected, original.length, durations[group]);
if (partitions.length) {
const shards = partitions.map((files) => {
const shard = {};
for (const file of files) (shard[projectByFile.get(file)] ??= []).push(file);
return shard;
});
manifest.lanes[group] = { shards };
for (let index = 0; index < shards.length; index++) {
const entry = { ...original[0], selection: true };
if (original[0].shard_count) {
Object.assign(entry, { shard_index: index, shard_count: shards.length });
if (entry.shard_label) entry.shard_label = `${index + 1}/${shards.length}`;
if (entry.group_label) entry.group_label = entry.group_label.replace(/\(\d+\/\d+\)/, `(${index + 1}/${shards.length})`);
}
matrix.push(entry);
}
}
stats.push({ group, total: projectByFile.size, selected: selected.length, beforeJobs: original.length, jobs: partitions.length });
}
result.jobs[job] = { ...result.jobs[job], run: matrix.length > 0, selected: matrix.length, proposed: matrix.length,
matrix: { include: matrix.length ? matrix : entries } };
}
validateSelection(manifest);
result.testSelection = { version: 1, head, hash: selectionHash(manifest), stats };
return { plan: result, manifest };
}
export function selectionDurations() {
return {
"general-server-without-chat": loadShardDurations("scripts/general-server-shard-durations.json"),
serialized: loadShardDurations("scripts/serialized-shard-durations.json"),
};
}
@@ -0,0 +1,56 @@
import assert from "node:assert/strict";
import { execFileSync, spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { selectProjectTests } from "./vitest-selection-graph.mjs";
import { selectionHash } from "./test-selection-manifest.mjs";
test("production runner catches the filesystem regression that raw Vitest selection misses", async (t) => {
const repo = process.cwd();
const root = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vr-")));
t.after(() => rmSync(root, { recursive: true, force: true }));
for (const directory of ["ui/src", "server/src/__tests__", "server/scripts"]) mkdirSync(path.join(root, directory), { recursive: true });
symlinkSync(path.join(repo, "node_modules"), path.join(root, "node_modules"), "dir");
const write = (file, body) => writeFileSync(path.join(root, file), body);
const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
write(".gitignore", "node_modules/\nselection.json\n");
write("package.json", '{"name":"selection-regression","private":true,"type":"module"}');
write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}');
write("ui/vitest.config.mjs", 'export default {test:{name:"@paperclipai/ui",include:["src/**/*.test.mjs"]}}');
write("ui/src/leaf.mjs", "export const value = 42;\n");
write("ui/src/import.test.mjs", 'import {test,expect} from "vitest"; import {value} from "./leaf.mjs"; test("value",()=>expect(value).toBe(42));');
write("ui/src/import.test.mjs.test.mjs", 'throw new Error("UNSELECTED_FILE: substring filters must not execute this file");');
write("ui/src/contract.test.mjs", 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; test("filesystem contract",()=>expect(readFileSync(new URL("./leaf.mjs",import.meta.url),"utf8")).not.toContain("FORBIDDEN_MARKER"));');
git("init", "-q"); git("add", ".");
const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture");
commit();
const run = (files) => {
const manifest = { version: 1, head: git("rev-parse", "HEAD"), lanes: {
"general-workspaces-a": { shards: [{ "@paperclipai/ui": files }] },
} };
write("selection.json", JSON.stringify(manifest));
return spawnSync(process.execPath, [path.join(repo, "scripts/run-vitest-stable.mjs"), "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1"], {
cwd: root, encoding: "utf8", timeout: 60000,
env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: path.join(root, "selection.json"),
PAPERCLIP_CI_SELECTION_HASH: selectionHash(manifest) },
});
};
const baseline = run(["ui/src/import.test.mjs", "ui/src/contract.test.mjs"]);
assert.equal(baseline.status, 0, baseline.stdout + baseline.stderr);
const missing = run(["ui/src/leaf.mjs"]);
assert.equal(missing.status, 1, "an existing source file is not an executable test specification");
assert.match(missing.stdout + missing.stderr, /Selected files must match/);
write("ui/src/leaf.mjs", `${readFileSync(path.join(root, "ui/src/leaf.mjs"), "utf8")}// FORBIDDEN_MARKER\n`);
commit();
const graph = await selectProjectTests({ root, project: "@paperclipai/ui", changed: ["ui/src/leaf.mjs"] });
assert.deepEqual(graph.related, ["ui/src/import.test.mjs"]);
const raw = run(graph.related);
assert.equal(raw.status, 0, raw.stdout + raw.stderr);
assert.ok(graph.selected.includes("ui/src/contract.test.mjs"));
const guarded = run(graph.selected);
assert.equal(guarded.status, 1, guarded.stdout + guarded.stderr);
assert.match(guarded.stdout + guarded.stderr, /filesystem contract/);
assert.match(guarded.stdout + guarded.stderr, /FORBIDDEN_MARKER/);
});
+100
View File
@@ -0,0 +1,100 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import vm from "node:vm";
import test from "node:test";
import { planTasks } from "../../.github/scripts/ci-plan.mjs";
const workflow = readFileSync(".github/workflows/pr-trusted.yml", "utf8");
const block = (job) => workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0];
const plan = planTasks({ config: JSON.parse(readFileSync("turbo.json", "utf8")),
lanes: JSON.parse(readFileSync(".github/ci-task-matrix.json", "utf8")), paths: ["ui/src/pages/Inbox.tsx"] });
const evaluate = (expression, needs, cancelled = false) => vm.runInNewContext(expression, {
needs, fromJSON: JSON.parse, toJSON: JSON.stringify, always: () => true, cancelled: () => cancelled,
});
test("failed, timed-out, skipped and unavailable selectors leave the original matrix runnable", () => {
for (const result of ["failure", "cancelled", "skipped", "success"]) {
const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } },
test_selection: { result, outputs: { plan: "" } } };
const job = block("general_tests");
const condition = job.match(/if: \$\{\{ (.*) \}\}/)[1];
const matrix = job.match(/matrix: \$\{\{ (.*) \}\}/)[1];
assert.equal(evaluate(condition, needs), true, result);
assert.deepEqual(JSON.parse(JSON.stringify(evaluate(matrix, needs))), plan.jobs.general_tests.matrix);
assert.equal(evaluate(condition, needs, true), false, "cancellation must not start more runners");
}
});
test("a published selection controls the matrix and the required aggregate", () => {
const selected = structuredClone(plan);
selected.testSelection = { version: 1, hash: "test" };
selected.jobs.general_tests.matrix.include = [selected.jobs.general_tests.matrix.include[0]];
selected.jobs.general_tests.selected = 1;
const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } },
test_selection: { result: "success", outputs: { plan: JSON.stringify(selected) } } };
const matrix = block("general_tests").match(/matrix: \$\{\{ (.*) \}\}/)[1];
assert.equal(evaluate(matrix, needs).include.length, 1);
const aggregate = block("verify");
const names = aggregate.match(/needs: \[([^\]]+)\]/)[1].split(", ");
const code = aggregate.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0]
.split("\n").map((line) => line.slice(10)).join("\n");
const results = Object.fromEntries(names.map((name) => [name, { result:
["gate", "policy", "test_selection"].includes(name) || selected.jobs[name]?.run ? "success" : "skipped" }]));
const run = () => spawnSync(process.execPath, ["--input-type=module", "-e", code], { encoding: "utf8",
env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(selected), CI_RESULTS: JSON.stringify(results) } });
assert.equal(run().status, 0);
for (const job of ["general_tests", "test_selection"]) {
for (const result of ["failure", "cancelled", "skipped"]) {
results[job].result = result;
assert.notEqual(run().status, 0, `${job}: ${result}`);
}
results[job].result = "success";
}
selected.jobs.general_tests.run = false;
results.general_tests.result = "skipped";
assert.equal(run().status, 0, "an explicitly empty selected lane may be skipped");
assert.ok(aggregate.includes("needs.test_selection.outputs.plan || needs.gate.outputs.plan"));
});
test("publication follows successful upload and manifests are downloaded without ignoring failures", () => {
const selection = block("test_selection");
assert.ok(selection.indexOf("Upload validated selection") < selection.indexOf("Publish the executable plan"));
assert.match(selection, /timeout-minutes: 5/);
assert.match(selection, /vars.CI_VITEST_SELECTION == '' \|\| vars.CI_VITEST_SELECTION == 'enabled'/);
for (const job of ["general_tests", "verify_serialized_server"]) {
assert.match(block(job), /PAPERCLIP_CI_SELECTION_HASH:.*testSelection.hash/);
assert.match(block(job), /Download selected test files\n if: matrix.selection\n uses: actions\/download-artifact@/);
assert.doesNotMatch(block(job), /continue-on-error: true\n.*download-artifact/);
}
});
test("failed-job retries use the original selector artifact, not the retry attempt", (t) => {
const directory = mkdtempSync(path.join(tmpdir(), "ci-retry-"));
t.after(() => rmSync(directory, { recursive: true, force: true }));
mkdirSync(path.join(directory, ".ci-test-selection"));
const output = path.join(directory, "output");
const publication = block("test_selection").split(" - name: Publish the executable plan")[1];
assert.match(publication, /ARTIFACT_ID: \$\{\{ steps.upload_selection.outputs.artifact-id \}\}/);
const code = publication.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0]
.split("\n").map((line) => line.slice(10)).join("\n");
const selected = { ...plan, testSelection: { version: 1, hash: "test" } };
const run = (value, artifactId) => {
writeFileSync(path.join(directory, ".ci-test-selection/plan.json"), JSON.stringify(value));
writeFileSync(output, "");
return spawnSync(process.execPath, ["--input-type=module", "-e", code], {
cwd: directory, encoding: "utf8", env: { ...process.env, ARTIFACT_ID: artifactId, GITHUB_OUTPUT: output },
});
};
assert.equal(run(selected, "12345").status, 0);
const savedPlan = readFileSync(output, "utf8").trim().slice("plan=".length);
for (const job of ["general_tests", "verify_serialized_server"]) {
const expression = block(job).match(/artifact-ids: \$\{\{ (.*) \}\}/)[1];
assert.equal(vm.runInNewContext(expression, { fromJSON: JSON.parse, github: { run_attempt: 2 },
needs: { test_selection: { outputs: { plan: savedPlan } } } }), "12345");
}
assert.notEqual(run(selected, "").status, 0, "a selection cannot be published without its uploaded artifact");
assert.equal(run(plan, "").status, 0, "fallback plans need no selection artifact");
});
@@ -0,0 +1,198 @@
import assert from "node:assert/strict";
import { execFileSync, spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { selectProjectTests } from "./vitest-selection-graph.mjs";
import { partitionSelection, refineTestPlan } from "./test-selection-plan.mjs";
import { selectionHash, selectedShard, validateSelection } from "./test-selection-manifest.mjs";
import { selectionEligibility } from "./select-tests.mjs";
import { mergeDiff } from "../../.github/scripts/ci-diff.mjs";
const root = process.cwd();
const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
test("real graph retains direct and indirect filesystem, process and opaque dependencies", async (t) => {
const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vg-")));
t.after(() => rmSync(directory, { recursive: true, force: true }));
mkdirSync(path.join(directory, "ui/src"), { recursive: true });
mkdirSync(path.join(directory, "ui/src/glob"));
symlinkSync(path.join(root, "node_modules"), path.join(directory, "node_modules"), "dir");
const write = (file, text) => writeFileSync(path.join(directory, file), text);
write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}');
write("ui/vitest.config.mjs", `export default {resolve:{alias:{"@leaf":${JSON.stringify(path.join(directory, "ui/src/leaf.ts"))}}},test:{name:"fixture",include:["src/**/*.test.ts"],setupFiles:["./setup.ts"]}}`);
write("ui/setup.ts", 'import "./src/environment.ts";');
write("ui/src/environment.ts", "export const environment = true;");
write("ui/src/leaf.ts", "export const value = 42;");
write("ui/src/bridge.ts", 'export {value} from "@leaf";');
write("ui/src/helper.ts", 'import {readFileSync} from "node:fs"; export const read = (file: string) => readFileSync(file, "utf8");');
write("ui/src/import.test.ts", 'import {value} from "./bridge"; console.log(value); throw new Error("selection must not execute tests");');
write("ui/src/reader.test.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));');
write("ui/src/indirect.test.ts", 'import {read} from "./helper"; read("input"); throw new Error("selection must not execute tests");');
write("ui/src/process.test.ts", 'import {execFileSync} from "node:child_process"; execFileSync("must-not-execute");');
write("ui/src/dynamic.test.ts", 'const load = (file: string) => import(file);');
write("ui/src/cjs.test.ts", 'const load = require;');
write("ui/src/builtin-loader.test.ts", 'process.getBuiltinModule("fs").readFileSync("input");');
write("ui/src/opaque.test.ts", 'import {parse} from "yaml"; console.log(parse("value: true"));');
write("ui/src/independent.test.ts", 'export const independent = true;');
write("ui/src/mocked.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); import {read} from "./helper"; read("input");');
write("ui/src/partial.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",async (original)=>await original()); import {read} from "./helper"; read("input");');
write("ui/src/automock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper"); import {read} from "./helper"; read("input");');
write("ui/src/bypass.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); await vi.importActual("./helper");');
write("ui/src/spy.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",{spy:true}); import {read} from "./helper"; read("input");');
write("ui/src/remock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); vi.doMock("./helper",async original=>await original());');
write("ui/src/arguments.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",function(){return arguments[0]();});');
write("ui/src/glob/one.ts", "export const value = 1;");
write("ui/src/globber.ts", 'export const modules = import.meta.glob("./glob/*.ts", {eager:true});');
write("ui/src/glob.test.ts", 'import {modules} from "./globber"; console.log(modules);');
execFileSync("git", ["init", "-q"], { cwd: directory });
execFileSync("git", ["add", "."], { cwd: directory });
const select = (changed) => selectProjectTests({ root: directory, project: "fixture", changed });
const result = await select(["ui/src/leaf.ts"]);
assert.deepEqual(result.related, ["ui/src/import.test.ts"]);
for (const file of ["reader", "indirect", "process", "dynamic", "cjs", "builtin-loader", "opaque", "partial", "automock", "bypass", "spy", "remock", "arguments"]) {
assert.ok(result.alwaysRun.includes(`ui/src/${file}.test.ts`), `${file} must never be dropped`);
}
assert.ok(!result.selected.includes("ui/src/independent.test.ts"));
assert.ok(!result.alwaysRun.includes("ui/src/mocked.test.ts"), "a completely replaced local dependency performs no real IO");
assert.ok(!result.alwaysRun.includes("ui/src/glob.test.ts"), "literal Vite globs use the real dependency graph");
assert.ok((await select(["ui/src/glob/one.ts"])).related.includes("ui/src/glob.test.ts"));
write("ui/src/glob/two.ts", "export const value = 2;");
execFileSync("git", ["add", "."], { cwd: directory });
assert.ok((await select(["ui/src/glob/two.ts"])).related.includes("ui/src/glob.test.ts"), "new glob members cannot be omitted");
const setup = await select(["ui/src/environment.ts"]);
assert.equal(setup.setupChanged, true);
assert.deepEqual(setup.selected, setup.inventory, "transitive setup edits affect every test");
write("ui/src/broken.test.ts", 'import "./missing";');
const broken = await select(["ui/src/leaf.ts"]);
assert.ok(broken.selected.includes("ui/src/broken.test.ts"), "an unresolved dependency must retain its test");
write("ui/src/generated.ts", "export const generated = 42;");
write("ui/src/generated.test.ts", 'import {generated} from "./generated"; console.log(generated);');
execFileSync("git", ["add", "ui/src/generated.test.ts"], { cwd: directory });
assert.ok((await select(["ui/src/leaf.ts"])).alwaysRun.includes("ui/src/generated.test.ts"),
"generated inputs have no trustworthy source-to-output graph and retain their consumers");
write("ui/src/environment.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));');
await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup builtin/, "setup filesystem reads can affect every test outside the import graph");
mkdirSync(path.join(directory, "server/src/__tests__/helpers"), { recursive: true });
for (const file of ["server/src/__tests__/setup-supertest.ts", "server/src/__tests__/helpers/isolated-test-home.ts"]) {
write(file, readFileSync(path.join(root, file), "utf8"));
}
write("ui/setup.ts", 'import "../server/src/__tests__/setup-supertest.ts";');
const reviewed = await select(["ui/src/leaf.ts"]);
assert.ok(!reviewed.selected.includes("ui/src/independent.test.ts"), "reviewed isolated setup permits file selection");
const helper = "server/src/__tests__/helpers/isolated-test-home.ts";
const changedSetup = await select([helper]);
assert.deepEqual(changedSetup.selected, changedSetup.inventory, "reviewed helpers still invalidate all tests when changed");
write(helper, `${readFileSync(path.join(directory, helper), "utf8")}\n// setup edited since review\n`);
await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup/, "a stale review cannot authorize new setup effects");
});
test("partitioning reduces runners and preserves an exact, non-overlapping file cover", () => {
const all = Array.from({ length: 120 }, (_, index) => `server/src/${index}.test.ts`);
const selected = all.slice(0, 15);
const shards = partitionSelection(all, selected, 12);
assert.equal(shards.length, 2);
assert.deepEqual(shards.flat().sort(), selected.sort());
assert.equal(new Set(shards.flat()).size, selected.length);
assert.deepEqual(partitionSelection(all, [], 12), []);
assert.equal(partitionSelection(all, all, 12).length, 12);
const durations = Object.fromEntries(all.map((file, index) => [file, index === 0 ? 1000 : 1]));
assert.equal(partitionSelection(all, [all[0]], 12, durations).length, 1);
assert.equal(partitionSelection(all, [all[0]], 12, Object.fromEntries(all.map((file) => [file, 0]))).length, 1);
});
function fixturePlan() {
const files = ["ui/src/pages/Inbox.test.tsx", "ui/src/pages/apps/chat/chat-ui-contract.test.ts"];
const entries = [0, 1].map((shard_index) => ({ group: "general-workspaces-a", group_label: `workspaces-a (${shard_index + 1}/2)`, shard_index, shard_count: 2, task: "ci:workspaces-a" }));
const plan = { version: 1, mode: "enabled", forceReason: null, changed: ["ui/src/pages/Inbox.tsx"], jobs: {
general_tests: { run: true, matrix: { include: entries }, selected: 2, total: 2 },
verify_serialized_server: { run: false, matrix: { include: [{ task: "ci:serialized" }] } },
} };
const owners = { "general-workspaces-a": { "@paperclipai/ui": files } };
const graphs = [{ project: "@paperclipai/ui", inventory: files, selected: [files[1]] }];
return { files, ...refineTestPlan({ plan, head, owners, graphs }) };
}
test("runner consumes the exact manifest, and rejects stale, missing, or mismatched manifests", (t) => {
const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vm-")));
t.after(() => rmSync(directory, { recursive: true, force: true }));
const { manifest, plan } = fixturePlan();
assert.equal(plan.jobs.general_tests.selected, 1);
const file = path.join(directory, "selection.json");
const run = (selection = manifest, hash = selectionHash(manifest)) => {
writeFileSync(file, JSON.stringify(selection));
return spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1", "--dry-run"], {
cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: hash },
});
};
const result = run();
assert.equal(result.status, 0, result.stderr);
assert.deepEqual(JSON.parse(result.stdout).selection, manifest.lanes["general-workspaces-a"].shards[0]);
assert.notEqual(run({ ...manifest, head: "0".repeat(40) }).status, 0);
assert.notEqual(run(manifest, "bad-hash").status, 0);
assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 2 }, ["@paperclipai/ui"]));
assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 1 }, ["@paperclipai/server"]));
const duplicate = structuredClone(manifest);
duplicate.lanes["general-workspaces-a"].shards.push(duplicate.lanes["general-workspaces-a"].shards[0]);
assert.throws(() => validateSelection(duplicate), /assigned twice/);
const wrongLane = { version: 1, head, lanes: { "general-server-without-chat": {
shards: [{ "@paperclipai/server": ["server/src/__tests__/issues-service.test.ts"] }],
} } };
writeFileSync(file, JSON.stringify(wrongLane));
const misplaced = spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-server-without-chat", "--shard-index", "0", "--shard-count", "1", "--dry-run"], {
cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: selectionHash(wrongLane) },
});
assert.notEqual(misplaced.status, 0);
assert.match(misplaced.stderr, /belongs to another server lane/);
});
test("forced modes, config, fixtures, deletions and unknown inputs cannot narrow CI", () => {
const plan = fixturePlan().plan;
assert.equal(selectionEligibility(plan), null);
for (const changed of [["ui/vitest.setup.ts"], ["ui/vite.config.ts"], ["package.json"], ["pnpm-lock.yaml"],
["docs/api/issues.md"], ["ui/src/missing-deleted.ts"], ["server/fixtures/input.json"], []]) {
assert.ok(selectionEligibility({ ...plan, changed }), JSON.stringify(changed));
}
for (const mode of ["full", "shadow"]) assert.ok(selectionEligibility({ ...plan, mode }));
assert.ok(selectionEligibility({ ...plan, forceReason: "full-ci" }));
});
test("the CLI publishes the complete coarse plan if setup mutates tracked inputs", (t) => {
const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vd-")));
t.after(() => rmSync(directory, { recursive: true, force: true }));
mkdirSync(path.join(directory, "ui/src"), { recursive: true });
const file = path.join(directory, "ui/src/input.ts");
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture");
git("init", "-q"); writeFileSync(file, "export const value = 1;"); git("add", "."); commit();
const base = git("rev-parse", "HEAD");
git("checkout", "-qb", "pr");
writeFileSync(file, "export const value = 2;"); commit();
const prHead = git("rev-parse", "HEAD");
git("checkout", "--detach", base); git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "merge", "--no-ff", "pr", "-m", "merge");
const merge = git("rev-parse", "HEAD");
writeFileSync(file, "export const value = 3; // generated during setup");
const plan = { version: 1, mode: "enabled", forceReason: null, ...mergeDiff(merge, prHead, directory),
jobs: { general_tests: { run: true, selected: 2, matrix: { include: [{ shard_index: 0 }, { shard_index: 1 }] } } } };
const result = spawnSync(process.execPath, [path.join(root, "scripts/ci/select-tests.mjs")], {
cwd: directory, encoding: "utf8", env: { ...process.env, CI_PLAN: JSON.stringify(plan), BASE_SHA: base,
GITHUB_SHA: merge, GITHUB_OUTPUT: "", GITHUB_STEP_SUMMARY: "" },
});
assert.equal(result.status, 0, result.stderr);
const fallback = JSON.parse(readFileSync(path.join(directory, ".ci-test-selection/plan.json"), "utf8"));
assert.match(fallback.testSelectionFallback, /Source inputs changed/);
assert.deepEqual(fallback.jobs, plan.jobs);
assert.equal(fallback.testSelection, undefined);
});
test("mixed audited documentation stays graph eligible and large complete source diffs are allowed", () => {
const plan = fixturePlan().plan;
for (const doc of ["doc/DEVELOPING.md", "doc/DATABASE.md", "doc/plans/2026-10-30-example.md"]) {
assert.equal(selectionEligibility({ ...plan, changed: [...plan.changed, doc] }), null);
}
assert.equal(selectionEligibility({ ...plan, changed: Array(350).fill(plan.changed[0]) }), null);
assert.ok(selectionEligibility({ ...plan, changed: ["doc/DEVELOPING.md"] }));
assert.ok(selectionEligibility({ ...plan, changed: [...plan.changed, "docs/api/issues.md"] }));
});
@@ -0,0 +1,38 @@
import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { measureVitestImpact } from "./vitest-impact-report.mjs";
test("real Vitest graph follows aliases/transitive imports but misses filesystem contracts", async (t) => {
const root = mkdtempSync(path.join(tmpdir(), "vitest-impact-"));
t.after(() => rmSync(root, { recursive: true, force: true }));
mkdirSync(path.join(root, "unit"));
const write = (file, text) => writeFileSync(path.join(root, file), text);
write("vitest.config.mjs", 'export default { test: { projects: ["unit"] } };');
write("unit/vitest.config.mjs", `export default {
resolve: { alias: { "@leaf": ${JSON.stringify(path.join(root, "unit/leaf.mjs"))} } },
test: { name: "fixture", include: ["**/*.test.mjs"] }
};`);
write("unit/leaf.mjs", "export const value = 42;");
write("unit/bridge.mjs", 'export { value } from "@leaf";');
// The graph must be collected without executing the test modules.
write("unit/related.test.mjs", 'import { value } from "./bridge.mjs"; throw new Error("must not execute");');
write("unit/dynamic.test.mjs", 'const load = () => import("./leaf.mjs"); throw new Error("must not execute");');
write("unit/independent.test.mjs", 'throw new Error("must not execute");');
write("unit/reader.test.mjs", 'import { readFileSync } from "node:fs"; const body = readFileSync(new URL("./leaf.mjs", import.meta.url), "utf8");');
const report = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/leaf.mjs"] });
assert.equal(report.mode, "report-only");
assert.equal(report.projects[0].totalFiles, 4);
assert.deepEqual(report.projects[0].selectedTests, ["unit/dynamic.test.mjs", "unit/related.test.mjs"]);
assert.deepEqual(report.projects[0].omittedTests, ["unit/independent.test.mjs", "unit/reader.test.mjs"]);
assert.ok(report.limitations.some((message) => message.includes("filesystem")));
const direct = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/independent.test.mjs"] });
assert.deepEqual(direct.projects[0].selectedTests, ["unit/independent.test.mjs"]);
const unmapped = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unmapped.md"] });
assert.equal(unmapped.projects[0].selectedFiles, 0);
assert.equal(unmapped.mode, "report-only", "an empty graph is not permission to skip CI");
});
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env node
import { execFileSync } from "node:child_process";
import { realpathSync, writeFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
function repoPath(root, file) {
const relative = path.relative(root, path.resolve(root, file));
if (!relative || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
throw new Error(`Expected a file inside the checkout: ${file}`);
}
return relative.split(path.sep).join("/");
}
// This is an experiment report, deliberately not a CI execution plan. Import
// graphs cannot describe filesystem reads, subprocesses, or external services.
export async function measureVitestImpact({ root = process.cwd(), projects, changedFiles }) {
root = realpathSync(root);
if (!projects?.length || !changedFiles?.length) {
throw new Error("Specify at least one project and one changed file");
}
const files = [...new Set(changedFiles.map((file) => repoPath(root, file)))].sort();
const { createVitest } = await import("vitest/node");
const results = [];
// Each project gets its own graph to bound memory usage. Cross-package imports
// are still followed, but projects omitted by the caller are NOT assessed.
for (const project of new Set(projects)) {
const started = performance.now();
const ctx = await createVitest({
root,
watch: false,
project: [project],
exclude: ["**/dist/**"],
reporters: [],
related: files.map((file) => path.resolve(root, file)),
});
try {
if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) {
throw new Error(`Expected exactly one project named ${project}`);
}
const inventory = await ctx.globTestSpecifications();
if (!inventory.length) throw new Error(`No test files found for ${project}`);
const ready = performance.now();
const selected = await ctx.getRelevantTestSpecifications();
const finished = performance.now();
const selectedTests = selected.map((spec) => repoPath(root, spec.moduleId)).sort();
const selectedSet = new Set(selectedTests);
results.push({
project,
totalFiles: inventory.length,
selectedFiles: selectedTests.length,
startupSeconds: (ready - started) / 1000,
selectionSeconds: (finished - ready) / 1000,
selectedTests,
omittedTests: inventory.map((spec) => repoPath(root, spec.moduleId))
.filter((file) => !selectedSet.has(file)).sort(),
});
} finally {
await ctx.close();
}
}
return {
mode: "report-only",
changedFiles: files,
limitations: [
"Only the explicitly requested projects were assessed.",
"Selection follows imports, not filesystem reads, subprocesses, or external services.",
"Zero selected files is not proof that a change needs no tests.",
"This report must not be used to omit required CI jobs.",
],
projects: results,
};
}
async function main() {
const { values } = parseArgs({ options: {
project: { type: "string", multiple: true },
file: { type: "string", multiple: true },
base: { type: "string" },
output: { type: "string" },
} });
if (!values.project?.length || (!!values.base === !!values.file?.length)) {
throw new Error("Usage: node scripts/ci/vitest-impact-report.mjs --project NAME [--project NAME] (--file PATH [--file PATH] | --base REF) [--output FILE]");
}
const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 });
const head = git("rev-parse", "--verify", "HEAD^{commit}").trim();
let comparison = { kind: "hypothetical", head };
let changedFiles = values.file;
if (values.base) {
// Vitest reads the working tree, so do not label dirty sources as HEAD.
if (git("status", "--porcelain", "--untracked-files=no").trim()) {
throw new Error("--base requires a clean tracked working tree; use --file for a hypothetical change");
}
const base = git("rev-parse", "--verify", "--end-of-options", `${values.base}^{commit}`).trim();
// Compare the actual trees, including both sides of a rename. The caller
// supplies the event base; HEAD should be the tested PR merge commit in CI.
changedFiles = git("diff", "--name-only", "--no-renames", "-z", base, head, "--")
.split("\0").filter(Boolean);
comparison = { kind: "git-diff", base, head };
}
process.env.NODE_ENV = "test";
process.env.VITEST = "true";
// Config/plugins are executable code. Run this only in a test job, never in
// the trusted metadata-only CI routing gate. Bound a failed graph traversal.
const timeout = setTimeout(() => {
console.error("Vitest impact analysis exceeded five minutes; report unavailable");
process.exit(1);
}, 300_000);
try {
const report = await measureVitestImpact({ projects: values.project, changedFiles });
const json = `${JSON.stringify({ ...report, comparison }, null, 2)}\n`;
if (values.output) writeFileSync(values.output, json);
else process.stdout.write(json);
for (const result of report.projects) {
console.error(`${result.project}: ${result.selectedFiles}/${result.totalFiles} files; `
+ `${result.selectionSeconds.toFixed(2)}s graph + ${result.startupSeconds.toFixed(2)}s startup`);
}
} finally {
clearTimeout(timeout);
}
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main().catch((error) => { console.error(error); process.exitCode = 1; });
}
+113
View File
@@ -0,0 +1,113 @@
# Vitest dependency selection experiment
The production selector is documented in `doc/DEVELOPING.md`. This probe remains
a diagnostic for the raw import graph, without the production retention rules.
This probe measures Vitest's real import graph against the current checkout.
It does not change the CI matrix or authorize skipping tests. It loads Vitest
configurations and Vite plugins, so run it in a normal test environment, never
in the trusted metadata-only routing job.
Install the workspace dependencies and prepare the same shared build outputs
used by CI:
```sh
pnpm install --frozen-lockfile
node scripts/ensure-plugin-build-deps.mjs
node --test scripts/ci/vitest-impact-report.integration.test.mjs
```
Probe a hypothetical change (the file need not actually be modified):
```sh
node scripts/ci/vitest-impact-report.mjs \
--project @paperclipai/ui --file ui/src/pages/Inbox.tsx \
--output /tmp/inbox-impact.json
```
Or compare a base revision with the checked-out HEAD. This requires a clean
tracked working tree. For a PR, check out the tested merge commit and supply
the event's base SHA, as the existing CI planner does:
```sh
node scripts/ci/vitest-impact-report.mjs \
--project @paperclipai/ui --project @paperclipai/server \
--base BASE_SHA --output /tmp/pr-impact.json
```
Every requested project is analyzed separately to bound memory usage. Imports
across package boundaries are followed within each graph, but unrequested
projects are **not assessed**. Runner's separate Vitest configurations, browser
E2E, Rust, typechecking, and builds are outside this experiment. The JSON records
selected and omitted test files, project inventory, startup and graph times,
and the comparison revisions. A failed or timed-out probe is not an empty plan.
## Initial observations
Measured on `5c807d11fa` with Vitest 5.0.3, macOS, and Node 26.4.0. These are
hypothetical single-file changes against that revision, not replays of historical
PR checkouts. CI uses Node 24 and different hardware.
| Changed file | Assessed project | Selected / total test files |
| --- | --- | ---: |
| `ui/src/pages/Inbox.tsx` | UI | 6 / 844 |
| `ui/src/components/MarkdownBody.tsx` | UI | 135 / 844 |
| `server/src/middleware/auth.ts` | Server | 45 / 1,428 |
| `server/src/services/run-failure-report.ts` | Server | 787 / 1,428 |
| `packages/shared/src/issue-references.ts` | Shared only | 5 / 96 |
| `docs/api/issues.md` | Server | 0 / 1,428 — known false negative |
Cold graph construction took 27.43 seconds for UI, 25.68 seconds for server,
and 2.79 seconds for shared. Subsequent queries on the same graph took about
0.05 seconds; fresh CI jobs must budget for cold construction. The six Inbox
files ran 81 assertions successfully in 20.92 seconds with four workers.
These figures exclude dependency installation and shared package builds.
For a local execution comparison, the complete UI project passed all 10,043
assertions in 844 files in 361.50 seconds with the same four-worker setting.
The initial Inbox graph plus selected execution took about 48 seconds, versus
six minutes for the full project. This is a local experiment, not a measured
change in GitHub CI latency. Later runs benefit from filesystem/transform caches.
The CLI's Git-diff path was also exercised against the actual `HEAD^..HEAD`
change for PR #18321. Across its eleven changed files, the server graph selected
1,138 / 1,428 files and the shared graph selected 3 / 96. Other projects were
not assessed. Broad shared/service imports can therefore preserve most of a
suite; the Inbox reduction should not be generalized to every PR.
## Why the raw graph is not yet a safe execution plan
There are concrete missing edges in this repository:
- `ui/src/pages/apps/chat/chat-ui-contract.test.ts` reads `Inbox.tsx` through
`readFileSync`. It is absent from the six-file selection even though changes
to Inbox can break its assertions.
- `server/src/__tests__/paperclip-skill-utils.test.ts` reads `docs/api/issues.md`
through `fs.readFile`. Changing that document selects zero server tests.
A deliberate negative control confirmed the first missing edge: temporarily
append a comment containing `reason: "retry_failed_run"` to `Inbox.tsx`, which
violates the contract test's source assertion without changing runtime behavior.
The fresh graph still selected six files. Running those six plus the omitted
contract produced six passing files and exactly one failing assertion, in
`chat-ui-contract.test.ts` (92 assertions passed, one failed). The original source
was restored immediately afterward. The full baseline had passed this contract.
The integration test uses actual Vitest graph traversal to verify transitive
imports, aliases, literal dynamic imports, direct test edits, and the missing
filesystem edge. Test modules deliberately throw if executed: graph discovery
must not execute the tests. Passing this test documents the limitation; it does
not prove that skipped repository tests are irrelevant.
A production rollout needs explicit dependencies or an always-run lane for
filesystem/subprocess contract tests; broad fallbacks for configuration, setup,
lockfiles, deleted files, unknown inputs, and selector failures; and comparison
against full-suite failures. Source-only changes also need those protections:
the Inbox example is not a documentation-only edge case. Once qualified, compute
the selected test list before constructing the matrix and partition that list,
preserving the existing serialized, chat, and native-runner ownership rules.
Verification for this experiment covers the real-graph integration test, CLI
probes, the complete UI project, and the deliberate negative control. Repository
wide typechecking, build, server execution, and the complete `test:run` suite
were not run; this is a report-only experiment, not a production CI rollout.
+264
View File
@@ -0,0 +1,264 @@
import { readFileSync, realpathSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import path from "node:path";
import { isBuiltin } from "node:module";
import { parseSync } from "oxc-parser";
import { createVitest } from "vitest/node";
// Everything else is opaque by default. Do not add IO clients, module loaders,
// filesystem helpers or process launchers here to improve selection counts.
const purePackages = new Set([
"vitest", "@vitest/expect", "@testing-library/react", "@testing-library/dom",
"@testing-library/user-event", "@testing-library/jest-dom", "react", "react-dom",
"react-router", "react-router-dom", "@tanstack/react-query", "@tanstack/query-core",
"zod", "lodash", "lodash-es", "date-fns", "clsx", "tailwind-merge", "class-variance-authority",
"lucide-react", "immer", "nanoid", "uuid", "semver",
"radix-ui", "motion", "@dnd-kit/core", "@dnd-kit/sortable", "@dnd-kit/utilities",
"i18next", "react-i18next",
]);
const pureBuiltins = new Set(["assert", "assert/strict", "path", "path/posix", "path/win32",
"url", "util", "util/types", "buffer", "events", "string_decoder", "crypto"]);
// Reviewed setup performs only temporary-home IO and Supertest socket setup,
// not reads of arbitrary repository inputs. Pin contents, not just paths: any
// edit disables this exception until its effects are reviewed again. Continue
// following local imports even for these files; helper changes affect all tests.
const reviewedSetup = new Map([
["server/src/__tests__/setup-supertest.ts", {
hash: "99d7193927468e98b41255ea9a5e37b1a681f02b259c304f3af1a52304eed2bb",
external: ["supertest/lib/test.js"],
}],
["server/src/__tests__/helpers/isolated-test-home.ts", {
hash: "bd5645dc590087f26b473beab06d5c3ddd7956a79949f0b3e9713f11897a0c1b",
external: [],
}],
]);
const packageName = (specifier) => specifier.startsWith("@")
? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0];
const inside = (root, file) => file.startsWith(`${root}${path.sep}`) && !file.includes(`${path.sep}node_modules${path.sep}`);
// Parse source, including imports erased by mocks/transforms. Unknown loaders
// retain their consumers. This is intentionally conservative, not a regex-based
// promise that arbitrary JavaScript has no effects.
export function sourceDependencies(file, source) {
const ast = parseSync(file, source);
const imports = new Set();
const reasons = new Set();
const mockBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest")
.flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name)).map((s) => s.local.name)));
const hoistedBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest")
.flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name)
&& s.local.name === s.imported.name).map((s) => s.local.name)));
const mocks = [];
if (ast.errors.length) reasons.add("unparsed source");
const literal = (node) => node?.type === "Literal" && typeof node.value === "string";
const importSource = (node) => {
if (literal(node)) imports.add(node.value);
else reasons.add("dynamic module name");
};
for (const statement of ast.program.body) {
const call = statement.type === "ExpressionStatement" ? statement.expression : null;
const callee = call?.type === "CallExpression" ? call.callee : null;
const factory = call?.arguments?.[1];
if (callee?.type === "MemberExpression" && !callee.computed && callee.property.name === "mock"
&& hoistedBindings.has(callee.object.name) && literal(call.arguments[0])
&& ["ArrowFunctionExpression", "FunctionExpression"].includes(factory?.type) && factory.params.length === 0) {
mocks.push(call.arguments[0].value);
}
}
function visit(node, parent = null) {
if (!node || typeof node !== "object") return;
if ((node.type === "ImportDeclaration" && node.importKind !== "type")
|| (["ExportNamedDeclaration", "ExportAllDeclaration"].includes(node.type) && node.exportKind !== "type")) {
if (node.source) importSource(node.source);
}
if (node.type === "ImportExpression") importSource(node.source);
if (node.type === "TSImportEqualsDeclaration") reasons.add("CommonJS import");
if (node.type === "Identifier" && /^(require|fetch|eval|Function|WebSocket|XMLHttpRequest|Worker)$/.test(node.name)) {
reasons.add("external input or dynamic code");
}
if ((node.type === "Identifier" && node.name === "getBuiltinModule")
|| (literal(node) && node.value === "getBuiltinModule")) reasons.add("dynamic builtin loader");
if ((node.type === "Identifier" && /^(importActual|unmock|doUnmock|doMock|arguments)$/.test(node.name))
|| (literal(node) && /^(importActual|unmock|doUnmock|doMock)$/.test(node.value))) reasons.add("mock bypass");
if (node.type === "Identifier" && mockBindings.has(node.name) && parent
&& parent.type !== "ImportSpecifier"
&& !(parent.type === "MemberExpression" && parent.object === node && !parent.computed)) {
reasons.add("escaped or dynamic mocking API");
}
if (node.type === "CallExpression") {
const name = source.slice(node.callee.start, node.callee.end);
if (/^(?:vi|vitest)\.(?:mock|doMock|importActual|importMock)$/.test(name)) {
importSource(node.arguments[0]);
}
if (/\brequire\b/.test(name)) {
reasons.add("CommonJS loader");
importSource(node.arguments[0]);
}
const viteGlob = name === "import.meta.glob" && (literal(node.arguments[0])
|| (node.arguments[0]?.type === "ArrayExpression" && node.arguments[0].elements.every(literal)));
// Vite expands literal globs into dependency edges. New files appear in
// that expansion; deleted/renamed and non-source inputs force full lanes.
if (!viteGlob && /\b(?:fetch|eval|Function|glob|globSync)\b/.test(name)) reasons.add("external input or dynamic code");
}
for (const value of Object.values(node)) {
if (Array.isArray(value)) value.forEach((child) => visit(child, node));
else if (value && typeof value === "object" && value.type) visit(value, node);
}
}
visit(ast.program);
return { imports: [...imports], reasons: [...reasons], mocks };
}
export async function selectProjectTests({ root = process.cwd(), project, changed, onlyOwnChanges = false }) {
root = realpathSync(root);
const started = performance.now();
const absoluteChanges = changed.map((file) => path.resolve(root, file));
const ctx = await createVitest({ root, watch: false, project: [project],
exclude: ["**/dist/**"], reporters: [], related: absoluteChanges });
try {
if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) throw new Error(`Unknown project: ${project}`);
const owner = ctx.projects[0];
const environment = owner.vite.environments.ssr;
const inventory = await ctx.globTestSpecifications();
if (!inventory.length) throw new Error(`Empty inventory: ${project}`);
const relative = (spec) => path.relative(root, spec.moduleId).split(path.sep).join("/");
if (onlyOwnChanges && !absoluteChanges.some((file) => file.startsWith(`${owner.config.root}${path.sep}`))) {
// Keep coarse dependency coverage, but avoid spending a graph traversal
// on an unchanged consumer project merely to retain most of its IO tests.
const files = inventory.map(relative).sort();
return { project, inventory: files, selected: files, related: [], alwaysRun: files, coveredChanges: [],
opaqueModules: {}, retainedReason: "Unchanged project retains complete lane coverage", seconds: (performance.now() - started) / 1000 };
}
const tracked = new Set(execFileSync("git", ["ls-files", "--cached", "-z"], {
cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
}).split("\0").filter(Boolean));
const audit = new Map();
const parsed = new Map();
const parse = (file) => {
if (!parsed.has(file)) parsed.set(file, sourceDependencies(file, readFileSync(file, "utf8")));
return parsed.get(file);
};
const resolve = async (specifier, importer) => {
try {
const resolved = await environment.pluginContainer.resolveId(specifier, importer);
if (!resolved || resolved.external || !path.isAbsolute(resolved.id) || resolved.id.includes("?")) return null;
return realpathSync(resolved.id);
} catch { return null; } // A conditional/require-only export is opaque.
};
// Setup dependencies are not test-import edges. A change anywhere in their
// local import closure invalidates the entire project, including helpers.
const setup = new Set();
async function setupClosure(file) {
if (setup.has(file)) return;
setup.add(file);
const source = parse(file);
const review = reviewedSetup.get(path.relative(root, file).split(path.sep).join("/"));
const reviewed = review?.hash === createHash("sha256").update(readFileSync(file)).digest("hex");
if (!reviewed && source.reasons.length) throw new Error(`Opaque setup dependency: ${file}`);
for (const specifier of source.imports) {
if (isBuiltin(specifier)) {
if (!reviewed && !pureBuiltins.has(specifier.replace(/^node:/, ""))) throw new Error(`Opaque setup builtin: ${specifier}`);
continue;
}
if (reviewed && review.external.includes(specifier)) continue;
const dependency = await resolve(specifier, file);
if (dependency && inside(root, dependency)) await setupClosure(dependency);
else if (!dependency) throw new Error(`Unresolved setup dependency: ${specifier}`);
else if (!purePackages.has(packageName(specifier))) throw new Error(`Opaque setup package: ${specifier}`);
}
}
for (const file of [...(owner.config.setupFiles ?? []), ...(owner.config.globalSetup ?? []),
...(owner.config.snapshotSerializers ?? [])]) {
const resolved = path.isAbsolute(file) ? file : await resolve(file, owner.vite.config.configFile);
if (resolved && inside(root, resolved)) await setupClosure(resolved);
else if (!resolved) throw new Error(`Unresolved setup module: ${file}`);
}
for (const file of owner.vite.config.configFileDependencies ?? []) setup.add(realpathSync(file));
const setupChanged = absoluteChanges.some((file) => setup.has(file));
const raw = await ctx.getRelevantTestSpecifications();
const modules = environment.moduleGraph.idToModuleMap;
const resolvedDependencies = new Map();
const localImports = new Map();
const dependencyPath = (dep) => {
if (!resolvedDependencies.has(dep)) {
const file = dep.startsWith("/@fs/") ? dep.slice(process.platform === "win32" ? 5 : 4)
: path.join(owner.config.root, dep);
try { resolvedDependencies.set(dep, realpathSync(file)); }
catch { resolvedDependencies.set(dep, null); }
}
return resolvedDependencies.get(dep);
};
for (const [id, module] of modules) {
if (!inside(root, id) || !/\.[cm]?[jt]sx?$/.test(id)) continue;
const source = parse(id);
const reasons = [...source.reasons];
if (!tracked.has(path.relative(root, id).split(path.sep).join("/"))) reasons.push("generated or untracked source");
const local = [];
localImports.set(id, local);
// Use the same resolved runtime edges as Vitest, including literal dynamic
// imports. Resolving the original syntax again uses the wrong conditions
// for require-only exports and invents edges for erased type imports.
const transformed = module.transformResult;
if (!Array.isArray(transformed?.deps)) reasons.push("unknown transform shape");
for (const dependencyId of [...(transformed?.deps ?? []), ...(transformed?.dynamicDeps ?? [])]) {
if (isBuiltin(dependencyId)) {
if (!pureBuiltins.has(dependencyId.replace(/^node:/, ""))) reasons.push(`builtin ${dependencyId}`);
continue;
}
const dependency = dependencyPath(dependencyId);
if (dependency && inside(root, dependency)) {
local.push(dependency);
// Vitest intentionally skips some edges (e.g. CJS, externalized
// workspaces). Never regard an untracked local edge as independent.
if (!modules.get(dependency)?.transformResult) reasons.push(`untracked dependency ${path.relative(root, dependency)}`);
} else {
const external = dependency?.split(`${path.sep}node_modules${path.sep}`).at(-1).split(path.sep).join("/");
if (!dependency || !dependency.includes(`${path.sep}node_modules${path.sep}`) || !purePackages.has(packageName(external))) {
reasons.push(`opaque dependency ${external ? packageName(external) : dependencyId}`);
}
}
}
if (!module.transformResult) reasons.push("untransformed module");
if (reasons.length) audit.set(id, [...new Set(reasons)]);
}
const retained = [];
for (const spec of inventory) {
const mocked = new Set();
// Honor only unconditional, explicitly supplied mock factories without an
// importOriginal parameter. Automocks, spies, unmock/importActual, dynamic
// mocking and escaped vi bindings stay conservative. The related-file
// graph remains uncut: edits to a mocked module can still select its tests.
if (owner.config.isolate !== false) {
for (const name of parse(spec.moduleId).mocks) {
const target = await resolve(name, spec.moduleId);
if (target && inside(root, target)) mocked.add(target);
}
}
const queue = [spec.moduleId], visited = new Set();
let opaque = setupChanged || owner.config.isolate === false || !localImports.has(spec.moduleId);
while (queue.length && !opaque) {
const file = queue.pop();
if (visited.has(file)) continue;
visited.add(file);
if (audit.has(file)) { opaque = true; break; }
for (const dependency of localImports.get(file) ?? []) if (!mocked.has(dependency)) queue.push(dependency);
}
if (opaque) retained.push(spec);
}
const selected = [...new Set([...raw, ...retained].map(relative))].sort();
return {
project,
inventory: inventory.map(relative).sort(),
selected,
related: raw.map(relative).sort(),
alwaysRun: retained.map(relative).sort(),
coveredChanges: changed.filter((file, index) => modules.has(absoluteChanges[index]) || setup.has(absoluteChanges[index])),
opaqueModules: Object.fromEntries([...audit].map(([file, reasons]) => [path.relative(root, file), reasons])),
setupChanged,
seconds: (performance.now() - started) / 1000,
};
} finally {
await ctx.close();
}
}
+34 -4
View File
@@ -8,6 +8,7 @@ import { loadShardDurations, selectGeneralServerShard } from "./general-server-s
import { invocationEnv } from "./ci-profile/record.mjs";
import { assertSelectedTests, partitionTestLines } from "./test-line-shard.mjs";
import { loadSelection, selectedShard } from "./ci/test-selection-manifest.mjs";
const repoRoot = process.cwd();
const scriptsDir = path.dirname(fileURLToPath(import.meta.url));
@@ -378,7 +379,7 @@ function selectSerializedSuites(routeTests, shardIndex, shardCount) {
return shardFiles.map((file) => byRepoPath.get(file));
}
async function runVitest(args, label, testShard = null) {
async function runVitest(args, label, testShard = null, exactFiles = false) {
console.log(`\n[test:run] ${label}`);
invocationIndex += 1;
const tempRootParent = process.platform === "win32" ? os.tmpdir() : "/tmp";
@@ -436,7 +437,9 @@ async function runVitest(args, label, testShard = null) {
args = [...args.filter((arg) => arg !== chatSuite), ...runFilters, "--allowOnly=false"];
}
await new Promise((resolve, reject) => {
const child = spawn("pnpm", ["exec", "vitest", "run", ...sourceOnlyVitestArgs, ...args], {
const command = exactFiles ? process.execPath : "pnpm";
const prefix = exactFiles ? [path.join(scriptsDir, "ci/run-selected-vitest.mjs")] : ["exec", "vitest", "run"];
const child = spawn(command, [...prefix, ...sourceOnlyVitestArgs, ...args], {
cwd: repoRoot,
env: invocationEnv("vitest", env),
stdio: "inherit",
@@ -615,6 +618,24 @@ const generalServerTestFiles = [
.sort((a, b) => a.localeCompare(b));
const options = parseCliOptions(process.argv.slice(2));
const selectionProjects = options.mode === serializedModeName || generalServerGroups.includes(options.group)
? ["@paperclipai/server"]
: options.group === generalWorkspacesAGroupName ? generalWorkspacesAProjects
: options.group === generalWorkspacesBGroupName ? generalWorkspacesBProjects
: options.group === generalWorkspacesCGroupName ? generalWorkspacesCProjects : [];
const selection = process.env.PAPERCLIP_CI_TEST_SELECTION
? selectedShard(loadSelection(), options, selectionProjects) : null;
if (selection?.["@paperclipai/server"]) {
const owned = new Set(options.mode === serializedModeName ? routeTests.map((test) => test.repoPath)
: generalServerTestFiles.filter((file) => !excludedServerSuites(options.group).includes(file)));
for (const file of selection["@paperclipai/server"]) {
if (!owned.has(file)) fail(`Selected test belongs to another server lane: ${file}`);
}
}
if (selection && options.dryRun) {
writeFileSync(1, JSON.stringify({ selection }));
process.exit(0);
}
if (options.dryRun) {
const serializedSuites =
options.mode === serializedModeName
@@ -670,7 +691,16 @@ if (options.dryRun) {
}
try {
if (options.mode === generalModeName || options.mode === allModeName) {
if (selection) {
for (const [project, files] of Object.entries(selection)) {
const serverArgs = project !== "@paperclipai/server" ? []
: options.mode === serializedModeName || process.env.PAPERCLIP_TEST_DATABASE_URL
? ["--pool=forks", "--isolate", ...serializedServerVitestArgs]
: ["--pool=forks", "--isolate", "--fileParallelism", "--maxWorkers=2"];
await runVitest(["--project", project, ...serverArgs, ...files],
`selected ${project}: ${files.length} files`, null, true);
}
} else if (options.mode === generalModeName || options.mode === allModeName) {
if (options.group) {
await runGeneralGroup(routeTests, options.group, options.shardIndex, options.shardCount);
} else {
@@ -678,7 +708,7 @@ try {
}
}
if (options.mode === serializedModeName || options.mode === allModeName) {
if (!selection && (options.mode === serializedModeName || options.mode === allModeName)) {
await runSerializedSuites(routeTests, options.shardIndex ?? 0, options.shardCount ?? 1);
}
} catch (error) {
@@ -300,7 +300,7 @@ describe("adapter management route authorization", () => {
mocks.reloadExternalAdapter.mockImplementation(async (type: string) =>
createAdapter(type),
);
}, 20_000);
}, 60_000);
afterEach(() => {
unregisterServerAdapter(EXTERNAL_ADAPTER_TYPE);
@@ -373,35 +373,31 @@ describe("adapter management route authorization", () => {
},
);
describe("cloud-managed adapter code install floor", () => {
beforeEach(() => {
process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "test-server-token";
describe("cloud-managed adapter code floor", () => {
afterEach(() => vi.unstubAllEnvs());
it.each(
["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"].flatMap((signal) =>
["install", "reinstall", "reload", "override", "delete"].map((routeName) => [signal, routeName]),
),
)("blocks %s adapter %s before package or registration access", async (signal, routeName) => {
seedInstalledExternalAdapter();
vi.clearAllMocks();
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-server-token");
const app = createApp(instanceAdmin);
const res = await sendMutatingRequest(app, routeName);
expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(403);
expect(res.body.details).toMatchObject({ code: "adapter_install_platform_managed" });
expect(mocks.execFile).not.toHaveBeenCalled();
expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
expect(mocks.reloadExternalAdapter).not.toHaveBeenCalled();
expect(mocks.getAdapterPluginByType).not.toHaveBeenCalled();
expect(mocks.addAdapterPlugin).not.toHaveBeenCalled();
expect(mocks.removeAdapterPlugin).not.toHaveBeenCalled();
expect(mocks.externalRecords.get(EXTERNAL_ADAPTER_TYPE)).toEqual(installedRecord());
});
afterEach(() => {
delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;
});
it.each(["install", "reinstall"] as const)(
"floors adapter %s off for instance admins on cloud-managed instances",
async (routeName) => {
resetInstalledExternalAdapterState();
if (routeName !== "install") {
seedInstalledExternalAdapter();
}
const app = createApp(instanceAdmin);
const res = await sendMutatingRequest(app, routeName);
expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(
403,
);
expect(res.body.details).toMatchObject({
code: "adapter_install_platform_managed",
});
expect(mocks.execFile).not.toHaveBeenCalled();
expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
},
);
});
describe("operator-hidden adapter management floor", () => {
@@ -103,6 +103,11 @@ vi.mock("../services/instance-settings.js", () => ({
instanceSettingsService: () => mockInstanceSettingsService,
}));
vi.mock("../services/adapter-settings.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../services/adapter-settings.js")>()),
adapterSettingsService: () => ({ getDisabledTypes: async () => [] }),
}));
vi.mock("../services/dot-runner-broker.js", () => ({
dotRunnerBroker: () => ({ enabled: async () => true, bindingForAgent: async () => null }),
}));
@@ -0,0 +1,86 @@
import { rmSync, writeFileSync, symlinkSync, mkdirSync, readFileSync } from "node:fs";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
import { readDistributionPluginCatalog } from "../services/distribution-plugin-catalog.js";
import boxd from "../../../packages/plugins/sandbox-providers/boxd/src/manifest.js";
import createos from "../../../packages/plugins/sandbox-providers/createos/src/manifest.js";
import cloudflare from "../../../packages/plugins/sandbox-providers/cloudflare/src/manifest.js";
import daytona from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
import e2b from "../../../packages/plugins/sandbox-providers/e2b/src/manifest.js";
import exe_dev from "../../../packages/plugins/sandbox-providers/exe-dev/src/manifest.js";
import kubernetes from "../../../packages/plugins/sandbox-providers/kubernetes/src/manifest.js";
import modal from "../../../packages/plugins/sandbox-providers/modal/src/manifest.js";
import novita from "../../../packages/plugins/sandbox-providers/novita/src/manifest.js";
const manifests = { boxd, createos, cloudflare, daytona, e2b, "exe-dev": exe_dev, kubernetes, modal, novita };
const roots: string[] = [];
function fixture() { const result = bundledImageFixture(); roots.push(result.root); return result; }
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
describe("bundled image package identity", () => {
it("resolves independent image roots from the same registration without any previous path", () => {
const first = fixture();
const second = fixture();
const registration = { ...first.registration, packagePath: "/lost/old/home/plugins/fixture" };
expect(first.resolver.resolve(registration)).toBe(first.packageRoot);
expect(second.resolver.resolve(registration)).toBe(second.packageRoot);
expect(first.resolver.inspect(registration, first.manifest)).toEqual(second.resolver.inspect(registration, second.manifest));
expect(registration.packagePath).toBe("/lost/old/home/plugins/fixture");
});
it("requires verified manifest evidence and blocks same-version manifest drift", () => {
const { resolver, registration, manifest } = fixture();
expect(resolver.inspect(registration)).toMatchObject({ compatible: false, code: "missing_manifest_evidence" });
expect(resolver.inspect(registration, { ...manifest, description: "changed" })).toMatchObject({ compatible: false, code: "incompatible_bundled_package", reason: expect.stringContaining("Approved plugin manifest") });
expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true, identity: { kind: "bundled_image", catalogKey: "fixture", pluginKey: manifest.id, version: "1.0.0" } });
});
it("rejects mismatched package names, versions, identity and escaping entrypoints", () => {
const { resolver, registration } = fixture();
for (const update of [
{ packageName: "@other/plugin" }, { version: "2.0.0" }, { pluginKey: "other.plugin" },
{ manifestJson: { ...registration.manifestJson, version: "2.0.0" } },
{ manifestJson: { ...registration.manifestJson, entrypoints: { worker: "../worker.js" } } },
]) expect(() => resolver.resolve({ ...registration, ...update })).toThrow();
});
it("fails explicitly for missing or deselected bundles even when a stale npm package exists", () => {
const { root, packageRoot, resolver, registration } = fixture();
const fallback = path.join(root, "home", "node_modules", registration.packageName);
mkdirSync(fallback, { recursive: true });
writeFileSync(path.join(fallback, "worker.js"), "throw new Error('unsafe fallback');");
rmSync(packageRoot, { recursive: true });
expect(() => resolver.resolve(registration)).toThrow(/ENOENT/);
expect(() => createBundledPluginPackageResolver([]).resolve(registration)).toThrow(/absent or not selected/);
});
it("rejects tampering before manifest import and symlink escapes before worker resolution", () => {
const { root, packageRoot, registration, resolver } = fixture();
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), "throw new Error('must never import');");
expect(() => readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG)).toThrow(/digest mismatch/);
rmSync(path.join(packageRoot, "dist", "worker.js"));
symlinkSync(path.join(root, "distribution", "catalog.json"), path.join(packageRoot, "dist", "worker.js"));
expect(() => resolver.resolve(registration)).toThrow(/escapes/);
});
it.each(BUNDLED_PLUGIN_CATALOG)("accepts the real shipped $key package and manifest contract", (entry) => {
const manifest = manifests[entry.key as keyof typeof manifests];
const pkg = JSON.parse(readFileSync(new URL(`../../../packages/plugins/${entry.relativePath}/package.json`, import.meta.url), "utf8"));
const { root } = fixture();
const packageRoot = path.join(root, entry.relativePath);
mkdirSync(path.join(packageRoot, "dist"), { recursive: true });
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(pkg));
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`);
writeFileSync(path.join(packageRoot, "dist", "worker.js"), "export default {};");
const installs = resolveBundledPluginInstalls([entry.key], { catalogRoot: root, enforceCatalogRoot: true, distributionPlugins: [], env: {} });
const resolver = createBundledPluginPackageResolver(installs);
const registration = { pluginKey: entry.pluginKey, packageName: pkg.name, version: manifest.version, manifestJson: manifest };
expect(pkg.version).toBe(manifest.version);
expect(resolver.resolve(registration)).toBe(packageRoot);
expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true });
});
});
@@ -0,0 +1,287 @@
import express from "express";
import request from "supertest";
import { execFileSync } from "node:child_process";
import { rmSync, mkdirSync, writeFileSync, cpSync, readFileSync, existsSync, mkdtempSync } from "node:fs";
import path from "node:path";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { applyPendingMigrations, closeRegisteredClients, companies, createDb, plugins } from "@paperclipai/db";
import { eq } from "drizzle-orm";
import type { PluginRecord } from "@paperclipai/shared";
import { createPluginWorkerManager } from "../services/plugin-worker-manager.js";
import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js";
import { linkSdkInto } from "../../../scripts/link-plugin-dev-sdk.mjs";
import { fileURLToPath } from "node:url";
import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
import { inspectCloudPackages } from "../services/cloud-package-qualification.js";
import { resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
import { pluginRegistryService } from "../services/plugin-registry.js";
import { pluginUiStaticRoutes } from "../routes/plugin-ui-static.js";
import { errorHandler } from "../middleware/error-handler.js";
import { derivePluginDatabaseNamespace } from "../services/plugin-database.js";
import { ensureBundledPlugins } from "../services/bundled-plugins.js";
import daytonaManifest from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const externalDatabaseUrl = process.env.PAPERCLIP_BUNDLED_PLUGIN_TEST_DATABASE_URL?.trim();
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describePostgres = support.supported ? describe : describe.skip;
describePostgres("bundled image registration in PostgreSQL", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
let peer: ReturnType<typeof createDb>;
const roots: string[] = [];
const fixture = () => { const result = bundledImageFixture(); roots.push(result.root); return result; };
beforeAll(async () => {
if (externalDatabaseUrl) {
await applyPendingMigrations(externalDatabaseUrl);
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) };
} else {
database = await startEmbeddedPostgresTestDatabase("bundled-image-registration-");
}
db = createDb(database.connectionString);
peer = createDb(database.connectionString);
}, 90_000);
afterEach(async () => {
await db.delete(plugins);
await db.delete(companies);
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});
afterAll(async () => { await database?.cleanup(); });
function runtime() {
const output: string[] = [];
const services = {
workerManager: {
startWorker: vi.fn(async (_id: string, options: { entrypointPath: string }) => { output.push(execFileSync(process.execPath, [options.entrypointPath, "--source"], { encoding: "utf8" })); }),
call: vi.fn(async () => ({})),
},
lifecycleManager: { markError: vi.fn() },
jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() },
toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) },
buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" },
};
return { services, output };
}
function loaderFor(image: ReturnType<typeof fixture>, databaseClient = db, services?: PluginRuntimeServices) {
return pluginLoader(databaseClient, {
localPluginDir: path.join(image.root, "lost-home"), prebuiltOnly: true,
resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation,
enableLocalFilesystem: false, enableNpmDiscovery: false,
}, services);
}
it("starts current-image workers after losing the old home and retains each company's config", async () => {
const old = fixture();
const first = fixture();
const second = fixture();
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
const registry = pluginRegistryService(db);
let row = (await registry.getByKey(old.manifest.id))!;
await registry.updateStatus(row.id, { status: "ready" });
const [companyA, companyB, unconfigured] = await db.insert(companies).values([
{ name: "Company A", issuePrefix: "CFA" }, { name: "Company B", issuePrefix: "CFB" }, { name: "Unconfigured", issuePrefix: "CFC" },
]).returning();
await registry.upsertConfig(row.id, companyA!.id, { configJson: { token: "only-A" } });
await registry.upsertConfig(row.id, companyB!.id, { configJson: { token: "only-B" } });
row = (await registry.getById(row.id))!;
rmSync(old.root, { recursive: true });
const original = { ...row };
for (const image of [first, second]) {
const current = runtime();
const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
const result = await loader.loadSingle(row.id);
expect(result.success).toBe(true);
expect(current.output).toEqual([expect.stringContaining(image.packageRoot)]);
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({
entrypointPath: path.join(image.packageRoot, "dist", "worker.js"), config: {}, proactiveCompanyScopes: [companyA!.id, companyB!.id].sort(),
}));
expect(current.services.workerManager.call.mock.calls).toEqual(expect.arrayContaining([
[row.id, "configChanged", { companyId: companyA!.id, config: { token: "only-A" } }],
[row.id, "configChanged", { companyId: companyB!.id, config: { token: "only-B" } }],
]));
expect(current.services.workerManager.call).toHaveBeenCalledTimes(2);
expect(await registry.getConfig(row.id, unconfigured!.id)).toBeNull();
const app = express();
app.use((req, _res, next) => {
req.actor = { type: "board", userId: "fixture-user", source: "session", isInstanceAdmin: false, companyIds: [companyA!.id] };
next();
});
app.use(pluginUiStaticRoutes(db, { localPluginDir: path.join(image.root, "lost-home"), resolvePackageRoot: image.resolver.resolve }));
app.use(errorHandler);
const served = await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyA!.id });
expect(served.status).toBe(200);
expect(served.text).toContain("current-image");
expect((await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyB!.id })).status).toBe(403);
}
expect(await registry.getById(row.id)).toEqual(original);
});
it("completes the real worker initialize and health protocol from a relocated image", async () => {
const old = fixture(); const image = fixture();
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
const registry = pluginRegistryService(db);
const row = (await registry.getByKey(old.manifest.id))!;
await registry.updateStatus(row.id, { status: "ready" });
rmSync(old.root, { recursive: true });
const manager = createPluginWorkerManager();
try {
const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
const result = await loaderFor(image, db, services).loadSingle(row.id);
expect(result.success).toBe(true);
expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(image.packageRoot), configs: {} } });
} finally { await manager.stopAll(); }
});
it("qualifies and initializes the current release SDK/shared exports through the baked loader", async () => {
const image = fixture();
const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
const provider = mkdtempSync(path.join(catalogRoot, "sandbox-providers", "image-sdk-proof-"));
roots.push(provider);
cpSync(image.packageRoot, provider, { recursive: true });
linkSdkInto(provider);
writeFileSync(path.join(provider, "dist", "worker.js"), 'import { definePlugin, runWorker } from "@paperclipai/plugin-sdk"; runWorker(definePlugin({ async setup() {}, async onHealth() { return { status: "ok", message: "Current image SDK/shared loaded" }; } }), import.meta.url);');
const installs = [{ key: "sdk-proof", pluginKey: image.manifest.id, localPath: provider }];
const resolver = createBundledPluginPackageResolver(installs);
const manager = createPluginWorkerManager();
const loader = pluginLoader(db, {
prebuiltOnly: true, resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
enableLocalFilesystem: false, enableNpmDiscovery: false,
}, { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices);
const registry = pluginRegistryService(db);
try {
await loader.installPlugin({ localPath: provider });
const installed = (await registry.getByKey(image.manifest.id))!;
await registry.updateStatus(installed.id, { status: "ready" });
const registration = (await registry.getById(installed.id))!;
expect((await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) })).blockers).toEqual([]);
expect((await loader.loadSingle(installed.id)).success).toBe(true);
expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Current image SDK/shared loaded" });
} finally { await manager.stopAll(); }
}, 90_000);
it.skipIf(!existsSync(fileURLToPath(new URL("../../../packages/plugins/sandbox-providers/daytona/dist/worker.js", import.meta.url))))("qualifies and initializes the actual prebuilt Daytona release package", async () => {
const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
const installs = resolveBundledPluginInstalls(["daytona"], { catalogRoot, env: {}, enforceCatalogRoot: true, distributionPlugins: [] });
const resolver = createBundledPluginPackageResolver(installs);
const manager = createPluginWorkerManager();
const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
const loader = pluginLoader(db, {
localPluginDir: "/lost/release/home/plugins", prebuiltOnly: true,
resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
enableLocalFilesystem: false, enableNpmDiscovery: false,
}, services);
const registry = pluginRegistryService(db);
vi.stubEnv("DAYTONA_API_KEY", "");
try {
await loader.installPlugin({ localPath: installs[0]!.localPath });
const installed = (await registry.getByKey(daytonaManifest.id))!;
await registry.update(installed.id, { packagePath: "/lost/release/home/daytona" });
await registry.updateStatus(installed.id, { status: "ready" });
const registration = (await registry.getById(installed.id))!;
const inspected = await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) });
expect(inspected.blockers).toEqual([]);
expect(inspected.packages[0]?.packageDigest).toMatch(/^sha256:/);
expect((await loader.loadSingle(installed.id)).success).toBe(true);
expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Daytona sandbox provider plugin healthy" });
} finally { await manager.stopAll(); vi.unstubAllEnvs(); }
}, 90_000);
it("registers the same package concurrently from two roots without replacing the winner", async () => {
const first = fixture(); const second = fixture();
await Promise.all([
loaderFor(first).installPlugin({ localPath: first.packageRoot }),
loaderFor(second, peer).installPlugin({ localPath: second.packageRoot }),
]);
const rows = await pluginRegistryService(db).listInstalled();
expect(rows).toHaveLength(1);
const original = rows[0]!;
expect([first.packageRoot, second.packageRoot]).toContain(original.packagePath);
await loaderFor(second, peer).installPlugin({ localPath: second.packageRoot });
expect(await pluginRegistryService(db).getById(original.id)).toEqual(original);
await expect(pluginRegistryService(peer).install({ packageName: original.packageName }, { ...first.manifest, description: "different package metadata" }, { idempotent: true })).rejects.toThrow(/already installed/);
});
it("preserves disabled registrations and avoids path writes during managed boot", async () => {
const old = fixture(); const image = fixture();
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
const registry = pluginRegistryService(db);
const row = (await registry.getByKey(old.manifest.id))!;
await db.update(plugins).set({ status: "disabled" }).where(eq(plugins.id, row.id));
const original = await registry.getById(row.id);
await ensureBundledPlugins(image.installs, { registry, loader: loaderFor(image), lifecycle: { load: vi.fn() }, logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }, { reinstallUninstalled: true, preservePackagePath: true });
expect(await registry.getById(row.id)).toEqual(original);
});
it("fails before worker startup for missing image bytes without using a home fallback or building", async () => {
const image = fixture();
await loaderFor(image).installPlugin({ localPath: image.packageRoot });
const registry = pluginRegistryService(db);
const row = (await registry.getByKey(image.manifest.id))!;
await registry.updateStatus(row.id, { status: "ready" });
const fallback = path.join(image.root, "lost-home", "node_modules", row.packageName);
mkdirSync(fallback, { recursive: true });
writeFileSync(path.join(fallback, "package.json"), JSON.stringify({ name: row.packageName }));
writeFileSync(path.join(fallback, "worker.js"), "throw new Error('must not start');");
rmSync(path.join(image.packageRoot, "dist", "worker.js"));
const current = runtime();
const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
const result = await loader.loadSingle(row.id);
expect(result.success).toBe(false);
expect(result.error).toContain("ENOENT");
expect(current.services.workerManager.startWorker).not.toHaveBeenCalled();
await expect(loader.installPlugin({ packageName: row.packageName })).rejects.toThrow(/current image catalog/);
await loader.cleanupInstallArtifacts(row as PluginRecord);
expect(image.resolver.inspect(row, image.manifest).compatible).toBe(false);
});
it("applies plugin migrations from the current image after the legacy package disappears", async () => {
const image = bundledImageFixture({
capabilities: ["database.namespace.migrate", "database.namespace.read", "database.namespace.write"],
database: { migrationsDir: "migrations" },
});
roots.push(image.root);
const [row] = await db.insert(plugins).values({
...image.registration, packagePath: "/lost/legacy/home/plugin", apiVersion: 1, categories: image.manifest.categories, status: "ready",
}).returning();
const current = runtime();
const result = await loaderFor(image, db, current.services as unknown as PluginRuntimeServices).loadSingle(row!.id);
expect(result.success).toBe(true);
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row!.id, expect.objectContaining({ databaseNamespace: derivePluginDatabaseNamespace(image.manifest.id) }));
await db.execute((await import("drizzle-orm")).sql.raw(`DROP SCHEMA "${derivePluginDatabaseNamespace(image.manifest.id)}" CASCADE`));
});
it("retains registry-install provider credentials for an ordinary npm package", async () => {
const npmManifest = { ...daytonaManifest, id: "paperclip.image-npm-fixture" };
const image = bundledImageFixture(npmManifest);
roots.push(image.root);
const localPluginDir = path.join(image.root, "npm-home");
const packageRoot = path.join(localPluginDir, "node_modules", "@paperclipai", "plugin-daytona");
mkdirSync(path.dirname(packageRoot), { recursive: true });
cpSync(image.packageRoot, packageRoot, { recursive: true });
const metadata = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8"));
metadata.name = "@paperclipai/plugin-daytona";
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(metadata));
const registry = pluginRegistryService(db);
const row = (await registry.install({ packageName: metadata.name }, npmManifest))!;
await registry.updateStatus(row.id, { status: "ready" });
vi.stubEnv("DAYTONA_API_KEY", "synthetic-provider-key");
try {
const current = runtime();
const loader = pluginLoader(db, { localPluginDir, enableNpmDiscovery: false, enableLocalFilesystem: false }, current.services as unknown as PluginRuntimeServices);
expect((await loader.loadSingle(row.id)).success).toBe(true);
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({ env: expect.objectContaining({ DAYTONA_API_KEY: "synthetic-provider-key" }) }));
} finally { vi.unstubAllEnvs(); }
});
it("keeps self-hosted local-path loading available without a managed resolver", async () => {
const image = fixture();
const loader = pluginLoader(db, { localPluginDir: path.join(image.root, "selfhost"), enableNpmDiscovery: false, enableLocalFilesystem: false });
await loader.installPlugin({ localPath: image.packageRoot });
const row = (await pluginRegistryService(db).getByKey(image.manifest.id))!;
expect(row.packagePath).toBe(image.packageRoot);
await expect(loader.installPlugin({ localPath: image.packageRoot })).rejects.toThrow(/already installed/);
});
});
@@ -0,0 +1,72 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
import { build, transform } from "esbuild";
import { bundleRuntime } from "../../scripts/bundle-runtime.mjs";
import { resolveBundledPluginRuntimeLoader } from "../services/bundled-plugin-runtime.js";
const roots: string[] = [];
function fixture() {
const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-plugin-runtime-")));
roots.push(root);
fs.mkdirSync(path.join(root, "server", "dist-runtime"), { recursive: true });
fs.writeFileSync(path.join(root, "server", "package.json"), JSON.stringify({ name: "@paperclipai/server", type: "module" }));
const loader = path.join(root, "cli", "node_modules", "tsx", "dist", "loader.mjs");
fs.mkdirSync(path.dirname(loader), { recursive: true });
fs.writeFileSync(loader, "export {};\n");
return { root, loader };
}
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
describe("current image plugin runtime loader", () => {
it.each(["src/services", "dist/services", "dist-runtime"])("resolves the owning release from server/%s", (location) => {
const { root, loader } = fixture();
const moduleDirectory = path.join(root, "server", location);
fs.mkdirSync(moduleDirectory, { recursive: true });
expect(resolveBundledPluginRuntimeLoader(moduleDirectory)).toBe(loader);
});
it("resolves a bundled entrypoint without preserved source module locations", async () => {
const { root, loader } = fixture();
const runtime = path.join(root, "server", "dist-runtime", "index.mjs");
await build({
stdin: { contents: `import { resolveBundledPluginRuntimeLoader } from ${JSON.stringify(fileURLToPath(new URL("../services/bundled-plugin-runtime.ts", import.meta.url)))}; process.stdout.write(resolveBundledPluginRuntimeLoader());`, resolveDir: process.cwd() },
outfile: runtime, bundle: true, platform: "node", format: "esm", target: "node24", logLevel: "silent",
});
expect(execFileSync(process.execPath, [runtime], { encoding: "utf8" })).toBe(loader);
});
it("uses the production bundler's preserved module location at the dist-runtime entrypoint", async () => {
const { root, loader } = fixture();
const service = path.join(root, "server", "dist", "services", "bundled-plugin-runtime.js");
fs.mkdirSync(path.dirname(service), { recursive: true });
const source = fs.readFileSync(new URL("../services/bundled-plugin-runtime.ts", import.meta.url), "utf8");
fs.writeFileSync(service, (await transform(source, { loader: "ts", format: "esm", target: "node24" })).code);
const entry = path.join(root, "server", "dist", "probe.js");
fs.writeFileSync(entry, 'import { resolveBundledPluginRuntimeLoader } from "./services/bundled-plugin-runtime.js"; process.stdout.write(resolveBundledPluginRuntimeLoader());');
const bundled = await bundleRuntime({ entry, outdir: path.join(root, "server", "dist-runtime") });
expect(execFileSync(process.execPath, [bundled.outfile], { encoding: "utf8" })).toBe(loader);
});
it("accepts the baked pnpm target inside the current image node_modules", () => {
const { root, loader } = fixture();
const physical = path.join(root, "node_modules", ".pnpm", "tsx@1", "node_modules", "tsx", "dist", "loader.mjs");
fs.mkdirSync(path.dirname(physical), { recursive: true });
fs.writeFileSync(physical, "export {};\n");
fs.rmSync(loader); fs.symlinkSync(path.relative(path.dirname(loader), physical), loader);
expect(resolveBundledPluginRuntimeLoader(path.join(root, "server", "dist-runtime"))).toBe(physical);
});
it("fails explicitly for missing or legacy-home loaders", () => {
const { root, loader } = fixture();
const moduleDirectory = path.join(root, "server", "dist-runtime");
fs.rmSync(loader);
expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("missing from the current image");
const outside = fixture().loader;
fs.symlinkSync(outside, loader);
expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("escapes the current image");
});
});
@@ -9,6 +9,7 @@ import {
ensureBundledPlugins,
resolveBundledCatalogRoot,
resolveBundledPluginInstalls,
resolveManagedBundledPluginKeys,
type BundledPluginProvisionerDeps,
type ResolvedBundledPlugin,
} from "../services/bundled-plugins.js";
@@ -30,6 +31,21 @@ afterAll(() => {
// Catalog-root resolution (fail-to-start allowlist)
// ---------------------------------------------------------------------------
describe("managed bundled plugin selection", () => {
it.each([
{ env: {}, selected: undefined, expected: null },
{ env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: undefined, expected: [] },
{ env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: " " }, selected: undefined, expected: null },
{ env: { PAPERCLIP_MANAGED_CONFIG: "" }, selected: undefined, expected: [] },
{ env: { PAPERCLIP_MANAGED_CONFIG: "{}" }, selected: null, expected: [] },
{ env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: ["daytona"], expected: ["daytona"] },
{ env: {}, selected: ["modal"], expected: ["modal"] },
{ env: {}, selected: [], expected: [] },
])("uses the canonical Cloud floor and preserves explicit selection: $env / $selected", ({ env, selected, expected }) => {
expect(resolveManagedBundledPluginKeys(selected, env)).toEqual(expected);
});
});
describe("resolveBundledPluginInstalls", () => {
it("resolves the optional CreateOS sandbox provider without making it a self-hosted default", () => {
expect(resolveBundledPluginInstalls(["createos"], {
@@ -0,0 +1,93 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js";
const roots: string[] = [];
function file(name: string, content = "export const value = 1;") {
fs.mkdirSync(path.dirname(name), { recursive: true }); fs.writeFileSync(name, content);
}
function link(target: string, name: string) {
fs.mkdirSync(path.dirname(name), { recursive: true }); fs.symlinkSync(path.relative(path.dirname(name), target), name);
}
function fixture() {
const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-package-graph-"))); roots.push(root);
const catalogRoot = path.join(root, "packages", "plugins");
const provider = path.join(catalogRoot, "sandbox-providers", "daytona");
const sdk = path.join(catalogRoot, "sdk");
const shared = path.join(root, "packages", "shared");
const dependency = path.join(root, "node_modules", ".pnpm", "dependency@1", "node_modules", "dependency");
const transitive = path.join(root, "node_modules", ".pnpm", "transitive@1", "node_modules", "transitive");
for (const directory of [provider, sdk, shared, dependency, transitive]) file(path.join(directory, "index.js"));
link(sdk, path.join(provider, "node_modules", "@paperclipai", "plugin-sdk"));
link(shared, path.join(sdk, "node_modules", "@paperclipai", "shared"));
link(dependency, path.join(shared, "node_modules", "dependency"));
link(transitive, path.join(path.dirname(dependency), "transitive"));
return { root, catalogRoot, provider, sdk, shared, dependency, transitive };
}
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
describe("bounded current-image package dependency inventory", () => {
it("hashes external SDK, shared, and pnpm sibling dependency bytes", () => {
const image = fixture();
const digest = () => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot });
let previous = digest();
for (const directory of [image.sdk, image.shared, image.dependency, image.transitive]) {
file(path.join(directory, "index.js"), "export const value = 2;");
const next = digest(); expect(next).not.toBe(previous); previous = next;
}
});
it("keeps the digest stable when an identical release moves to a new root", () => {
const original = fixture(); const replacement = fixture();
expect(cloudBundledPackageDigest(original.provider, { catalogRoot: original.catalogRoot }))
.toBe(cloudBundledPackageDigest(replacement.provider, { catalogRoot: replacement.catalogRoot }));
});
it("rejects legacy-home links and preserves package-contained defaults", () => {
const image = fixture();
expect(() => cloudBundledPackageDigest(image.provider)).toThrow("escapes its package");
const outside = fixture();
link(path.join(outside.sdk, "index.js"), path.join(image.shared, "legacy-home.js"));
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("authorized image roots");
});
it("deduplicates cycles without scanning unrelated image node_modules", () => {
const image = fixture();
link(image.sdk, path.join(image.shared, "node_modules", "cycle"));
const unrelated = path.join(image.root, "node_modules", "unrelated", "special");
file(unrelated);
const first = cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot });
file(unrelated, "changed unrelated code");
expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toBe(first);
});
it.each(["", ".pnpm", ".pnpm/node_modules"])("rejects aggregate image dependency targets: %s", (aggregate) => {
const image = fixture();
const target = path.join(image.root, "node_modules", aggregate);
fs.mkdirSync(target, { recursive: true });
link(target, path.join(image.sdk, "node_modules", "aggregate"));
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("select an individual package");
});
it("accepts the shipped provider's observed entry class", () => {
const image = fixture();
const directory = path.join(image.provider, "many"); fs.mkdirSync(directory);
for (let index = 0; index < 16_000; index++) fs.writeFileSync(path.join(directory, `${index}.js`), "");
expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toMatch(/^sha256:/);
});
it("rejects oversized metadata and excessive depth", () => {
const image = fixture();
file(path.join(image.sdk, "package.json"), " ".repeat(128 * 1024 + 1));
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("metadata");
fs.rmSync(path.join(image.sdk, "package.json"));
const large = path.join(image.shared, "large.js");
fs.closeSync(fs.openSync(large, "w")); fs.truncateSync(large, 256 * 1024 * 1024 + 1);
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("byte limit");
fs.rmSync(large);
file(path.join(image.sdk, ...Array.from({ length: 33 }, () => "deep"), "file.js"));
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("depth limit");
});
});
@@ -0,0 +1,325 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, plugins, type Db } from "@paperclipai/db";
import { eq, inArray, sql } from "drizzle-orm";
import { paperclipConfigSchema } from "@paperclipai/shared";
import { adapterSettingsService, initializeAdapterSettings } from "../services/adapter-settings.js";
import { cloudRuntimeConfigService, initializeCloudRuntimeConfig } from "../services/cloud-runtime-config.js";
import { cloudConfigurationMigrationService } from "../services/cloud-configuration-migration.js";
import { qualifyCloudConfigurationForCurrentImage } from "../services/cloud-configuration-qualification.js";
import { runCloudConfigurationQualification } from "../qualify-cloud-configuration.js";
import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js";
import { pluginRegistryService } from "../services/plugin-registry.js";
import { createPluginWorkerManager } from "../services/plugin-worker-manager.js";
import { ensureBundledPlugins } from "../services/bundled-plugins.js";
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const externalDatabaseUrl = process.env.PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL?.trim();
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const keys = ["adapter-settings:v1", "cloud-runtime-config:v1", "cloud-configuration-migration:v1"];
(support.supported ? describe : describe.skip)("Cloud configuration migration and current-image qualification", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: Db;
let peer: Db;
let homeA: string;
let homeB: string;
const roots: string[] = [];
const fixture = (version = "1.0.0") => { const image = bundledImageFixture({ version }); roots.push(image.root); return image; };
const env = (home = homeA) => ({ PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "test-cloud-token", PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: path.join(home, "config.json") });
const migration = (client = db, home = homeA) => cloudConfigurationMigrationService(client, { runtimeEnv: env(home) });
beforeAll(async () => {
if (externalDatabaseUrl) {
await applyPendingMigrations(externalDatabaseUrl);
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) };
} else database = await startEmbeddedPostgresTestDatabase("cloud-configuration-migration-");
db = createDb(database.connectionString);
peer = createDb(database.connectionString);
}, 90_000);
beforeEach(async () => {
homeA = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-legacy-A-"));
homeB = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-replacement-B-"));
roots.push(homeA, homeB);
await db.delete(instanceSettings).where(inArray(instanceSettings.singletonKey, keys));
await db.delete(plugins);
});
afterEach(async () => {
vi.restoreAllMocks();
for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true });
});
afterAll(async () => { await database?.cleanup(); });
async function legacy() {
const config = paperclipConfigSchema.parse({
$meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" },
database: { mode: "postgres" }, logging: { mode: "cloud" },
server: { port: 3199 }, telemetry: { enabled: false },
});
await fs.writeFile(path.join(homeA, "config.json"), JSON.stringify(config));
await fs.writeFile(path.join(homeA, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local", "future_adapter"] }));
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]");
return config;
}
function loaderFor(image: ReturnType<typeof fixture>, client = db, services?: PluginRuntimeServices) {
return pluginLoader(client, {
localPluginDir: path.join(homeB, "plugins"), prebuiltOnly: true,
resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation,
enableLocalFilesystem: false, enableNpmDiscovery: false,
}, services);
}
async function install(image: ReturnType<typeof fixture>) {
await loaderFor(image).installPlugin({ localPath: image.packageRoot });
const registry = pluginRegistryService(db);
const row = (await registry.getByKey(image.manifest.id))!;
await registry.updateStatus(row.id, { status: "ready" });
return row;
}
function qualify(image: ReturnType<typeof fixture>, client = db, home = homeA, expectedPackageEvidenceDigest?: string) {
const loader = loaderFor(image, client);
return migration(client, home).qualify({ installs: image.installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest });
}
it("imports A atomically, retains later DB revisions, and starts B's real image worker after A disappears", async () => {
await legacy();
const oldImage = fixture();
const newImage = fixture();
const row = await install(oldImage);
const first = await migration().migrate();
expect(first.migration.adapter.authority).toBe("legacy_file");
expect(first.migration.config.authority).toBe("legacy_file");
const original = await qualify(oldImage);
expect(original.configurationPackagesEligible).toBe(true);
expect(original.samePackageOverlapEligible).toBe(false);
await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true);
const config = (await cloudRuntimeConfigService(peer).inspectStored())!;
expect(config.config).not.toBeNull();
await cloudRuntimeConfigService(peer).replace({ ...config.config!, server: { ...config.config!.server, port: 3299 } }, config.revision);
await fs.rm(homeA, { recursive: true });
await fs.rm(oldImage.root, { recursive: true });
expect((await migration(peer, homeB).migrate()).migration).toEqual(first.migration);
await initializeAdapterSettings(peer, { runtimeEnv: env(homeB) });
const restored = await initializeCloudRuntimeConfig(peer, { runtimeEnv: env(homeB) });
expect(restored?.current.config?.server.port).toBe(3299);
expect(await adapterSettingsService(peer, { runtimeEnv: env(homeB) }).getDisabledTypes()).toEqual(["claude_local", "codex_local", "future_adapter"]);
const replacement = await qualify(newImage, peer, homeB, original.packageEvidenceDigest);
expect(replacement).toMatchObject({ adapterRevision: 1, configRevision: 1, configurationPackagesEligible: true, samePackageOverlapEligible: true });
expect(replacement.configurationDigest).not.toBe(original.configurationDigest);
const manager = createPluginWorkerManager();
try {
const services = {
workerManager: manager, lifecycleManager: { markError: vi.fn() },
jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() },
toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) },
buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "qualification-fixture", hostVersion: "1.0.0" },
} as unknown as PluginRuntimeServices;
expect((await loaderFor(newImage, peer, services).loadSingle(row.id)).success).toBe(true);
expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(newImage.packageRoot) } });
} finally { await manager.stopAll(); }
});
it("serializes concurrent attempts and never imports stale inputs over later DB changes", async () => {
await legacy();
const [first, second] = await Promise.all([migration().migrate(), migration(peer).migrate()]);
expect(first).toEqual(second);
await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("claude_local", false);
await fs.writeFile(path.join(homeA, "config.json"), "invalid stale file");
await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid stale file");
expect(await migration().migrate()).toEqual(first);
expect(await adapterSettingsService(db, { runtimeEnv: env() }).getDisabledTypes()).toEqual(["future_adapter"]);
});
it("rolls back both seeds and the receipt if completion fails, then retries after the crash boundary", async () => {
await legacy();
await db.execute(sql`CREATE FUNCTION reject_cloud_migration_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.singleton_key = 'cloud-configuration-migration:v1' THEN RAISE EXCEPTION 'test receipt failure'; END IF; RETURN NEW; END $$`);
await db.execute(sql`CREATE TRIGGER reject_cloud_migration_receipt BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_cloud_migration_receipt()`);
try {
await expect(migration().migrate()).rejects.toMatchObject({ cause: { message: expect.stringContaining("test receipt failure") } });
expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]);
} finally {
await db.execute(sql`DROP TRIGGER reject_cloud_migration_receipt ON instance_settings`);
await db.execute(sql`DROP FUNCTION reject_cloud_migration_receipt()`);
}
expect((await migration(peer).migrate()).revision).toBe(0);
expect((await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).length).toBe(3);
});
it("fails without partial authority for malformed legacy config and preserves already-authoritative rows", async () => {
await legacy();
await fs.writeFile(path.join(homeA, "config.json"), "invalid");
await expect(migration().migrate()).rejects.toThrow("Legacy Cloud runtime configuration");
expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]);
await adapterSettingsService(db, { runtimeEnv: env() }).seedIfAbsent({ disabledTypes: ["codex_local"] });
await cloudRuntimeConfigService(db).seedIfAbsent(null);
await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid");
const receipt = await migration(peer).migrate();
expect(receipt.migration.adapter.authority).toBe("database");
expect(receipt.migration.config.authority).toBe("database");
expect((await cloudRuntimeConfigService(peer).inspectStored())?.config).toBeNull();
});
it("returns database errors instead of falling back to files or a completed receipt", async () => {
await legacy();
await migration().migrate();
const broken = createDb(database.connectionString);
await broken.$client.end({ timeout: 0 });
await expect(migration(broken).migrate()).rejects.toThrow();
const image = fixture();
await expect(qualify(image, broken)).rejects.toThrow();
});
it("keeps external registration blockers durable when B has no legacy home and requires deliberate original-source resolution", async () => {
await legacy();
const sentinel = path.join(homeA, "external-code-executed");
await fs.writeFile(path.join(homeA, "external.js"), `require('node:fs').writeFileSync(${JSON.stringify(sentinel)}, 'executed');`);
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), JSON.stringify([{ type: "legacy_external", packageName: "legacy-adapter", localPath: path.join(homeA, "external.js") }]));
const image = fixture();
await install(image);
await migration().migrate();
expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "unsupported_external_adapters" }));
await expect(fs.stat(sentinel)).rejects.toMatchObject({ code: "ENOENT" });
await migration(peer, homeB).migrate();
const replacement = await qualify(image, peer, homeB);
expect(replacement.configurationPackagesEligible).toBe(false);
await expect(migration(peer, homeB).recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("original legacy source");
await fs.unlink(path.join(homeA, "adapter-plugins.json"));
await expect(migration().recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("present, valid registration file");
expect((await migration().inspectStored())?.migration.externalRegistrations.blocker).toBe("unsupported_external_adapters");
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]");
await expect(migration().recheckLegacyExternalRegistrations(0)).rejects.toThrow("revision conflict");
await migration().recheckLegacyExternalRegistrations(replacement.receiptRevision);
expect((await qualify(image)).configurationPackagesEligible).toBe(true);
});
it("retains a metadata blocker for malformed or oversized registration files", async () => {
await legacy();
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), Buffer.alloc(65_537));
const receipt = await migration().migrate();
expect(receipt.migration.externalRegistrations.blocker).toBe("invalid_external_registration_metadata");
await fs.rm(homeA, { recursive: true });
expect((await migration(peer, homeB).migrate()).migration.externalRegistrations).toEqual(receipt.migration.externalRegistrations);
});
it("records image mismatches before reconciliation, permits maintenance upgrades, and requires explicit fresh overlap evidence", async () => {
await legacy();
const oldImage = fixture();
const nextImage = fixture("1.1.0");
await install(oldImage);
await migration().migrate();
const old = await qualify(oldImage);
const mismatch = await qualify(nextImage, peer, homeB, old.packageEvidenceDigest);
expect(mismatch.configurationPackagesEligible).toBe(false);
expect(mismatch.samePackageOverlapEligible).toBe(false);
expect(mismatch.blockers).toContainEqual(expect.objectContaining({ code: "incompatible_bundled_package", registeredVersion: "1.0.0", imageVersion: "1.1.0" }));
const registry = pluginRegistryService(db);
await ensureBundledPlugins(nextImage.installs, {
registry, loader: loaderFor(nextImage), lifecycle: { load: vi.fn() },
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}, { reinstallUninstalled: true, preservePackagePath: true });
const current = await qualify(nextImage);
expect(current.configurationPackagesEligible).toBe(true);
expect(current.samePackageOverlapEligible).toBe(false);
expect((await migration().inspectStored())?.lastPackageMismatch?.packageEvidenceDigest).toBe(mismatch.packageEvidenceDigest);
expect((await qualify(nextImage, peer, homeB, current.packageEvidenceDigest)).samePackageOverlapEligible).toBe(true);
});
it("fails qualification for missing package or manifest bytes without loading a home fallback", async () => {
await legacy();
const image = fixture();
await install(image);
await migration().migrate();
const good = await qualify(image);
await fs.rm(path.join(image.packageRoot, "dist", "manifest.js"));
const broken = await qualify(image, peer, homeB, good.packageEvidenceDigest);
expect(broken.configurationPackagesEligible).toBe(false);
expect(broken.samePackageOverlapEligible).toBe(false);
expect(broken.blockers.length).toBeGreaterThan(0);
await fs.rm(image.packageRoot, { recursive: true });
expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "missing_bundled_package" }));
});
it("rejects changed image bytes before manifest import and never uses stale completed qualification", async () => {
await legacy();
const image = fixture();
await install(image);
await migration().migrate();
const original = await qualify(image);
await fs.appendFile(path.join(image.packageRoot, "dist", "worker.js"), "\nprocess.exit(99);\n");
const loadManifest = vi.fn();
const current = await migration(peer, homeB).qualify({ installs: image.installs, loadManifest, expectedPackageEvidenceDigest: original.packageEvidenceDigest });
expect(current.configurationPackagesEligible).toBe(false);
expect(current.samePackageOverlapEligible).toBe(false);
expect(current.blockers).toContainEqual(expect.objectContaining({ code: "package_integrity_mismatch" }));
expect(loadManifest).not.toHaveBeenCalled();
});
it("allows an administrator update while manifest inspection waits and refuses to certify the stale snapshot", async () => {
await legacy();
const image = fixture();
await install(image);
await migration().migrate();
let started!: () => void;
let resume!: () => void;
const entered = new Promise<void>((resolve) => { started = resolve; });
const release = new Promise<void>((resolve) => { resume = resolve; });
const loader = loaderFor(image);
const pending = migration().qualify({ installs: image.installs, loadManifest: async (root) => { started(); await release; return loader.loadManifest(root); } });
await entered;
try {
const change = adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true);
let timer: ReturnType<typeof setTimeout> | undefined;
try {
expect(await Promise.race([change, new Promise<never>((_resolve, reject) => { timer = setTimeout(() => reject(new Error("Admin update was blocked by image inspection")), 2_000); })])).toBe(true);
} finally { clearTimeout(timer); }
} finally { resume(); }
const result = await pending;
expect(result).toMatchObject({ configurationPackagesEligible: false, adapterRevision: 1 });
expect(result.blockers).toContainEqual(expect.objectContaining({ code: "configuration_changed_during_inspection" }));
expect((await qualify(image)).configurationPackagesEligible).toBe(true);
});
it("uses the current managed catalog through the operator command and durably records catalog failure", async () => {
await legacy();
const image = fixture();
await install(image);
await migration().migrate();
const runtimeEnv = {
...env(homeB), DATABASE_URL: database.connectionString, PAPERCLIP_BUNDLED_PLUGIN_ROOT: image.root,
PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "fixture", features: {}, plugins: { autoInstall: ["fixture"] } }),
};
const initial = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv });
expect(initial?.configurationPackagesEligible).toBe(true);
const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
try {
expect(await runCloudConfigurationQualification(["--expected-package-evidence", initial!.packageEvidenceDigest], runtimeEnv)).toBe(0);
const report = stdout.mock.calls.map((call) => String(call[0])).find((text) => text.startsWith('{\n "receiptRevision"'));
expect(report && JSON.parse(report).samePackageOverlapEligible).toBe(true);
} finally { stdout.mockRestore(); }
const missing = await qualifyCloudConfigurationForCurrentImage(peer, { runtimeEnv: { ...runtimeEnv, PAPERCLIP_MANAGED_CONFIG: runtimeEnv.PAPERCLIP_MANAGED_CONFIG.replace('"fixture"]', '"absent"]') } });
expect(missing?.configurationPackagesEligible).toBe(false);
expect(missing?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" }));
expect((await migration().inspectStored())?.lastPackageMismatch?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" }));
});
it.skipIf(process.platform === "win32")("detects changed pnpm dependency bytes in a builtin package overlap comparison", async () => {
await legacy();
const image = fixture();
await install(image);
await migration().migrate();
const dependency = path.join(image.packageRoot, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep");
await fs.mkdir(dependency, { recursive: true });
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;");
await fs.symlink(path.relative(path.join(image.packageRoot, "node_modules"), dependency), path.join(image.packageRoot, "node_modules", "dep"));
const installs = image.installs.map(({ distribution: _distribution, ...install }) => install);
const loader = loaderFor(image);
const original = await migration().qualify({ installs, loadManifest: (root) => loader.loadManifest(root) });
expect(original.configurationPackagesEligible).toBe(true);
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;");
const changed = await migration(peer, homeB).qualify({ installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: original.packageEvidenceDigest });
expect(changed.configurationPackagesEligible).toBe(true);
expect(changed.samePackageOverlapEligible).toBe(false);
expect(changed.overlapBlockers).toContainEqual(expect.objectContaining({ code: "package_evidence_changed" }));
});
});
@@ -0,0 +1,48 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { readLegacyAdapterSettings } from "../services/adapter-settings.js";
import { inspectLegacyExternalRegistrations } from "../services/cloud-configuration-evidence.js";
import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js";
vi.mock("../adapters/plugin-loader.js", () => ({ buildExternalAdapters: async () => [] }));
describe("bounded Cloud legacy readers and image inventory", () => {
let root: string;
beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-config-reader-")); });
afterEach(async () => { await fs.rm(root, { recursive: true, force: true }); });
it("distinguishes an absent adapter file from a saved empty preference", async () => {
expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toBeNull();
await fs.writeFile(path.join(root, "adapter-settings.json"), '{"disabledTypes":[]}');
expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toEqual({ disabledTypes: [] });
});
it.skipIf(process.platform === "win32")("rejects symlinks and FIFOs without waiting for a writer", async () => {
const target = path.join(root, "target.json");
const settings = path.join(root, "adapter-settings.json");
await fs.writeFile(target, '{"disabledTypes":[]}');
await fs.symlink(target, settings);
await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toMatchObject({ code: "ELOOP" });
expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("unreadable_external_registration_metadata");
await fs.unlink(settings);
execFileSync("mkfifo", [settings]);
await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toThrow("regular file");
expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("invalid_external_registration_metadata");
});
it.skipIf(process.platform === "win32")("hashes pnpm-style contained links and the physical dependency bytes", async () => {
const dependency = path.join(root, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep");
await fs.mkdir(dependency, { recursive: true });
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;");
await fs.symlink(path.relative(path.join(root, "node_modules"), dependency), path.join(root, "node_modules", "dep"));
const first = cloudBundledPackageDigest(root);
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;");
expect(cloudBundledPackageDigest(root)).not.toBe(first);
await fs.unlink(path.join(root, "node_modules", "dep"));
await fs.symlink("/", path.join(root, "node_modules", "dep"));
expect(() => cloudBundledPackageDigest(root)).toThrow("escapes its package");
});
});
@@ -0,0 +1,302 @@
import fs from "node:fs";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { fileURLToPath, pathToFileURL } from "node:url";
import os from "node:os";
import path from "node:path";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, type Db } from "@paperclipai/db";
import { eq } from "drizzle-orm";
import { paperclipConfigSchema } from "@paperclipai/shared";
import { loadCloudBootstrapConfig, loadConfig } from "../config.js";
import { readConfigFile } from "../config-file.js";
import {
CLOUD_RUNTIME_CONFIG_KEY,
cloudRuntimeConfigService,
initializeCloudRuntimeConfig,
readLegacyCloudRuntimeConfig,
} from "../services/cloud-runtime-config.js";
import {
getCloudRuntimeConfigSnapshot,
projectCloudRuntimeConfig,
setCloudRuntimeConfigSnapshot,
} from "../services/cloud-runtime-config-snapshot.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const externalDatabaseUrl = process.env.PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL;
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describePostgres = support.supported ? describe : describe.skip;
let home: string;
let configPath: string;
function legacyConfig() {
return {
$meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" },
llm: { provider: "openai", apiKey: "legacy-openai-secret" },
database: {
mode: "postgres",
connectionString: "postgres://legacy:database-secret@example.test/paperclip",
backup: { enabled: false, intervalMinutes: 73, retentionDays: 11, dir: "/tmp/legacy-backups" },
},
logging: { mode: "cloud", logDir: "/tmp/legacy-logs" },
server: { deploymentMode: "authenticated", exposure: "private", port: 3210, serveUi: false, allowedHostnames: ["saved.example.test"] },
auth: { baseUrlMode: "explicit", publicBaseUrl: "https://saved.example.test", disableSignUp: true },
storage: { provider: "s3", s3: { bucket: "saved-bucket", region: "eu-west-1", prefix: "saved/", forcePathStyle: true } },
secrets: { provider: "local_encrypted", strictMode: true, localEncrypted: { keyFilePath: "/tmp/legacy-master.key" } },
telemetry: { enabled: false },
updates: { checkEnabled: false },
unsupportedSecret: "unknown-secret-value",
};
}
beforeEach(() => {
home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-runtime-config-"));
configPath = path.join(home, "config.json");
vi.stubEnv("PAPERCLIP_HOME", home);
vi.stubEnv("PAPERCLIP_CONFIG", configPath);
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "test-token");
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
vi.stubEnv("DATABASE_MIGRATION_URL", undefined);
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
vi.stubEnv("PAPERCLIP_AUTH_PUBLIC_BASE_URL", undefined);
vi.stubEnv("PAPERCLIP_AUTH_BASE_URL_MODE", undefined);
vi.stubEnv("PAPERCLIP_PUBLIC_URL", undefined);
vi.stubEnv("BETTER_AUTH_URL", undefined);
vi.stubEnv("BETTER_AUTH_BASE_URL", undefined);
vi.stubEnv("PAPERCLIP_ALLOWED_HOSTNAMES", undefined);
vi.stubEnv("PAPERCLIP_TAILNET_BIND_HOST", "100.100.0.1");
vi.stubEnv("OPENAI_API_KEY", "environment-api-secret");
setCloudRuntimeConfigSnapshot(null);
});
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllEnvs();
setCloudRuntimeConfigSnapshot(null);
fs.rmSync(home, { recursive: true, force: true });
});
describe("Cloud configuration file boundary", () => {
it.each(["null", "{", JSON.stringify({ unexpected: true })])("rejects malformed present legacy data %s", async (contents) => {
fs.writeFileSync(configPath, contents);
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/Legacy Cloud runtime configuration/);
});
it("rejects oversized and linked legacy sources", async () => {
fs.writeFileSync(configPath, " ".repeat(65_537));
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/);
fs.rmSync(configPath);
const target = path.join(home, "target.json");
fs.writeFileSync(target, JSON.stringify(legacyConfig()));
fs.symlinkSync(target, configPath);
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/);
});
it("requires an environment credential for the legacy OpenAI consumer", async () => {
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
vi.stubEnv("OPENAI_API_KEY", undefined);
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/OPENAI_API_KEY/);
expect(() => readConfigFile()).toThrow(/not been initialized/);
});
it.each(["https://user:secret@saved.example.test", "https://saved.example.test?key=secret", "https://saved.example.test#secret"])("rejects secret-bearing URL %s", async (url) => {
const config = legacyConfig();
config.auth.publicBaseUrl = url;
fs.writeFileSync(configPath, JSON.stringify(config));
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/configuration is invalid/);
});
it("loads the minimal Cloud bootstrap from environment only", () => {
fs.writeFileSync(configPath, "malformed");
vi.stubEnv("DATABASE_URL", "postgres://bootstrap:bootstrap-secret@127.0.0.1:5432/runtime");
expect(loadCloudBootstrapConfig()).toMatchObject({ databaseMode: "postgres", databaseUrl: process.env.DATABASE_URL });
vi.stubEnv("DATABASE_URL", undefined);
expect(() => loadCloudBootstrapConfig()).toThrow(/requires DATABASE_URL/);
vi.stubEnv("DATABASE_URL", "sqlite:///tmp/db");
expect(() => loadCloudBootstrapConfig()).toThrow(/must use PostgreSQL/);
});
it.each(["", " "])("treats an empty migration URL as absent (%j)", (migrationUrl) => {
vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime");
vi.stubEnv("DATABASE_MIGRATION_URL", migrationUrl);
expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBeUndefined();
});
it("normalizes and validates an explicit migration URL", () => {
vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime");
vi.stubEnv("DATABASE_MIGRATION_URL", " postgres://migration:password@127.0.0.1/runtime ");
expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBe("postgres://migration:password@127.0.0.1/runtime");
vi.stubEnv("DATABASE_MIGRATION_URL", "https://wrong.example.test");
expect(() => loadCloudBootstrapConfig()).toThrow("DATABASE_MIGRATION_URL must use PostgreSQL");
});
it("preserves local config file behavior without managed signals", () => {
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
expect(readConfigFile()?.llm?.apiKey).toBe("legacy-openai-secret");
expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" });
});
});
describePostgres("Cloud runtime configuration in PostgreSQL", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let first: Db;
let second: Db;
beforeAll(async () => {
if (externalDatabaseUrl) {
await applyPendingMigrations(externalDatabaseUrl);
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl, { scope: "database" }) };
} else {
database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-config-db-");
}
first = createDb(database.connectionString);
second = createDb(database.connectionString);
}, 90_000);
beforeEach(async () => {
await first.delete(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY));
vi.stubEnv("DATABASE_URL", database.connectionString);
});
afterAll(async () => { await database?.cleanup(); });
it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])("seeds a fresh database with %s", async (signal) => {
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token");
const result = await initializeCloudRuntimeConfig(first);
expect(result).toMatchObject({ seeded: true, current: { version: 1, revision: 0, config: null } });
expect(await cloudRuntimeConfigService(second).inspectStored()).toEqual(result?.current);
expect(fs.existsSync(configPath)).toBe(false);
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", "authenticated");
vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://environment.example.test");
expect(loadConfig()).toMatchObject({ databaseMode: "postgres", authBaseUrlMode: "explicit", secretsStrictMode: true, telemetryEnabled: true });
});
it("loads the saved authenticated mode before validating an environment host bind", async () => {
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
vi.stubEnv("HOST", "0.0.0.0");
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
expect(loadCloudBootstrapConfig()).toEqual({
databaseMode: "postgres",
databaseUrl: database.connectionString,
databaseMigrationUrl: process.env.DATABASE_MIGRATION_URL,
});
await initializeCloudRuntimeConfig(first);
expect(loadConfig()).toMatchObject({ deploymentMode: "authenticated", host: "0.0.0.0", port: 3210 });
});
it("imports existing choices without credentials and restores them from a second client and new home", async () => {
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
const initial = await initializeCloudRuntimeConfig(first);
expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, databaseBackupEnabled: false, storageS3Bucket: "saved-bucket", authDisableSignUp: true });
const [row] = await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY));
const persisted = JSON.stringify(row.general);
for (const secret of ["legacy-openai-secret", "database-secret", "unknown-secret-value", "environment-api-secret", database.connectionString]) {
expect(persisted).not.toContain(secret);
}
const replacementHome = fs.mkdtempSync(path.join(home, "replacement-"));
const replacementConfigDir = path.join(replacementHome, ".paperclip");
fs.mkdirSync(replacementConfigDir);
const envPath = path.join(replacementConfigDir, ".env");
const envContents = "PORT=9999\nPAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL=should-not-load\n";
fs.writeFileSync(envPath, envContents);
vi.stubEnv("PAPERCLIP_HOME", replacementHome);
vi.stubEnv("PAPERCLIP_CONFIG", path.join(replacementConfigDir, "config.json"));
setCloudRuntimeConfigSnapshot(null);
const files = vi.spyOn(fs, "readFileSync");
expect(await initializeCloudRuntimeConfig(second)).toEqual({ seeded: false, current: initial?.current });
expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" });
expect(files.mock.calls.map(([file]) => String(file)).filter(file => file.startsWith(replacementHome))).toEqual([]);
files.mockRestore();
const root = fileURLToPath(new URL("../../../", import.meta.url));
const childScript = path.join(home, "restore.mts");
fs.writeFileSync(childScript, `
import { createDb } from ${JSON.stringify(pathToFileURL(path.join(root, "packages/db/src/index.ts")).href)};
import { initializeCloudRuntimeConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/services/cloud-runtime-config.ts")).href)};
import { loadConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/config.ts")).href)};
const db = createDb(process.env.DATABASE_URL);
await initializeCloudRuntimeConfig(db);
const config = loadConfig();
console.log(JSON.stringify({ port: config.port, telemetryEnabled: config.telemetryEnabled, storageS3Bucket: config.storageS3Bucket,
envSentinel: process.env.PAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL ?? null }));
await db.$client.end({ timeout: 1 });
`);
const child = await promisify(execFile)(process.execPath, [path.join(root, "server/node_modules/tsx/dist/cli.mjs"), childScript], {
cwd: replacementHome,
env: { ...process.env },
timeout: 30_000,
});
expect(JSON.parse(child.stdout.trim())).toEqual({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket", envSentinel: null });
expect(fs.readFileSync(envPath, "utf8")).toBe(envContents);
}, 45_000);
it("keeps environment overlays out of saved settings and ignores corrupt old files after seed", async () => {
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
await initializeCloudRuntimeConfig(first);
vi.stubEnv("PORT", "4567");
vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", "operator-bucket");
vi.stubEnv("PAPERCLIP_AUTH_DISABLE_SIGN_UP", "false");
fs.writeFileSync(configPath, "malformed legacy contents");
await initializeCloudRuntimeConfig(second);
expect(loadConfig()).toMatchObject({ port: 4567, storageS3Bucket: "operator-bucket", authDisableSignUp: false, telemetryEnabled: false });
expect((await cloudRuntimeConfigService(first).inspectStored())?.config).toMatchObject({ server: { port: 3210 }, storage: { s3: { bucket: "saved-bucket" } }, auth: { disableSignUp: true } });
});
it("keeps one committed seed when two clients propose different initial values", async () => {
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
const results = await Promise.all([
cloudRuntimeConfigService(first).seedIfAbsent({ ...config, telemetry: { enabled: true } }),
cloudRuntimeConfigService(second).seedIfAbsent({ ...config, telemetry: { enabled: false } }),
]);
expect(results.filter(result => result.seeded)).toHaveLength(1);
expect(results[0].current).toEqual(results[1].current);
expect(await cloudRuntimeConfigService(first).inspectStored()).toEqual(results[0].current);
});
it("rejects unknown or credential fields on the direct mutation path", async () => {
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
await cloudRuntimeConfigService(first).seedIfAbsent(config);
await expect(cloudRuntimeConfigService(first).replace({ ...config, llm: { provider: "openai", apiKey: "must-not-store" } } as never, 0)).rejects.toThrow(/configuration is invalid/);
expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(0);
});
it("serializes revisions and rejects competing stale writes from two database clients", async () => {
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
await cloudRuntimeConfigService(first).seedIfAbsent(config);
const results = await Promise.allSettled([
cloudRuntimeConfigService(first).replace({ ...config, telemetry: { enabled: true } }, 0),
cloudRuntimeConfigService(second).replace({ ...config, telemetry: { enabled: false } }, 0),
]);
expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1);
expect(results.filter(result => result.status === "rejected")).toHaveLength(1);
expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(1);
});
it("composes a seed with the caller transaction and rolls back both on failure", async () => {
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
await expect(first.transaction(async tx => {
await cloudRuntimeConfigService(first).seedIfAbsent(config, tx);
await tx.insert(instanceSettings).values({ singletonKey: "runtime-config-test-receipt", general: {} });
throw new Error("injected migration failure");
})).rejects.toThrow(/injected migration failure/);
expect(await cloudRuntimeConfigService(second).inspectStored()).toBeNull();
expect(await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, "runtime-config-test-receipt"))).toEqual([]);
});
it("fails closed on invalid database settings without consulting a valid legacy file", async () => {
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
await first.insert(instanceSettings).values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: { version: 99, config: { secret: "hidden" } } });
await expect(initializeCloudRuntimeConfig(second)).rejects.toThrow(/Persisted Cloud runtime configuration is invalid/);
expect(() => getCloudRuntimeConfigSnapshot()).toThrow(/not been initialized/);
});
it("does not fall back to a file or old snapshot when the database fails", async () => {
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
await initializeCloudRuntimeConfig(first);
const unavailable = createDb(database.connectionString);
await unavailable.$client.end({ timeout: 1 });
await expect(initializeCloudRuntimeConfig(unavailable)).rejects.toThrow();
expect(() => readConfigFile()).toThrow(/not been initialized/);
});
});
@@ -0,0 +1,50 @@
import { mkdtempSync, mkdirSync, realpathSync, writeFileSync } from "node:fs";
import path from "node:path";
import os from "node:os";
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../../services/bundled-plugins.js";
import { distributionBundleDigest, readDistributionPluginCatalog } from "../../services/distribution-plugin-catalog.js";
import { derivePluginDatabaseNamespace } from "../../services/plugin-database.js";
import { createBundledPluginPackageResolver } from "../../services/bundled-plugin-package.js";
export function bundledImageFixture(manifestOverrides: Partial<PaperclipPluginManifestV1> = {}) {
const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "bundled-image-")));
const packageRoot = path.join(root, "distribution", "fixture");
const manifest: PaperclipPluginManifestV1 = {
id: "paperclip.image-fixture", apiVersion: 1, version: "1.0.0", displayName: "Image Fixture",
description: "Image package relocation fixture", author: "Paperclip", categories: ["automation"], capabilities: ["issues.read"],
entrypoints: { worker: "dist/worker.js", ui: "dist/ui" }, ...manifestOverrides,
};
mkdirSync(path.join(packageRoot, "dist", "ui"), { recursive: true });
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify({
name: "@paperclipai/plugin-image-fixture", version: manifest.version, type: "module",
paperclipPlugin: { manifest: "dist/manifest.js", ...manifest.entrypoints },
}));
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`);
writeFileSync(path.join(packageRoot, "dist", "worker.js"), `import readline from "node:readline";
if (process.argv.includes("--source")) process.stdout.write(import.meta.url);
else {
const configs = {};
readline.createInterface({ input: process.stdin }).on("line", line => {
const request = JSON.parse(line);
let result = {};
if (request.method === "initialize") result = { ok: true, supportedMethods: ["configChanged", "health"] };
if (request.method === "configChanged") configs[request.params.companyId] = request.params.config;
if (request.method === "health") result = { status: "ok", details: { source: import.meta.url, configs } };
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result }) + "\\n");
if (request.method === "shutdown") setImmediate(() => process.exit(0));
});
}`);
writeFileSync(path.join(packageRoot, "dist", "ui", "index.js"), "export const source = 'current-image';");
if (manifest.database) {
mkdirSync(path.join(packageRoot, manifest.database.migrationsDir), { recursive: true });
writeFileSync(path.join(packageRoot, manifest.database.migrationsDir, "001_fixture.sql"), `CREATE TABLE ${derivePluginDatabaseNamespace(manifest.id)}.image_rows (id uuid PRIMARY KEY);`);
}
const entry = { key: "fixture", pluginKey: manifest.id, version: manifest.version, directory: "fixture", digest: distributionBundleDigest(packageRoot) };
writeFileSync(path.join(root, "distribution", "catalog.json"), JSON.stringify({ schemaVersion: 1, plugins: [entry] }));
const catalog = readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG);
const installs = resolveBundledPluginInstalls(["fixture"], { catalogRoot: root, enforceCatalogRoot: true, env: {}, distributionPlugins: catalog });
const resolver = createBundledPluginPackageResolver(installs);
const registration = { pluginKey: manifest.id, packageName: "@paperclipai/plugin-image-fixture", version: manifest.version, manifestJson: manifest };
return { root, packageRoot, manifest, catalog, installs, resolver, registration };
}
@@ -20,6 +20,22 @@ import { gzipSync } from "node:zlib";
import { tmpdir } from "node:os";
import path from "node:path";
const cloudRuntimeStartupMocks = vi.hoisted(() => ({
initialize: vi.fn(async () => null),
bootstrap: vi.fn(),
telemetry: vi.fn(),
}));
vi.mock("../services/adapter-settings.js", async () => ({
...await vi.importActual<typeof import("../services/adapter-settings.js")>("../services/adapter-settings.js"),
initializeAdapterSettings: vi.fn(async () => null),
}));
vi.mock("../services/cloud-runtime-config.js", () => ({ initializeCloudRuntimeConfig: cloudRuntimeStartupMocks.initialize }));
const cloudMigrationMocks = vi.hoisted(() => ({ migrate: vi.fn(async () => null), qualify: vi.fn(async () => null) }));
vi.mock("../services/cloud-configuration-migration.js", () => ({ migrateCloudConfiguration: cloudMigrationMocks.migrate }));
vi.mock("../services/cloud-configuration-qualification.js", () => ({ qualifyCloudConfigurationForCurrentImage: cloudMigrationMocks.qualify }));
vi.mock("../telemetry.js", () => ({ initTelemetry: cloudRuntimeStartupMocks.telemetry, getTelemetryClient: vi.fn(() => null) }));
const ORIGINAL_PAPERCLIP_API_URL = process.env.PAPERCLIP_API_URL;
const ORIGINAL_PAPERCLIP_RUNTIME_API_URL = process.env.PAPERCLIP_RUNTIME_API_URL;
const ORIGINAL_PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON;
@@ -343,6 +359,7 @@ vi.mock("../services/native-runtime/native-safe-replacement.js", () => ({
vi.mock("../config.js", () => ({
loadConfig: loadConfigMock,
loadCloudBootstrapConfig: cloudRuntimeStartupMocks.bootstrap,
}));
vi.mock("../middleware/logger.js", () => ({
@@ -586,6 +603,8 @@ describe("startServer feedback export wiring", () => {
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
process.env.PAPERCLIP_AGENT_JWT_SECRET = "0123456789abcdef0123456789abcdef";
loadConfigMock.mockReturnValue(buildTestConfig());
cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
resolveHeartbeatSchedulingSuppressionMock.mockReturnValue({
suppressed: false,
reason: null,
@@ -595,6 +614,64 @@ describe("startServer feedback export wiring", () => {
process.env.BETTER_AUTH_SECRET = "test-secret";
});
it("awaits saved Cloud configuration before telemetry, authentication and app construction", async () => {
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
vi.stubEnv("HOST", "0.0.0.0");
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
let release: (() => void) | undefined;
cloudRuntimeStartupMocks.initialize.mockImplementationOnce(() => new Promise<null>(resolve => {
release = () => {
loadConfigMock.mockReturnValue(buildTestConfig({ telemetryEnabled: false, authDisableSignUp: true, port: 4321, host: "0.0.0.0", bind: "all" }));
resolve(null);
};
}));
try {
const starting = startServer();
for (let index = 0; index < 30 && !release; index += 1) await new Promise<void>(resolve => setImmediate(resolve));
expect(release).toBeTypeOf("function");
expect(cloudMigrationMocks.migrate.mock.invocationCallOrder[0]).toBeLessThan(cloudRuntimeStartupMocks.initialize.mock.invocationCallOrder[0]!);
expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
expect(createBetterAuthInstanceMock).not.toHaveBeenCalled();
expect(createAppMock).not.toHaveBeenCalled();
expect(loadConfigMock).not.toHaveBeenCalled();
release!();
const started = await starting;
await settleStartupHousekeeping();
expect(cloudRuntimeStartupMocks.telemetry).toHaveBeenCalledExactlyOnceWith({ enabled: false });
expect(createBetterAuthInstanceMock).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ authDisableSignUp: true, port: 4321 }), expect.anything());
expect(started.listenPort).toBe(4321);
expect(cloudMigrationMocks.qualify.mock.invocationCallOrder[0]).toBeLessThan(createAppMock.mock.invocationCallOrder[0]!);
} finally {
release?.();
vi.unstubAllEnvs();
}
});
it("does not create telemetry or the app when saved Cloud configuration fails", async () => {
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
cloudRuntimeStartupMocks.initialize.mockRejectedValueOnce(new Error("persisted config unavailable"));
try {
await expect(startServer()).rejects.toThrow("persisted config unavailable");
expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
expect(createBetterAuthInstanceMock).not.toHaveBeenCalled();
expect(createAppMock).not.toHaveBeenCalled();
} finally {
vi.unstubAllEnvs();
}
});
it("stops before configuration consumers when the atomic Cloud migration fails", async () => {
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
cloudMigrationMocks.migrate.mockRejectedValueOnce(new Error("migration did not commit"));
try {
await expect(startServer()).rejects.toThrow("migration did not commit");
expect(cloudRuntimeStartupMocks.initialize).not.toHaveBeenCalled();
expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
expect(createAppMock).not.toHaveBeenCalled();
} finally { vi.unstubAllEnvs(); }
});
it.each([false, true])("registers runner evidence independently from the backup scan (%s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled }));
await startServer(); await settleStartupHousekeeping();
@@ -1509,6 +1586,8 @@ describe("startServer authenticated auth origin setup", () => {
vi.clearAllMocks();
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
loadConfigMock.mockReturnValue(buildTestConfig());
cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
createBetterAuthInstanceMock.mockReturnValue({});
deriveAuthTrustedOriginsMock.mockReturnValue([]);
process.env.BETTER_AUTH_SECRET = "test-secret";
@@ -1570,6 +1649,8 @@ describe("startServer PAPERCLIP_API_URL handling", () => {
vi.clearAllMocks();
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
loadConfigMock.mockReturnValue(buildTestConfig());
cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
process.env.BETTER_AUTH_SECRET = "test-secret";
delete process.env.PAPERCLIP_API_URL;
});
@@ -0,0 +1,136 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
let home: string;
let localPackage: string;
let npmPackage: string;
let storePath: string;
let sentinel: string;
const parserSource = "export function parseStdoutLine() { return { message: 'external parser' }; }";
const managedError = { status: 403, details: { code: "adapter_install_platform_managed" } };
function writePackage(packageDir: string, type: string): void {
fs.mkdirSync(packageDir, { recursive: true });
fs.writeFileSync(path.join(packageDir, "package.json"), JSON.stringify({
name: `fixture-${type}`,
type: "module",
exports: { ".": "./index.mjs", "./ui-parser": "./ui-parser.mjs" },
paperclip: { adapterUiParser: "1.0.0" },
}));
fs.writeFileSync(path.join(packageDir, "index.mjs"), `
import fs from "node:fs";
fs.appendFileSync(${JSON.stringify(sentinel)}, ${JSON.stringify(type + "\n")});
export function createServerAdapter() {
return {
type: ${JSON.stringify(type)},
agentConfigurationDoc: "External fixture",
models: [],
execute: async () => ({ exitCode: 41, signal: null, timedOut: false }),
testEnvironment: async () => ({ status: "pass", checks: [] }),
};
}
`);
fs.writeFileSync(path.join(packageDir, "ui-parser.mjs"), parserSource);
}
beforeEach(() => {
vi.resetModules();
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-cloud-adapters-"));
vi.stubEnv("PAPERCLIP_HOME", home);
sentinel = path.join(home, "external-code-executed");
localPackage = path.join(home, "local-package");
npmPackage = path.join(home, "adapter-plugins", "node_modules", "fixture-npm");
storePath = path.join(home, "adapter-plugins.json");
writePackage(localPackage, "codex_local");
writePackage(npmPackage, "fixture_npm");
fs.writeFileSync(storePath, JSON.stringify([
{ packageName: "fixture-local", localPath: localPackage, type: "codex_local", installedAt: "2026-01-01" },
{ packageName: "fixture-npm", type: "fixture_npm", installedAt: "2026-01-01" },
]));
});
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllEnvs();
fs.rmSync(home, { recursive: true, force: true });
});
describe("Cloud image adapter boundary", () => {
it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])(
"cold boots built-ins without reading or executing persisted packages with %s",
async (signal) => {
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token");
const registrationBefore = fs.readFileSync(storePath, "utf8");
const reads = vi.spyOn(fs, "readFileSync");
const registry = await import("./registry.js");
await registry.waitForExternalAdapters();
const loader = await import("./plugin-loader.js");
const store = await import("../services/adapter-plugin-store.js");
expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture");
expect(registry.requireServerAdapter("process").execute).toBeTypeOf("function");
expect(registry.findServerAdapter("fixture_npm")).toBeNull();
expect(registry.hasActiveAdapterOverride("codex_local")).toBe(false);
expect(await loader.buildExternalAdapters()).toEqual([]);
expect(store.listAdapterPlugins()).toEqual([]);
expect(store.getAdapterPluginByType("codex_local")).toBeUndefined();
expect(loader.getUiParserSource("codex_local")).toBeUndefined();
expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined();
await expect(loader.loadExternalAdapterPackage("fixture-local", localPackage)).rejects.toMatchObject(managedError);
await expect(loader.loadExternalAdapterPackage("fixture-npm")).rejects.toMatchObject(managedError);
await expect(loader.reloadExternalAdapter("codex_local")).rejects.toMatchObject(managedError);
expect(() => registry.registerServerAdapter(registry.requireServerAdapter("codex_local"))).toThrow();
expect(() => registry.unregisterServerAdapter("codex_local")).toThrow();
expect(() => registry.setOverridePaused("codex_local", false)).toThrow();
expect(() => store.addAdapterPlugin({ packageName: "fixture", type: "fixture", installedAt: "2026-01-01" })).toThrow();
expect(() => store.removeAdapterPlugin("codex_local")).toThrow();
expect(() => store.getAdapterPluginsDir()).toThrow();
expect(store.getAdapterPluginStorePath()).toBe(storePath);
expect(reads.mock.calls.map(([file]) => String(file)).filter(file =>
file === storePath || file.startsWith(localPackage + path.sep) || file.startsWith(npmPackage + path.sep),
)).toEqual([]);
reads.mockRestore();
expect(fs.existsSync(sentinel)).toBe(false);
expect(fs.readFileSync(storePath, "utf8")).toBe(registrationBefore);
},
);
it("preserves local and npm package loading, overrides, reload, and lazy UI parsing on self-hosted instances", async () => {
const loader = await import("./plugin-loader.js");
expect(loader.getOrExtractUiParserSource("codex_local")).toBe(parserSource);
const registry = await import("./registry.js");
await registry.waitForExternalAdapters();
const override = registry.requireServerAdapter("codex_local");
expect(override.agentConfigurationDoc).toBe("External fixture");
expect(registry.hasActiveAdapterOverride("codex_local")).toBe(true);
expect(registry.requireServerAdapter("fixture_npm").agentConfigurationDoc).toBe("External fixture");
expect(await override.execute({} as never)).toMatchObject({ exitCode: 41 });
expect(fs.readFileSync(sentinel, "utf8").trim().split("\n")).toEqual(["codex_local", "fixture_npm"]);
expect(loader.getUiParserSource("fixture_npm")).toBe(parserSource);
const reloaded = await loader.reloadExternalAdapter("codex_local");
expect(reloaded?.type).toBe("codex_local");
expect(await reloaded?.execute({} as never)).toMatchObject({ exitCode: 41 });
expect(registry.setOverridePaused("codex_local", true)).toBe(true);
expect(registry.findActiveServerAdapter("codex_local")?.agentConfigurationDoc).not.toBe("External fixture");
expect(registry.setOverridePaused("codex_local", false)).toBe(true);
registry.unregisterServerAdapter("codex_local");
expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture");
});
it("does not expose a cached registration or parser after a managed signal becomes present", async () => {
const loader = await import("./plugin-loader.js");
const store = await import("../services/adapter-plugin-store.js");
await loader.loadExternalAdapterPackage("fixture-local", localPackage);
expect(loader.getUiParserSource("codex_local")).toBe(parserSource);
expect(store.listAdapterPlugins()).toHaveLength(2);
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", "{}");
expect(store.listAdapterPlugins()).toEqual([]);
expect(loader.getUiParserSource("codex_local")).toBeUndefined();
expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined();
});
});
+7
View File
@@ -15,6 +15,8 @@ import { pathToFileURL } from "node:url";
import type { ServerAdapterModule } from "./types.js";
import { validateAdapterLoginCapability } from "@paperclipai/adapter-utils";
import { logger } from "../middleware/logger.js";
import { isCloudManagedInstance } from "../services/cloud-instance.js";
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
import {
listAdapterPlugins,
@@ -30,6 +32,7 @@ import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js";
const uiParserCache = new Map<string, string>();
export function getUiParserSource(adapterType: string): string | undefined {
if (isCloudManagedInstance()) return undefined;
return uiParserCache.get(adapterType);
}
@@ -38,6 +41,7 @@ export function getUiParserSource(adapterType: string): string | undefined {
* Makes the ui-parser.js endpoint self-healing.
*/
export function getOrExtractUiParserSource(adapterType: string): string | undefined {
if (isCloudManagedInstance()) return undefined;
const cached = uiParserCache.get(adapterType);
if (cached) return cached;
@@ -182,6 +186,7 @@ export async function loadExternalAdapterPackage(
packageName: string,
localPath?: string,
): Promise<ServerAdapterModule> {
assertExternalAdapterCodeAllowed();
const packageDir = localPath
? path.resolve(localPath)
: path.resolve(getAdapterPluginsDir(), "node_modules", packageName);
@@ -221,6 +226,7 @@ async function loadFromRecord(record: AdapterPluginRecord): Promise<ServerAdapte
export async function reloadExternalAdapter(
type: string,
): Promise<ServerAdapterModule | null> {
assertExternalAdapterCodeAllowed();
const record = getAdapterPluginByType(type);
if (!record) return null;
@@ -271,6 +277,7 @@ export async function reloadExternalAdapter(
* Build all external adapter modules from the plugin store.
*/
export async function buildExternalAdapters(): Promise<ServerAdapterModule[]> {
if (isCloudManagedInstance()) return [];
const results: ServerAdapterModule[] = [];
const storeRecords = listAdapterPlugins();
+7 -8
View File
@@ -136,7 +136,8 @@ import {
import { agentConfigurationDoc as piAgentConfigurationDoc } from "@paperclipai/adapter-pi-local";
import { BUILTIN_ADAPTER_TYPES } from "./builtin-adapter-types.js";
import { buildExternalAdapters } from "./plugin-loader.js";
import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js";
import { isCloudManagedInstance } from "../services/cloud-instance.js";
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
import { NativeEnvironmentProbeCleanupError } from "../vendor/paperclip-runner/index.js";
import { processAdapter } from "./process/index.js";
import { httpAdapter } from "./http/index.js";
@@ -997,11 +998,6 @@ registerBuiltInAdapters();
// applies the registry fallback for externals overriding a built-in type.
// ---------------------------------------------------------------------------
/** Cached sync wrapper — the store is a simple JSON file read, safe to call frequently. */
function getDisabledAdapterTypesFromStore(): string[] {
return getDisabledAdapterTypes();
}
/**
* Merge an external adapter module with host-provided session management.
*
@@ -1034,6 +1030,7 @@ export function resolveExternalAdapterRegistration(
* and avoid racing against the loading window.
*/
const externalAdaptersReady: Promise<void> = (async () => {
if (isCloudManagedInstance()) return;
try {
const externalAdapters = await buildExternalAdapters();
for (const externalAdapter of externalAdapters) {
@@ -1069,6 +1066,7 @@ export function waitForExternalAdapters(): Promise<void> {
}
export function registerServerAdapter(adapter: ServerAdapterModule): void {
assertExternalAdapterCodeAllowed();
if (BUILTIN_ADAPTER_TYPES.has(adapter.type) && !builtinFallbacks.has(adapter.type)) {
const existing = adaptersByType.get(adapter.type);
if (existing) {
@@ -1079,6 +1077,7 @@ export function registerServerAdapter(adapter: ServerAdapterModule): void {
}
export function unregisterServerAdapter(type: string): void {
assertExternalAdapterCodeAllowed();
if (type === processAdapter.type || type === httpAdapter.type) return;
if (builtinFallbacks.has(type)) {
pausedOverrides.delete(type);
@@ -1186,8 +1185,7 @@ export function listServerAdapters(): ServerAdapterModule[] {
* Used for menus and agent creation flows — disabled adapters remain
* functional for existing agents but hidden from selection.
*/
export function listEnabledServerAdapters(): ServerAdapterModule[] {
const disabled = getDisabledAdapterTypesFromStore();
export function listEnabledServerAdapters(disabled: string[]): ServerAdapterModule[] {
const disabledSet = disabled.length > 0 ? new Set(disabled) : null;
return disabledSet
? Array.from(adaptersByType.values()).filter((a) => !disabledSet.has(a.type))
@@ -1227,6 +1225,7 @@ export async function detectAdapterModel(
* an override or was already in the requested state.
*/
export function setOverridePaused(type: string, paused: boolean): boolean {
assertExternalAdapterCodeAllowed();
if (!builtinFallbacks.has(type)) return false;
const wasPaused = pausedOverrides.has(type);
if (paused && !wasPaused) {
+13 -4
View File
@@ -183,7 +183,9 @@ import {
ensureBundledPlugins,
resolveBundledCatalogRoot,
resolveBundledPluginInstalls,
resolveManagedBundledPluginKeys,
} from "./services/bundled-plugins.js";
import { createBundledPluginPackageResolver } from "./services/bundled-plugin-package.js";
import { readDistributionPluginCatalog, distributionPluginActivationGuard } from "./services/distribution-plugin-catalog.js";
import {
createPluginWorkerManager,
@@ -623,7 +625,7 @@ export async function createApp(
res.set("Cache-Control", "no-store"); res.set("Referrer-Policy", "no-referrer");
res.redirect(303, await chatChannels.githubWizard.identityCallback(String(req.query.state ?? ""), String(req.query.code ?? ""), req.actor));
});
const managedAutoInstallKeys = opts.managedPluginAutoInstall ?? null;
const managedAutoInstallKeys = resolveManagedBundledPluginKeys(opts.managedPluginAutoInstall);
const bundledCatalogRoot =
opts.bundledPluginCatalogRoot ?? resolveBundledCatalogRoot(process.env);
const distributionPlugins = readDistributionPluginCatalog(bundledCatalogRoot, BUNDLED_PLUGIN_CATALOG);
@@ -636,6 +638,7 @@ export async function createApp(
distributionPlugins,
},
);
const bundledPackageResolver = managedAutoInstallKeys === null ? undefined : createBundledPluginPackageResolver(bundledPluginInstalls);
const managedBundledPluginKeys =
managedAutoInstallKeys !== null
? bundledPluginInstalls.map((install) => install.pluginKey)
@@ -906,7 +909,11 @@ export async function createApp(
{
localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
migrationDb: opts.pluginMigrationDb,
assertPackageActivation: distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys),
assertPackageActivation: bundledPackageResolver?.assertActivation ?? distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys),
resolvePackageRoot: bundledPackageResolver?.resolve,
prebuiltOnly: managedAutoInstallKeys !== null,
enableLocalFilesystem: managedAutoInstallKeys === null,
enableNpmDiscovery: managedAutoInstallKeys === null,
},
{
workerManager,
@@ -965,6 +972,7 @@ export async function createApp(
);
api.use(
adapterRoutes({
db,
resolveRunnerTarget: (companyId, environmentId) => resolveAgentRunnerTargetForCompany(db, companyId, environmentId),
getNativeRunnerEnabled: async () =>
(await instanceSettingsService(db).getExperimental())
@@ -1014,6 +1022,7 @@ export async function createApp(
app.use(
pluginUiStaticRoutes(db, {
localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
resolvePackageRoot: bundledPackageResolver?.resolve,
}),
);
@@ -1253,7 +1262,7 @@ export async function createApp(
void toolDispatcher.initialize().catch((err) => {
logger.error({ err }, "Failed to initialize plugin tool dispatcher");
});
const devWatcher = createPluginDevWatcher(
const devWatcher = managedAutoInstallKeys !== null ? null : createPluginDevWatcher(
lifecycle,
async (pluginId) =>
(await pluginRegistry.getById(pluginId))?.packagePath ?? null,
@@ -1299,7 +1308,7 @@ export async function createApp(
// owns provisioning); self-hosted leaves an operator's uninstall alone.
// Disabled plugins never start automatically. Added distribution permissions
// still enter upgrade_pending so enabling them requires an operator decision.
{ reinstallUninstalled: managedAutoInstallKeys !== null },
{ reinstallUninstalled: managedAutoInstallKeys !== null, preservePackagePath: managedAutoInstallKeys !== null },
)
.then(() => loader.loadAll())
.then((result) => {
+3
View File
@@ -8,6 +8,8 @@ import {
} from "@paperclipai/shared";
import { ZodError } from "zod";
import { resolvePaperclipConfigPath } from "./paths.js";
import { isCloudManagedInstance } from "./services/cloud-instance.js";
import { getCloudRuntimeConfigSnapshot, toPaperclipConfig } from "./services/cloud-runtime-config-snapshot.js";
function formatConfigValidationError(error: ZodError): string {
return error.issues
@@ -19,6 +21,7 @@ function formatConfigValidationError(error: ZodError): string {
}
export function readConfigFile(): PaperclipConfig | null {
if (isCloudManagedInstance()) return toPaperclipConfig(getCloudRuntimeConfigSnapshot().config);
const configPath = resolvePaperclipConfigPath();
if (!fs.existsSync(configPath)) return null;
+46 -22
View File
@@ -1,4 +1,7 @@
import { readConfigFile, resolveDeploymentMode } from "./config-file.js";
import { isCloudManagedInstance } from "./services/cloud-instance.js";
import { StartupRefusalError } from "./startup-refusals.js";
import type { PaperclipConfig } from "@paperclipai/shared";
import { parseChatWebhookPublicBaseUrl } from "./chat-webhook-public-url.js";
import { execFileSync } from "node:child_process";
import { existsSync, realpathSync } from "node:fs";
@@ -32,27 +35,29 @@ import {
resolveHomeAwarePath,
} from "./home-paths.js";
const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath();
if (existsSync(PAPERCLIP_ENV_FILE_PATH)) {
loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true });
}
if (!isCloudManagedInstance()) {
const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath();
if (existsSync(PAPERCLIP_ENV_FILE_PATH)) {
loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true });
}
const CWD_ENV_PATH = resolve(process.cwd(), ".env");
const cwdEnvExists = existsSync(CWD_ENV_PATH);
const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH)
? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH)
: CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH;
if (shouldLoadWorkingDirectoryEnv({
cwdEnvExists,
isPaperclipEnvFile: isSameFile,
})) {
loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true });
}
const CWD_ENV_PATH = resolve(process.cwd(), ".env");
const cwdEnvExists = existsSync(CWD_ENV_PATH);
const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH)
? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH)
: CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH;
if (shouldLoadWorkingDirectoryEnv({
cwdEnvExists,
isPaperclipEnvFile: isSameFile,
})) {
loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true });
}
if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") {
applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH)));
if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") {
applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH)));
}
maybeRepairLegacyWorktreeConfigAndEnvFiles();
}
maybeRepairLegacyWorktreeConfigAndEnvFiles();
const TAILSCALE_DETECT_TIMEOUT_MS = 3000;
@@ -147,13 +152,12 @@ function detectTailnetBindHost(): string | undefined {
}
}
export function loadConfig(): Config {
const fileConfig = readConfigFile();
export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()): Config {
const fileDatabaseMode =
(fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode;
(isCloudManagedInstance() || fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode;
const fileDbUrl =
fileDatabaseMode === "postgres"
!isCloudManagedInstance() && fileDatabaseMode === "postgres"
? fileConfig?.database.connectionString
: undefined;
const fileDatabaseBackup = fileConfig?.database.backup;
@@ -460,3 +464,23 @@ export function loadConfig(): Config {
announcementsFeedUrl: process.env.PAPERCLIP_ANNOUNCEMENTS_FEED_URL?.trim() || "https://pages.paperclip.ing/announcements/v1/current.json",
};
}
export type CloudBootstrapConfig = {
databaseMode: "postgres";
databaseUrl: string;
databaseMigrationUrl: string | undefined;
};
export function loadCloudBootstrapConfig(): CloudBootstrapConfig {
if (!isCloudManagedInstance()) throw new Error("Cloud bootstrap configuration requires a managed instance");
const url = process.env.DATABASE_URL?.trim();
if (!url) throw new StartupRefusalError("database-contract-unmet", "Cloud bootstrap requires DATABASE_URL from the environment");
const migrationUrl = process.env.DATABASE_MIGRATION_URL?.trim() || undefined;
for (const [name, value] of [["DATABASE_URL", url], ["DATABASE_MIGRATION_URL", migrationUrl]]) {
if (!value) continue;
let parsed: URL;
try { parsed = new URL(value); } catch { throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`); }
if (!["postgres:", "postgresql:"].includes(parsed.protocol)) throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`);
}
return { databaseMode: "postgres", databaseUrl: url, databaseMigrationUrl: migrationUrl };
}
+42 -21
View File
@@ -104,7 +104,8 @@ import {
} from "@paperclipai/db";
import detectPort from "detect-port";
import { createApp } from "./app.js";
import { loadConfig } from "./config.js";
import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js";
import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js";
import { logger } from "./middleware/logger.js";
import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js";
import { setStartupRecoveryPhase } from "./startup-recovery-state.js";
@@ -186,6 +187,9 @@ import {
loadWithoutCoordinatedShutdownSignalHooks,
} from "./shutdown.js";
import { initializeCloudRuntimeIdentity } from "./services/cloud-runtime-identity.js";
import { initializeAdapterSettings } from "./services/adapter-settings.js";
import { migrateCloudConfiguration } from "./services/cloud-configuration-migration.js";
import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js";
import { systemdNotify } from "./services/systemd-notify.js";
import { flushInFlightRunLogMirrors } from "./services/run-log-store.js";
import {
@@ -251,6 +255,19 @@ async function endDatabaseClient(client: unknown, timeoutSeconds: number): Promi
await sql.end({ timeout: timeoutSeconds });
}
function initializeRuntimeConfigConsumers(config: ReturnType<typeof loadConfig>): void {
initTelemetry({ enabled: config.telemetryEnabled });
if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
}
if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) {
process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false";
}
if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) {
process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath;
}
}
export async function startServer(): Promise<StartedServer> {
const startupDatabase: StartupDatabaseTeardown = { close: null };
try {
@@ -278,17 +295,8 @@ async function startServerWithDatabaseTeardown(
// sentry.ts.
await sentryReady;
ensureDecisionSigningSecret();
let config = loadConfig();
initTelemetry({ enabled: config.telemetryEnabled });
if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
}
if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) {
process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false";
}
if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) {
process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath;
}
const startupConfig = isCloudManagedInstance() ? loadCloudBootstrapConfig() : loadConfig();
if ("deploymentMode" in startupConfig) initializeRuntimeConfigConsumers(startupConfig);
type MigrationSummary =
| "skipped"
@@ -390,7 +398,7 @@ async function startServerWithDatabaseTeardown(
}
}
function assertCloudDatabaseContract(): void {
function assertCloudDatabaseContract(config: Config): void {
if (config.deploymentMode !== "authenticated" || config.deploymentExposure !== "public") {
return;
}
@@ -483,17 +491,19 @@ async function startServerWithDatabaseTeardown(
let startupDbInfo:
| { mode: "external-postgres"; connectionString: string }
| { mode: "embedded-postgres"; dataDir: string; port: number };
assertCloudDatabaseContract();
if (config.databaseUrl) {
const migrationUrl = config.databaseMigrationUrl ?? config.databaseUrl;
if ("deploymentMode" in startupConfig) assertCloudDatabaseContract(startupConfig);
if (startupConfig.databaseUrl) {
const migrationUrl = startupConfig.databaseMigrationUrl ?? startupConfig.databaseUrl;
migrationSummary = await ensureMigrations(migrationUrl, "PostgreSQL");
db = createDb(config.databaseUrl);
pluginMigrationDb = config.databaseMigrationUrl ? createDb(config.databaseMigrationUrl) : db;
db = createDb(startupConfig.databaseUrl);
pluginMigrationDb = startupConfig.databaseMigrationUrl ? createDb(startupConfig.databaseMigrationUrl) : db;
logger.info("Using external PostgreSQL via DATABASE_URL/config");
activeDatabaseConnectionString = config.databaseUrl;
startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
activeDatabaseConnectionString = startupConfig.databaseUrl;
startupDbInfo = { mode: "external-postgres", connectionString: startupConfig.databaseUrl };
} else {
if (!("deploymentMode" in startupConfig)) throw new Error("Cloud bootstrap requires an external PostgreSQL connection");
const config = startupConfig;
const moduleName = "embedded-postgres";
let EmbeddedPostgres: EmbeddedPostgresCtor;
try {
@@ -711,12 +721,19 @@ async function startServerWithDatabaseTeardown(
};
startupDatabase.close = closeDatabaseClients;
setSentryRunLinkResolver(createSentryRunLinkResolver(db));
await migrateCloudConfiguration(db as any);
await initializeAdapterSettings(db as any);
await initializeCloudRuntimeConfig(db as any);
let config = "deploymentMode" in startupConfig ? startupConfig : loadConfig();
// A claimed warm-pool stack may restart while its provider environment still
// names the pool host. Restore the signed, durable identity before Better
// Auth, routes, or child-runtime configuration capture any public URL.
const restoredCloudRuntimeIdentity = await initializeCloudRuntimeIdentity(db as any);
if (restoredCloudRuntimeIdentity) config = loadConfig();
if (isCloudManagedInstance()) initializeRuntimeConfigConsumers(config);
const isWarmStandby = await createCloudWarmStandby(db as any);
if (config.deploymentMode === "local_trusted" && !isLoopbackHost(config.host)) {
@@ -963,6 +980,10 @@ async function startServerWithDatabaseTeardown(
// document parsed fail-closed above (`plugins.autoInstall`). Absent env means
// self-hosted: createApp falls back to its built-in kubernetes-only default.
const managedPluginAutoInstall = managedConfig?.plugins.autoInstall ?? null;
const cloudQualification = await qualifyCloudConfigurationForCurrentImage(db as any);
if (cloudQualification && !cloudQualification.configurationPackagesEligible) {
logger.warn({ scope: cloudQualification.scope, blockerCodes: cloudQualification.blockers.map((blocker) => blocker.code) }, "Cloud configuration and package qualification has blockers");
}
const app = await createApp(db as any, {
prepareIdleDatabaseBackup: idleDatabaseBackupEnabled
? async () => (await runServerDatabaseBackup("idle")) !== null
+2
View File
@@ -1,6 +1,7 @@
import fs from "node:fs";
import path from "node:path";
import { resolveDefaultConfigPath } from "./home-paths.js";
import { isCloudManagedInstance } from "./services/cloud-instance.js";
const PAPERCLIP_CONFIG_BASENAME = "config.json";
const PAPERCLIP_ENV_FILENAME = ".env";
@@ -26,6 +27,7 @@ function findConfigFileFromAncestors(startDir: string): string | null {
export function resolvePaperclipConfigPath(overridePath?: string): string {
if (overridePath) return path.resolve(overridePath);
if (process.env.PAPERCLIP_CONFIG) return path.resolve(process.env.PAPERCLIP_CONFIG);
if (isCloudManagedInstance()) return resolveDefaultConfigPath();
return findConfigFileFromAncestors(process.cwd()) ?? resolveDefaultConfigPath();
}
+47
View File
@@ -0,0 +1,47 @@
import path from "node:path";
import { pathToFileURL } from "node:url";
import { createDb } from "@paperclipai/db";
import { cloudConfigurationMigrationService } from "./services/cloud-configuration-migration.js";
import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js";
export async function runCloudConfigurationQualification(args: string[], env: NodeJS.ProcessEnv = process.env): Promise<number> {
const allowed = new Set(["--migrate", "--expected-package-evidence", "--recheck-external-revision"]);
let migrate = false;
let expectedPackageEvidenceDigest: string | undefined;
let recheckRevision: number | undefined;
for (let index = 0; index < args.length; index += 1) {
const argument = args[index];
if (!allowed.has(argument)) throw new Error(`Unknown qualification argument: ${argument}`);
if (argument === "--migrate") migrate = true;
if (argument === "--expected-package-evidence") {
expectedPackageEvidenceDigest = args[++index];
if (!expectedPackageEvidenceDigest || !/^sha256:[a-f0-9]{64}$/.test(expectedPackageEvidenceDigest)) throw new Error("A SHA-256 package evidence digest is required");
}
if (argument === "--recheck-external-revision") {
const value = args[++index];
if (!value || !/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error("A nonnegative receipt revision is required");
recheckRevision = Number(value);
}
}
const url = env.DATABASE_URL?.trim();
if (!url) throw new Error("DATABASE_URL is required for Cloud configuration qualification");
const db = createDb(url);
try {
const service = cloudConfigurationMigrationService(db, { runtimeEnv: env });
if (migrate) await service.migrate();
if (recheckRevision !== undefined) await service.recheckLegacyExternalRegistrations(recheckRevision);
const report = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv: env, expectedPackageEvidenceDigest });
if (!report) throw new Error("Cloud configuration qualification requires a managed instance");
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
return report.configurationPackagesEligible && (!expectedPackageEvidenceDigest || report.samePackageOverlapEligible) ? 0 : 2;
} finally {
await db.$client.end({ timeout: 5 });
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
runCloudConfigurationQualification(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : "Cloud configuration qualification failed"}\n`);
process.exitCode = 1;
});
}
+37 -27
View File
@@ -19,11 +19,13 @@ import { readFile } from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import { Router } from "express";
import { companies, type Db } from "@paperclipai/db";
import { adapterSettingsService } from "../services/adapter-settings.js";
import { logActivity, publishActivity, type ActivityPublication } from "../services/activity-log.js";
import {
listServerAdapters,
findServerAdapter,
findActiveServerAdapter,
listEnabledServerAdapters,
registerServerAdapter,
resolveExternalAdapterRegistration,
unregisterServerAdapter,
@@ -37,8 +39,6 @@ import {
removeAdapterPlugin,
getAdapterPluginByType,
getAdapterPluginsDir,
getDisabledAdapterTypes,
setAdapterDisabled,
} from "../services/adapter-plugin-store.js";
import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js";
import type { ServerAdapterModule, AdapterConfigSchema } from "../adapters/types.js";
@@ -49,9 +49,9 @@ import type {
import { loadExternalAdapterPackage, getUiParserSource, getOrExtractUiParserSource, reloadExternalAdapter } from "../adapters/plugin-loader.js";
import { logger } from "../middleware/logger.js";
import { badRequest, forbidden } from "../errors.js";
import { isCloudManagedInstance } from "../services/cloud-instance.js";
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
import { getHiddenSettings } from "../services/settings-visibility.js";
import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin } from "./authz.js";
import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin, getActorInfo } from "./authz.js";
import { BUILTIN_ADAPTER_TYPES } from "../adapters/builtin-adapter-types.js";
import { agentRunnerAvailability, type RunnerTarget } from "../services/agent-runner-selection.js";
import type { AgentRunnerAvailability } from "@paperclipai/shared";
@@ -60,22 +60,6 @@ import { resolveAntigravityCandidateAdmission } from "../services/native-runtime
const execFileAsync = promisify(execFile);
let warnedMalformedAntigravityAdmission = false;
/**
* Floor: on cloud-managed instances adapter code is bundled into the platform
* image; fetching and loading external adapter packages at runtime stays off
* for every actor, including instance admins. Adapter code executes in the
* server process, so a runtime install would let an instance admin read the
* platform trust anchors from the process environment (mirrors the
* bundled-only plugin install floor in plugin-install-guard.ts).
*/
function assertAdapterCodeInstallAllowed() {
if (isCloudManagedInstance()) {
throw forbidden("Adapter installation is platform-managed on cloud-managed instances", {
code: "adapter_install_platform_managed",
});
}
}
/**
* Floor: when the hosting operator hides the Adapters settings surface
* (`instance.adapters` in PAPERCLIP_HIDDEN_SETTINGS), adapter management
@@ -209,7 +193,7 @@ function buildAdapterInfo(adapter: ServerAdapterModule, externalRecord: AdapterP
loaded: true, // If it's in the registry, it's loaded
disabled: disabledSet.has(adapter.type),
capabilities: buildAdapterCapabilities(adapter),
...agentRunnerAvailability(adapter.type, target),
...agentRunnerAvailability(adapter.type, target, [...disabledSet]),
...(adapter.acp ? { acp: adapter.acp } : {}),
overriddenBuiltin: externalRecord ? BUILTIN_ADAPTER_TYPES.has(adapter.type) : undefined,
overridePaused: BUILTIN_ADAPTER_TYPES.has(adapter.type) ? isOverridePaused(adapter.type) : undefined,
@@ -265,11 +249,13 @@ function registerWithSessionManagement(adapter: ServerAdapterModule): void {
// ---------------------------------------------------------------------------
export function adapterRoutes(options: {
db?: Db;
getNativeRunnerEnabled?: () => Promise<boolean>;
getOpenAiDotEnabled?: () => Promise<boolean>;
resolveRunnerTarget?: (companyId: string, environmentId?: string) => Promise<RunnerTarget>;
} = {}) {
const router = Router();
const settings = adapterSettingsService(options.db);
/**
* GET /api/adapters
@@ -303,7 +289,7 @@ export function adapterRoutes(options: {
const externalRecords = new Map(
listAdapterPlugins().map((r) => [r.type, r]),
);
const disabledSet = new Set(getDisabledAdapterTypes());
const disabledSet = new Set(await settings.getDisabledTypes());
// One shared implementation, with independent provider rollouts. Explicit
// adapter-admin disabling still applies to both choices.
@@ -343,7 +329,7 @@ export function adapterRoutes(options: {
*/
router.post("/adapters/install", async (req, res) => {
assertInstanceAdmin(req);
assertAdapterCodeInstallAllowed();
assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
const { packageName, isLocalPath = false, version } = req.body as AdapterInstallRequest;
@@ -477,7 +463,7 @@ export function adapterRoutes(options: {
}
const externalRecord = getAdapterPluginByType(adapterType);
const disabledSet = new Set(getDisabledAdapterTypes());
const disabledSet = new Set(await settings.getDisabledTypes());
res.json(buildAdapterInfo(adapter, externalRecord, disabledSet));
});
@@ -509,7 +495,28 @@ export function adapterRoutes(options: {
return;
}
const changed = setAdapterDisabled(adapterType, disabled);
const publications: ActivityPublication[] = [];
const changed = await settings.setDisabled(adapterType, disabled, async (tx, current) => {
const actor = getActorInfo(req);
const companyIds = await tx.select({ id: companies.id }).from(companies);
for (const company of companyIds) {
await logActivity(tx as Db, {
companyId: company.id,
...actor,
action: "instance.adapter.updated",
entityType: "instance_settings",
entityId: "adapter-settings:v1",
details: { adapterType, disabled, revision: current.revision },
}, publications);
}
});
for (const publication of publications) {
try {
publishActivity(publication);
} catch (error) {
logger.error({ err: error, companyId: publication.companyId }, "failed to publish adapter settings activity");
}
}
if (changed) {
logger.info({ type: adapterType, disabled }, "Adapter enabled/disabled");
@@ -528,6 +535,7 @@ export function adapterRoutes(options: {
*/
router.patch("/adapters/:type/override", async (req, res) => {
assertInstanceAdmin(req);
assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
@@ -558,6 +566,7 @@ export function adapterRoutes(options: {
*/
router.delete("/adapters/:type", async (req, res) => {
assertInstanceAdmin(req);
assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
const adapterType = req.params.type;
@@ -635,6 +644,7 @@ export function adapterRoutes(options: {
*/
router.post("/adapters/:type/reload", async (req, res) => {
assertInstanceAdmin(req);
assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
const type = req.params.type;
@@ -688,7 +698,7 @@ export function adapterRoutes(options: {
// package name, but without the risk of losing the store record.
router.post("/adapters/:type/reinstall", async (req, res) => {
assertInstanceAdmin(req);
assertAdapterCodeInstallAllowed();
assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
const type = req.params.type;
+2 -11
View File
@@ -154,7 +154,7 @@ import type {
} from "@paperclipai/adapter-utils";
import { evaluateCodexCredentialReadiness } from "@paperclipai/adapter-codex-local/server";
import type { AdapterAuthSignal, AdapterAuthSignalResponse, CodexAccountBindingClaim } from "@paperclipai/shared";
import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js";
import { adapterSettingsService } from "../services/adapter-settings.js";
import { skillVersionSelectionMap } from "../services/runtime-skill-selections.js";
import { isFixedClaudeOAuthBinding, secretService } from "../services/secrets.js";
import { authorizationDeniedDetails } from "../services/authorization.js";
@@ -1859,15 +1859,6 @@ export function agentRoutes(
* Adapter validation for the paths that CHOOSE a harness for a new agent
* (hire + create), as opposed to the paths that operate on an existing one.
*
* A disabled adapter is one this instance cannot run — most often because a
* declarative registry (PAPERCLIP_ADAPTERS) curated it out, which
* reconcileAdapterAvailability turns into a disabled type at boot. Registered
* but disabled still passes assertKnownAdapterType, so an agent could be
* created on it and then fail EVERY run at lease time with
* `Adapter "..." is not in the configured adapter registry` — an error that
* arrives minutes later, in a run log, with no way back to the choice that
* caused it. Refuse at selection time instead, and name what can be chosen.
*
* Existing agents on a now-disabled adapter are deliberately untouched
* (listEnabledServerAdapters documents the same rule: hidden from selection,
* still functional for agents that already use them).
@@ -1902,7 +1893,7 @@ export function agentRoutes(
);
}
}
const disabled = new Set(getDisabledAdapterTypes());
const disabled = new Set(await adapterSettingsService(db).getDisabledTypes());
if (!disabled.has(adapterType)) return adapterType;
const available = listServerAdapters()
.map((a) => a.type)
+12 -7
View File
@@ -32,6 +32,7 @@ import path from "node:path";
import fs from "node:fs";
import crypto from "node:crypto";
import type { Db } from "@paperclipai/db";
import type { PluginRecord } from "@paperclipai/shared";
import { pluginRegistryService } from "../services/plugin-registry.js";
import { logger } from "../middleware/logger.js";
import { assertCompanyAccess } from "./authz.js";
@@ -192,6 +193,7 @@ export interface PluginUiStaticRouteOptions {
* Defaults to the standard `~/.paperclip/plugins/` location.
*/
localPluginDir: string;
resolvePackageRoot?: (plugin: PluginRecord) => string;
}
/**
@@ -304,7 +306,7 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions
if (typeof devUiUrl === "string" && devUiUrl.length > 0) {
// Dev proxy is only available in development mode
if (process.env.NODE_ENV === "production") {
if (options.resolvePackageRoot || process.env.NODE_ENV === "production") {
log.warn(
{ pluginId: plugin.id },
"plugin-ui-static: devUiUrl ignored in production",
@@ -405,12 +407,15 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions
}
// Step 3: Resolve the plugin's UI directory
const uiDir = resolvePluginUiDir(
options.localPluginDir,
plugin.packageName,
manifest.entrypoints.ui,
plugin.packagePath,
);
let uiDir: string | null;
try {
uiDir = options.resolvePackageRoot
? path.resolve(options.resolvePackageRoot(plugin), manifest.entrypoints.ui)
: resolvePluginUiDir(options.localPluginDir, plugin.packageName, manifest.entrypoints.ui, plugin.packagePath);
} catch {
res.status(503).json({ error: "Plugin package is unavailable in the current image" });
return;
}
if (!uiDir) {
log.warn(
@@ -0,0 +1,10 @@
import { forbidden } from "../errors.js";
import { isCloudManagedInstance } from "./cloud-instance.js";
export function assertExternalAdapterCodeAllowed(): void {
if (isCloudManagedInstance()) {
throw forbidden("Adapter installation is platform-managed on cloud-managed instances", {
code: "adapter_install_platform_managed",
});
}
}
+12 -2
View File
@@ -15,6 +15,8 @@
import fs from "node:fs";
import path from "node:path";
import { resolvePaperclipHomeDir } from "../home-paths.js";
import { isCloudManagedInstance } from "./cloud-instance.js";
import { assertExternalAdapterCodeAllowed } from "./adapter-code-policy.js";
// ---------------------------------------------------------------------------
// Types
@@ -43,8 +45,8 @@ interface AdapterSettings {
// Paths
// ---------------------------------------------------------------------------
function adapterPluginPaths() {
const paperclipDir = resolvePaperclipHomeDir();
function adapterPluginPaths(homeOverride?: string) {
const paperclipDir = resolvePaperclipHomeDir(homeOverride);
return {
adapterPluginsDir: path.join(paperclipDir, "adapter-plugins"),
adapterPluginsStorePath: path.join(paperclipDir, "adapter-plugins.json"),
@@ -79,6 +81,7 @@ function ensureDirs(): string {
}
function readStore(): AdapterPluginRecord[] {
if (isCloudManagedInstance()) return [];
const { adapterPluginsStorePath } = adapterPluginPaths();
if (storeCache?.path === adapterPluginsStorePath) return storeCache.records;
try {
@@ -130,11 +133,16 @@ function writeSettings(settings: AdapterSettings): void {
// Public API
// ---------------------------------------------------------------------------
export function getAdapterPluginStorePath(homeOverride?: string): string {
return adapterPluginPaths(homeOverride).adapterPluginsStorePath;
}
export function listAdapterPlugins(): AdapterPluginRecord[] {
return readStore();
}
export function addAdapterPlugin(record: AdapterPluginRecord): void {
assertExternalAdapterCodeAllowed();
const store = [...readStore()];
const idx = store.findIndex((r) => r.type === record.type);
if (idx >= 0) {
@@ -146,6 +154,7 @@ export function addAdapterPlugin(record: AdapterPluginRecord): void {
}
export function removeAdapterPlugin(type: string): boolean {
assertExternalAdapterCodeAllowed();
const store = [...readStore()];
const idx = store.findIndex((r) => r.type === type);
if (idx < 0) return false;
@@ -159,6 +168,7 @@ export function getAdapterPluginByType(type: string): AdapterPluginRecord | unde
}
export function getAdapterPluginsDir(): string {
assertExternalAdapterCodeAllowed();
return ensureDirs();
}
@@ -32,8 +32,7 @@ describe("reconcileAdapterAvailability", () => {
]);
expect(result.enabled).toEqual(["opencode_local"]);
expect(result.disabled.sort()).toEqual(["claude_local", "pi_local"]);
expect(setAdapterDisabled).toHaveBeenCalledWith("claude_local", true);
expect(setAdapterDisabled).toHaveBeenCalledWith("opencode_local", false);
expect(setAdapterDisabled).not.toHaveBeenCalled();
});
it("throws when a declared adapter has no installed implementation", () => {
@@ -1,73 +1,8 @@
/**
* Declarative adapter-registry bootstrap.
*
* One source (`PAPERCLIP_ADAPTERS` inline JSON, or `PAPERCLIP_ADAPTERS_FILE` a
* path to a JSON file) feeds two consumers:
* 1. Availability: reconcile the file-backed disabled-set so the picker shows
* exactly the declared, enabled set (runs for any instance).
* 2. k8s runtime: the same registry rides on the Kubernetes environment config
* (see execution-policy-bootstrap) so the plugin resolves runtime
* image/envKeys/allowFqdns/probe/defaultEnv from it.
*
* Parsing is pure + fails loud on malformed/invalid config, mirroring
* execution-policy-bootstrap.
*/
import fs from "node:fs";
import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared";
import type { AdapterRegistryEntryParsed } from "@paperclipai/shared";
import { logger } from "../middleware/logger.js";
import { listServerAdapters } from "../adapters/registry.js";
import { setAdapterDisabled } from "./adapter-plugin-store.js";
export { parseAdapterRegistryEnv, type AdapterRegistryEnv } from "./adapter-registry-config.js";
export type AdapterRegistryEnv = Record<string, string | undefined>;
/**
* Parse the declarative registry from env. Returns null when unconfigured
* (built-in defaults; local/OSS unchanged). Throws on malformed/invalid input.
*/
export function parseAdapterRegistryEnv(
env: AdapterRegistryEnv = process.env,
): AdapterRegistryEntryParsed[] | null {
const inline = env.PAPERCLIP_ADAPTERS?.trim();
const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim();
if (!inline && !filePath) return null;
let rawText: string;
if (inline) {
rawText = inline;
} else {
try {
rawText = fs.readFileSync(filePath as string, "utf-8");
} catch (err) {
throw new Error(
`PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`,
);
}
}
let parsed: unknown;
try {
parsed = JSON.parse(rawText);
} catch (err) {
throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`);
}
const result = adapterRegistrySchema.safeParse(parsed);
if (!result.success) {
throw new Error(
`PAPERCLIP_ADAPTERS failed validation: ${result.error.issues
.map((i) => `${i.path.join(".")}: ${i.message}`)
.join("; ")}`,
);
}
return result.data;
}
/**
* Reconcile availability: every known server adapter NOT enabled in the declared
* registry is disabled; declared+enabled ones are enabled. Throws if a declared
* adapterType has no installed adapter (cannot offer a harness with no
* implementation). No-op when `registry` is null.
*/
export function reconcileAdapterAvailability(
registry: AdapterRegistryEntryParsed[] | null,
): { enabled: string[]; disabled: string[] } {
@@ -88,10 +23,9 @@ export function reconcileAdapterAvailability(
for (const type of knownTypes) {
const entry = declared.get(type);
const shouldEnable = entry !== undefined && entry.enabled !== false;
setAdapterDisabled(type, !shouldEnable);
(shouldEnable ? enabled : disabled).push(type);
}
logger.info({ enabled, disabled }, "reconciled adapter availability from PAPERCLIP_ADAPTERS");
logger.info({ enabled, disabled }, "validated adapter availability from PAPERCLIP_ADAPTERS");
return { enabled, disabled };
}
@@ -0,0 +1,47 @@
import fs from "node:fs";
import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared";
export type AdapterRegistryEnv = Record<string, string | undefined>;
let cached: { inline: string | undefined; filePath: string | undefined; registry: AdapterRegistryEntryParsed[] } | undefined;
export function parseAdapterRegistryEnv(
env: AdapterRegistryEnv = process.env,
): AdapterRegistryEntryParsed[] | null {
const inline = env.PAPERCLIP_ADAPTERS?.trim();
const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim();
if (!inline && !filePath) return null;
if (cached && cached.inline === inline && cached.filePath === filePath) return structuredClone(cached.registry);
let rawText: string;
if (inline) {
rawText = inline;
} else {
try {
rawText = fs.readFileSync(filePath as string, "utf-8");
} catch (err) {
throw new Error(
`PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`,
);
}
}
let parsed: unknown;
try {
parsed = JSON.parse(rawText);
} catch (err) {
throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`);
}
const result = adapterRegistrySchema.safeParse(parsed);
if (!result.success) {
throw new Error(
`PAPERCLIP_ADAPTERS failed validation: ${result.error.issues
.map((i) => `${i.path.join(".")}: ${i.message}`)
.join("; ")}`,
);
}
cached = { inline, filePath, registry: result.data };
return structuredClone(result.data);
}
@@ -0,0 +1,87 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("../adapters/registry.js", () => ({ listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }] }));
import { adapterSettingsService, type AdapterSettingsDb } from "./adapter-settings.js";
import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js";
import { parseAdapterRegistryEnv } from "./adapter-registry-config.js";
describe("local adapter settings compatibility", () => {
let directory: string;
beforeEach(() => {
directory = fs.mkdtempSync(path.join(os.tmpdir(), "adapter-settings-local-"));
vi.stubEnv("PAPERCLIP_HOME", directory);
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
vi.stubEnv("PAPERCLIP_ADAPTERS", undefined);
vi.stubEnv("PAPERCLIP_ADAPTERS_FILE", undefined);
});
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllEnvs();
fs.rmSync(directory, { recursive: true, force: true });
});
it("reads and writes the existing file without a database", async () => {
fs.writeFileSync(path.join(directory, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local"] }));
const service = adapterSettingsService(undefined);
expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
expect(await service.setDisabled("codex_local", true)).toBe(true);
expect(getDisabledAdapterTypes()).toEqual(["claude_local", "codex_local"]);
expect(setAdapterDisabled("claude_local", false)).toBe(true);
expect(await service.getDisabledTypes()).toEqual(["codex_local"]);
expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: ["codex_local"] });
});
it("loads an operator file once and leaves the saved file intact", async () => {
setAdapterDisabled("codex_local", true);
const file = path.join(directory, "operator-adapters.json");
fs.writeFileSync(file, JSON.stringify([{ adapterType: "codex_local", enabled: true }]));
const env = { PAPERCLIP_ADAPTERS_FILE: file };
parseAdapterRegistryEnv(env);
const read = vi.spyOn(fs, "readFileSync");
const service = adapterSettingsService(undefined, { runtimeEnv: env });
expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
expect(read).not.toHaveBeenCalled();
expect(getDisabledAdapterTypes()).toEqual(["codex_local"]);
});
it("restores the saved choice when its audit fails and audits a later retry", async () => {
const transaction = vi.fn(async (callback: (tx: AdapterSettingsDb) => Promise<unknown>) => callback(db));
const db = { transaction } as unknown as AdapterSettingsDb;
const service = adapterSettingsService(db);
const audit = vi.fn().mockRejectedValueOnce(new Error("audit unavailable")).mockResolvedValue(undefined);
await expect(service.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable");
expect(await service.getDisabledTypes()).toEqual([]);
expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: [] });
expect(await service.setDisabled("codex_local", true, audit)).toBe(true);
expect(await service.getDisabledTypes()).toEqual(["codex_local"]);
expect(audit).toHaveBeenCalledTimes(2);
expect(transaction).toHaveBeenCalledTimes(2);
});
it("keeps a failed audit rollback from overwriting a concurrent save", async () => {
const db = { transaction: async (callback: (tx: AdapterSettingsDb) => Promise<unknown>) => callback(db) } as unknown as AdapterSettingsDb;
const first = adapterSettingsService(db);
const second = adapterSettingsService(db);
let failAudit!: (reason: Error) => void;
const auditWait = new Promise<void>((_resolve, reject) => { failAudit = reject; });
const audit = vi.fn(() => auditWait);
const rejected = expect(first.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable");
await vi.waitFor(() => expect(audit).toHaveBeenCalledOnce());
const nextAudit = vi.fn(async () => undefined);
const next = second.setDisabled("codex_local", true, nextAudit);
await Promise.resolve();
expect(nextAudit).not.toHaveBeenCalled();
failAudit(new Error("audit unavailable"));
await rejected;
expect(await next).toBe(true);
expect(await first.getDisabledTypes()).toEqual(["codex_local"]);
expect(nextAudit).toHaveBeenCalledOnce();
});
});
@@ -0,0 +1,228 @@
import express from "express";
import request from "supertest";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { activityLog, applyPendingMigrations, companies, createDb, instanceSettings, type Db } from "@paperclipai/db";
import { eq, sql } from "drizzle-orm";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
const legacy = vi.hoisted(() => ({ get: vi.fn(() => [] as string[]), set: vi.fn() }));
vi.mock("./adapter-plugin-store.js", () => ({
getDisabledAdapterTypes: legacy.get,
setAdapterDisabled: legacy.set,
listAdapterPlugins: () => [],
getAdapterPluginByType: () => undefined,
}));
vi.mock("../adapters/registry.js", () => ({
listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }, { type: "paperclip_runner" }],
findServerAdapter: (type: string) => ({ type }),
findActiveServerAdapter: (type: string) => ({ type }),
isOverridePaused: () => false,
hasActiveAdapterOverride: () => false,
waitForExternalAdapters: async () => {},
}));
vi.mock("../adapters/plugin-loader.js", () => ({}));
import { adapterSettingsService, initializeAdapterSettings } from "./adapter-settings.js";
import { adapterRoutes } from "../routes/adapters.js";
import { errorHandler } from "../middleware/error-handler.js";
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
import * as activities from "./activity-log.js";
const externalDatabaseUrl = process.env.PAPERCLIP_ADAPTER_SETTINGS_TEST_DATABASE_URL;
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("durable managed adapter settings", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: Db;
let peer: Db;
let directory: string;
const managed = { runtimeEnv: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "synthetic-test-token" } };
const service = () => adapterSettingsService(db, managed);
beforeAll(async () => {
database = externalDatabaseUrl
? { connectionString: externalDatabaseUrl, cleanup: async () => {} }
: await startEmbeddedPostgresTestDatabase("adapter-settings-");
if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl);
db = createDb(database.connectionString);
peer = createDb(database.connectionString);
await db.insert(companies).values({ name: "Adapter settings test", issuePrefix: "AST" });
}, 90_000);
afterAll(async () => {
await Promise.all([db?.$client.end({ timeout: 0 }), peer?.$client.end({ timeout: 0 })]);
await database?.cleanup();
}, 60_000);
beforeEach(async () => {
directory = await fs.mkdtemp(path.join(os.tmpdir(), "adapter-settings-import-"));
await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, "adapter-settings:v1"));
legacy.get.mockClear();
legacy.set.mockClear();
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "synthetic-test-token");
});
afterEach(async () => {
vi.restoreAllMocks();
vi.unstubAllEnvs();
await fs.rm(directory, { recursive: true, force: true });
});
function app() {
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
req.actor = { type: "board", source: "local_implicit", userId: "local-board", isInstanceAdmin: true };
next();
});
app.use(adapterRoutes({ db }));
app.use(errorHandler);
return app;
}
it("persists across independent connections and reconstructed stores without file reads", async () => {
await expect(service().getDisabledTypes()).rejects.toThrow("not been initialized");
await service().seedIfAbsent({ disabledTypes: [] });
expect(await service().setDisabled("codex_local", true)).toBe(true);
expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]);
await db.$client.end({ timeout: 0 });
db = createDb(database.connectionString);
expect(await service().inspectStored()).toEqual({ version: 1, revision: 1, disabledTypes: ["codex_local"] });
expect(legacy.get).not.toHaveBeenCalled();
expect(legacy.set).not.toHaveBeenCalled();
});
it("acknowledges the committed save when its live activity event fails", async () => {
await service().seedIfAbsent({ disabledTypes: [] });
const publication = vi.spyOn(activities, "publishActivity").mockImplementation(() => { throw new Error("event unavailable"); });
const response = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
expect(response.status, JSON.stringify(response.body)).toBe(200);
expect(publication).toHaveBeenCalled();
expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]);
const events = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated"));
expect(events.some((event) => event.details?.adapterType === "codex_local")).toBe(true);
});
it("preserves simultaneous first writes and concurrent removals from other connections", async () => {
const writers = [service(), adapterSettingsService(peer, managed)];
await Promise.all(Array.from({ length: 12 }, (_, index) => writers[index % 2].setDisabled(`adapter_${index}`, true)));
expect((await service().inspectStored())?.disabledTypes).toHaveLength(12);
await Promise.all([
writers[0].setDisabled("adapter_0", false),
writers[1].setDisabled("adapter_1", false),
writers[0].setDisabled("extra", true),
]);
const stored = await service().inspectStored();
expect(stored?.revision).toBe(15);
expect(stored?.disabledTypes).not.toContain("adapter_0");
expect(stored?.disabledTypes).not.toContain("adapter_1");
expect(stored?.disabledTypes).toContain("extra");
expect(await writers[1].setDisabled("extra", true)).toBe(false);
expect((await service().inspectStored())?.revision).toBe(15);
});
it("imports legacy choices once before admission and ignores stale files after the DB record exists", async () => {
const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory };
const file = path.join(directory, "adapter-settings.json");
await fs.writeFile(file, JSON.stringify({ disabledTypes: ["codex_local", "future_adapter"] }));
const results = await Promise.all([
initializeAdapterSettings(db, { runtimeEnv }),
initializeAdapterSettings(peer, { runtimeEnv }),
]);
expect(results.filter((result) => result?.seeded)).toHaveLength(1);
expect(await service().getDisabledTypes()).toEqual(["codex_local", "future_adapter"]);
await service().setDisabled("codex_local", false);
await fs.writeFile(file, "invalid stale JSON");
expect((await initializeAdapterSettings(peer, { runtimeEnv }))?.seeded).toBe(false);
expect(await service().getDisabledTypes()).toEqual(["future_adapter"]);
});
it("uses defaults only for an absent legacy file and rejects malformed or oversized input", async () => {
const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory };
const file = path.join(directory, "adapter-settings.json");
await fs.writeFile(file, JSON.stringify({ disabledTypes: [42] }));
await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("Persisted adapter settings are invalid");
expect(await service().inspectStored()).toBeNull();
await fs.writeFile(file, Buffer.alloc(65_537));
await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("size limit");
expect(await service().inspectStored()).toBeNull();
await fs.unlink(file);
expect((await initializeAdapterSettings(db, { runtimeEnv }))?.current.disabledTypes).toEqual([]);
expect(await service().getDisabledTypes()).toEqual([]);
});
it("composes atomic import and receipts with the caller transaction", async () => {
await expect(db.transaction(async (tx) => {
expect(await service().seedIfAbsent({ disabledTypes: ["codex_local"] }, tx)).toEqual({
seeded: true, current: { version: 1, revision: 0, disabledTypes: ["codex_local"] },
});
throw new Error("receipt failed");
})).rejects.toThrow("receipt failed");
expect(await service().inspectStored()).toBeNull();
await Promise.all([
service().seedIfAbsent({ disabledTypes: ["codex_local"] }),
adapterSettingsService(peer, managed).seedIfAbsent({ disabledTypes: ["claude_local"] }),
]).then((results) => {
expect(results.filter((result) => result.seeded)).toHaveLength(1);
expect(results[0].current).toEqual(results[1].current);
});
});
it("applies operator restrictions without rewriting saved choices", async () => {
await service().setDisabled("codex_local", true);
const env = { ...managed.runtimeEnv, PAPERCLIP_ADAPTERS: JSON.stringify([{ adapterType: "codex_local", enabled: true }]) };
const overlay = adapterSettingsService(peer, { runtimeEnv: env });
expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]);
expect(await overlay.inspectStored()).toEqual(await service().inspectStored());
await expect(overlay.setDisabled("claude_local", false)).rejects.toThrow("hosting operator");
await expect(overlay.setDisabled("codex_local", true)).rejects.toThrow("hosting operator");
expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]);
expect(await service().getDisabledTypes()).toEqual(["codex_local"]);
});
it("refreshes the disabled harness at each new-agent admission", async () => {
await service().seedIfAbsent({ disabledTypes: [] });
const input = { adapterType: "codex_local", runner: "legacy" as const };
expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local");
await adapterSettingsService(peer, managed).setDisabled("codex_local", true);
await expect(resolveNewAgentRunnerForCompany(db, "company", input)).rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } });
await adapterSettingsService(peer, managed).setDisabled("codex_local", false);
expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local");
});
it("rolls back the setting when its audit write fails", async () => {
await expect(service().setDisabled("codex_local", true, async () => { throw new Error("audit failed"); })).rejects.toThrow("audit failed");
expect(await service().inspectStored()).toBeNull();
await expect(adapterSettingsService(peer, managed).getDisabledTypes()).rejects.toThrow("not been initialized");
});
it("publishes catalog changes across stores and records the instance mutation", async () => {
const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
expect(res.status).toBe(200);
expect(res.body).toEqual({ type: "codex_local", disabled: true, changed: true });
const catalog = await request(app()).get("/adapters");
expect(catalog.status).toBe(200);
expect(catalog.body.find((entry: { type: string }) => entry.type === "codex_local").disabled).toBe(true);
const [audit] = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated"));
expect(audit.details).toMatchObject({ adapterType: "codex_local", disabled: true, revision: 1 });
});
it("returns failure when PostgreSQL rejects the update and keeps the durable choice", async () => {
await service().seedIfAbsent({ disabledTypes: [] });
await db.execute(sql`CREATE FUNCTION reject_adapter_settings() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test adapter settings failure'; END $$`);
await db.execute(sql`CREATE TRIGGER reject_adapter_settings BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_adapter_settings()`);
try {
const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
expect(res.status).toBe(500);
expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual([]);
} finally {
await db.execute(sql`DROP TRIGGER reject_adapter_settings ON instance_settings`);
await db.execute(sql`DROP FUNCTION reject_adapter_settings()`);
}
});
it("fails closed on malformed stored settings and unavailable database", async () => {
await db.insert(instanceSettings).values({ singletonKey: "adapter-settings:v1", general: { version: 2, disabledTypes: [] } });
await expect(service().getDisabledTypes()).rejects.toThrow("Persisted adapter settings are invalid");
await expect(adapterSettingsService(undefined, managed).getDisabledTypes()).rejects.toThrow("database is required");
expect(legacy.get).not.toHaveBeenCalled();
});
});
+149
View File
@@ -0,0 +1,149 @@
import type { Db } from "@paperclipai/db";
import { instanceSettings } from "@paperclipai/db";
import { eq } from "drizzle-orm";
import { open } from "node:fs/promises";
import { constants } from "node:fs";
import path from "node:path";
import { z } from "zod";
import { listServerAdapters } from "../adapters/registry.js";
import { forbidden } from "../errors.js";
import { resolvePaperclipHomeDir } from "../home-paths.js";
import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js";
import { parseAdapterRegistryEnv } from "./adapter-registry-config.js";
import { isCloudManagedInstance } from "./cloud-instance.js";
const SINGLETON_KEY = "adapter-settings:v1";
let localSettingsMutation: Promise<unknown> = Promise.resolve();
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
export type AdapterSettingsDb = Pick<Db | Transaction, "select" | "insert" | "update" | "transaction">;
export type StoredAdapterSettings = { version: 1; revision: number; disabledTypes: string[] };
const storedSettingsSchema = z.object({
version: z.literal(1),
revision: z.number().int().nonnegative(),
disabledTypes: z.array(z.string().refine((type) => type.trim().length > 0)),
});
function parseStored(value: Record<string, unknown>): StoredAdapterSettings {
const parsed = storedSettingsSchema.safeParse(value);
if (!parsed.success) throw new Error("Persisted adapter settings are invalid");
return { ...parsed.data, disabledTypes: [...new Set(parsed.data.disabledTypes)].sort() };
}
export function effectiveDisabledAdapterTypes(saved: string[], knownTypes: string[] | (() => string[]), env: Record<string, string | undefined> = process.env): string[] {
const registry = parseAdapterRegistryEnv(env);
if (!registry) return [...saved];
const enabled = new Set(registry.filter((entry) => entry.enabled !== false).map((entry) => entry.adapterType));
const types = typeof knownTypes === "function" ? knownTypes() : knownTypes;
return [...new Set(types.filter((type) => !enabled.has(type)))].sort();
}
export function adapterSettingsService(db: AdapterSettingsDb | undefined, options: { runtimeEnv?: Record<string, string | undefined> } = {}) {
const env = options.runtimeEnv ?? process.env;
const managed = isCloudManagedInstance(env);
function requireDb(): AdapterSettingsDb {
if (!db) throw new Error("A database is required for managed adapter settings");
return db;
}
async function inspectStored(handle: AdapterSettingsDb = requireDb()): Promise<StoredAdapterSettings | null> {
const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings)
.where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).limit(1);
return row ? parseStored(row.general) : null;
}
async function seedIfAbsent(value: { disabledTypes: string[] }, handle: AdapterSettingsDb = requireDb()) {
const snapshot = parseStored({ version: 1, revision: 0, disabledTypes: value.disabledTypes });
const inserted = await handle.insert(instanceSettings).values({ singletonKey: SINGLETON_KEY, general: snapshot })
.onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
const current = await inspectStored(handle);
if (!current) throw new Error("Adapter settings seed was not persisted");
return { seeded: inserted.length > 0, current };
}
return {
inspectStored,
seedIfAbsent,
async getDisabledTypes(): Promise<string[]> {
const current = managed ? await inspectStored() : null;
if (managed && !current) throw new Error("Managed adapter settings have not been initialized");
const stored = managed ? current!.disabledTypes : getDisabledAdapterTypes();
return effectiveDisabledAdapterTypes(stored, () => listServerAdapters().map((adapter) => adapter.type), env);
},
async setDisabled(type: string, disabled: boolean, afterChange?: (handle: AdapterSettingsDb, current: StoredAdapterSettings) => Promise<void>): Promise<boolean> {
if (!type.trim()) throw new Error("Adapter type is required");
const registry = parseAdapterRegistryEnv(env);
if (registry && disabled === registry.some((entry) => entry.adapterType === type && entry.enabled !== false)) {
throw forbidden("Adapter availability is managed by the hosting operator", { code: "settings_operator_managed" });
}
if (!managed) {
const mutation = localSettingsMutation.then(async () => {
const previous = getDisabledAdapterTypes().includes(type);
const changed = setAdapterDisabled(type, disabled);
if (!changed || !db || !afterChange) return changed;
try {
await db.transaction(async (tx) => {
await afterChange(tx, { version: 1, revision: 0, disabledTypes: getDisabledAdapterTypes() });
});
} catch (error) {
setAdapterDisabled(type, previous);
throw error;
}
return true;
});
localSettingsMutation = mutation.catch(() => undefined);
return mutation;
}
return requireDb().transaction(async (tx) => {
await seedIfAbsent({ disabledTypes: [] }, tx);
const [row] = await tx.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).for("update");
if (!row) throw new Error("Adapter settings are unavailable");
const current = parseStored(row.general);
const types = new Set(current.disabledTypes);
if (types.has(type) === disabled) return false;
if (disabled) types.add(type); else types.delete(type);
const next: StoredAdapterSettings = { version: 1, revision: current.revision + 1, disabledTypes: [...types].sort() };
await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id));
await afterChange?.(tx, next);
return true;
});
},
};
}
export type AdapterSettingsService = Pick<ReturnType<typeof adapterSettingsService>, "getDisabledTypes" | "setDisabled">;
export async function initializeAdapterSettings(db: AdapterSettingsDb, options: { runtimeEnv?: Record<string, string | undefined> } = {}) {
const env = options.runtimeEnv ?? process.env;
if (!isCloudManagedInstance(env)) return null;
const service = adapterSettingsService(db, options);
const current = await service.inspectStored();
if (current) return { seeded: false, current };
return service.seedIfAbsent(await readLegacyAdapterSettings(env) ?? { disabledTypes: [] });
}
export async function readLegacyAdapterSettings(env: Record<string, string | undefined> = process.env): Promise<{ disabledTypes: string[] } | null> {
const filePath = path.join(resolvePaperclipHomeDir(env.PAPERCLIP_HOME), "adapter-settings.json");
try {
const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
try {
const stat = await file.stat();
if (!stat.isFile() || stat.size > 65_536) throw new Error("Legacy adapter settings exceed the size limit or are not a regular file");
const buffer = Buffer.alloc(65_537);
let total = 0;
while (total < buffer.length) {
const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
if (bytesRead === 0) break;
total += bytesRead;
}
if (total > 65_536) throw new Error("Legacy adapter settings exceed the size limit");
const value = JSON.parse(buffer.subarray(0, total).toString("utf8"));
return { disabledTypes: parseStored({ version: 1, revision: 0, disabledTypes: value?.disabledTypes }).disabledTypes };
} finally {
await file.close();
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
return null;
}
}
@@ -3,6 +3,10 @@ import { agentHarnessType, agentRunner, paperclipRunnerProfileForHarness, paperc
const state = vi.hoisted(() => ({ disabled: [] as string[], overridden: new Set<string>(), settings: {} as Record<string, unknown>, getEnvironment: vi.fn(), bindings: vi.fn(), activeTemplate: vi.fn().mockResolvedValue(null), secretPaths: vi.fn().mockResolvedValue(new Set(["apiKey"])), managedEnvironment: vi.fn(), kubernetesEnvironment: vi.fn(), ensureKubernetesEnvironment: vi.fn(), ssh: vi.fn(), resolveEnvironment: vi.fn(), sandboxQualification: vi.fn(), pluginByKey: vi.fn() }));
vi.mock("../adapters/registry.js", () => ({ findActiveServerAdapter: (type: string) => ({ type }), hasActiveAdapterOverride: (type: string) => state.overridden.has(type), waitForExternalAdapters: async () => {} }));
vi.mock("./adapter-plugin-store.js", () => ({ getDisabledAdapterTypes: () => state.disabled }));
vi.mock("./adapter-settings.js", async (original) => ({
...await original(),
adapterSettingsService: () => ({ getDisabledTypes: async () => state.disabled }),
}));
vi.mock("./instance-settings.js", () => ({ instanceSettingsService: () => ({ get: async () => state.settings }) }));
vi.mock("./environments.js", () => ({ environmentService: () => ({ getById: state.getEnvironment, listBoundCompanyIds: state.bindings, findManagedSandboxEnvironment: state.managedEnvironment, findKubernetesEnvironment: state.kubernetesEnvironment, ensureKubernetesEnvironment: state.ensureKubernetesEnvironment }) }));
vi.mock("./environment-custom-images.js", () => ({ environmentCustomImageService: () => ({ getActiveTemplate: state.activeTemplate }) }));
@@ -351,7 +355,7 @@ describe("server-owned Codex runner selection", () => {
state.settings = { general: { executionMode: "kubernetes" } };
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company");
expect(selected).toEqual({ driver: "sandbox" });
expect(agentRunnerAvailability("codex_local", selected)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
expect(agentRunnerAvailability("codex_local", selected, state.disabled)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
expect(state.kubernetesEnvironment).toHaveBeenCalledWith("company");
expect(state.ensureKubernetesEnvironment).not.toHaveBeenCalled();
expect(state.resolveEnvironment).not.toHaveBeenCalled();
@@ -477,14 +481,14 @@ describe("server-owned Codex runner selection", () => {
config: { provider: "daytona", snapshot: "replacement-snapshot" }, metadata: { managedByPaperclip: true } });
state.resolveEnvironment.mockRejectedValue(new Error("Saved provider secret is unavailable."));
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("paperclip");
expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("paperclip");
expect(await resolveNewAgentRunnerForCompany({} as never, "company", {
adapterType, defaultEnvironmentId: "sandbox", adapterConfig: { model: adapterType === "opencode_local" ? "provider/model" : undefined },
})).toMatchObject({ adapterType: "paperclip_runner" });
expect(state.sandboxQualification).not.toHaveBeenCalled();
expect(state.resolveEnvironment).not.toHaveBeenCalled();
state.disabled.push("paperclip_runner");
expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("legacy");
expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("legacy");
});
it.each(["self-hosted", "custom-environment", "captured-template", "stale-marker"])("keeps qualification for a %s sandbox", async kind => {
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", kind === "self-hosted" ? undefined : JSON.stringify({ ...managedCloudConfig,
@@ -495,7 +499,7 @@ describe("server-owned Codex runner selection", () => {
metadata: { managedByPaperclip: kind !== "custom-environment" } });
state.activeTemplate.mockResolvedValue(kind === "captured-template" ? { id: "template", provider: "daytona", templateRef: "custom-snapshot" } : null);
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
expect(agentRunnerAvailability("codex_local", selected).defaultRunner).toBe("legacy");
expect(agentRunnerAvailability("codex_local", selected, state.disabled).defaultRunner).toBe("legacy");
expect(state.sandboxQualification).toHaveBeenCalled();
});
it("ignores a stale captured image and preserves a matching custom image without resolving credentials", async () => {
@@ -507,17 +511,17 @@ describe("server-owned Codex runner selection", () => {
sourceEnvironmentConfigFingerprint: fingerprint({ ...config, snapshot: "previous-snapshot" }, { excludePaths: ["apiKey"] }) };
state.activeTemplate.mockResolvedValue(template);
const current = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
expect(agentRunnerAvailability("codex_local", current).defaultRunner).toBe("paperclip");
expect(agentRunnerAvailability("codex_local", current, state.disabled).defaultRunner).toBe("paperclip");
expect(await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "codex_local", runner: "paperclip", defaultEnvironmentId: "sandbox" }))
.toMatchObject({ adapterType: "paperclip_runner" });
expect(state.sandboxQualification).not.toHaveBeenCalled();
state.activeTemplate.mockResolvedValue({ ...template, sourceEnvironmentConfigFingerprint: fingerprint(config, { excludePaths: ["apiKey"] }) });
const custom = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
expect(agentRunnerAvailability("codex_local", custom).defaultRunner).toBe("legacy");
expect(agentRunnerAvailability("codex_local", custom, state.disabled).defaultRunner).toBe("legacy");
expect(state.sandboxQualification).toHaveBeenCalled();
state.getEnvironment.mockResolvedValue({ id: "sandbox", driver: "sandbox", status: "active", config: { ...config, timeoutMs: "invalid" }, metadata: { managedByPaperclip: true } });
const invalid = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
expect(agentRunnerAvailability("codex_local", invalid).defaultRunner).toBe("legacy");
expect(agentRunnerAvailability("codex_local", invalid, state.disabled).defaultRunner).toBe("legacy");
expect(state.resolveEnvironment).not.toHaveBeenCalled();
});
it("checks company access before trusting a managed Cloud image", async () => {
+23 -10
View File
@@ -5,6 +5,9 @@ import { resolveClaudeModel } from "@paperclipai/adapter-claude-local";
import { DEFAULT_GROK_LOCAL_MODEL } from "@paperclipai/adapter-grok-local";
import { findActiveServerAdapter, hasActiveAdapterOverride, waitForExternalAdapters } from "../adapters/registry.js";
import { getDisabledAdapterTypes } from "./adapter-plugin-store.js";
import { adapterSettingsService, effectiveDisabledAdapterTypes } from "./adapter-settings.js";
import { listServerAdapters } from "../adapters/registry.js";
import { isCloudManagedInstance } from "./cloud-instance.js";
import { claudeRunnerUnsupportedSettings, PaperclipRunnerProviderProfileError, resolvePaperclipRunnerNativeProviderInput } from "./native-runtime/provider-profile.js";
import { forbidden, unprocessable } from "../errors.js";
import { runSshCommand } from "@paperclipai/adapter-utils/ssh";
@@ -24,6 +27,7 @@ export type RunnerTarget = {
export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi";
export interface AgentRunnerSelection {
adapterType?: string | null;
disabledTypes?: string[];
adapterConfig?: Record<string, unknown>;
runner?: AgentRunnerChoice;
target?: RunnerTarget;
@@ -32,8 +36,13 @@ export interface AgentRunnerSelection {
onConfigurationWarnings?: (warnings: AgentConfigurationWarning[]) => void;
}
function runnerAdaptersAvailable(harness: string, driver: string): boolean {
const disabled = new Set(getDisabledAdapterTypes());
function localDisabledTypes(): string[] {
if (isCloudManagedInstance()) throw new Error("Managed runner selection requires a fresh adapter settings snapshot");
return effectiveDisabledAdapterTypes(getDisabledAdapterTypes(), () => listServerAdapters().map((adapter) => adapter.type));
}
function runnerAdaptersAvailable(harness: string, driver: string, disabledTypes: string[]): boolean {
const disabled = new Set(disabledTypes);
return !!paperclipRunnerProfileForHarness(harness)
&& !disabled.has(harness) && !disabled.has("paperclip_runner")
&& !!findActiveServerAdapter(harness) && !!findActiveServerAdapter("paperclip_runner")
@@ -41,13 +50,13 @@ function runnerAdaptersAvailable(harness: string, driver: string): boolean {
&& isEnvironmentDriverSupportedForAdapter("paperclip_runner", driver);
}
export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget()): AgentRunnerAvailability {
export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget(), disabledTypes: string[] = localDisabledTypes()): AgentRunnerAvailability {
if (target.runnerTasks || target.managedCloudImage) {
return runnerAdaptersAvailable(harness, target.driver)
return runnerAdaptersAvailable(harness, target.driver, disabledTypes)
? { supportedRunners: target.runnerTasks ? ["paperclip"] : ["paperclip", "legacy"], defaultRunner: "paperclip" }
: { supportedRunners: ["legacy"], defaultRunner: "legacy" };
}
const supported = runnerAdaptersAvailable(harness, target.driver)
const supported = runnerAdaptersAvailable(harness, target.driver, disabledTypes)
&& (target.driver !== "sandbox" || Boolean(target.platform && target.architecture
&& (target.platform !== "linux" || (target.libc === "glibc" && target.glibcVersion))))
&& (["claude_local", "grok_local"].includes(harness)
@@ -166,11 +175,12 @@ export function resolveNewAgentRunner(input: AgentRunnerSelection): { adapterTyp
function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string; adapterConfig: Record<string, unknown> } {
const adapterType = input.adapterType ?? "process";
let config = { ...input.adapterConfig };
const disabledTypes = input.disabledTypes ?? localDisabledTypes();
const harness = agentHarnessType(adapterType, config);
const native = adapterType === "paperclip_runner";
if (paperclipRunnerProfileForHarness(adapterType) && getDisabledAdapterTypes().includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" });
if (paperclipRunnerProfileForHarness(adapterType) && disabledTypes.includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" });
if (native && (input.runner === undefined || input.runner === "paperclip" || (harness !== "codex_local" && input.runner !== "legacy"))) return { adapterType, adapterConfig: normalizeLegacyRunnerProvider(config) };
const availability = agentRunnerAvailability(harness, input.target);
const availability = agentRunnerAvailability(harness, input.target, disabledTypes);
const choice = input.runner ?? "auto";
const useNative = choice === "paperclip" || (choice === "auto" && availability.defaultRunner === "paperclip");
if (!useNative) {
@@ -318,13 +328,14 @@ function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string;
}
export async function resolveNewAgentRunnerForCompany(db: Db, companyId: string, input: AgentRunnerSelection & { defaultEnvironmentId?: string | null }) {
input = { ...input, disabledTypes: await adapterSettingsService(db).getDisabledTypes() };
if (!paperclipRunnerProfileForHarness(agentHarnessType(input.adapterType ?? "process", input.adapterConfig))
|| input.runner === "legacy" || (input.adapterType === "paperclip_runner" && (input.runner === undefined || input.runner === "paperclip"))) return resolveNewAgentRunner(input);
const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId);
const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId, input.disabledTypes);
return resolveNewAgentRunner({ ...input, target });
}
export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null): Promise<RunnerTarget> {
export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null, disabledTypes?: string[]): Promise<RunnerTarget> {
await waitForExternalAdapters();
const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId);
let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget()
@@ -356,7 +367,9 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
const qualified = await readSandboxPlatformQualification(db, environment, companyId);
if (qualified) target = { driver, ...qualified };
}
if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver))) {
const disabledForSsh = driver === "ssh" && environment
? disabledTypes ?? await adapterSettingsService(db).getDisabledTypes() : [];
if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver, disabledForSsh))) {
const parsed = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment);
if (parsed.driver !== "ssh") throw unprocessable("The selected SSH environment is unavailable.");
let output: string;
@@ -0,0 +1,108 @@
import fs from "node:fs";
import path from "node:path";
import { isDeepStrictEqual } from "node:util";
import type { PaperclipPluginManifestV1, PluginRecord } from "@paperclipai/shared";
import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
import { assertDistributionManifestCapabilities } from "./distribution-plugin-catalog.js";
export type BundledPluginRegistration = Pick<PluginRecord, "pluginKey" | "packageName" | "version" | "manifestJson">;
export interface BundledPluginPackageIdentity {
kind: "bundled_image";
catalogKey: string;
pluginKey: string;
packageName: string;
version: string;
digest?: string;
}
export type BundledPluginPackageInspection =
| { compatible: true; identity: BundledPluginPackageIdentity }
| { compatible: false; pluginKey: string; code: "incompatible_bundled_package" | "missing_manifest_evidence"; reason: string };
function containedEntrypoint(root: string, declared: unknown, directory: boolean): string {
const relative = typeof declared === "string" ? declared.replace(/^\.\//, "").replace(/\/$/, "") : "";
if (!relative || path.isAbsolute(relative) || path.win32.isAbsolute(relative) || relative.includes("\\") || relative.split("/").length > 32 || relative.split("/").some((part) => !part || part === "." || part === "..")) {
throw new Error("Bundled plugin entrypoint must stay inside its package");
}
const target = fs.realpathSync(path.join(root, relative));
const resolvedRelative = path.relative(root, target);
if (!resolvedRelative || resolvedRelative.startsWith("..") || path.isAbsolute(resolvedRelative)) {
throw new Error("Bundled plugin entrypoint escapes its package");
}
const stat = fs.statSync(target);
if (directory ? !stat.isDirectory() : !stat.isFile()) throw new Error("Bundled plugin entrypoint is not prebuilt");
return relative;
}
export function createBundledPluginPackageResolver(installs: readonly ResolvedBundledPlugin[]) {
const byKey = new Map(installs.map((install) => [install.pluginKey, install]));
function read(install: ResolvedBundledPlugin) {
const root = fs.realpathSync(install.localPath);
if (root !== path.resolve(install.localPath)) throw new Error("Bundled plugin package root changed after catalog resolution");
containedEntrypoint(root, "package.json", false);
const metadataPath = path.join(root, "package.json");
if (fs.statSync(metadataPath).size > 128 * 1024) throw new Error("Bundled plugin package metadata exceeds verification limits");
const pkg = JSON.parse(fs.readFileSync(metadataPath, "utf8"));
if (typeof pkg.name !== "string" || typeof pkg.version !== "string" || !pkg.paperclipPlugin) {
throw new Error("Bundled plugin package identity or entrypoints are missing");
}
if (install.distribution && pkg.version !== install.distribution.version) throw new Error("Bundled plugin package version does not match the catalog");
const manifest = containedEntrypoint(root, pkg.paperclipPlugin.manifest, false);
const worker = containedEntrypoint(root, pkg.paperclipPlugin.worker, false);
const ui = pkg.paperclipPlugin.ui === undefined ? undefined : containedEntrypoint(root, pkg.paperclipPlugin.ui, true);
if (install.distribution && (worker !== install.distribution.entrypoints.worker || ui !== install.distribution.entrypoints.ui)) {
throw new Error("Bundled plugin entrypoints do not match the verified package");
}
return { root, pkg, manifest, worker, ui };
}
function assertManifest(install: ResolvedBundledPlugin, manifest: PaperclipPluginManifestV1, previousManifest?: PaperclipPluginManifestV1) {
const bundle = read(install);
if (manifest.id !== install.pluginKey || manifest.version !== bundle.pkg.version) throw new Error("Bundled plugin manifest identity/version does not match the image");
const worker = containedEntrypoint(bundle.root, manifest.entrypoints.worker, false);
const ui = manifest.entrypoints.ui === undefined ? undefined : containedEntrypoint(bundle.root, manifest.entrypoints.ui, true);
if (worker !== bundle.worker || ui !== bundle.ui) throw new Error("Bundled plugin manifest entrypoints do not match the image");
assertDistributionManifestCapabilities(manifest);
if (previousManifest) {
const approved = new Set(previousManifest.capabilities);
if (manifest.capabilities.some((capability) => !approved.has(capability))) throw new Error("Bundled plugin capabilities require approval");
}
return bundle;
}
function resolve(plugin: BundledPluginRegistration): string {
const install = byKey.get(plugin.pluginKey);
if (!install) throw new Error(`Bundled plugin is absent or not selected in this deployment: ${plugin.pluginKey}`);
const bundle = assertManifest(install, plugin.manifestJson);
if (plugin.packageName !== bundle.pkg.name || plugin.version !== bundle.pkg.version) throw new Error("Bundled plugin registration identity/version does not match the image");
return bundle.root;
}
return {
resolve,
inspect(plugin: BundledPluginRegistration, verifiedManifest?: PaperclipPluginManifestV1): BundledPluginPackageInspection {
try {
resolve(plugin);
const install = byKey.get(plugin.pluginKey)!;
if (!verifiedManifest) return { compatible: false, pluginKey: plugin.pluginKey, code: "missing_manifest_evidence", reason: "Image manifest evidence is required" };
assertManifest(install, verifiedManifest);
if (!isDeepStrictEqual(verifiedManifest, plugin.manifestJson)) throw new Error("Approved plugin manifest does not match the image");
return { compatible: true, identity: {
kind: "bundled_image", catalogKey: install.key, pluginKey: plugin.pluginKey,
packageName: plugin.packageName, version: plugin.version,
...(install.distribution ? { digest: install.distribution.digest } : {}),
} };
} catch (error) {
return { compatible: false, pluginKey: plugin.pluginKey, code: "incompatible_bundled_package", reason: error instanceof Error ? error.message : String(error) };
}
},
assertActivation(input: { pluginKey?: string; packageRoot: string; manifest?: PaperclipPluginManifestV1; previousManifest?: PaperclipPluginManifestV1 }) {
const install = input.pluginKey ? byKey.get(input.pluginKey) : installs.find((candidate) => candidate.localPath === input.packageRoot);
if (!install || path.resolve(input.packageRoot) !== install.localPath) throw new Error("Bundled plugin is absent or not selected in this deployment");
read(install);
if (input.manifest) assertManifest(install, input.manifest, input.previousManifest);
},
};
}
@@ -0,0 +1,37 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
export function resolveBundledPluginRuntimeLoader(
moduleDirectory = path.dirname(fileURLToPath(import.meta.url)),
): string {
let directory = fs.realpathSync(moduleDirectory);
let imageRoot: string | undefined;
for (let depth = 0; depth < 5; depth++) {
const metadataPath = path.join(directory, "package.json");
const metadata = fs.lstatSync(metadataPath, { throwIfNoEntry: false });
if (metadata?.isFile() && metadata.size <= 128 * 1024
&& JSON.parse(fs.readFileSync(metadataPath, "utf8")).name === "@paperclipai/server") {
imageRoot = path.dirname(directory);
break;
}
const parent = path.dirname(directory);
if (parent === directory) break;
directory = parent;
}
if (!imageRoot) throw new Error("Managed plugin server package is missing from the current image");
const declared = path.join(imageRoot, "cli/node_modules/tsx/dist/loader.mjs");
let loader: string;
try {
loader = fs.realpathSync(declared);
if (!fs.statSync(loader).isFile()) throw new Error("not a file");
} catch {
throw new Error("Managed plugin runtime loader is missing from the current image");
}
const roots = [path.join(imageRoot, "cli"), path.join(imageRoot, "node_modules")];
if (!roots.some((root) => {
const relative = path.relative(root, loader);
return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
})) throw new Error("Managed plugin runtime loader escapes the current image");
return loader;
}
+13 -4
View File
@@ -1,6 +1,7 @@
import path from "node:path";
import fs from "node:fs";
import { isDeepStrictEqual } from "node:util";
import { isCloudManagedInstance } from "./cloud-instance.js";
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { assertDistributionManifestCapabilities, readDistributionPluginCatalog, type DistributionPlugin } from "./distribution-plugin-catalog.js";
@@ -118,6 +119,13 @@ export const BUNDLED_PLUGIN_CATALOG: readonly BundledPluginCatalogEntry[] = [
*/
export const SELF_HOSTED_AUTO_INSTALL_KEYS: readonly string[] = ["kubernetes"];
export function resolveManagedBundledPluginKeys(
keys: readonly string[] | null | undefined,
env: Record<string, string | undefined> = process.env,
): readonly string[] | null {
return keys ?? (isCloudManagedInstance(env) ? [] : null);
}
export function resolveBundledCatalogRoot(
env: Record<string, string | undefined>,
): string {
@@ -203,7 +211,7 @@ export function resolveBundledPluginInstalls(
`bundled plugin "${key}" resolves to "${localPath}", outside the bundled catalog root "${opts.catalogRoot}"; refusing to start`,
);
}
resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath });
resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath: opts.enforceCatalogRoot ? canonicalize(localPath) : localPath });
}
return resolved;
}
@@ -269,6 +277,7 @@ async function reconcileBundledPluginManifest(
deps: BundledPluginProvisionerDeps,
bundleManifestExists: (localPath: string) => boolean,
verifiedManifest?: PaperclipPluginManifestV1,
preservePackagePath = false,
): Promise<"upgrade_pending" | undefined> {
try {
// Managed reinstalls take the install path instead; this branch means the
@@ -283,7 +292,7 @@ async function reconcileBundledPluginManifest(
const approved = new Set(existing.manifestJson.capabilities ?? []);
requiresApproval = bundleManifest.capabilities.some((capability) => !approved.has(capability));
}
const rebindPackage = install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled";
const rebindPackage = !preservePackagePath && install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled";
const refreshDistribution = install.distribution && !isDeepStrictEqual(bundleManifest, existing.manifestJson);
if (bundleManifest.version === existing.version && !rebindPackage && !requiresApproval && !refreshDistribution) return;
await deps.registry.update(existing.id, {
@@ -389,7 +398,7 @@ async function reenableErroredBundledPlugin(
export async function ensureBundledPlugins(
installs: readonly ResolvedBundledPlugin[],
deps: BundledPluginProvisionerDeps,
opts: { reinstallUninstalled: boolean },
opts: { reinstallUninstalled: boolean; preservePackagePath?: boolean },
): Promise<void> {
const bundleManifestExists = deps.bundleManifestExists ?? defaultBundleManifestExists;
for (const install of installs) {
@@ -411,7 +420,7 @@ export async function ensureBundledPlugins(
// existing install, because the auto-install below skips a present
// plugin. Distribution entries also replace a legacy/npm package path
// before loadAll resolves the worker. Configuration and status stay put.
const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest);
const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest, opts.preservePackagePath);
if (reconciledStatus === "upgrade_pending") continue;
if (existing.status === "error") {
await reenableErroredBundledPlugin(existing, install, deps);
@@ -0,0 +1,95 @@
import fs from "node:fs";
import path from "node:path";
import { createHash } from "node:crypto";
import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js";
export function cloudBundledPackageDigest(packageRoot: string, options: { catalogRoot?: string } = {}): string {
const root = fs.realpathSync(packageRoot);
const roots = [{ key: "package", directory: root }];
let imageNodeModules: string | undefined;
if (options.catalogRoot) {
const catalogRoot = fs.realpathSync(options.catalogRoot);
const relative = path.relative(catalogRoot, root);
if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("Image package is outside the selected catalog root");
imageNodeModules = path.resolve(catalogRoot, "../../node_modules");
roots.push(
{ key: "plugin-sdk", directory: path.join(catalogRoot, "sdk") },
{ key: "shared", directory: path.resolve(catalogRoot, "../shared") },
{ key: "image-node-modules", directory: imageNodeModules },
);
}
roots.sort((left, right) => right.directory.length - left.directory.length);
const inventory = new Map<string, [string, string, string, number]>();
const directories = new Set<string>();
const files = new Set<string>();
const buffer = Buffer.alloc(65_536);
let bytes = 0;
let entries = 0;
function logical(file: string): string {
for (const candidate of roots) {
const relative = path.relative(candidate.directory, file);
if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) {
if (relative.split(path.sep).length > 32) throw new Error("Image package exceeds the directory depth limit");
return `${candidate.key}/${relative.split(path.sep).join("/")}`;
}
}
throw new Error(options.catalogRoot ? "Image dependency escapes its authorized image roots" : "Image package symlink escapes its package");
}
function follow(target: string) {
logical(target);
if (imageNodeModules) {
const relative = path.relative(imageNodeModules, target).split(path.sep);
if ((relative.length === 1 && relative[0] === "")
|| (relative[0] === ".pnpm" && (relative.length === 1 || (relative.length === 2 && relative[1] === "node_modules")))) {
throw new Error("Image dependency must select an individual package");
}
if (relative[0] === ".pnpm" && relative.length > 2 && relative[1] && relative[1] !== "node_modules") {
walk(path.join(imageNodeModules, ".pnpm", relative[1]));
return;
}
}
walk(target);
}
function walk(file: string) {
const name = logical(file);
const stat = fs.lstatSync(file);
if (stat.isSymbolicLink()) {
const target = fs.realpathSync(file);
inventory.set(name, [name, "link", logical(target), stat.mode & 0o777]);
follow(target);
} else if (stat.isDirectory()) {
const real = fs.realpathSync(file);
logical(real);
if (directories.has(real)) return;
directories.add(real);
for (const child of fs.readdirSync(real).sort()) {
if (++entries > 50_000) throw new Error("Image package exceeds the entry limit");
walk(path.join(real, child));
}
} else if (stat.isFile()) {
const real = fs.realpathSync(file);
if (files.has(real)) return;
files.add(real);
if (path.basename(file) === "package.json" && stat.size > 128 * 1024) throw new Error("Image package metadata exceeds the byte limit");
if (bytes + stat.size > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit");
const handle = fs.openSync(file, "r");
const hash = createHash("sha256");
try {
while (true) {
const count = fs.readSync(handle, buffer);
if (!count) break;
bytes += count;
if (bytes > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit");
hash.update(buffer.subarray(0, count));
}
} finally { fs.closeSync(handle); }
inventory.set(name, [name, "file", `sha256:${hash.digest("hex")}`, stat.mode & 0o777]);
} else throw new Error("Image package contains a special file");
}
walk(root);
if (!inventory.size) throw new Error("Image package is empty");
return cloudConfigurationDigest([...inventory.values()].sort((left, right) => left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
}
@@ -0,0 +1,74 @@
import { createHash } from "node:crypto";
import { constants } from "node:fs";
import { open } from "node:fs/promises";
import { z } from "zod";
function canonical(value: unknown): unknown {
if (Array.isArray(value)) return value.map(canonical);
if (value && typeof value === "object") {
return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([key, child]) => [key, canonical(child)]));
}
return value;
}
export function cloudConfigurationDigest(value: unknown): string {
return `sha256:${createHash("sha256").update(JSON.stringify(canonical(value))).digest("hex")}`;
}
export const externalRegistrationEvidenceSchema = z.object({
sourceLocatorDigest: z.string(),
sourceDigest: z.string(),
state: z.enum(["missing", "parsed", "invalid", "unreadable"]),
adapterTypes: z.array(z.string()),
blocker: z.enum(["unsupported_external_adapters", "invalid_external_registration_metadata", "unreadable_external_registration_metadata"]).nullable(),
}).strict();
export type ExternalRegistrationEvidence = z.infer<typeof externalRegistrationEvidenceSchema>;
const legacyRegistrationSchema = z.array(z.object({
type: z.string().regex(/^[a-zA-Z0-9_.-]{1,128}$/),
packageName: z.string().min(1).max(1024),
version: z.string().max(128).optional(),
localPath: z.string().max(4096).optional(),
disabled: z.boolean().optional(),
}).passthrough()).max(100);
export async function inspectLegacyExternalRegistrations(filePath: string): Promise<ExternalRegistrationEvidence> {
const sourceLocatorDigest = cloudConfigurationDigest(filePath);
try {
const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
let raw: unknown;
try {
const stat = await file.stat();
if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid legacy registration file");
const buffer = Buffer.alloc(65_537);
let total = 0;
while (total < buffer.length) {
const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
if (!bytesRead) break;
total += bytesRead;
}
if (total > 65_536) throw new Error("oversized legacy registration file");
raw = JSON.parse(buffer.subarray(0, total).toString("utf8"));
} finally {
await file.close();
}
const records = legacyRegistrationSchema.safeParse(raw);
if (!records.success) {
return { sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state: "invalid" }), state: "invalid", adapterTypes: [], blocker: "invalid_external_registration_metadata" };
}
const metadata = records.data.map((record) => ({ type: record.type, packageName: record.packageName, version: record.version, disabled: record.disabled, hasLocalPath: record.localPath !== undefined }));
return {
sourceLocatorDigest, sourceDigest: cloudConfigurationDigest(metadata), state: "parsed",
adapterTypes: [...new Set(metadata.map((record) => record.type))].sort(),
blocker: metadata.length ? "unsupported_external_adapters" : null,
};
} catch (error) {
const absent = (error as NodeJS.ErrnoException).code === "ENOENT";
const invalid = error instanceof SyntaxError || (error instanceof Error && /legacy registration file/.test(error.message));
const state = absent ? "missing" : invalid ? "invalid" : "unreadable";
return {
sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state }), state, adapterTypes: [],
blocker: absent ? null : invalid ? "invalid_external_registration_metadata" : "unreadable_external_registration_metadata",
};
}
}
@@ -0,0 +1,193 @@
import { instanceSettings, plugins, type Db } from "@paperclipai/db";
import { eq, inArray, ne } from "drizzle-orm";
import { z } from "zod";
import { adapterSettingsService, readLegacyAdapterSettings } from "./adapter-settings.js";
import { getAdapterPluginStorePath } from "./adapter-plugin-store.js";
import { cloudRuntimeConfigService, readLegacyCloudRuntimeConfig } from "./cloud-runtime-config.js";
import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
import { cloudConfigurationDigest, externalRegistrationEvidenceSchema, inspectLegacyExternalRegistrations } from "./cloud-configuration-evidence.js";
import { cloudPackageEvidenceSchema, inspectCloudPackages, qualificationBlockerSchema } from "./cloud-package-qualification.js";
import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
const RECEIPT_KEY = "cloud-configuration-migration:v1";
const CONFIGURATION_KEYS = ["adapter-settings:v1", "cloud-runtime-config:v1"];
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
export type CloudConfigurationDb = Pick<Db | Transaction, "select" | "insert" | "update" | "transaction">;
const sourceSchema = z.object({
authority: z.enum(["database", "legacy_file", "default"]), sourceDigest: z.string(),
importedRevision: z.number().int().nonnegative(),
}).strict();
const qualificationSchema = z.object({
scope: z.literal("configuration_and_packages"), checkedAt: z.string(),
adapterRevision: z.number().int().nonnegative(), configRevision: z.number().int().nonnegative(),
configurationDigest: z.string(), packageEvidenceDigest: z.string(),
packageInspectionDurationMs: z.number().nonnegative(),
expectedPackageEvidenceDigest: z.string().nullable(),
configurationPackagesEligible: z.boolean(), samePackageOverlapEligible: z.boolean(),
packages: z.array(cloudPackageEvidenceSchema), blockers: z.array(qualificationBlockerSchema),
overlapBlockers: z.array(qualificationBlockerSchema),
}).strict();
const receiptSchema = z.object({
version: z.literal(1), revision: z.number().int().nonnegative(),
migration: z.object({ completedAt: z.string(), adapter: sourceSchema, config: sourceSchema, externalRegistrations: externalRegistrationEvidenceSchema }).strict(),
qualification: qualificationSchema.nullable(), lastPackageMismatch: qualificationSchema.nullable(),
}).strict();
export type CloudConfigurationReceipt = z.infer<typeof receiptSchema>;
function parseReceipt(value: unknown): CloudConfigurationReceipt {
const result = receiptSchema.safeParse(value);
if (!result.success) throw new Error("Persisted Cloud configuration migration receipt is invalid");
return result.data;
}
export function cloudConfigurationMigrationService(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv; now?: () => Date } = {}) {
const env = options.runtimeEnv ?? process.env;
const now = () => (options.now?.() ?? new Date()).toISOString();
const adapters = adapterSettingsService(db, { runtimeEnv: env });
const config = cloudRuntimeConfigService(db);
function assertManaged() {
if (!isCloudManagedInstance(env)) throw new Error("Cloud configuration qualification requires a managed instance");
}
async function readRow(handle: CloudConfigurationDb, lock = false) {
const query = handle.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, RECEIPT_KEY));
const [row] = lock ? await query.for("update") : await query;
return row ?? null;
}
async function currentConfiguration(handle: CloudConfigurationDb) {
await handle.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS))
.orderBy(instanceSettings.singletonKey).for("update");
const adapter = await adapters.inspectStored(handle);
const runtime = await config.inspectStored(handle);
if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete");
return { adapter, runtime };
}
async function registrationSnapshot(handle: CloudConfigurationDb, lock = false) {
const query = handle.select({
pluginKey: plugins.pluginKey, packageName: plugins.packageName, version: plugins.version,
manifestJson: plugins.manifestJson, status: plugins.status,
}).from(plugins).where(ne(plugins.status, "uninstalled")).orderBy(plugins.pluginKey);
return lock ? query.for("share") : query;
}
async function save(handle: CloudConfigurationDb, id: string, receipt: CloudConfigurationReceipt) {
const validated = parseReceipt(receipt);
const updated = await handle.update(instanceSettings).set({ general: validated, updatedAt: new Date() }).where(eq(instanceSettings.id, id)).returning({ id: instanceSettings.id });
if (updated.length !== 1) throw new Error("Cloud configuration migration receipt was not persisted");
return validated;
}
return {
async inspectStored(): Promise<CloudConfigurationReceipt | null> {
const row = await readRow(db);
return row ? parseReceipt(row.general) : null;
},
async migrate(): Promise<CloudConfigurationReceipt> {
assertManaged();
return db.transaction(async (tx) => {
const inserted = await tx.insert(instanceSettings).values({ singletonKey: RECEIPT_KEY, general: { version: 1 } })
.onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
const row = await readRow(tx, true);
if (!row) throw new Error("Cloud configuration migration receipt is unavailable");
if (!inserted.length) {
const receipt = parseReceipt(row.general);
await currentConfiguration(tx);
return receipt;
}
await tx.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS))
.orderBy(instanceSettings.singletonKey).for("update");
const storedAdapter = await adapters.inspectStored(tx);
const storedConfig = await config.inspectStored(tx);
const adapterCandidate = storedAdapter ? null : await readLegacyAdapterSettings(env);
const configCandidate = storedConfig ? null : await readLegacyCloudRuntimeConfig(env);
const adapterSeed = storedAdapter ? null : await adapters.seedIfAbsent({ disabledTypes: adapterCandidate?.disabledTypes ?? [] }, tx);
const configSeed = storedConfig ? null : await config.seedIfAbsent(configCandidate, tx);
const adapter = storedAdapter ?? adapterSeed!.current;
const runtime = storedConfig ?? configSeed!.current;
const adapterFromDatabase = Boolean(storedAdapter || !adapterSeed?.seeded);
const configFromDatabase = Boolean(storedConfig || !configSeed?.seeded);
const receipt: CloudConfigurationReceipt = {
version: 1, revision: 0,
migration: {
completedAt: now(),
adapter: { authority: adapterFromDatabase ? "database" : adapterCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(adapterFromDatabase ? { disabledTypes: adapter.disabledTypes } : adapterCandidate), importedRevision: adapter.revision },
config: { authority: configFromDatabase ? "database" : configCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(configFromDatabase ? runtime.config : configCandidate), importedRevision: runtime.revision },
externalRegistrations: await inspectLegacyExternalRegistrations(getAdapterPluginStorePath(env.PAPERCLIP_HOME)),
},
qualification: null, lastPackageMismatch: null,
};
return save(tx, row.id, receipt);
});
},
async recheckLegacyExternalRegistrations(expectedRevision: number): Promise<CloudConfigurationReceipt> {
assertManaged();
return db.transaction(async (tx) => {
const row = await readRow(tx, true);
if (!row) throw new Error("Cloud configuration migration has not completed");
const receipt = parseReceipt(row.general);
if (receipt.revision !== expectedRevision) throw new Error("Cloud configuration migration receipt revision conflict");
const filePath = getAdapterPluginStorePath(env.PAPERCLIP_HOME);
if (cloudConfigurationDigest(filePath) !== receipt.migration.externalRegistrations.sourceLocatorDigest) {
throw new Error("External registration recheck requires the original legacy source location");
}
const evidence = await inspectLegacyExternalRegistrations(filePath);
if (evidence.state !== "parsed") throw new Error("External registration recheck requires a present, valid registration file");
receipt.migration.externalRegistrations = evidence;
receipt.revision += 1;
receipt.qualification = null;
return save(tx, row.id, receipt);
});
},
async qualify(input: {
installs: readonly ResolvedBundledPlugin[];
catalogRoot?: string;
loadManifest: (packageRoot: string) => Promise<PaperclipPluginManifestV1 | null>;
expectedPackageEvidenceDigest?: string;
imageCatalogUnavailable?: boolean;
}) {
assertManaged();
const adapter = await adapters.inspectStored();
const runtime = await config.inspectStored();
if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete");
const configurationBefore = cloudConfigurationDigest({ adapter, runtime });
const registrationsBefore = await registrationSnapshot(db);
const evidence = await inspectCloudPackages({ ...input, registrations: registrationsBefore });
return db.transaction(async (tx) => {
const row = await readRow(tx, true);
if (!row) throw new Error("Cloud configuration migration has not completed");
const receipt = parseReceipt(row.general);
const current = await currentConfiguration(tx);
const registrations = await registrationSnapshot(tx, true);
const blockers = [...evidence.blockers];
if (configurationBefore !== cloudConfigurationDigest(current) || cloudConfigurationDigest(registrationsBefore) !== cloudConfigurationDigest(registrations)) {
blockers.push({ code: "configuration_changed_during_inspection", reason: "Configuration or plugin registration changed during image inspection; run qualification again" });
}
if (input.imageCatalogUnavailable) blockers.push({ code: "current_image_catalog_unavailable", reason: "The current image package catalog could not be verified" });
const external = receipt.migration.externalRegistrations;
if (external.blocker) blockers.push({ code: external.blocker, reason: "Legacy external adapter registration metadata requires an explicit operator resolution" });
const overlapBlockers = input.expectedPackageEvidenceDigest && input.expectedPackageEvidenceDigest !== evidence.packageEvidenceDigest
? [{ code: "package_evidence_changed", reason: "Current packages do not match the expected package evidence digest" }] : [];
const qualification: z.infer<typeof qualificationSchema> = {
scope: "configuration_and_packages", checkedAt: now(),
adapterRevision: current.adapter.revision, configRevision: current.runtime.revision,
configurationDigest: cloudConfigurationDigest(current), packageEvidenceDigest: evidence.packageEvidenceDigest,
packageInspectionDurationMs: evidence.packageInspectionDurationMs,
expectedPackageEvidenceDigest: input.expectedPackageEvidenceDigest ?? null,
configurationPackagesEligible: blockers.length === 0,
samePackageOverlapEligible: Boolean(input.expectedPackageEvidenceDigest && blockers.length === 0 && overlapBlockers.length === 0),
packages: evidence.packages, blockers, overlapBlockers,
};
receipt.revision += 1;
receipt.qualification = qualification;
if (evidence.blockers.length || input.imageCatalogUnavailable || overlapBlockers.length) receipt.lastPackageMismatch = qualification;
await save(tx, row.id, receipt);
return { receiptRevision: receipt.revision, ...qualification };
});
},
};
}
export async function migrateCloudConfiguration(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) {
if (!isCloudManagedInstance(options.runtimeEnv ?? process.env)) return null;
return cloudConfigurationMigrationService(db, options).migrate();
}
@@ -0,0 +1,33 @@
import type { Db } from "@paperclipai/db";
import { BUNDLED_PLUGIN_CATALOG, resolveBundledCatalogRoot, resolveBundledPluginInstalls, resolveManagedBundledPluginKeys, type ResolvedBundledPlugin } from "./bundled-plugins.js";
import { readDistributionPluginCatalog } from "./distribution-plugin-catalog.js";
import { createBundledPluginPackageResolver } from "./bundled-plugin-package.js";
import { cloudConfigurationMigrationService } from "./cloud-configuration-migration.js";
import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
import { getManagedInstanceConfig } from "./managed-config.js";
import { pluginLoader } from "./plugin-loader.js";
export async function qualifyCloudConfigurationForCurrentImage(db: Db, options: {
runtimeEnv?: CloudInstanceEnv;
expectedPackageEvidenceDigest?: string;
} = {}) {
const env = options.runtimeEnv ?? process.env;
if (!isCloudManagedInstance(env)) return null;
const service = cloudConfigurationMigrationService(db, { runtimeEnv: env });
const catalogRoot = resolveBundledCatalogRoot(env);
let installs: ResolvedBundledPlugin[];
try {
const distribution = readDistributionPluginCatalog(catalogRoot, BUNDLED_PLUGIN_CATALOG);
installs = resolveBundledPluginInstalls(resolveManagedBundledPluginKeys(getManagedInstanceConfig(env)?.plugins.autoInstall, env) ?? [], {
catalogRoot, env, enforceCatalogRoot: true, distributionPlugins: distribution,
});
} catch {
return service.qualify({ installs: [], loadManifest: async () => null, imageCatalogUnavailable: true, expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest });
}
const resolver = createBundledPluginPackageResolver(installs);
const loader = pluginLoader(db, {
prebuiltOnly: true, enableLocalFilesystem: false, enableNpmDiscovery: false,
assertPackageActivation: resolver.assertActivation, resolvePackageRoot: resolver.resolve,
});
return service.qualify({ installs, catalogRoot, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest });
}
@@ -0,0 +1,72 @@
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { z } from "zod";
import { createBundledPluginPackageResolver, type BundledPluginRegistration } from "./bundled-plugin-package.js";
import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
import { distributionBundleDigest } from "./distribution-plugin-catalog.js";
import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js";
import { cloudBundledPackageDigest } from "./cloud-bundled-package-digest.js";
export const qualificationBlockerSchema = z.object({
code: z.string(), pluginKey: z.string().optional(), reason: z.string(),
registeredVersion: z.string().optional(), imageVersion: z.string().optional(),
}).strict();
export type QualificationBlocker = z.infer<typeof qualificationBlockerSchema>;
export const cloudPackageEvidenceSchema = z.object({
catalogKey: z.string(), pluginKey: z.string(), packageDigest: z.string().nullable(),
imageManifestDigest: z.string().nullable(), approvedManifestDigest: z.string().nullable(),
}).strict();
export type CloudPackageEvidence = z.infer<typeof cloudPackageEvidenceSchema>;
export async function inspectCloudPackages(input: {
installs: readonly ResolvedBundledPlugin[];
registrations: readonly BundledPluginRegistration[];
catalogRoot?: string;
loadManifest: (packageRoot: string) => Promise<PaperclipPluginManifestV1 | null>;
}) {
const started = performance.now();
const resolver = createBundledPluginPackageResolver(input.installs);
const registrations = new Map(input.registrations.map((record) => [record.pluginKey, record]));
const selected = new Set(input.installs.map((install) => install.pluginKey));
const blockers: QualificationBlocker[] = input.registrations.filter((record) => !selected.has(record.pluginKey)).map((record) => ({
code: "unsupported_plugin_registration", pluginKey: record.pluginKey, reason: "An installed plugin is not selected from the current image", registeredVersion: record.version,
}));
const packages: CloudPackageEvidence[] = [];
for (const install of [...input.installs].sort((left, right) => left.pluginKey < right.pluginKey ? -1 : left.pluginKey > right.pluginKey ? 1 : 0)) {
const registration = registrations.get(install.pluginKey);
const evidence: CloudPackageEvidence = {
catalogKey: install.key, pluginKey: install.pluginKey, packageDigest: null,
imageManifestDigest: null, approvedManifestDigest: registration ? cloudConfigurationDigest(registration.manifestJson) : null,
};
packages.push(evidence);
let stage = "missing_bundled_package";
try {
resolver.assertActivation({ packageRoot: install.localPath });
stage = "unverified_package_integrity";
evidence.packageDigest = install.distribution ? distributionBundleDigest(install.localPath) : cloudBundledPackageDigest(install.localPath, { catalogRoot: input.catalogRoot });
if (install.distribution && evidence.packageDigest !== install.distribution.digest) {
blockers.push({ code: "package_integrity_mismatch", pluginKey: install.pluginKey, reason: "Current package bytes do not match the verified distribution digest" });
continue;
}
stage = "missing_manifest_evidence";
const manifest = await input.loadManifest(install.localPath);
if (!manifest) {
blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image has no verified plugin manifest" });
continue;
}
evidence.imageManifestDigest = cloudConfigurationDigest(manifest);
if (!registration) {
blockers.push({ code: "missing_plugin_registration", pluginKey: install.pluginKey, reason: "The current image package is not registered in the database", imageVersion: manifest.version });
continue;
}
const inspected = resolver.inspect(registration, manifest);
if (!inspected.compatible) blockers.push({
code: inspected.code, pluginKey: install.pluginKey, reason: inspected.reason,
registeredVersion: registration.version, imageVersion: manifest.version,
});
} catch {
blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image package could not be verified" });
}
}
const packageEvidenceDigest = cloudConfigurationDigest(packages.map(({ approvedManifestDigest: _approved, ...evidence }) => evidence));
return { packages, blockers, packageEvidenceDigest, packageInspectionDurationMs: Math.round(performance.now() - started) };
}
@@ -0,0 +1,129 @@
import {
authConfigSchema,
databaseBackupConfigSchema,
llmConfigSchema,
loggingConfigSchema,
paperclipConfigSchema,
secretsConfigSchema,
secretsLocalEncryptedConfigSchema,
serverConfigSchema,
storageConfigSchema,
storageLocalDiskConfigSchema,
storageS3ConfigSchema,
type PaperclipConfig,
} from "@paperclipai/shared";
import { z } from "zod";
const publicUrl = z.string().url().refine((value) => {
const url = new URL(value);
return !url.username && !url.password && !url.search && !url.hash;
});
export const cloudRuntimeConfigSchema = z.object({
server: serverConfigSchema.strict(),
auth: authConfigSchema.extend({ publicBaseUrl: publicUrl.optional() }).strict(),
backup: databaseBackupConfigSchema.strict(),
storage: storageConfigSchema.extend({
localDisk: storageLocalDiskConfigSchema.strict(),
s3: storageS3ConfigSchema.extend({ endpoint: publicUrl.optional() }).strict(),
}).strict(),
secrets: secretsConfigSchema.extend({
localEncrypted: secretsLocalEncryptedConfigSchema.strict(),
}).strict(),
telemetry: z.object({ enabled: z.boolean() }).strict(),
logging: loggingConfigSchema.strict(),
updates: z.object({ checkEnabled: z.boolean() }).strict().optional(),
llm: llmConfigSchema.omit({ apiKey: true }).strict().optional(),
}).strict();
export type CloudRuntimeConfig = z.infer<typeof cloudRuntimeConfigSchema>;
export type StoredCloudRuntimeConfig = { version: 1; revision: number; config: CloudRuntimeConfig | null };
const storedSchema = z.object({
version: z.literal(1),
revision: z.number().int().nonnegative(),
config: cloudRuntimeConfigSchema.nullable(),
}).strict();
export function parseStoredCloudRuntimeConfig(value: unknown): StoredCloudRuntimeConfig {
const parsed = storedSchema.safeParse(value);
if (!parsed.success || Buffer.byteLength(JSON.stringify(parsed.data), "utf8") > 65_536) {
throw new Error("Persisted Cloud runtime configuration is invalid");
}
const full = parsed.data.config === null ? { success: true } : paperclipConfigSchema.safeParse(toPaperclipConfig(parsed.data.config));
if (!full.success) throw new Error("Persisted Cloud runtime configuration is invalid");
return parsed.data;
}
export function toPaperclipConfig(config: CloudRuntimeConfig | null): PaperclipConfig | null {
if (config === null) return null;
const { backup, ...settings } = structuredClone(config);
return {
$meta: { version: 1, updatedAt: "1970-01-01T00:00:00.000Z", source: "configure" },
database: {
mode: "postgres",
embeddedPostgresDataDir: "",
embeddedPostgresPort: 54329,
backup,
},
...settings,
};
}
export function projectCloudRuntimeConfig(source: PaperclipConfig): CloudRuntimeConfig {
const raw = {
server: {
deploymentMode: source.server.deploymentMode,
exposure: source.server.exposure,
bind: source.server.bind,
customBindHost: source.server.customBindHost,
host: source.server.host,
port: source.server.port,
allowedHostnames: source.server.allowedHostnames,
serveUi: source.server.serveUi,
},
auth: {
baseUrlMode: source.auth.baseUrlMode,
publicBaseUrl: source.auth.publicBaseUrl,
disableSignUp: source.auth.disableSignUp,
},
backup: {
enabled: source.database.backup.enabled,
intervalMinutes: source.database.backup.intervalMinutes,
retentionDays: source.database.backup.retentionDays,
dir: source.database.backup.dir,
},
storage: {
provider: source.storage.provider,
localDisk: { baseDir: source.storage.localDisk.baseDir },
s3: {
bucket: source.storage.s3.bucket,
region: source.storage.s3.region,
endpoint: source.storage.s3.endpoint,
prefix: source.storage.s3.prefix,
forcePathStyle: source.storage.s3.forcePathStyle,
},
},
secrets: {
provider: source.secrets.provider,
strictMode: source.secrets.strictMode,
localEncrypted: { keyFilePath: source.secrets.localEncrypted.keyFilePath },
},
telemetry: { enabled: source.telemetry.enabled },
logging: { mode: source.logging.mode, logDir: source.logging.logDir },
updates: source.updates ? { checkEnabled: source.updates.checkEnabled } : undefined,
llm: source.llm ? { provider: source.llm.provider } : undefined,
};
return parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config: raw }).config!;
}
let snapshot: StoredCloudRuntimeConfig | null = null;
export function setCloudRuntimeConfigSnapshot(value: StoredCloudRuntimeConfig | null): void {
snapshot = value ? parseStoredCloudRuntimeConfig(value) : null;
}
export function getCloudRuntimeConfigSnapshot(): StoredCloudRuntimeConfig {
if (!snapshot) throw new Error("Cloud runtime configuration has not been initialized");
return structuredClone(snapshot);
}
+105
View File
@@ -0,0 +1,105 @@
import { constants } from "node:fs";
import { open } from "node:fs/promises";
import path from "node:path";
import { eq } from "drizzle-orm";
import { instanceSettings, type Db } from "@paperclipai/db";
import { paperclipConfigSchema } from "@paperclipai/shared";
import { resolvePaperclipConfigPathForInstance } from "@paperclipai/shared/home-paths";
import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
import {
parseStoredCloudRuntimeConfig,
projectCloudRuntimeConfig,
setCloudRuntimeConfigSnapshot,
type CloudRuntimeConfig,
type StoredCloudRuntimeConfig,
} from "./cloud-runtime-config-snapshot.js";
export const CLOUD_RUNTIME_CONFIG_KEY = "cloud-runtime-config:v1";
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
export type CloudRuntimeConfigDb = Pick<Db | Transaction, "select" | "insert" | "update" | "transaction">;
export function cloudRuntimeConfigService(db: CloudRuntimeConfigDb) {
async function inspectStored(handle: CloudRuntimeConfigDb = db): Promise<StoredCloudRuntimeConfig | null> {
const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings)
.where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).limit(1);
return row ? parseStoredCloudRuntimeConfig(row.general) : null;
}
async function seedIfAbsent(config: CloudRuntimeConfig | null, handle: CloudRuntimeConfigDb = db) {
const next = parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config });
const inserted = await handle.insert(instanceSettings)
.values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: next })
.onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
const current = await inspectStored(handle);
if (!current) throw new Error("Cloud runtime configuration seed was not persisted");
return { seeded: inserted.length > 0, current };
}
return {
inspectStored,
seedIfAbsent,
async replace(config: CloudRuntimeConfig | null, expectedRevision: number, handle: CloudRuntimeConfigDb = db) {
const validated = parseStoredCloudRuntimeConfig({ version: 1, revision: expectedRevision, config });
return handle.transaction(async (tx) => {
const [row] = await tx.select().from(instanceSettings)
.where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).for("update");
if (!row) throw new Error("Cloud runtime configuration has not been initialized");
const current = parseStoredCloudRuntimeConfig(row.general);
if (current.revision !== expectedRevision) throw new Error("Cloud runtime configuration revision conflict");
const next = { ...validated, revision: current.revision + 1 };
await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id));
return next;
});
},
};
}
export function legacyCloudRuntimeConfigPath(env: CloudInstanceEnv = process.env): string {
return env.PAPERCLIP_CONFIG
? path.resolve(env.PAPERCLIP_CONFIG)
: resolvePaperclipConfigPathForInstance({ homeDir: env.PAPERCLIP_HOME, instanceId: env.PAPERCLIP_INSTANCE_ID });
}
export async function readLegacyCloudRuntimeConfig(env: CloudInstanceEnv = process.env): Promise<CloudRuntimeConfig | null> {
let raw: unknown;
try {
const file = await open(legacyCloudRuntimeConfigPath(env), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
try {
const stat = await file.stat();
if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid file");
const buffer = Buffer.alloc(65_537);
let total = 0;
while (total < buffer.length) {
const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
if (bytesRead === 0) break;
total += bytesRead;
}
if (total > 65_536) throw new Error("oversized file");
raw = JSON.parse(buffer.subarray(0, total).toString("utf8"));
} finally {
await file.close();
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw new Error("Legacy Cloud runtime configuration could not be read safely");
return null;
}
const parsed = paperclipConfigSchema.safeParse(raw);
if (!parsed.success) throw new Error("Legacy Cloud runtime configuration is invalid");
if (parsed.data.llm?.provider === "openai" && parsed.data.llm.apiKey?.trim() && !env.OPENAI_API_KEY?.trim()) {
throw new Error("Move the legacy OpenAI API key to OPENAI_API_KEY before initializing Cloud runtime configuration");
}
return projectCloudRuntimeConfig(parsed.data);
}
export async function initializeCloudRuntimeConfig(db: CloudRuntimeConfigDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) {
const env = options.runtimeEnv ?? process.env;
if (!isCloudManagedInstance(env)) return null;
setCloudRuntimeConfigSnapshot(null);
const service = cloudRuntimeConfigService(db);
const current = await service.inspectStored();
const result = current
? { seeded: false, current }
: await service.seedIfAbsent(await readLegacyCloudRuntimeConfig(env));
setCloudRuntimeConfigSnapshot(result.current);
return result;
}
+21 -17
View File
@@ -1,3 +1,4 @@
import { resolveBundledPluginRuntimeLoader } from "./bundled-plugin-runtime.js";
import { finishServiceShutdown } from "../shutdown.js";
/**
* PluginLoader — discovery, installation, and runtime activation of plugins.
@@ -31,7 +32,7 @@ import { execFile } from "node:child_process";
import os from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { isDeepStrictEqual, promisify } from "node:util";
import type { Db } from "@paperclipai/db";
import { PLUGIN_RPC_ERROR_CODES } from "@paperclipai/plugin-sdk";
import type {
@@ -271,6 +272,8 @@ function getDeclaredPageRoutePaths(manifest: PaperclipPluginManifestV1): string[
* Options for the plugin loader service.
*/
export interface PluginLoaderOptions {
resolvePackageRoot?: (plugin: PluginRecord) => string;
prebuiltOnly?: boolean;
/** Image-owned deployment policy, checked before importing code and starting workers. */
assertPackageActivation?: (input: {
pluginKey?: string;
@@ -1157,6 +1160,8 @@ export function pluginLoader(
enableLocalFilesystem = true,
enableNpmDiscovery = true,
assertPackageActivation,
resolvePackageRoot,
prebuiltOnly = false,
} = options;
const registry = pluginRegistryService(db);
@@ -1211,6 +1216,7 @@ export function pluginLoader(
installOptions: PluginInstallOptions,
): Promise<DiscoveredPlugin> {
const { packageName, localPath, version, installDir } = installOptions;
if (prebuiltOnly && !localPath) throw new Error("Managed plugins must come from the current image catalog");
if (!packageName && !localPath) {
throw new Error("Either packageName or localPath must be provided");
@@ -1286,7 +1292,7 @@ export function pluginLoader(
const pkgJson = await readPackageJson(resolvedPackagePath);
if (!pkgJson) throw new Error(`Missing package.json at ${resolvedPackagePath}`);
if (localPath) {
if (localPath && !prebuiltOnly) {
await ensureLocalPluginBuilt(resolvedPackagePath, pkgJson);
}
@@ -1398,7 +1404,7 @@ export function pluginLoader(
}
assertPackageActivation?.({ packageRoot, installedPackagePath: plugin.packagePath, pluginKey: plugin.pluginKey, manifest, previousManifest: plugin.manifestJson });
if (JSON.stringify(manifest) === JSON.stringify(plugin.manifestJson)) {
if (isDeepStrictEqual(manifest, plugin.manifestJson)) {
return plugin;
}
@@ -1749,6 +1755,7 @@ export function pluginLoader(
packagePath: discovered.source === "local-filesystem" ? discovered.packagePath : undefined,
},
manifest,
{ idempotent: prebuiltOnly },
);
if (!installed) {
@@ -1803,12 +1810,7 @@ export function pluginLoader(
newManifest: PaperclipPluginManifestV1;
discovered: DiscoveredPlugin;
}> {
const plugin = (await registry.getById(pluginId)) as {
id: string;
packageName: string;
packagePath: string | null;
manifestJson: PaperclipPluginManifestV1;
} | null;
const plugin = await registry.getById(pluginId);
if (!plugin) throw new Error(`Plugin not found: ${pluginId}`);
const oldManifest = plugin.manifestJson;
@@ -1817,7 +1819,7 @@ export function pluginLoader(
// For local-path installs, fall back to the stored packagePath so
// `upgradePlugin` can re-read the manifest from disk without needing
// the caller to re-supply the path every time.
localPath = plugin.packagePath ?? undefined,
localPath = resolvePackageRoot ? resolvePackageRoot(plugin) : plugin.packagePath ?? undefined,
version,
} = upgradeOptions;
@@ -1887,6 +1889,7 @@ export function pluginLoader(
// -----------------------------------------------------------------------
async cleanupInstallArtifacts(plugin: PluginRecord): Promise<void> {
if (prebuiltOnly) return;
const managedTargets = new Set<string>();
const managedNodeModulesDir = resolveManagedInstallPackageDir(localPluginDir, plugin.packageName);
const directManagedDir = path.join(localPluginDir, plugin.packageName);
@@ -2262,12 +2265,14 @@ export function pluginLoader(
// ------------------------------------------------------------------
// 1. Resolve worker entrypoint
// ------------------------------------------------------------------
const packageRoot = resolvePluginPackageRoot(activePlugin, localPluginDir);
const packageRoot = resolvePackageRoot ? resolvePackageRoot(activePlugin) : resolvePluginPackageRoot(activePlugin, localPluginDir);
assertPackageActivation?.({ pluginKey, packageRoot, installedPackagePath: activePlugin.packagePath });
activePlugin = await refreshPluginManifestFromPackage(activePlugin, packageRoot);
manifest = activePlugin.manifestJson;
assertPackageActivation?.({ pluginKey, packageRoot, installedPackagePath: activePlugin.packagePath, manifest });
const workerEntrypoint = resolveWorkerEntrypoint(activePlugin, localPluginDir);
const workerEntrypoint = resolvePackageRoot
? path.resolve(packageRoot, manifest.entrypoints.worker)
: resolveWorkerEntrypoint(activePlugin, localPluginDir);
// ------------------------------------------------------------------
// 2. Apply restricted database migrations before worker startup
@@ -2334,7 +2339,7 @@ export function pluginLoader(
env: buildPluginWorkerEnv({
manifest,
packageName: activePlugin.packageName,
packagePath: activePlugin.packagePath,
packagePath: resolvePackageRoot ? packageRoot : activePlugin.packagePath,
instanceInfo,
}),
// Authorize the worker to act on each configured company from its
@@ -2345,10 +2350,9 @@ export function pluginLoader(
proactiveCompanyScopes: configRows.map((row) => row.companyId),
};
// Repo-local plugin installs can resolve workspace TS sources at runtime
// (for example @paperclipai/shared exports). Run those workers through
// the tsx loader so first-party example plugins work in development.
if (activePlugin.packagePath && existsSync(DEV_TSX_LOADER_PATH)) {
if (prebuiltOnly) {
workerOptions.execArgv = ["--import", resolveBundledPluginRuntimeLoader()];
} else if (activePlugin.packagePath && existsSync(DEV_TSX_LOADER_PATH)) {
workerOptions.execArgv = ["--import", DEV_TSX_LOADER_PATH];
}
+17 -6
View File
@@ -1,3 +1,4 @@
import { isDeepStrictEqual } from "node:util";
import { writePluginJobChange } from "./plugin-job-work-notifications.js";
import { asc, eq, isNull, ne, sql, and } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -31,6 +32,10 @@ import type {
import { conflict, notFound } from "../errors.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
function sameBundledPluginRegistration(existing: typeof plugins.$inferSelect, packageName: string, manifest: PaperclipPluginManifestV1): boolean {
return existing.pluginKey === manifest.id && existing.packageName === packageName && existing.version === manifest.version && isDeepStrictEqual(existing.manifestJson, manifest);
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
@@ -140,10 +145,11 @@ export function pluginRegistryService(db: Db) {
* manifest from the package. This method persists the plugin row and
* assigns the next install order.
*/
install: async (input: InstallPlugin, manifest: PaperclipPluginManifestV1) => {
install: async (input: InstallPlugin, manifest: PaperclipPluginManifestV1, options: { idempotent?: boolean } = {}) => {
const existing = await getByKey(manifest.id);
if (existing) {
if (existing.status !== "uninstalled") {
if (options.idempotent && sameBundledPluginRegistration(existing, input.packageName, manifest)) return existing;
throw conflict(`Plugin already installed: ${manifest.id}`);
}
@@ -153,7 +159,7 @@ export function pluginRegistryService(db: Db) {
.update(plugins)
.set({
packageName: input.packageName,
packagePath: input.packagePath ?? null,
packagePath: options.idempotent ? existing.packagePath : input.packagePath ?? null,
version: manifest.version,
apiVersion: manifest.apiVersion,
categories: manifest.categories,
@@ -170,7 +176,7 @@ export function pluginRegistryService(db: Db) {
const installOrder = await nextInstallOrder();
try {
const rows = await db
const insertion = db
.insert(plugins)
.values({
pluginKey: manifest.id,
@@ -182,9 +188,14 @@ export function pluginRegistryService(db: Db) {
status: "installed" as PluginStatus,
installOrder,
packagePath: input.packagePath ?? null,
})
.returning();
return rows[0];
});
const rows = await (options.idempotent
? insertion.onConflictDoNothing({ target: plugins.pluginKey })
: insertion).returning();
if (rows[0]) return rows[0];
const concurrent = await getByKey(manifest.id);
if (concurrent && concurrent.status !== "uninstalled" && sameBundledPluginRegistration(concurrent, input.packageName, manifest)) return concurrent;
throw conflict(`Plugin registration differs from the current image: ${manifest.id}`);
} catch (error) {
if (isPluginKeyConflict(error)) {
throw conflict(`Plugin already installed: ${manifest.id}`);
+24
View File
@@ -27,6 +27,7 @@ never start a server or depend on a tenant filesystem.
| `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload |
| `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload |
| `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run |
| `task-search-filter` | Company, signed-in board user, task-write capability; zero model runs | Search three owned target/control tasks through the task list, require exact matching identities, filter Backlog to exclude the completed control, reload and return from the exact saved task with search/filter context intact |
| `plan-revise-accept` | Native agent; three work runs | Reuse the classic Plan story: canonical draft, browser revision request, new revision-bound confirmation, approval and exact saved final response |
| `build-revise` | Native agent; two work runs; qualified Docker artifact oracle | Browser-download both ZIP versions, independently exercise source/CLI behavior, verify different hashes and preserve the original download |
| `protected-actions` | Native agent; four work runs; authenticated reachable review-service fixture | Approve one action and decline another on separate owned tasks; verify zero calls before either decision, one after approval, zero after decline and a truthful saved explanation |
@@ -51,6 +52,29 @@ The shared Studio helper waits for the original saved file body before typing.
An editor control can mount before the file query completes; editing that empty
draft can otherwise lose the original content when the file arrives.
The search/filter journey creates only campaign-owned, board-assigned tasks through
the public API. The target and completed control share a unique search marker;
a second Backlog control has a different marker. Search and status selection use
the production controls. The oracle compares the complete visible task identity
set at each step, verifies the exact saved company, owner, title, description and
status, and checks all three tasks have no native runs. It reopens the target after
reload and browser Back, and leaves the target visible for the harness screenshot.
The combined Tasks shell reapplies its named view's status preset on mount, so
the journey first proves the checkbox filter, then selects the normal Backlog
view and enters the same search again before checking reload/Back persistence.
Coordinator and legacy task lists keep their supported saved-filter behavior.
The status selector stays inside the Status group. Legacy status glyphs contribute
their label to the checkbox's accessible name, producing `BacklogBacklog`; the
refreshed panel supplies the explicit `Backlog` name. A credential-free browser
regression covers both forms and a separate assignee called Backlog.
Unexpected browser/API errors retain only a closed `task_search_<stage>_failed`
code. Stage names come from the journey source; raw exception text, URLs, form
values, and response bodies are not added to reporting. Existing owned evidence
failures keep their specific codes and the screenshot privacy guard stays in place.
It does not depend on row order or an otherwise empty company. This selection is
manual until a separate target owner records a fresh pass and admits it to a daily
plan; adding the shared journey does not qualify any deployed target.
Supported acquisitions are managed self-hosted (`local`), registered existing
Cloud (`local` or `daytona` when present), and new Cloud. New Cloud must start
with onboarding. Its context can then satisfy the identical task-properties
+1 -1
View File
@@ -59,7 +59,7 @@ describe("runner E2E catalog", () => {
});
it("exports deployment journeys separately from the model matrix and default selection", () => {
expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "question-answer-resume", "task-properties", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]);
expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]);
expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false);
expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false);
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false);
+1 -1
View File
@@ -19,7 +19,7 @@ function associate(suites: readonly string[], cases: Record<string, readonly str
}
associate(["product-journeys"], { "task-create": ["tasks/create"], "ask-question": ["questions-and-approvals/questions"],
"task-properties": ["tasks/properties"], "cloud-onboarding": ["onboarding/first-task"] });
"task-properties": ["tasks/properties"], "task-search-filter": ["tasks/organize"], "cloud-onboarding": ["onboarding/first-task"] });
associate(["antigravity-agent-journey"], {
"saved-subscription-two-turn": ["agents/hire", "agents/instructions", "runs-adapters/validation", "runs-adapters/session",
"tasks/create", "tasks/completion", "documents-artifacts/attachments", "documents-artifacts/work-products"],
@@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test";
/** Browser-only harness regressions: no Paperclip instance or provider credentials. */
export default defineConfig({
testDir: ".",
testMatch: ["skill-studio-flow.spec.ts", "native-task-reply.spec.ts", "screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts"],
testMatch: ["skill-studio-flow.spec.ts", "native-task-reply.spec.ts", "screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts", "task-search-filter.spec.ts"],
workers: 1,
retries: 0,
timeout: 10_000,
@@ -31,13 +31,14 @@ vi.mock("../dashboard-regenerate.js", () => ({ regenerateRunnerDashboard: vi.fn(
if(input.outputDirectory){const {writeFile}=await import("node:fs/promises");await Promise.all(["index.html","dashboard.html"].map(name=>writeFile(path.join(input.outputDirectory!,name),"dashboard")));}
}) }));
vi.mock("./task-properties.js", () => ({taskProperties: async (_context:unknown,_session:unknown,_id:unknown,evidence:any) => {evidence.checks.push({id:"assignment-persisted",passed:true},{id:"no-agent-work",passed:true});}}));
vi.mock("./task-search-filter.js", () => ({taskSearchFilter: async (_context:unknown,_session:unknown,_id:unknown,evidence:any) => {evidence.checks.push(...["task-search-results-exact","task-status-filter-excludes-controls","task-search-filter-reloaded","task-search-result-reopened","no-agent-work"].map(id=>({id,passed:true})));}}));
import { chromium } from "@playwright/test";
import { launchProductJourneys } from "./launch.js";
import { regenerateRunnerDashboard } from "../dashboard-regenerate.js";
const roots: string[]=[];
afterEach(async()=>{vi.useRealTimers(); vi.restoreAllMocks(); persistenceFault.kind=null; persistenceFault.active=false; persistenceFault.latestReport=null; process.exitCode=0; await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));});
type Hook = "collect" | "release" | "stop" | "finalizeEvidence" | "checkpointEvidence";
async function run(options:{authFailure?:boolean;onboardingFailure?:boolean;preAcceptanceFailure?:boolean;missingArtifact?:boolean;drift?:boolean;cleanupFailure?:boolean;collectionFailure?:boolean;mini?:boolean;missingReceipt?:boolean;zeroRequests?:boolean;missingCheck?:boolean;finalizationFailure?:boolean;persistenceFailure?:"write"|"rename";persistenceAfter?:"collection"|"release";stallHook?:Hook;onHook?:(hook:Hook,signal?:AbortSignal)=>void}={}) {
async function run(options:{search?:boolean;authFailure?:boolean;onboardingFailure?:boolean;preAcceptanceFailure?:boolean;missingArtifact?:boolean;drift?:boolean;cleanupFailure?:boolean;collectionFailure?:boolean;mini?:boolean;missingReceipt?:boolean;zeroRequests?:boolean;missingCheck?:boolean;finalizationFailure?:boolean;persistenceFailure?:"write"|"rename";persistenceAfter?:"collection"|"release";stallHook?:Hook;onHook?:(hook:Hook,signal?:AbortSignal)=>void}={}) {
const root=await mkdtemp(path.join(os.tmpdir(),"journey-campaign-test-")); roots.push(root);
const module=path.join(root,"provider.mjs");
const proof = createConnectionProof();
@@ -61,7 +62,7 @@ const report=JSON.parse(await readFile(input.outputDir+'/normalized-results.json
if(report.results.some(r=>r.cleanup!=='failed'))throw Error('failure checkpoint preceded failure persistence');
await writeFile(input.outputDir+'/failure-checkpoint.json',JSON.stringify(report));
}}}};`);
const config={version:1,id:"qa",mode:"new-cloud",adapterModule:module,journeys:["cloud-onboarding","task-properties"],executionEnvironment:"daytona",baseline:{features:{},connections:[]},limits:{durationMs:60000,modelBudgetCents:1000,maxRuns:1},...(options.mini?{modelProfileId:"runner-codex-mini"}:{})};
const config={version:1,id:"qa",mode:"new-cloud",adapterModule:module,journeys:["cloud-onboarding",options.search?"task-search-filter":"task-properties"],executionEnvironment:"daytona",baseline:{features:{},connections:[]},limits:{durationMs:60000,modelBudgetCents:1000,maxRuns:1},...(options.mini?{modelProfileId:"runner-codex-mini"}:{})};
const file=path.join(root,"target.json"),output=path.join(root,"report"); await writeFile(file,JSON.stringify(config));
const lifecycle: string[] = [];
let artifactBeforeRelease: unknown;
@@ -86,6 +87,13 @@ await writeFile(input.outputDir+'/failure-checkpoint.json',JSON.stringify(report
failureCheckpoint: await readFile(path.join(output,"failure-checkpoint.json"),"utf8").then(JSON.parse).catch(()=>null) };
}
describe("product journey campaign boundary outcomes",()=>{
it("dispatches task search separately and reports it without model work", async () => {
const campaign = await run({search:true});
const search = campaign.results[1];
expect(search).toMatchObject({caseId:"task-search-filter",profileId:"browser",model:null,provider:null,runtimeMode:"none",status:"passed"});
expect(search.runIds ?? []).toEqual([]);
expect(search.productJourney.checks).toHaveLength(5);
});
it("accepts the conventional pnpm separator in a credential-free preview", async () => {
const root=await mkdtemp(path.join(os.tmpdir(),"journey-preview-test-")); roots.push(root);
const file=path.join(root,"target.json");
@@ -6,6 +6,7 @@ export const productJourneys = [
{ id: "ask-question", label: "Ask an agent a question and read its answer", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true },
{ id: "question-answer-resume", label: "Answer agent questions and verify the saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true },
{ id: "task-properties", label: "Assign a task to the signed-in user", requires: ["company", "board-user", "task-write", "backlog-task"], capabilities: ["tasks/properties"], paid: false },
{ id: "task-search-filter", label: "Search and filter tasks, then reopen the exact saved result", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/organize"], paid: false },
{ id: "plan-revise-accept", label: "Revise and approve a saved Plan, then complete the task", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/plan-review"], paid: true },
{ id: "build-revise", label: "Download and independently verify both artifact versions", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["documents-artifacts/attachments"], paid: true },
{ id: "protected-actions", label: "Approve and decline protected service actions", requires: ["company", "board-user", "task-write", "agent", "review-service"], capabilities: ["questions-and-approvals/tool-review"], paid: true },
Loaded 100 of 113 files, more files were not shown because too many files have changed in this diff. Show more