mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-02 16:58:52 +01:00
Merge master and preserve question and search journeys
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
04f34f278e
113 files changed
+6319
-466
No files matched your search
@@ -0,0 +1,22 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
|
||||
// Read trees only. The routing job fetches Git objects into a bare repository;
|
||||
// no pull request code, hooks, filters or package scripts run here.
|
||||
export function mergeDiff(merge, prHead, cwd = process.cwd()) {
|
||||
assert.match(merge ?? "", /^[a-f0-9]{40}$/);
|
||||
assert.match(prHead ?? "", /^[a-f0-9]{40}$/);
|
||||
const git = (...args) => execFileSync("git", args, {
|
||||
cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
|
||||
}).trim();
|
||||
const header = git("--no-replace-objects", "cat-file", "-p", merge).split("\n\n", 1)[0];
|
||||
const parents = [...header.matchAll(/^parent ([a-f0-9]{40})$/gm)].map((match) => match[1]);
|
||||
assert.equal(parents.length, 2, "Expected a GitHub pull request merge commit");
|
||||
assert.equal(parents[1], prHead, "Merge commit does not contain the expected PR head");
|
||||
const base = parents[0];
|
||||
const changed = execFileSync("git", ["diff", "--no-ext-diff", "--no-textconv", "--no-renames",
|
||||
"--name-only", "-z", base, merge, "--"], {
|
||||
cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
|
||||
}).split("\0").filter(Boolean).sort();
|
||||
return { diff: { source: "git-merge-parent", base, head: merge, prHead }, changed };
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
// Audited prose inputs, shared by routing, Vitest selection and task hashing.
|
||||
// Do not broaden this to all Markdown: docs/api and bundled skills are runtime inputs.
|
||||
export function documentationInput(file) {
|
||||
if (file === "doc/DEVELOPING.md" || file === "doc/DATABASE.md") return "guidance";
|
||||
if (/^doc\/plans\/(?:[^/]+\/)*[^/]+\.md$/.test(file)
|
||||
&& !file.split("/").includes("..")) return "historical";
|
||||
return null;
|
||||
}
|
||||
|
||||
export const productInputs = (files) => files.filter((file) => !documentationInput(file));
|
||||
@@ -2,6 +2,8 @@ import { appendFileSync, readFileSync, realpathSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { matchesGlob } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { documentationInput, productInputs } from "./ci-inputs.mjs";
|
||||
import { mergeDiff } from "./ci-diff.mjs";
|
||||
|
||||
const releaseNote = /^releases\/(beta\/)?v[^/]+\.md$/;
|
||||
const validPath = (file) => typeof file === "string" && file.length > 0
|
||||
@@ -52,28 +54,33 @@ export function planTasks({ config, lanes, paths, mode = "enabled", forceReason
|
||||
const changed = [...new Set(paths)];
|
||||
if (!changed.length || changed.some((file) => !validPath(file))) forceReason ??= "Missing or invalid diff";
|
||||
const matches = (file, globs) => globs.some((glob) => matchesGlob(file, glob));
|
||||
const globalChanges = changed.filter((file) => matches(file, config.globalDependencies));
|
||||
const sourceChanges = productInputs(changed);
|
||||
const globalChanges = sourceChanges.filter((file) => matches(file, config.globalDependencies));
|
||||
if (globalChanges.length) forceReason ??= `Global CI input changed: ${globalChanges[0]}`;
|
||||
const unknown = changed.filter((file) => !releaseNote.test(file)
|
||||
const unknown = sourceChanges.filter((file) => !releaseNote.test(file)
|
||||
&& !matches(file, config.globalDependencies) && ![...inputs.values()].some((globs) => matches(file, globs)));
|
||||
if (unknown.length) forceReason ??= `Unclassified path: ${unknown[0]}`;
|
||||
if (mode === "full") forceReason ??= "Full CI requested";
|
||||
const reasons = Object.fromEntries([...inputs].map(([task, globs]) => [task,
|
||||
forceReason ? [forceReason] : [...new Set([...globalChanges, ...changed.filter((file) => matches(file, globs))])],
|
||||
forceReason ? [forceReason] : [...new Set([...globalChanges, ...sourceChanges.filter((file) => matches(file, globs))])],
|
||||
]));
|
||||
// This uncached lane owns the repository-wide CLI guidance check.
|
||||
if (reasons["ci:typecheck"]) reasons["ci:typecheck"].push(...changed.filter((file) => documentationInput(file) === "guidance"));
|
||||
const jobs = Object.fromEntries(Object.entries(lanes).map(([job, entries]) => {
|
||||
const affected = entries.filter((entry) => reasons[entry.task].length > 0);
|
||||
const selected = mode === "shadow" ? entries : affected;
|
||||
return [job, { run: selected.length > 0, matrix: { include: selected.length ? selected : entries },
|
||||
selected: selected.length, proposed: affected.length, total: entries.length }];
|
||||
}));
|
||||
return { version: 1, mode, forceReason, changed, reasons, jobs };
|
||||
return { version: 1, mode, forceReason, changed, documentation: changed.filter(documentationInput), reasons, jobs };
|
||||
}
|
||||
|
||||
export function planSummary(plan) {
|
||||
const rows = Object.entries(plan.jobs).map(([job, lane]) =>
|
||||
`| ${job} | ${lane.proposed}/${lane.total} | ${lane.selected}/${lane.total} |`);
|
||||
return ["### Incremental CI", "", `Mode: **${plan.mode}**${plan.forceReason ? ` — ${plan.forceReason}` : ""}`,
|
||||
...(plan.diff ? [`Diff: \`${plan.diff.base}\` → \`${plan.diff.head}\` (complete Git tree comparison)`, ""] : []),
|
||||
`Documentation inputs: ${(plan.documentation ?? []).join(", ") || "none"}`,
|
||||
"", "| Suite | Affected jobs | Scheduled jobs |", "| --- | ---: | ---: |", ...rows,
|
||||
"", "<details><summary>Selection reasons</summary>", "", "```json", JSON.stringify(plan.reasons, null, 2),
|
||||
"```", "</details>", ""].join("\n");
|
||||
@@ -83,9 +90,13 @@ if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.me
|
||||
try {
|
||||
const config = JSON.parse(readFileSync(process.env.CI_TASK_CONFIG ?? "turbo.json", "utf8"));
|
||||
const lanes = JSON.parse(readFileSync(process.env.CI_TASK_MATRIX ?? ".github/ci-task-matrix.json", "utf8"));
|
||||
let paths = [], forceReason = process.env.CI_FORCE_REASON || null;
|
||||
let paths = [], diff, forceReason = process.env.CI_FORCE_REASON || null;
|
||||
try {
|
||||
if (process.env.CI_COMPARISON) {
|
||||
if (process.env.CI_DIFF_REPOSITORY) {
|
||||
const result = mergeDiff(process.env.EVENT_MERGE_SHA, process.env.EVENT_HEAD_SHA, process.env.CI_DIFF_REPOSITORY);
|
||||
paths = result.changed;
|
||||
diff = result.diff;
|
||||
} else if (process.env.CI_COMPARISON) {
|
||||
paths = comparisonPaths(JSON.parse(readFileSync(process.env.CI_COMPARISON, "utf8")));
|
||||
} else {
|
||||
const [base, head = "HEAD"] = process.argv.slice(2);
|
||||
@@ -95,6 +106,7 @@ if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.me
|
||||
}
|
||||
} catch (error) { forceReason ??= `Diff unavailable: ${error.message}`; }
|
||||
const plan = planTasks({ config, lanes, paths, mode: process.env.CI_INCREMENTAL_MODE || "enabled", forceReason });
|
||||
if (diff) plan.diff = diff;
|
||||
if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `plan=${JSON.stringify(plan)}\n`);
|
||||
else console.log(JSON.stringify(plan, null, 2));
|
||||
if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, planSummary(plan));
|
||||
|
||||
@@ -5,6 +5,7 @@ import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { comparisonPaths, planTasks } from "../ci-plan.mjs";
|
||||
import { mergeDiff } from "../ci-diff.mjs";
|
||||
|
||||
const root = new URL("../../../", import.meta.url);
|
||||
const config = JSON.parse(readFileSync(new URL("turbo.json", root), "utf8"));
|
||||
@@ -18,7 +19,8 @@ test("trusted workflow fallbacks preserve every declared partition and command",
|
||||
const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8");
|
||||
for (const [job, entries] of Object.entries(lanes)) {
|
||||
const block = workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0];
|
||||
assert.ok(block.includes(`fromJSON(needs.gate.outputs.plan).jobs.${job}.run`), job);
|
||||
const normalized = block.replaceAll("(needs.test_selection.outputs.plan || needs.gate.outputs.plan)", "needs.gate.outputs.plan");
|
||||
assert.ok(normalized.includes(`fromJSON(needs.gate.outputs.plan).jobs.${job}.run`), job);
|
||||
if (!block.includes(" matrix:")) { assert.equal(entries.length, 1); continue; }
|
||||
const fallback = block.match(/matrix:.*\|\| '(.*)'\) \}\}/)?.[1];
|
||||
assert.ok(fallback, `${job}: missing full fallback`);
|
||||
@@ -115,7 +117,7 @@ test("cache-affecting definition changes force execution across successive PR pu
|
||||
}
|
||||
}
|
||||
const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8");
|
||||
assert.equal((workflow.match(/force:.*fromJSON\(needs.gate.outputs.plan\).forceReason != null/g) ?? []).length, 3);
|
||||
assert.equal((workflow.match(/force:.*\.forceReason != null/g) ?? []).length, 3);
|
||||
const action = readFileSync(new URL(".github/actions/run-ci-task/action.yml", root), "utf8");
|
||||
assert.match(action, /CI_FORCE_TASKS: \$\{\{ inputs.force \}\}/);
|
||||
assert.equal((action.match(/if:.*inputs.force != 'true'/g) ?? []).length, 2);
|
||||
@@ -169,6 +171,9 @@ test("required aggregates reject omitted, failed and cancelled selected work", (
|
||||
env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(resultPlan), CI_RESULTS: JSON.stringify(results) } });
|
||||
assert.equal(run().status, 0);
|
||||
for (const name of names) {
|
||||
// Optional analysis can fail or be skipped; the unchanged task plan
|
||||
// still has to pass. Selected-plan cases are exercised separately.
|
||||
if (name === "test_selection") continue;
|
||||
const previous = results[name].result;
|
||||
for (const bad of previous === "success" ? ["failure", "cancelled", "skipped"] : ["success"]) {
|
||||
results[name].result = bad;
|
||||
@@ -185,6 +190,30 @@ test("the real routing step uses pinned trusted definitions and defaults to full
|
||||
const shell = block.split(" run: |\n")[1].split("\n").map((line) => line.slice(10)).join("\n");
|
||||
const directory = mkdtempSync(path.join(tmpdir(), "ci-routing-plan-"));
|
||||
try {
|
||||
const repository = path.join(directory, "source");
|
||||
mkdirSync(repository);
|
||||
const gitBin = execFileSync("which", ["git"], { encoding: "utf8" }).trim();
|
||||
const git = (...args) => execFileSync(gitBin, args, { cwd: repository, encoding: "utf8" }).trim();
|
||||
git("init", "-q", "-b", "main"); git("config", "user.name", "CI"); git("config", "user.email", "ci@example.invalid");
|
||||
writeFileSync(path.join(repository, "base"), "base"); git("add", "."); git("commit", "-qm", "base");
|
||||
git("checkout", "-qb", "pr");
|
||||
mkdirSync(path.join(repository, "ui/src/components"), { recursive: true });
|
||||
writeFileSync(path.join(repository, "ui/src/components/Sidebar.tsx"), "change");
|
||||
git("add", "."); git("commit", "-qm", "PR");
|
||||
const prHead = git("rev-parse", "HEAD");
|
||||
git("checkout", "-q", "main");
|
||||
writeFileSync(path.join(repository, "unrelated-master"), "already merged");
|
||||
git("add", "."); git("commit", "-qm", "advance base");
|
||||
const base = git("rev-parse", "HEAD");
|
||||
git("merge", "--no-ff", "pr", "-m", "tested merge");
|
||||
const merge = git("rev-parse", "HEAD");
|
||||
writeFileSync(path.join(directory, "git"), `#!/usr/bin/env node
|
||||
const {spawnSync}=require('node:child_process');
|
||||
if(process.env.TEST_GIT_FAILURE==='true')process.exit(1);
|
||||
const args=process.argv.slice(2).map(a=>a==='https://github.com/paperclipai/paperclip.git'?process.env.TEST_GIT_REPOSITORY:a);
|
||||
const r=spawnSync(${JSON.stringify(gitBin)},args,{stdio:'inherit'});process.exit(r.status??1);
|
||||
`);
|
||||
chmodSync(path.join(directory, "git"), 0o755);
|
||||
const gh = path.join(directory, "gh");
|
||||
writeFileSync(gh, `#!/usr/bin/env node
|
||||
const fs = require('node:fs'), path = require('node:path');
|
||||
@@ -195,27 +224,77 @@ else if (endpoint?.includes('/contents/')) {
|
||||
const [file, query] = endpoint.split('/contents/')[1].split('?');
|
||||
if (query !== 'ref=' + 'a'.repeat(40)) process.exit(2);
|
||||
process.stdout.write(fs.readFileSync(path.join(process.env.TEST_TRUSTED_ROOT, file)));
|
||||
} else if (endpoint?.includes('/compare/')) console.log(JSON.stringify({ files: [{filename:'ui/src/components/Sidebar.tsx'}] }));
|
||||
}
|
||||
else process.exit(3);
|
||||
`);
|
||||
chmodSync(gh, 0o755);
|
||||
for (const overrides of [{}, { TEST_FETCH_FAILURE: "true" }, { PROFILE_CI: "true" }, { STACK_JSON: "{}" }, { FULL_CI: "false" }]) {
|
||||
for (const overrides of [{}, { TEST_FETCH_FAILURE: "true" }, { TEST_GIT_FAILURE: "true" }, { PROFILE_CI: "true" }, { STACK_JSON: "{}" }, { FULL_CI: "false" }]) {
|
||||
const output = path.join(directory, "output");
|
||||
writeFileSync(output, "");
|
||||
const result = spawnSync("bash", ["-c", shell], { encoding: "utf8", env: {
|
||||
...process.env, PATH: `${directory}:${process.env.PATH}`, TEST_TRUSTED_ROOT: root.pathname,
|
||||
RUNNER_TEMP: directory, FULL_CI: "true", PROFILE_CI: "false", FORCE_FULL_CI: "false",
|
||||
CI_INCREMENTAL_MODE: "enabled", STACK_JSON: "null", EVENT_BASE_SHA: "b".repeat(40), EVENT_MERGE_SHA: "c".repeat(40),
|
||||
CI_INCREMENTAL_MODE: "enabled", STACK_JSON: "null", EVENT_BASE_SHA: "b".repeat(40), EVENT_HEAD_SHA: prHead, EVENT_MERGE_SHA: merge,
|
||||
TEST_GIT_REPOSITORY: repository,
|
||||
GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: "", ...overrides,
|
||||
} });
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const value = readFileSync(output, "utf8").trim();
|
||||
if (overrides.TEST_FETCH_FAILURE || overrides.FULL_CI === "false") assert.equal(value, "plan=");
|
||||
if (overrides.TEST_FETCH_FAILURE || overrides.TEST_GIT_FAILURE || overrides.FULL_CI === "false") assert.equal(value, "plan=");
|
||||
else {
|
||||
assert.notEqual(value, "plan=", `${JSON.stringify(overrides)}: ${result.stderr}`);
|
||||
const resultPlan = JSON.parse(value.slice("plan=".length));
|
||||
assert.equal(resultPlan.jobs.general_tests.selected, overrides.PROFILE_CI || overrides.STACK_JSON ? 19 : 2);
|
||||
assert.equal(resultPlan.diff.base, base);
|
||||
assert.deepEqual(resultPlan.changed, ["ui/src/components/Sidebar.tsx"]);
|
||||
}
|
||||
}
|
||||
} finally { rmSync(directory, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
|
||||
test("audited prose does not widen source selection, but guidance still runs its uncached guard", () => {
|
||||
const source = "ui/src/components/Sidebar.tsx";
|
||||
const sourcePlan = plan([source]);
|
||||
for (const doc of ["doc/DEVELOPING.md", "doc/DATABASE.md", "doc/plans/2026-10-30-example.md"]) {
|
||||
const mixed = plan([source, doc]);
|
||||
assert.equal(mixed.forceReason, null);
|
||||
assert.deepEqual(mixed.jobs, sourcePlan.jobs);
|
||||
assert.deepEqual(mixed.documentation, [doc]);
|
||||
}
|
||||
const guidance = plan(["doc/DEVELOPING.md"]);
|
||||
assert.deepEqual([...selectedTasks(guidance)], ["ci:typecheck"]);
|
||||
assert.equal(config.tasks["//#ci:typecheck"].cache, false);
|
||||
assert.deepEqual([...selectedTasks(plan(["doc/plans/2026-10-30-example.md"]))], []);
|
||||
for (const file of ["docs/api/issues.md", "doc/unknown.md", "doc/plans/tool.sh", "skills/paperclip/SKILL.md",
|
||||
"server/src/onboarding-assets/default/AGENTS.md", "tests/fixtures/example.md"]) {
|
||||
assert.ok(selectedTasks(plan([source, file])).has("ci:server"), file);
|
||||
}
|
||||
});
|
||||
|
||||
test("complete merge trees handle more than 300 files, base advancement, renames and deletions", (t) => {
|
||||
const directory = mkdtempSync(path.join(tmpdir(), "ci-merge-diff-"));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
|
||||
const write = (file, value) => { mkdirSync(path.dirname(path.join(directory, file)), { recursive: true }); writeFileSync(path.join(directory, file), value); };
|
||||
const commit = () => { git("add", "."); git("commit", "-qm", "fixture"); return git("rev-parse", "HEAD"); };
|
||||
git("init", "-q", "-b", "main"); git("config", "user.name", "CI"); git("config", "user.email", "ci@example.invalid");
|
||||
write("ui/src/deleted.ts", "old"); write("ui/src/renamed.ts", "rename"); const staleBase = commit();
|
||||
git("checkout", "-qb", "pr");
|
||||
for (let index = 0; index < 350; index++) write(`ui/src/component-${index}.ts`, "new");
|
||||
git("rm", "ui/src/deleted.ts");
|
||||
mkdirSync(path.join(directory, "doc/plans"), { recursive: true });
|
||||
git("mv", "ui/src/renamed.ts", "doc/plans/2026-10-30-moved.md");
|
||||
const prHead = commit();
|
||||
git("checkout", "-q", "main"); write(".github/workflows/unrelated.yml", "already merged CI change"); const base = commit();
|
||||
git("merge", "--no-ff", "pr", "-m", "tested merge"); const head = git("rev-parse", "HEAD");
|
||||
const result = mergeDiff(head, prHead, directory);
|
||||
assert.equal(result.diff.base, base); assert.notEqual(result.diff.base, staleBase);
|
||||
assert.equal(result.changed.length, 353);
|
||||
assert.ok(result.changed.includes("ui/src/deleted.ts")); assert.ok(result.changed.includes("ui/src/renamed.ts"));
|
||||
assert.ok(!result.changed.includes(".github/workflows/unrelated.yml"));
|
||||
assert.equal(plan(result.changed).forceReason, null);
|
||||
assert.throws(() => mergeDiff(head, base, directory), /expected PR head/);
|
||||
assert.throws(() => mergeDiff(prHead, prHead, directory), /merge commit/);
|
||||
assert.throws(() => mergeDiff("f".repeat(40), prHead, directory));
|
||||
});
|
||||
@@ -7,6 +7,8 @@ on:
|
||||
- turbo.json
|
||||
- .github/ci-task-matrix.json
|
||||
- .github/scripts/ci-plan.mjs
|
||||
- .github/scripts/ci-diff.mjs
|
||||
- .github/scripts/ci-inputs.mjs
|
||||
- .github/scripts/tests/ci-plan.test.mjs
|
||||
- .github/actions/run-ci-task/**
|
||||
- .github/workflows/ci-routing.yml
|
||||
@@ -22,6 +24,50 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
qualify_test_selection:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86
|
||||
with:
|
||||
version: 9.15.4
|
||||
- uses: ./.github/actions/setup-ci-dependencies
|
||||
- name: Prove conservative graphs, exact partitions, manifests and failure fallbacks
|
||||
run: node --test scripts/ci/test-selection*.test.mjs scripts/ci/vitest-impact-report.integration.test.mjs
|
||||
- name: Qualify selection against the real Paperclip workspace
|
||||
env:
|
||||
NODE_ENV: test
|
||||
VITEST: 'true'
|
||||
run: |
|
||||
node --input-type=module <<'NODE'
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { planTasks } from './.github/scripts/ci-plan.mjs';
|
||||
import { createTestSelection } from './scripts/ci/select-tests.mjs';
|
||||
const config = JSON.parse(readFileSync('turbo.json', 'utf8'));
|
||||
const lanes = JSON.parse(readFileSync('.github/ci-task-matrix.json', 'utf8'));
|
||||
const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
|
||||
for (const [file, project, related] of [
|
||||
['ui/src/pages/Inbox.tsx', '@paperclipai/ui', 'ui/src/pages/Inbox.test.tsx'],
|
||||
['server/src/middleware/auth.ts', '@paperclipai/server', 'server/src/__tests__/agent-auth-middleware.test.ts'],
|
||||
]) {
|
||||
const selected = await createTestSelection(planTasks({ config, lanes, paths: [file, "doc/DEVELOPING.md", "doc/plans/2026-10-30-ci-proof.md"] }), head);
|
||||
const graph = selected.graphs.find(graph => graph.project === project);
|
||||
assert.ok(graph.related.includes(related));
|
||||
assert.ok(graph.alwaysRun.length > 0, 'Filesystem contracts must remain covered');
|
||||
assert.ok(graph.selected.length < graph.inventory.length, 'The real workspace must support narrowing');
|
||||
console.log(file, JSON.stringify(selected.plan.testSelection.stats, null, 2));
|
||||
}
|
||||
NODE
|
||||
|
||||
qualify:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
@@ -350,24 +350,27 @@ jobs:
|
||||
FORCE_FULL_CI: ${{ contains(github.event.pull_request.labels.*.name, 'full-ci') }}
|
||||
CI_INCREMENTAL_MODE: ${{ vars.CI_INCREMENTAL_MODE || 'enabled' }}
|
||||
STACK_JSON: ${{ toJSON(github.event.pull_request.stack) }}
|
||||
EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
EVENT_MERGE_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
fallback() { echo 'plan=' >> "$GITHUB_OUTPUT"; exit 0; }
|
||||
[[ "$FULL_CI" == 'true' ]] || fallback
|
||||
[[ "$EVENT_BASE_SHA" =~ ^[0-9a-f]{40}$ && "$EVENT_MERGE_SHA" =~ ^[0-9a-f]{40}$ ]] || fallback
|
||||
[[ "$EVENT_HEAD_SHA" =~ ^[0-9a-f]{40}$ && "$EVENT_MERGE_SHA" =~ ^[0-9a-f]{40}$ ]] || fallback
|
||||
trusted="$(gh api /repos/paperclipai/paperclip/git/ref/heads/master --jq .object.sha)" || fallback
|
||||
[[ "$trusted" =~ ^[0-9a-f]{40}$ ]] || fallback
|
||||
directory="$(mktemp -d)"
|
||||
trap 'rm -rf "$directory"' EXIT
|
||||
for file in .github/scripts/ci-plan.mjs .github/ci-task-matrix.json turbo.json; do
|
||||
for file in .github/scripts/ci-plan.mjs .github/scripts/ci-inputs.mjs .github/scripts/ci-diff.mjs .github/ci-task-matrix.json turbo.json; do
|
||||
gh api -H 'Accept: application/vnd.github.raw+json' \
|
||||
"/repos/paperclipai/paperclip/contents/$file?ref=$trusted" \
|
||||
> "$directory/${file##*/}" || fallback
|
||||
done
|
||||
gh api "/repos/paperclipai/paperclip/compare/$EVENT_BASE_SHA...$EVENT_MERGE_SHA" \
|
||||
> "$directory/comparison.json" || fallback
|
||||
# Read the exact tested merge and both parents. The event's base SHA can
|
||||
# lag behind its actual first parent. A bare tree diff has no API file cap.
|
||||
git init --bare "$directory/repository.git" >/dev/null || fallback
|
||||
git -C "$directory/repository.git" fetch --quiet --no-tags --filter=blob:none --depth=2 \
|
||||
https://github.com/paperclipai/paperclip.git "$EVENT_MERGE_SHA" || fallback
|
||||
if [[ "$PROFILE_CI" == 'true' || "$FORCE_FULL_CI" == 'true' ]]; then
|
||||
export CI_FORCE_REASON='Full suite requested by label'
|
||||
elif [[ "$STACK_JSON" != 'null' && -n "$STACK_JSON" ]]; then
|
||||
@@ -375,5 +378,5 @@ jobs:
|
||||
fi
|
||||
export CI_TASK_CONFIG="$directory/turbo.json"
|
||||
export CI_TASK_MATRIX="$directory/ci-task-matrix.json"
|
||||
export CI_COMPARISON="$directory/comparison.json"
|
||||
export CI_DIFF_REPOSITORY="$directory/repository.git"
|
||||
node "$directory/ci-plan.mjs" || fallback
|
||||
@@ -24,6 +24,82 @@ jobs:
|
||||
gate:
|
||||
uses: paperclipai/paperclip/.github/workflows/ci-routing.yml@master
|
||||
|
||||
test_selection:
|
||||
name: Select Vitest files
|
||||
needs: [gate]
|
||||
if: >-
|
||||
needs.gate.outputs.full_ci == 'true' && needs.gate.outputs.plan != '' &&
|
||||
fromJSON(needs.gate.outputs.plan).mode == 'enabled' &&
|
||||
fromJSON(needs.gate.outputs.plan).forceReason == null &&
|
||||
(vars.CI_VITEST_SELECTION == '' || vars.CI_VITEST_SELECTION == 'enabled')
|
||||
runs-on: ${{ needs.gate.outputs.runner }}
|
||||
timeout-minutes: 10
|
||||
# An unavailable optimization must not suppress the original test matrix.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
plan: ${{ steps.publish.outputs.plan }}
|
||||
env:
|
||||
CI_PLAN: ${{ needs.gate.outputs.plan }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
- name: Check source-diff eligibility
|
||||
id: eligible
|
||||
run: |
|
||||
if [[ -f scripts/ci/select-tests.mjs ]]; then
|
||||
node scripts/ci/select-tests.mjs --eligible
|
||||
else
|
||||
echo 'enabled=false' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86
|
||||
if: steps.eligible.outputs.enabled == 'true'
|
||||
with:
|
||||
version: 9.15.4
|
||||
- name: Restore dependencies for graph analysis
|
||||
if: steps.eligible.outputs.enabled == 'true'
|
||||
uses: ./.github/actions/setup-ci-dependencies
|
||||
- name: Select files and rebalance test lanes
|
||||
id: select
|
||||
if: steps.eligible.outputs.enabled == 'true'
|
||||
timeout-minutes: 5
|
||||
run: node scripts/ci/select-tests.mjs
|
||||
- name: Upload validated selection and audit
|
||||
id: upload_selection
|
||||
if: steps.select.outputs.selected == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
|
||||
with:
|
||||
name: vitest-selection-${{ github.run_attempt }}
|
||||
path: |
|
||||
.ci-test-selection/manifest.json
|
||||
.ci-test-selection/audit.json
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
# Publish only after upload succeeds. Missing/failed/timed-out selectors
|
||||
# produce no plan, and dependent jobs use the original gate's matrix.
|
||||
- name: Publish the executable plan
|
||||
id: publish
|
||||
if: steps.eligible.outputs.enabled == 'true'
|
||||
env:
|
||||
ARTIFACT_ID: ${{ steps.upload_selection.outputs.artifact-id }}
|
||||
run: |
|
||||
node --input-type=module <<'NODE'
|
||||
import assert from 'node:assert/strict';
|
||||
import { appendFileSync, readFileSync } from 'node:fs';
|
||||
const plan = JSON.parse(readFileSync('.ci-test-selection/plan.json', 'utf8'));
|
||||
if (plan.testSelection) {
|
||||
assert.match(process.env.ARTIFACT_ID ?? '', /^[0-9]+$/);
|
||||
plan.testSelection.artifactId = process.env.ARTIFACT_ID;
|
||||
}
|
||||
appendFileSync(process.env.GITHUB_OUTPUT, `plan=${JSON.stringify(plan)}\n`);
|
||||
NODE
|
||||
|
||||
policy:
|
||||
needs: [gate]
|
||||
runs-on: ${{ needs.gate.outputs.runner }}
|
||||
@@ -358,15 +434,17 @@ jobs:
|
||||
|
||||
general_tests:
|
||||
name: General tests (${{ matrix.group_label }})
|
||||
needs: [gate]
|
||||
if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.general_tests.run) }}
|
||||
needs: [gate, test_selection]
|
||||
if: ${{ always() && !cancelled() && needs.gate.result == 'success' && needs.gate.outputs.full_ci == 'true' && ((needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.general_tests.run) }}
|
||||
runs-on: ${{ needs.gate.outputs.runner }}
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.general_tests.matrix) || '{"include":[{"group":"general-server-without-chat","group_label":"server (1/12)","shard_index":0,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (2/12)","shard_index":1,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (3/12)","shard_index":2,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (4/12)","shard_index":3,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (5/12)","shard_index":4,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (6/12)","shard_index":5,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (7/12)","shard_index":6,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (8/12)","shard_index":7,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (9/12)","shard_index":8,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (10/12)","shard_index":9,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (11/12)","shard_index":10,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (12/12)","shard_index":11,"shard_count":12,"task":"ci:server"},{"group":"general-chat","group_label":"chat (1/3)","shard_index":0,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (2/3)","shard_index":1,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (3/3)","shard_index":2,"shard_count":3,"task":"ci:chat"},{"group":"general-workspaces-a","group_label":"workspaces-a (1/2)","shard_index":0,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-a","group_label":"workspaces-a (2/2)","shard_index":1,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-b","group_label":"workspaces-b","task":"ci:workspaces-b"},{"group":"general-workspaces-c","group_label":"workspaces-c","task":"ci:db"}]}') }}
|
||||
matrix: ${{ fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan) != '' && toJSON(fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.general_tests.matrix) || '{"include":[{"group":"general-server-without-chat","group_label":"server (1/12)","shard_index":0,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (2/12)","shard_index":1,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (3/12)","shard_index":2,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (4/12)","shard_index":3,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (5/12)","shard_index":4,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (6/12)","shard_index":5,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (7/12)","shard_index":6,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (8/12)","shard_index":7,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (9/12)","shard_index":8,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (10/12)","shard_index":9,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (11/12)","shard_index":10,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (12/12)","shard_index":11,"shard_count":12,"task":"ci:server"},{"group":"general-chat","group_label":"chat (1/3)","shard_index":0,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (2/3)","shard_index":1,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (3/3)","shard_index":2,"shard_count":3,"task":"ci:chat"},{"group":"general-workspaces-a","group_label":"workspaces-a (1/2)","shard_index":0,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-a","group_label":"workspaces-a (2/2)","shard_index":1,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-b","group_label":"workspaces-b","task":"ci:workspaces-b"},{"group":"general-workspaces-c","group_label":"workspaces-c","task":"ci:db"}]}') }}
|
||||
|
||||
env:
|
||||
PAPERCLIP_CI_TEST_SELECTION: ${{ matrix.selection && '.ci-test-selection/manifest.json' || '' }}
|
||||
PAPERCLIP_CI_SELECTION_HASH: ${{ matrix.selection && fromJSON(needs.test_selection.outputs.plan).testSelection.hash || '' }}
|
||||
PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }}
|
||||
PAPERCLIP_CI_PROFILE_JOB: General tests (${{ matrix.group_label }})
|
||||
|
||||
@@ -376,6 +454,13 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download selected test files
|
||||
if: matrix.selection
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
|
||||
with:
|
||||
artifact-ids: ${{ fromJSON(needs.test_selection.outputs.plan).testSelection.artifactId }}
|
||||
path: .ci-test-selection
|
||||
|
||||
- name: Initialize CI profile
|
||||
continue-on-error: true
|
||||
if: env.PAPERCLIP_CI_PROFILE == 'true'
|
||||
@@ -438,7 +523,7 @@ jobs:
|
||||
task: ${{ matrix.task }}
|
||||
args: ${{ matrix.shard_count && format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) || '[]' }}
|
||||
partition: ${{ strategy.job-index || 0 }}
|
||||
force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }}
|
||||
force: ${{ needs.gate.outputs.profile_ci == 'true' || (needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).mode != 'enabled' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).forceReason != null }}
|
||||
|
||||
- name: Run grouped general test suites
|
||||
if: hashFiles('turbo.json') == ''
|
||||
@@ -521,14 +606,14 @@ jobs:
|
||||
verify:
|
||||
name: verify
|
||||
if: ${{ always() }}
|
||||
needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run]
|
||||
needs: [gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run]
|
||||
runs-on: ${{ needs.gate.outputs.runner }}
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Validate every planned suite result
|
||||
env:
|
||||
FULL_CI: ${{ needs.gate.outputs.full_ci }}
|
||||
CI_PLAN: ${{ needs.gate.outputs.plan }}
|
||||
CI_PLAN: ${{ needs.test_selection.outputs.plan || needs.gate.outputs.plan }}
|
||||
CI_RESULTS: ${{ toJSON(needs) }}
|
||||
run: |
|
||||
node --input-type=module <<'NODE'
|
||||
@@ -539,6 +624,7 @@ jobs:
|
||||
assert.ok(['true', 'false'].includes(process.env.FULL_CI), 'Valid scope required');
|
||||
const plan = process.env.CI_PLAN ? JSON.parse(process.env.CI_PLAN) : null;
|
||||
if (plan) assert.equal(plan.version, 1, 'Supported task plan required');
|
||||
if (plan?.testSelection) assert.equal(results.test_selection.result, 'success', 'Published selection must succeed');
|
||||
for (const job of ["typecheck_release_registry","general_tests","verify_paperclip_runner","build","docker_context_integrity","verify_serialized_server","canary_dry_run"]) {
|
||||
if (plan) assert.equal(typeof plan.jobs[job]?.run, 'boolean', 'Missing suite decision: ' + job);
|
||||
const selected = process.env.FULL_CI === 'true' && (!plan || plan.jobs[job].run);
|
||||
@@ -1021,15 +1107,17 @@ jobs:
|
||||
|
||||
verify_serialized_server:
|
||||
name: Verify serialized server suites (${{ matrix.shard_label }})
|
||||
needs: [gate]
|
||||
if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.run) }}
|
||||
needs: [gate, test_selection]
|
||||
if: ${{ always() && !cancelled() && needs.gate.result == 'success' && needs.gate.outputs.full_ci == 'true' && ((needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.verify_serialized_server.run) }}
|
||||
runs-on: ${{ needs.gate.outputs.runner }}
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.matrix) || '{"include":[{"shard_index":0,"shard_count":9,"shard_label":"1/9","task":"ci:serialized"},{"shard_index":1,"shard_count":9,"shard_label":"2/9","task":"ci:serialized"},{"shard_index":2,"shard_count":9,"shard_label":"3/9","task":"ci:serialized"},{"shard_index":3,"shard_count":9,"shard_label":"4/9","task":"ci:serialized"},{"shard_index":4,"shard_count":9,"shard_label":"5/9","task":"ci:serialized"},{"shard_index":5,"shard_count":9,"shard_label":"6/9","task":"ci:serialized"},{"shard_index":6,"shard_count":9,"shard_label":"7/9","task":"ci:serialized"},{"shard_index":7,"shard_count":9,"shard_label":"8/9","task":"ci:serialized"},{"shard_index":8,"shard_count":9,"shard_label":"9/9","task":"ci:serialized"}]}') }}
|
||||
matrix: ${{ fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan) != '' && toJSON(fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.verify_serialized_server.matrix) || '{"include":[{"shard_index":0,"shard_count":9,"shard_label":"1/9","task":"ci:serialized"},{"shard_index":1,"shard_count":9,"shard_label":"2/9","task":"ci:serialized"},{"shard_index":2,"shard_count":9,"shard_label":"3/9","task":"ci:serialized"},{"shard_index":3,"shard_count":9,"shard_label":"4/9","task":"ci:serialized"},{"shard_index":4,"shard_count":9,"shard_label":"5/9","task":"ci:serialized"},{"shard_index":5,"shard_count":9,"shard_label":"6/9","task":"ci:serialized"},{"shard_index":6,"shard_count":9,"shard_label":"7/9","task":"ci:serialized"},{"shard_index":7,"shard_count":9,"shard_label":"8/9","task":"ci:serialized"},{"shard_index":8,"shard_count":9,"shard_label":"9/9","task":"ci:serialized"}]}') }}
|
||||
|
||||
env:
|
||||
PAPERCLIP_CI_TEST_SELECTION: ${{ matrix.selection && '.ci-test-selection/manifest.json' || '' }}
|
||||
PAPERCLIP_CI_SELECTION_HASH: ${{ matrix.selection && fromJSON(needs.test_selection.outputs.plan).testSelection.hash || '' }}
|
||||
PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }}
|
||||
PAPERCLIP_CI_PROFILE_JOB: Verify serialized server suites (${{ matrix.shard_label }})
|
||||
|
||||
@@ -1039,6 +1127,13 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download selected test files
|
||||
if: matrix.selection
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
|
||||
with:
|
||||
artifact-ids: ${{ fromJSON(needs.test_selection.outputs.plan).testSelection.artifactId }}
|
||||
path: .ci-test-selection
|
||||
|
||||
- name: Initialize CI profile
|
||||
continue-on-error: true
|
||||
if: env.PAPERCLIP_CI_PROFILE == 'true'
|
||||
@@ -1101,7 +1196,7 @@ jobs:
|
||||
task: ci:serialized
|
||||
args: ${{ format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) }}
|
||||
partition: ${{ strategy.job-index || 0 }}
|
||||
force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }}
|
||||
force: ${{ needs.gate.outputs.profile_ci == 'true' || (needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).mode != 'enabled' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).forceReason != null }}
|
||||
|
||||
- name: Run serialized server test shard
|
||||
if: hashFiles('turbo.json') == ''
|
||||
|
||||
@@ -99,6 +99,7 @@ test-results/
|
||||
|
||||
# Opt-in CI timing artifacts
|
||||
.ci-profile/
|
||||
.ci-test-selection/
|
||||
|
||||
# Content-addressed CI task results
|
||||
.turbo/
|
||||
@@ -71,6 +71,22 @@
|
||||
"packages/paperclip-runner/src/**",
|
||||
"packages/paperclip-runner/runner/**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"path": "rules/acpx-profile-identity.mdc",
|
||||
"description": "ACPX provider identity model: provider digests attest upstream releases, the pack contract digest covers Runner source and the shared ACPX layer, and profile versions change only for deliberate resets.",
|
||||
"scope": [
|
||||
"packages/paperclip-runner/acpx-profiles.json",
|
||||
"packages/paperclip-runner/cursor-contract.json",
|
||||
"packages/paperclip-runner/cursor-distributions.json",
|
||||
"packages/paperclip-runner/antigravity-distributions.json",
|
||||
"packages/paperclip-runner/scripts/acpx-release-attestations.mjs",
|
||||
"packages/paperclip-runner/scripts/generate-acpx-profiles.mjs",
|
||||
"packages/paperclip-runner/scripts/contracts/**",
|
||||
"packages/paperclip-runner/src/drivers/acpx/**",
|
||||
"packages/paperclip-runner/src/providers/**",
|
||||
"packages/paperclip-runner/protocol/**"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -41,6 +41,10 @@ For `server/src/services/chat-channels.ts` and everything under
|
||||
For `server/src/services/heartbeat.ts`, `server/src/services/heartbeat/`,
|
||||
heartbeat-prefixed sibling services, and heartbeat regression tests, read
|
||||
[`rules/heartbeat-execution-boundaries.mdc`](rules/heartbeat-execution-boundaries.mdc).
|
||||
For ACPX provider profiles, their release attestations and pins, the provider-pack
|
||||
contract registry, and ACPX driver and provider code under
|
||||
`packages/paperclip-runner/`, read
|
||||
[`rules/acpx-profile-identity.mdc`](rules/acpx-profile-identity.mdc).
|
||||
See [`rules/README.md`](rules/README.md) for rule authoring and Greptile setup.
|
||||
|
||||
## 3. Repo Map
|
||||
|
||||
@@ -722,6 +722,60 @@ connection, before enabling the new server and UI.
|
||||
- The repo keeps only the newest 5 snapshots. `generate` runs `prune:snapshots` afterwards to delete older ones. Drizzle only reads the newest snapshot, and each snapshot is a full copy of the schema (over 1 MB each). Older snapshots are still in git history.
|
||||
- `packages/db/src/migration-snapshot-drift.test.ts` is the enforcement backstop. It repeats the diff that `generate` performs and fails when the newest snapshot no longer matches `packages/db/src/schema/`.
|
||||
|
||||
## Managed adapter settings
|
||||
|
||||
Cloud-managed instances keep adapter selection preferences in the private
|
||||
`instance_settings` singleton `adapter-settings:v1`. Its `general` JSON contains
|
||||
`version: 1`, a monotonic `revision`, and `disabledTypes`. It is separate from
|
||||
the public `default` settings row and never appears in the instance settings API.
|
||||
Each update locks this row in a transaction. The update and company activity
|
||||
records commit together. Discovery and new-agent admission read the database
|
||||
at their asynchronous operation boundaries; no polling or mutable settings cache
|
||||
is used. Disabled adapters remain available to existing agents.
|
||||
|
||||
`PAPERCLIP_ADAPTERS` and `PAPERCLIP_ADAPTERS_FILE` declare the effective operator
|
||||
adapter list. The declaration is parsed once for a process configuration and
|
||||
validated at startup. It does not rewrite stored preferences. A conflicting
|
||||
adapter toggle returns 403. Removing the declaration restores saved choices.
|
||||
Operator file edits require a restart. Self-hosted instances continue to use
|
||||
`adapter-settings.json` for saved preferences.
|
||||
|
||||
The server-only adapter settings service exposes `inspectStored()` and
|
||||
`seedIfAbsent({ disabledTypes }, transaction?)` for migration tooling. Seeding
|
||||
uses the unique singleton key, returns `{ seeded, current }`, and never replaces
|
||||
an existing record. The caller can use the same transaction for its import
|
||||
receipt. Before admission, `initializeAdapterSettings` checks the durable row.
|
||||
When absent, it imports a valid legacy settings file once, retaining unknown
|
||||
adapter types, or seeds empty preferences if no file exists. It rejects files
|
||||
larger than 64 KiB, malformed input, and read failures. An existing durable row
|
||||
wins over every stale file. Ordinary Cloud reads do not read a legacy file.
|
||||
|
||||
The focused PostgreSQL suite can use an owned disposable PostgreSQL database via
|
||||
`PAPERCLIP_ADAPTER_SETTINGS_TEST_DATABASE_URL` when embedded PostgreSQL is
|
||||
unavailable. It applies migrations and inserts test data, so never point this
|
||||
variable at an existing application database.
|
||||
|
||||
## Cloud configuration migration receipt
|
||||
|
||||
The private `instance_settings` singleton `cloud-configuration-migration:v1`
|
||||
records atomic import completion, semantic source digests, imported revisions,
|
||||
and metadata-only external adapter blockers. It composes the adapter and
|
||||
startup-config owners without replacing an existing authoritative row. Receipt
|
||||
locking and the import transaction handle concurrent attempts and retries; no
|
||||
local completion marker participates in conflict resolution.
|
||||
|
||||
Qualification records current configuration revisions and selected image package
|
||||
evidence before normal plugin reconciliation. Image inspection does not hold
|
||||
admin write locks; configuration and registration snapshots are rechecked in a
|
||||
short transaction before the report is saved. A historical successful report
|
||||
never grants current eligibility. External metadata blockers survive loss of the
|
||||
legacy home and require a deliberate revision-checked recheck of a present valid
|
||||
original source file. Package mismatches remain recorded after maintenance
|
||||
reconciliation. See the [bounded migration and qualification procedure](cloud-configuration-migration.md)
|
||||
for the stopped-writer prerequisite, commands, retry semantics, integrity limits,
|
||||
and the explicit exclusion of secret bytes, whole-server zero downtime, and
|
||||
volume removal.
|
||||
|
||||
## Cloud runtime identity singleton
|
||||
|
||||
The private `instance_settings` row whose singleton key is
|
||||
@@ -1490,3 +1544,12 @@ It uses disposable embedded PostgreSQL by default. A disposable local database
|
||||
named `review_wait_benchmark...` can be supplied through
|
||||
`PAPERCLIP_TEST_DATABASE_URL` to compare another PostgreSQL major version.
|
||||
DataRow bytes exclude response metadata and transport overhead.
|
||||
|
||||
## Cloud runtime configuration singleton
|
||||
|
||||
The private `instance_settings` key `cloud-runtime-config:v1` holds a strictly
|
||||
validated, versioned nonsecret configuration and revision. It is separate from
|
||||
the ordinary instance settings API. Seed inserts preserve an existing winner;
|
||||
replacements lock the row and reject stale revisions. Database/transaction
|
||||
handles permit atomic migration receipts. Connection URLs, API keys and unknown
|
||||
fields are excluded. See [field ownership and startup order](cloud-runtime-configuration.md).
|
||||
+81
-5
@@ -171,10 +171,12 @@ does not include the lower PRs' code.
|
||||
|
||||
For ordinary full-scope PRs, the routing job further selects affected suites
|
||||
using `turbo.json` and `.github/ci-task-matrix.json`. It downloads the planner
|
||||
and both definitions from one canonical master revision, without executing PR
|
||||
code. The comparison covers the tested merge commit. Both rename paths count;
|
||||
failed, empty, malformed, or potentially truncated comparisons select the full
|
||||
suite. Unclassified files also select full CI. Stack verification retains the
|
||||
and its helpers and definitions from one canonical master revision, without executing PR
|
||||
code. It fetches the exact tested merge commit and its parents into a temporary
|
||||
bare Git repository, validates the PR head, and compares the merge tree against
|
||||
its first parent. The event's base SHA can be stale. Git tree comparisons have
|
||||
no GitHub compare-API 300-file cap. Renames count as deletion plus addition;
|
||||
failed, empty, or malformed comparisons select the full suite. Unclassified files also select full CI. Stack verification retains the
|
||||
existing scope policy and runs all suites whenever that policy requires them.
|
||||
Changes to global inputs, including the task graph, package scripts, workflows,
|
||||
and execution helpers, force full execution and disable result-cache restore and
|
||||
@@ -193,6 +195,69 @@ canary dry run.
|
||||
The uncached typecheck job also runs the repository-wide CLI guidance safety
|
||||
guard. It still scans UI changes when unrelated server shards are omitted.
|
||||
|
||||
The shared policy in `.github/scripts/ci-inputs.mjs` exempts only audited prose:
|
||||
`doc/DEVELOPING.md`, `doc/DATABASE.md`, and Markdown under `doc/plans/`.
|
||||
The first two still select the uncached typecheck/guidance lane; historical plans
|
||||
need only the always-running policy checks. These inputs do not widen product
|
||||
test selection or enter product task cache hashes. All other docs, bundled
|
||||
skills, fixtures and unknown paths keep their existing conservative coverage.
|
||||
A source change plus exempt prose remains graph eligible. When adding a reader
|
||||
of exempt prose, update this policy as well as the task inputs.
|
||||
|
||||
For ordinary source PRs (including the audited prose inputs above), a separate `Select Vitest files` job refines the
|
||||
general and serialized test lanes before their runners are allocated. It runs
|
||||
on the gate-approved test fleet, loads the real Vitest projects, and follows
|
||||
imports across workspace packages. Only projects with their own source changes
|
||||
are narrowed; unchanged consumer projects retain their complete coarse lane
|
||||
coverage without paying for another graph traversal. Filesystem, subprocess, network, CommonJS,
|
||||
dynamic-loader, unresolved, and unapproved external-package consumers are
|
||||
retained conservatively, including indirect consumers through local helpers.
|
||||
The selector explicitly tracks setup/configuration dependencies as well as test
|
||||
imports. The small pure-dependency allowlist lives in
|
||||
`scripts/ci/vitest-selection-graph.mjs`; new external packages remain opaque
|
||||
until reviewed. This is deliberately broader than `vitest related` alone.
|
||||
For conservative retention, explicit complete local mock factories cut the
|
||||
mocked implementation's I/O edges. Partial mocks, spies, automocks, mock bypasses,
|
||||
and dynamic or escaped mocking APIs do not. Edits to a mocked module still use
|
||||
Vitest's unmodified related-file graph. Non-isolated projects run in full.
|
||||
Literal Vite globs use Vite's expanded dependency edges, including newly added
|
||||
members. Consumers of generated or untracked modules remain in the conservative
|
||||
set because the import graph does not describe their build inputs.
|
||||
The server's temporary-home and Supertest setup has a reviewed, content-hashed
|
||||
exception. It still follows local helper imports. Any change to the reviewed
|
||||
contents disables the exception until reviewed again; setup dependency changes
|
||||
invalidate the whole project. Unreviewed setup I/O uses the full task plan.
|
||||
|
||||
Selected files are repartitioned within their existing owners using the current
|
||||
per-runner budgets and server timing weights. Small selections allocate fewer
|
||||
jobs; empty lanes allocate none. Serialized files remain serialized, and chat's
|
||||
line shards, native Runner lanes, Rust, builds, typechecking and E2E retain their
|
||||
existing selection/execution rules. The job summary reports files and jobs
|
||||
selected per lane, and the `vitest-selection` artifact records the manifest and
|
||||
dependency audit. Selected jobs execute exact Vitest test specifications, so
|
||||
similarly named `.test.ts` and `.test.tsx` files cannot overlap between shards.
|
||||
|
||||
Missing/deleted/renamed source files, non-exempt non-source inputs, unrecognized graph inputs,
|
||||
setup/config changes, failed dependency installs, graph errors and timeouts use
|
||||
the complete original task plan. Source mutations during dependency setup also
|
||||
disable narrowing. A published manifest must match the checked-out
|
||||
merge SHA and parent diff, its plan hash and the expected shard/project ownership. Missing or
|
||||
invalid manifests fail selected jobs and the required aggregate; they cannot
|
||||
turn reduced matrices green. Failed-job retries download the original artifact
|
||||
ID saved with the plan, even when GitHub increments the run attempt. Turbo hashes the selected file assignment through
|
||||
`PAPERCLIP_CI_SELECTION_HASH`, so a passing subset cannot satisfy a full run or a
|
||||
different selection.
|
||||
|
||||
Set the repository variable `CI_VITEST_SELECTION=off` to disable file selection
|
||||
while keeping the existing task-level optimization. Empty or `enabled` enables
|
||||
it; other values disable it. `full-ci`, profiling, and task-level `full`/`shadow`
|
||||
modes also bypass file selection. Qualification uses real Vitest graphs and real
|
||||
Turbo cache entries, including a filesystem contract omitted by the raw graph:
|
||||
|
||||
```sh
|
||||
node --test scripts/ci/test-selection*.test.mjs scripts/ci/cache.integration.test.mjs
|
||||
```
|
||||
|
||||
General and serialized test suites and the standalone Rust suite use Turbo's
|
||||
content-addressed cache. An execution adapter maps selected lanes to the existing
|
||||
package scripts and their declared inputs. It selects tracked and non-ignored
|
||||
@@ -1488,7 +1553,9 @@ Paperclip also persists an empty `OPENAI_API_KEY` override for those agents so a
|
||||
|
||||
If the `codex` CLI is not installed or not on `PATH`, `codex_local` agent runs fail at execution time with a clear adapter error. Quota polling uses a short-lived `codex app-server` subprocess: when `codex` cannot be spawned, that provider reports `ok: false` in aggregated quota results and the API server keeps running (it must not exit on a missing binary).
|
||||
|
||||
Local adapters require their corresponding CLI/session setup on the machine running Paperclip. External adapters are installed through the adapter/plugin flow and should not require hardcoded imports in `server/` or `ui/`.
|
||||
Local adapters require their corresponding CLI/session setup on the machine running Paperclip. Self-hosted instances can install external adapters through the adapter/plugin flow without hardcoded imports in `server/` or `ui/`.
|
||||
|
||||
Cloud-managed instances use only adapter implementations supplied by the release image. A tenant server token or the presence of `PAPERCLIP_MANAGED_CONFIG` enforces this boundary. Startup and UI parser requests do not read external adapter registrations or packages. Install, reinstall, reload, remove, and override operations return `403` with `adapter_install_platform_managed`, including for instance admins. Legacy `adapter-plugins.json` files stay untouched and do not change the built-in registry.
|
||||
|
||||
### Filesystem access boundary (2026-10-20)
|
||||
|
||||
@@ -3815,3 +3882,12 @@ in `_test` for `PAPERCLIP_MCP_SLEEP_TEST_DATABASE_URL`,
|
||||
`PAPERCLIP_DOT_RUNNER_TEST_DATABASE_URL`. The interaction-service regression suite
|
||||
also accepts `PAPERCLIP_INTERACTIONS_TEST_DATABASE_URL`. The suites migrate and write to those
|
||||
URLs; the sleep suite truncates company data between tests.
|
||||
|
||||
## Cloud startup configuration
|
||||
|
||||
Cloud stores nonsecret startup choices in a private PostgreSQL singleton.
|
||||
Bootstrap connection credentials stay in the supplied environment. A bounded
|
||||
one-time config-file seed preserves existing choices, then later starts use the
|
||||
database without loading home `.env` files or repairing worktree config. See
|
||||
[Cloud runtime configuration](cloud-runtime-configuration.md) for field ownership,
|
||||
secret handling, initialization order and migration interfaces.
|
||||
@@ -0,0 +1,148 @@
|
||||
# Cloud configuration migration and package qualification
|
||||
|
||||
This procedure covers saved adapter preferences, the allowlisted nonsecret
|
||||
startup configuration, and selected bundled plugin packages. The result is a
|
||||
point-in-time report. It does not authorize a deployment, guarantee zero
|
||||
downtime, establish general Railway admission, or prove that a volume can be
|
||||
removed. Secret and key bytes, database credentials, artifacts, logs, local
|
||||
storage, workspaces, and plugin state outside PostgreSQL need their own storage
|
||||
and replacement checks. This includes the decision signing key and local
|
||||
encryption keys. Their byte authority is unchanged by this procedure.
|
||||
Runtime environment and secret equality, plus an actual HTTP handoff between
|
||||
application processes, remain separate rollout prerequisites. The package digest
|
||||
does not establish those conditions.
|
||||
|
||||
## Migration prerequisite
|
||||
|
||||
Stop every legacy file writer before the first import. This includes old
|
||||
application replicas and operator commands that can write configuration or
|
||||
adapter settings. One maintenance window is sufficient. The importer cannot
|
||||
make an old file writer participate in its database transaction. Keep those
|
||||
writers stopped after import; later file edits do not update database authority.
|
||||
|
||||
Keep the normal database backup and recovery procedure for the instance. Use the
|
||||
new release with the same managed environment and database connection. The
|
||||
server runs migration before constructing configuration-dependent services.
|
||||
For an explicit source-checkout invocation, use:
|
||||
|
||||
```sh
|
||||
pnpm exec tsx server/src/qualify-cloud-configuration.ts --migrate
|
||||
```
|
||||
|
||||
The command uses `DATABASE_URL` and the normal Cloud environment. It does not
|
||||
accept credentials as arguments. It writes the receipt to PostgreSQL and prints
|
||||
a JSON report; no report file or local completion marker is required. Exit code
|
||||
0 means the requested configuration/package check passed, 2 means the report has
|
||||
blockers, and 1 means the operation failed. A fresh installation can report a
|
||||
missing plugin registration before normal boot installation completes. Run the
|
||||
check again after that installation settles.
|
||||
|
||||
## Database authority and retry behavior
|
||||
|
||||
The private `instance_settings` singleton `cloud-configuration-migration:v1`
|
||||
contains the versioned receipt. The transaction locks that row and composes the
|
||||
adapter and startup-config owners' atomic seed methods. Completion and both
|
||||
seeds commit together. A failed write or crash before commit leaves no completed
|
||||
receipt. A retry uses committed database state and never replaces a later
|
||||
settings revision. Earlier independently seeded owner records remain authoritative.
|
||||
|
||||
The receipt records source kind, a digest of the nonsecret semantic candidate,
|
||||
the imported revision, and bounded legacy external-registration evidence.
|
||||
An absent startup configuration remains an authoritative `config: null`; it is
|
||||
distinct from an uninitialized database record. No raw secret values are copied.
|
||||
Once the receipt exists, ordinary boots do not reread legacy source files.
|
||||
A replacement home can be empty or absent without changing saved authority.
|
||||
|
||||
All legacy readers have a 64 KiB limit. They require regular files, reject
|
||||
symlinks and FIFOs, and use bounded reads. External registration inspection reads
|
||||
metadata only. It never loads a registered module or follows a registered package
|
||||
path. A nonempty, invalid, or unreadable registration file creates a durable
|
||||
blocker. Removing the old home does not clear that blocker.
|
||||
|
||||
To resolve an external-registration blocker, stop its writer, deliberately
|
||||
remove the unsupported registration metadata at the original source location,
|
||||
and leave a present valid empty JSON array file. Read the current receipt revision
|
||||
from the report, then run:
|
||||
|
||||
```sh
|
||||
pnpm exec tsx server/src/qualify-cloud-configuration.ts --recheck-external-revision 3
|
||||
```
|
||||
|
||||
Use the actual current revision in place of `3`. A changed revision, a different
|
||||
source location, or a missing/invalid source file rejects the recheck. Reusing
|
||||
the same container path with an empty home is insufficient. This is an explicit
|
||||
operator resolution; qualification never clears the blocker automatically.
|
||||
|
||||
## Current packages and overlap comparison
|
||||
|
||||
Every qualification reads current settings revisions and plugin registrations.
|
||||
It resolves selected packages from this image with the same bundled package
|
||||
identity rules as normal loading. It verifies package bytes before importing a
|
||||
trusted image manifest, then compares that manifest with the approved database
|
||||
registration. Missing packages, missing evidence, incompatible versions,
|
||||
unselected registrations, and integrity failures prevent qualification. Legacy
|
||||
package paths and npm discovery cannot supply missing image bytes.
|
||||
|
||||
Image inspection happens outside database write locks. A short transaction then
|
||||
locks and rereads the current configuration and registration records. A change
|
||||
during inspection produces a blocker and requires a new check. The report gives
|
||||
`adapterRevision`, `configRevision`, `configurationDigest`, and
|
||||
`packageEvidenceDigest`. A historical completed receipt is evidence, not current
|
||||
authorization. Call qualification again for a current result.
|
||||
|
||||
The server records this report before ordinary bundled-plugin reconciliation.
|
||||
It retains `lastPackageMismatch` after later successful checks, so reconciliation
|
||||
cannot erase the evidence seen before an upgrade. The report does not block
|
||||
normal maintenance upgrades or replace capability approval. After an upgrade
|
||||
settles, its current packages can qualify normally.
|
||||
|
||||
For a same-package comparison, capture the current release's
|
||||
`packageEvidenceDigest` and pass it explicitly on the replacement:
|
||||
|
||||
```sh
|
||||
pnpm exec tsx server/src/qualify-cloud-configuration.ts --expected-package-evidence sha256:REPLACE_WITH_THE_64_HEX_DIGEST
|
||||
```
|
||||
|
||||
`configurationPackagesEligible` describes current database/configuration/package
|
||||
compatibility. `samePackageOverlapEligible` is true only when an explicit expected
|
||||
digest also matches. There is no permanent first-release baseline that would
|
||||
prevent future maintenance releases from establishing their own evidence.
|
||||
|
||||
The package evidence digest covers selected packages, not container image bytes.
|
||||
Distribution packages reuse and reverify their catalog digest. Builtin packages
|
||||
have a bounded inventory of physical file bytes, modes, and logical symlink
|
||||
targets, including standalone pnpm dependency trees. With the selected catalog
|
||||
root, inventory may follow links only into that provider, the catalog SDK,
|
||||
`packages/shared`, and the current image's root `node_modules`. Reached pnpm
|
||||
package units include their sibling dependency links; unrelated root packages
|
||||
are not scanned. Canonical directory/file deduplication bounds cycles, and
|
||||
logical root labels preserve the digest when identical image bytes relocate.
|
||||
Dangling or escaping links, special files, more than 50,000 entries, depth above
|
||||
32, metadata above 128 KiB, or more than 256 MiB per package fail verification.
|
||||
The measured cached Cloud Daytona closure has about 22,253 entries and 130 MB;
|
||||
that artifact predates this change and is not a complete current release proof.
|
||||
Inventory work is synchronous and linear in selected package bytes. It runs only during startup before the application is constructed
|
||||
or in the explicit operator process, with no idle polling. The report includes
|
||||
`packageInspectionDurationMs`. Packages must remain immutable throughout a check.
|
||||
|
||||
## Verification scope
|
||||
|
||||
The focused PostgreSQL tests use two independent clients and separate home/image
|
||||
roots. They import legacy settings, change saved revisions, remove the old roots,
|
||||
reconstruct from the database, and start a real plugin worker from the replacement
|
||||
image fixture. They also cover concurrent migration, commit failure and retry,
|
||||
database failure, stale sources, durable external blockers, package mismatch,
|
||||
maintenance reconciliation, and changes during manifest inspection.
|
||||
|
||||
The bundled-plugin registration suite also has an actual prebuilt Daytona case.
|
||||
Build its standalone package with the normal SDK linker before running that case;
|
||||
it is skipped when the provider build is absent. It qualifies the real linked
|
||||
SDK/shared package graph and uses the production loader and worker manager for
|
||||
credential-free initialize/health calls. The runtime-path tests also invoke the
|
||||
production server bundler to verify its preserved module locations.
|
||||
|
||||
Set `PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL` only to an owned disposable
|
||||
PostgreSQL database when embedded PostgreSQL is unavailable. The suite applies
|
||||
migrations and writes/deletes test records. A complete release image, hosted
|
||||
replacement, overlapping production processes, secret-byte migration, and volume
|
||||
removal are not established by these local fixtures.
|
||||
@@ -0,0 +1,118 @@
|
||||
# Cloud runtime configuration
|
||||
|
||||
Cloud-managed servers store nonsecret startup and application choices in the
|
||||
private `instance_settings` singleton `cloud-runtime-config:v1`. The row contains
|
||||
`version: 1`, an increasing `revision`, and a strict allowlist of configuration
|
||||
fields. It is separate from the normal instance settings row and is not exposed
|
||||
by the instance settings API.
|
||||
|
||||
The canonical managed-instance predicate applies: a nonblank Cloud tenant token
|
||||
or the presence of `PAPERCLIP_MANAGED_CONFIG`, including a blank value, selects
|
||||
this configuration path. Invalid managed feature documents still retain their
|
||||
existing validation at application startup.
|
||||
|
||||
## Startup and authority
|
||||
|
||||
1. The hosting environment supplies `DATABASE_URL` and, if needed,
|
||||
`DATABASE_MIGRATION_URL`. Cloud requires a PostgreSQL URL and never falls back
|
||||
to embedded PostgreSQL. Credentials, trust keys, provider configuration and
|
||||
deployment wiring needed before the database connection remain supplied by
|
||||
the environment or the existing secret system. This phase parses only the
|
||||
database connection record. Runtime mode, host binding and authentication
|
||||
checks wait until the saved snapshot has loaded.
|
||||
2. After schema migration and database connection, startup reads the private
|
||||
runtime configuration row. An existing invalid row or a database error stops
|
||||
startup. Neither condition selects a legacy file or an old process snapshot.
|
||||
3. If no row exists, startup imports the legacy config once. It uses
|
||||
`PAPERCLIP_CONFIG` when supplied, otherwise the instance home config path.
|
||||
It does not search working-directory ancestors. A truly absent file saves
|
||||
an authoritative `config: null` marker. This retains the existing dynamic
|
||||
environment/image defaults, including authentication and secret strictness.
|
||||
The marker never enables a later file fallback. A present malformed, linked, nonregular, or oversized file
|
||||
stops initialization. Reads and stored snapshots are limited to 64 KiB.
|
||||
4. The seed uses an insert that does not replace an existing row. Concurrent
|
||||
initializers read the committed winner. Later starts do not inspect the
|
||||
legacy file, even when it is absent, corrupt, or changed.
|
||||
5. Startup publishes a detached configuration snapshot, restores the signed
|
||||
Cloud runtime identity, then initializes telemetry, secret-provider defaults,
|
||||
authentication and application services. A saved telemetry opt-out therefore
|
||||
takes effect before a telemetry client can be created.
|
||||
|
||||
Existing operator environment overlays keep precedence over saved values.
|
||||
Reading an overlaid value does not save it back to the database. Credentials and
|
||||
connection URLs are never copied into this JSONB row.
|
||||
|
||||
Cloud does not load instance or working-directory `.env` files and does not run
|
||||
legacy worktree configuration repair. The hosting system must supply those
|
||||
bootstrap variables directly. In particular, a database connection kept only
|
||||
in `.env` cannot bootstrap this path. Self-hosted configuration and dotenv
|
||||
behavior remain unchanged.
|
||||
|
||||
## Field ownership
|
||||
|
||||
The table lists every field consumed from the existing config file and the
|
||||
remaining known schema fields. Environment-only `Config` fields, such as
|
||||
heartbeat maintenance flags and feedback-export credentials, retain their
|
||||
existing environment sources and are not frozen into the saved snapshot.
|
||||
|
||||
| Config file fields | Cloud owner |
|
||||
| --- | --- |
|
||||
| `server.deploymentMode`, `exposure`, `bind`, `customBindHost`, `host`, `port`, `allowedHostnames`, `serveUi` | Saved runtime configuration, with existing environment overlays. |
|
||||
| `auth.baseUrlMode`, `publicBaseUrl`, `disableSignUp` | Saved runtime configuration, with environment overlays and the signed Cloud identity restoration. Saved URLs cannot contain userinfo, query parameters, or fragments. |
|
||||
| `database.backup.enabled`, `intervalMinutes`, `retentionDays`, `dir` | Saved runtime configuration, with existing backup environment overlays. |
|
||||
| `database.mode`, `connectionString` | Cloud bootstrap requires the environment `DATABASE_URL`. The legacy URL is never saved. |
|
||||
| `database.embeddedPostgresDataDir`, `embeddedPostgresPort` | Local embedded-database settings. Cloud does not use embedded PostgreSQL or these paths. |
|
||||
| `storage.provider`, `localDisk.baseDir`, `s3.bucket`, `s3.region`, `s3.endpoint`, `s3.prefix`, `s3.forcePathStyle` | Saved runtime configuration, with existing environment overlays. Saved endpoints cannot contain userinfo, query parameters, or fragments. |
|
||||
| `secrets.provider`, `strictMode`, `localEncrypted.keyFilePath` | Saved nonsecret provider choice and locator, with existing environment overlays. No key bytes are imported or saved. |
|
||||
| `telemetry.enabled` | Saved choice, passed to the existing telemetry initialization and opt-out policy before client creation. |
|
||||
| `llm.provider` | Saved provider metadata. |
|
||||
| `llm.apiKey` | Never saved. The existing OpenAI model-discovery consumer uses `OPENAI_API_KEY` first. A legacy OpenAI key requires that environment credential before import, or startup stops. Claude key metadata has no server consumer. |
|
||||
| `logging.mode`, `logDir`, `updates.checkEnabled` | Nonsecret legacy/CLI metadata is preserved. Server logging continues to use its existing environment configuration. |
|
||||
| `$meta` | CLI file-write metadata is not application authority. The compatibility view synthesizes valid metadata. |
|
||||
| Unknown extension keys | No server config consumer exists. They are excluded from the strict saved allowlist, including any unknown credential fields. |
|
||||
|
||||
A saved path is a locator. This change does not make local storage objects,
|
||||
backup bytes, logging files, or secret-key bytes durable. Their storage owners
|
||||
and existing environment/secret mechanisms retain responsibility for those
|
||||
bytes. Restart tests prove configuration restoration under a different home;
|
||||
they do not prove durable file, secret, or backup storage.
|
||||
|
||||
The decision signing service still uses `PAPERCLIP_DECISION_SIGNING_SECRET` when
|
||||
supplied and otherwise reads or creates its instance key file before the database
|
||||
connection. The local encrypted secret provider still uses its environment key
|
||||
or existing key-file mechanism. Those byte-storage fallbacks are outside this
|
||||
configuration migration. Qualification must account for them separately; this
|
||||
PR does not establish file-free secret storage on Cloud.
|
||||
|
||||
## Internal update and migration interface
|
||||
|
||||
`cloudRuntimeConfigService(db)` provides `inspectStored(handle?)`,
|
||||
`seedIfAbsent(config, handle?)`, and `replace(config, expectedRevision, handle?)`.
|
||||
Each method accepts a database or transaction handle. A migration can compose a
|
||||
config seed, related settings and a receipt in one transaction.
|
||||
|
||||
Replacement validates the same allowlist, locks the singleton row, checks the
|
||||
expected revision, and increases it. A stale revision fails. There is no new
|
||||
public configuration endpoint or settings UI. Startup consumers use a snapshot;
|
||||
a saved replacement takes effect on the next initialization/restart.
|
||||
|
||||
`readLegacyCloudRuntimeConfig(env?)` returns the validated nonsecret candidate
|
||||
without writing it. `initializeCloudRuntimeConfig` performs the standalone
|
||||
one-time seed and publishes its committed snapshot. Integrated migration
|
||||
receipts and deployed Cloud qualification are separate consumers of this
|
||||
interface.
|
||||
|
||||
## Verification
|
||||
|
||||
`server/src/__tests__/cloud-runtime-config.test.ts` exercises real PostgreSQL
|
||||
clients, concurrent writes, transaction rollback, changed homes, a fresh Node
|
||||
process, environment precedence, secret exclusion, invalid data and database
|
||||
failure. It uses embedded PostgreSQL by default. A dedicated disposable database
|
||||
can be selected with `PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL` when the host
|
||||
cannot start embedded PostgreSQL. That database must be reserved for this test.
|
||||
The test applies the current schema before use.
|
||||
|
||||
The startup orchestration controls in
|
||||
`server-startup-feedback-export.test.ts` hold configuration initialization and
|
||||
verify that telemetry, authentication and app construction wait for it. These
|
||||
local controls do not claim a live Cloud rollout.
|
||||
@@ -34,6 +34,42 @@ Current limitations to keep in mind:
|
||||
|
||||
In practice, that means the current implementation is a good fit for local development and self-hosted persistent deployments, but not yet for multi-instance cloud plugin distribution.
|
||||
|
||||
### Managed image packages
|
||||
|
||||
Managed Cloud provisions only plugins selected from the current release image
|
||||
catalog. A Cloud tenant token or managed configuration keeps this image-only
|
||||
policy active. Without an explicit selected package list, Cloud selects no
|
||||
packages and cannot fall back to local installs. An explicit operator-managed
|
||||
list also keeps this policy active outside Cloud. The database registration's
|
||||
plugin key, package name, version and
|
||||
approved manifest identify that package. Its legacy `packagePath` remains for
|
||||
local/self-hosted compatibility; Cloud never uses that path to locate code.
|
||||
Workers, plugin migrations and UI files resolve through the same current-image
|
||||
package resolver. Cloud does not install npm dependencies, build packages, use
|
||||
the development UI proxy, or fall back to a server home directory.
|
||||
|
||||
Prebuilt workers use the current release's baked CLI `tsx` loader for the SDK's
|
||||
workspace TypeScript dependencies. The loader is located from the owning server
|
||||
package in both regular and bundled server layouts. Missing loaders and loader
|
||||
links outside the release's CLI or root `node_modules` fail activation. This
|
||||
loads image-supplied code without installing dependencies or building packages.
|
||||
|
||||
Equivalent packages at different image roots reuse the existing registration,
|
||||
configuration, approval status and company scopes without rewriting its stored
|
||||
path. Concurrent first registration of the same package retains one plugin ID.
|
||||
Conflicting package identities fail explicitly. Normal maintenance boot retains
|
||||
the existing bundled manifest/version reconciliation and approval gates.
|
||||
|
||||
`createBundledPluginPackageResolver(...).inspect(registration, verifiedManifest)`
|
||||
is a read-only qualification hook. Resolve the current image path, load its
|
||||
manifest with the validated plugin loader, and inspect the registration before
|
||||
normal boot reconciliation. Missing manifest evidence and incompatible packages
|
||||
produce explicit blockers. Distribution packages retain their catalog digest
|
||||
verification; built-in packages trust the immutable release image and must bump
|
||||
their version when their code changes. Qualification supports identical package
|
||||
sets only. This package-resolution boundary does not coordinate workers,
|
||||
background jobs, or deployment overlap between servers.
|
||||
|
||||
## 1. Scope
|
||||
|
||||
This spec covers:
|
||||
|
||||
@@ -74,6 +74,9 @@ API: `present` — representative operation IDs:
|
||||
- `PATCH /api/instance/settings`.
|
||||
- `GET /api/instance/settings/experimental`.
|
||||
- `PATCH /api/instance/settings/experimental`.
|
||||
- `GET /api/adapters`.
|
||||
- `GET /api/adapters/{type}`.
|
||||
- `PATCH /api/adapters/{type}`.
|
||||
|
||||
Agent access: `restricted` — Board/admin instance settings; CLI configure/env; local
|
||||
configuration files.
|
||||
@@ -88,9 +91,38 @@ Verification: `not-run` — assert the documented agent denial with an owned tes
|
||||
verify any listed agent request/result path separately. A delegated operation needs its
|
||||
own policy and access review.
|
||||
|
||||
Cloud runtime configuration uses a separate private PostgreSQL singleton and
|
||||
adds no public read or mutation endpoint. Existing instance settings APIs do not
|
||||
expose it. Bootstrap database credentials and trust keys remain in the supplied
|
||||
environment; the strict saved allowlist excludes API keys and connection URLs.
|
||||
Internal configuration updates require a matching revision and transaction lock.
|
||||
[Database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) cover
|
||||
seed/restart, credential exclusion, conflicts and rollback. These are local
|
||||
controls, not agent authorization or deployed Cloud qualification.
|
||||
|
||||
Gaps: Settings APIs cover parts of configuration; pre-start database/binding/storage
|
||||
configuration is local operator tooling.
|
||||
|
||||
Adapter toggles require instance-admin authority and remain blocked when
|
||||
`instance.adapters` is hidden. Ordinary board members can read the catalog;
|
||||
agent keys cannot toggle adapters. Managed Cloud preferences are stored in a
|
||||
private database singleton. The operator adapter declaration sets the effective
|
||||
list and rejects conflicting toggles. [PostgreSQL regression coverage](../../server/src/services/adapter-settings-postgres.test.ts)
|
||||
exercises persistence, concurrent writers, admission freshness, activity audit,
|
||||
and failed writes; [authorization tests](../../server/src/__tests__/adapter-routes-authz.test.ts)
|
||||
cover admin and hidden-setting gates. These are local tests, not deployed proof.
|
||||
|
||||
Cloud configuration migration and package qualification are operator commands,
|
||||
not agent-accessible REST capabilities. They require the managed environment and
|
||||
database operator access. The [command](../../server/src/qualify-cloud-configuration.ts)
|
||||
composes transaction-safe owner seeds and writes a private receipt. Every
|
||||
qualification rereads current revisions and image package evidence; a completed
|
||||
historical receipt grants no deployment or execution authority. [PostgreSQL coverage](../../server/src/__tests__/cloud-configuration-migration-postgres.test.ts)
|
||||
uses independent clients, lost-home reconstruction, a real replacement fixture
|
||||
worker, rollback/retry, and durable blockers. [Reader coverage](../../server/src/__tests__/cloud-configuration-readers.test.ts)
|
||||
checks bounded sources and pnpm-style link integrity. These are local fixtures;
|
||||
see the [scope and operator procedure](../../doc/cloud-configuration-migration.md).
|
||||
|
||||
Cloud stack Admins may update experimental settings through the trusted Cloud
|
||||
actor without receiving other instance-admin powers. This uses the current
|
||||
stack role on each request; company membership alone does not grant it.
|
||||
|
||||
@@ -55,7 +55,14 @@ Policy: Company model discovery requires company access; account-backed discover
|
||||
also enforces its account/pool and target-agent rules. Instance adapter inventory
|
||||
and installation retain board/admin gates. Delegated managed
|
||||
AI-account login operations are enumerated under `model-accounts`; they do not
|
||||
grant raw host administration or execution-session reset.
|
||||
grant raw host administration or execution-session reset. On Cloud-managed
|
||||
instances, external adapter install, reinstall, reload, removal and override
|
||||
operations return `403` with `adapter_install_platform_managed`, including for
|
||||
instance admins. Built-in metadata remains available to board users. Startup
|
||||
and UI parser reads do not consume saved external registrations or package code.
|
||||
[Local fixture controls](../../server/src/adapters/cloud-external-adapters.test.ts)
|
||||
and [route controls](../../server/src/__tests__/adapter-routes-authz.test.ts)
|
||||
cover this boundary; they do not establish live agent or provider qualification.
|
||||
|
||||
Evidence: `source-only` — [implementation](../../server/src/routes/agents.ts#L6608) at
|
||||
the reviewed commit above.
|
||||
|
||||
@@ -63,8 +63,38 @@ unfinished application write must still prevent sleep after its client leaves.
|
||||
|
||||
### `settings-backup`
|
||||
|
||||
Cloud startup reads nonsecret application choices from a private, versioned
|
||||
PostgreSQL singleton after a bounded one-time legacy config seed. A fresh
|
||||
process and different home restore the saved choices without consulting a stale
|
||||
`.env` file. Invalid saved data or an unavailable database prevents startup;
|
||||
saved telemetry opt-out applies before client creation. The
|
||||
[database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) and
|
||||
[startup-order controls](../../server/src/__tests__/server-startup-feedback-export.test.ts)
|
||||
cover these local boundaries. See [field ownership](../../doc/cloud-runtime-configuration.md).
|
||||
These controls do not qualify a deployed Cloud rollout or durable file storage.
|
||||
|
||||
Automated: [experimental settings](../../ui/src/pages/InstanceExperimentalSettings.test.tsx) and [database backup routes](../../server/src/__tests__/instance-database-backups-routes.test.ts) cover their UI/API contracts.
|
||||
|
||||
Cloud migration: in one maintenance window, stop legacy file writers and run the
|
||||
[configuration/package qualification command](../../doc/cloud-configuration-migration.md).
|
||||
Inspect its database receipt, update a saved preference, then repeat against a
|
||||
second owned application/image root with the first home unavailable. Use an
|
||||
explicit package evidence digest for same-package comparison. Confirm stale
|
||||
qualification and missing legacy files cannot clear blockers. This recipe covers
|
||||
configuration/packages and does not qualify whole-server zero downtime, secret
|
||||
bytes, or volume removal. [Local PostgreSQL evidence path](../../server/src/__tests__/cloud-configuration-migration-postgres.test.ts)
|
||||
includes a real fixture worker; it does not establish a hosted replacement.
|
||||
|
||||
Managed adapters: change a visible adapter preference as an instance admin,
|
||||
then use a second application process against the same disposable database to
|
||||
confirm catalog and new-agent selection reflect it. Verify saved agents remain
|
||||
usable. Restart with an operator adapter declaration, confirm its effective
|
||||
list, and reject conflicting toggles. Remove the declaration and confirm saved
|
||||
preferences return. [Database tests](../../server/src/services/adapter-settings-postgres.test.ts)
|
||||
cover local persistence, concurrent writers, audit rollback, and failure responses;
|
||||
[local compatibility tests](../../server/src/services/adapter-settings-local.test.ts)
|
||||
cover the legacy file. No deployed walkthrough is claimed.
|
||||
|
||||
Manual: Change a harmless experimental setting, reload, and confirm the affected route honors it. Back up a disposable instance, inspect the reported file and metadata, and restore using the documented procedure into a second isolated target. Verify a known record after restoration.
|
||||
|
||||
Cloud staging: On a managed `*.staging.paperclip.app` instance, verify experiments
|
||||
|
||||
@@ -170,6 +170,15 @@ controls do not establish live provider acceptance.
|
||||
|
||||
### `adapter-setup`
|
||||
|
||||
Cloud-managed instances load adapters from the release image. Saved external
|
||||
registrations cannot replace built-ins or supply UI parser code. Install,
|
||||
reinstall, reload, remove, and override operations remain platform-managed.
|
||||
The [fixture loader controls](../../server/src/adapters/cloud-external-adapters.test.ts)
|
||||
exercise real packages, import side effects, both managed-instance signals and
|
||||
self-hosted compatibility. The [route authorization controls](../../server/src/__tests__/adapter-routes-authz.test.ts)
|
||||
check managed denials before package or registration access. These are local
|
||||
controls, not live Cloud provider qualification.
|
||||
|
||||
Automated: [adapter validation routes](../../server/src/__tests__/agent-adapter-validation-routes.test.ts) and [model refresh](../../server/src/__tests__/adapter-model-refresh-routes.test.ts) cover server behavior.
|
||||
|
||||
Claude native setup uses the existing Anthropic API-key or subscription connection
|
||||
|
||||
@@ -27,6 +27,12 @@ checks only assignment to the signed-in user and persistence after reload. Its
|
||||
source is reusable across self-hosted and Cloud; Cloud live qualification is
|
||||
separate. It does not cover other property operations or agent authorization.
|
||||
|
||||
The [Product task-search-filter journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md)
|
||||
checks `tasks/organize` with owned matching and nonmatching controls, exact task
|
||||
identities, saved task content, and search/status persistence after reload and
|
||||
detail/back navigation. It is model-free and reusable across deployment targets.
|
||||
Live Cloud qualification is separate from the journey's assertion unit tests.
|
||||
|
||||
## How to get to it (user POV)
|
||||
|
||||
Cold collection loads use a lightweight index and batched hierarchy context,
|
||||
|
||||
@@ -101,6 +101,7 @@
|
||||
"agentmail": "^0.5.31",
|
||||
"cross-env": "^10.1.0",
|
||||
"esbuild": "^0.28.2",
|
||||
"oxc-parser": "0.127.0",
|
||||
"typescript": "^7.0.2",
|
||||
"vitest": "^5.0.3",
|
||||
"yaml": "^2.8.1",
|
||||
|
||||
@@ -552,6 +552,13 @@ pnpm --filter @paperclipai/paperclip-runner verify
|
||||
The verification command requires a stable Rust toolchain with `cargo` on
|
||||
`PATH`, in addition to Node.js 24.11+ and pnpm 9+.
|
||||
|
||||
Rust development builds use `debug = "line-tables-only"` to reduce build storage
|
||||
while retaining file names and line numbers in backtraces. The default test
|
||||
profile inherits this setting. Incremental compilation remains enabled, and
|
||||
release profiles keep their existing settings. For full debugger support,
|
||||
including local variables and types, set `CARGO_PROFILE_DEV_DEBUG=2` and
|
||||
`CARGO_PROFILE_TEST_DEBUG=2` for the build or test command.
|
||||
|
||||
Minimal Debian/Ubuntu hosts without root access can extract the required
|
||||
Playwright browser libraries into a user-owned cache and run the same acceptance
|
||||
sequence with:
|
||||
|
||||
@@ -73,17 +73,30 @@ real transport; it does not qualify a hosted OpenCode or ACPX deployment.
|
||||
## Profile identity
|
||||
|
||||
Every ACPX provider's `commandDigest` attests only its own upstream release.
|
||||
For Copilot (profile 92), Grok (20) and Pi (37) it binds the upstream package
|
||||
versions, the platform closure pins and, for Grok, the launcher bytes.
|
||||
`scripts/acpx-release-attestations.mjs` computes these digests and
|
||||
`generate:acpx-profiles` writes them. Runner source changes do not change them.
|
||||
Every ACPX harness binds the shared ACPX layer the same way: saved inputs must
|
||||
match the current ACPX version, and the provider pack's `contractDigest` binds
|
||||
the Runner's ACPX patch with the other Runner contracts.
|
||||
Saved inputs with any profile version from 1 through the current one decode, so
|
||||
they reach recovery. Launch and recovery admission require the exact current
|
||||
profile and digest, and refuse the rest. A profile identity does not establish
|
||||
hosted harness qualification.
|
||||
Each provider declares how in `ACPX_RELEASE_ATTESTATIONS`
|
||||
(`scripts/acpx-release-attestations.mjs`):
|
||||
|
||||
| Kind | Providers | The digest is |
|
||||
| --- | --- | --- |
|
||||
| `installed-command` | Claude, Codex | the upstream package's launch command, hashed at installation |
|
||||
| `launcher` | Grok | the SHA-256 of the launcher shipped verbatim |
|
||||
| `closure` | Copilot, Pi | upstream versions and the complete platform closure pins |
|
||||
| `declaration` | Cursor, Antigravity | a checked-in release declaration, as canonical JSON |
|
||||
|
||||
`generate:acpx-profiles` writes the computed digests and `check:acpx-profiles`
|
||||
refuses a stale one, so a digest changes only with its upstream release. Every
|
||||
ACPX harness binds the shared ACPX layer the same way: saved inputs must match
|
||||
the current ACPX version, and the provider pack's `contractDigest` binds the
|
||||
Runner's ACPX patch with the other Runner contracts. Runner source changes
|
||||
change no provider digest.
|
||||
|
||||
A profile version changes only to reset every saved session for that provider
|
||||
on purpose; an upstream release keeps it, and a new provider starts at 1. Saved
|
||||
inputs with any version from 1 through the current one decode, so they reach
|
||||
recovery. Launch and recovery admission require the exact current profile and
|
||||
digest, and refuse the rest. A profile identity does not establish hosted
|
||||
harness qualification. [`rules/acpx-profile-identity.mdc`](../../../rules/acpx-profile-identity.mdc)
|
||||
summarizes these practices for contributors.
|
||||
|
||||
## External controller checkpoints
|
||||
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
members = ["crates/runner-core"]
|
||||
resolver = "2"
|
||||
|
||||
[profile.dev]
|
||||
debug = "line-tables-only"
|
||||
|
||||
[workspace.package]
|
||||
version = "0.0.0"
|
||||
edition = "2021"
|
||||
|
||||
@@ -3,13 +3,34 @@ import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
// Copilot, Grok and Pi profile identities attest upstream releases, like Claude,
|
||||
// Codex, Cursor and Antigravity. They bind upstream versions and the bytes a
|
||||
// repackager copies verbatim: complete platform closures and Grok's launcher.
|
||||
// Never bind Paperclip source, the shared ACPX layer, absolute paths, the
|
||||
// provider-pack manifest digest or install layout. The profile's acpxVersion and
|
||||
// the pack contract digest bind ACPX and Runner source for every harness alike.
|
||||
export const ATTESTED_ACPX_AGENTS = Object.freeze(["copilot", "grok", "pi"]);
|
||||
/**
|
||||
* How each ACPX provider's `commandDigest` attests its upstream release. Every
|
||||
* profile in acpx-profiles.json declares one kind here:
|
||||
*
|
||||
* - `installed-command`: installation hashes the upstream package's launch
|
||||
* command and compares it with the digest (Claude, Codex).
|
||||
* - `launcher`: the SHA-256 of a launcher shipped verbatim (Grok).
|
||||
* - `closure`: upstream versions and the complete platform closure pins.
|
||||
* - `declaration`: a checked-in release declaration, hashed as canonical JSON.
|
||||
*
|
||||
* A digest changes only when the upstream release changes. It never binds
|
||||
* Paperclip source, the shared ACPX layer, absolute paths, the provider-pack
|
||||
* manifest digest or install layout: the profile's acpxVersion and the pack
|
||||
* contract digest bind ACPX and Runner source for every provider alike.
|
||||
* See rules/acpx-profile-identity.mdc.
|
||||
*/
|
||||
export const ACPX_RELEASE_ATTESTATIONS = Object.freeze({
|
||||
claude: { kind: "installed-command" },
|
||||
codex: { kind: "installed-command" },
|
||||
grok: { kind: "launcher", path: "src/providers/grok/launcher.cjs" },
|
||||
copilot: { kind: "closure", pins: { closure: ["copilot-closure-pins.ts", "COPILOT_CLOSURE_SHA256"] } },
|
||||
pi: { kind: "closure", pins: {
|
||||
nodeVersion: ["pi-node-pins.ts", "PI_NODE_VERSION"],
|
||||
closure: ["pi-closure-pins.ts", "PI_DISTRIBUTION_CLOSURE_SHA256"],
|
||||
} },
|
||||
cursor: { kind: "declaration", path: "cursor-contract.json", digestField: "commandDigest" },
|
||||
antigravity: { kind: "declaration", path: "antigravity-distributions.json" },
|
||||
});
|
||||
|
||||
const sha256 = bytes => createHash("sha256").update(bytes).digest("hex");
|
||||
const canonicalJson = value => value && typeof value === "object"
|
||||
@@ -17,28 +38,42 @@ const canonicalJson = value => value && typeof value === "object"
|
||||
: `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
|
||||
: JSON.stringify(value);
|
||||
|
||||
/** The release attestation for one provider, computed from a runner package root. */
|
||||
/**
|
||||
* The release attestation for one provider, computed from a runner package
|
||||
* root, or null when installation verifies the digest instead.
|
||||
*/
|
||||
export async function acpxReleaseAttestation(agent, packageRoot) {
|
||||
const manifest = JSON.parse(readFileSync(resolve(packageRoot, "acpx-profiles.json"), "utf8"));
|
||||
const profile = manifest.profiles[agent];
|
||||
if (!ATTESTED_ACPX_AGENTS.includes(agent) || !profile) throw new Error(`No upstream release attestation for ${agent}`);
|
||||
// Grok keeps its physical byte-digest contract: admission hashes the launcher.
|
||||
if (agent === "grok") {
|
||||
return { commandDigest: `sha256:${sha256(readFileSync(resolve(packageRoot, "src/providers/grok/launcher.cjs")))}` };
|
||||
const attestation = Object.hasOwn(ACPX_RELEASE_ATTESTATIONS, agent) ? ACPX_RELEASE_ATTESTATIONS[agent] : undefined;
|
||||
if (!profile || !attestation) {
|
||||
throw new Error(`${agent} needs a release attestation: declare how its commandDigest attests the upstream release `
|
||||
+ "in ACPX_RELEASE_ATTESTATIONS (scripts/acpx-release-attestations.mjs). See rules/acpx-profile-identity.mdc.");
|
||||
}
|
||||
switch (attestation.kind) {
|
||||
case "installed-command":
|
||||
return null;
|
||||
case "launcher":
|
||||
return { commandDigest: `sha256:${sha256(readFileSync(resolve(packageRoot, attestation.path)))}` };
|
||||
case "declaration": {
|
||||
const { [attestation.digestField]: _digest, ...declaration } = JSON.parse(readFileSync(resolve(packageRoot, attestation.path), "utf8"));
|
||||
return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
|
||||
}
|
||||
case "closure": {
|
||||
const declaration = {
|
||||
schema: "paperclip.acpx-release-attestation.v1", agent,
|
||||
agentProfileVersion: profile.agentProfileVersion,
|
||||
agentServerPackage: profile.agentServerPackage, agentServerVersion: profile.agentServerVersion,
|
||||
agentRuntimePackage: profile.agentRuntimePackage, agentRuntimeVersion: profile.agentRuntimeVersion,
|
||||
};
|
||||
for (const [field, [file, name]] of Object.entries(attestation.pins)) {
|
||||
// Key each import on content so a process never reuses pins from edited bytes.
|
||||
const path = resolve(packageRoot, "src/drivers/acpx", file);
|
||||
declaration[field] = (await import(`${pathToFileURL(path).href}?sha256=${sha256(readFileSync(path))}`))[name];
|
||||
}
|
||||
return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
|
||||
}
|
||||
default:
|
||||
throw new Error(`${agent} declares an unknown release attestation kind: ${attestation.kind}`);
|
||||
}
|
||||
// Key each import on content so a process never reuses pins from edited bytes.
|
||||
const pins = async file => {
|
||||
const path = resolve(packageRoot, "src/drivers/acpx", file);
|
||||
return import(`${pathToFileURL(path).href}?sha256=${sha256(readFileSync(path))}`);
|
||||
};
|
||||
const declaration = {
|
||||
schema: "paperclip.acpx-release-attestation.v1", agent,
|
||||
agentProfileVersion: profile.agentProfileVersion,
|
||||
agentServerPackage: profile.agentServerPackage, agentServerVersion: profile.agentServerVersion,
|
||||
agentRuntimePackage: profile.agentRuntimePackage, agentRuntimeVersion: profile.agentRuntimeVersion,
|
||||
...agent === "copilot"
|
||||
? { closure: (await pins("copilot-closure-pins.ts")).COPILOT_CLOSURE_SHA256 }
|
||||
: { nodeVersion: (await pins("pi-node-pins.ts")).PI_NODE_VERSION, closure: (await pins("pi-closure-pins.ts")).PI_DISTRIBUTION_CLOSURE_SHA256 },
|
||||
};
|
||||
return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
|
||||
}
|
||||
@@ -5,10 +5,12 @@ import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test from "node:test";
|
||||
import { acpxReleaseAttestation, ATTESTED_ACPX_AGENTS } from "./acpx-release-attestations.mjs";
|
||||
import { ACPX_RELEASE_ATTESTATIONS, acpxReleaseAttestation } from "./acpx-release-attestations.mjs";
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const manifest = JSON.parse(readFileSync(join(packageRoot, "acpx-profiles.json"), "utf8"));
|
||||
// Providers whose digest the generator computes; installation verifies the rest.
|
||||
const COMPUTED = Object.keys(manifest.profiles).filter(agent => ACPX_RELEASE_ATTESTATIONS[agent]?.kind !== "installed-command");
|
||||
const packageFiles = [
|
||||
"acpx-profiles.json", "package.json", "cursor-distributions.json", "cursor-contract.json", "antigravity-distributions.json",
|
||||
"scripts/acpx-release-attestations.mjs", "scripts/generate-acpx-profiles.mjs",
|
||||
@@ -33,15 +35,27 @@ function relocatedPackage(t) {
|
||||
const generate = (root, ...args) => spawnSync(process.execPath, [join(root, "scripts/generate-acpx-profiles.mjs"), ...args],
|
||||
{ cwd: root, env: { PATH: process.env.PATH }, encoding: "utf8", timeout: 20_000 });
|
||||
|
||||
test("the manifest carries each provider's current release attestation", async () => {
|
||||
for (const agent of ATTESTED_ACPX_AGENTS) {
|
||||
test("every profile declares how its digest attests the upstream release", async () => {
|
||||
assert.deepEqual(Object.keys(ACPX_RELEASE_ATTESTATIONS).sort(), Object.keys(manifest.profiles).sort());
|
||||
assert.deepEqual(COMPUTED.sort(), ["antigravity", "copilot", "cursor", "grok", "pi"]);
|
||||
for (const agent of COMPUTED) {
|
||||
assert.equal(manifest.profiles[agent].commandDigest, (await acpxReleaseAttestation(agent, packageRoot)).commandDigest, agent);
|
||||
}
|
||||
});
|
||||
|
||||
test("a new provider is told to declare its release attestation", async t => {
|
||||
const relocated = relocatedPackage(t);
|
||||
const path = join(relocated, "acpx-profiles.json"); const value = JSON.parse(readFileSync(path, "utf8"));
|
||||
value.profiles.newagent = { ...value.profiles.codex };
|
||||
writeFileSync(path, JSON.stringify(value));
|
||||
const result = generate(relocated, "--check");
|
||||
assert.notEqual(result.status, 0);
|
||||
assert.match(result.stderr, /newagent needs a release attestation: declare how its commandDigest attests the upstream release/u);
|
||||
});
|
||||
|
||||
test("a relocated copy of the provider inputs computes the same attestations", async t => {
|
||||
const relocated = relocatedPackage(t);
|
||||
for (const agent of ATTESTED_ACPX_AGENTS) {
|
||||
for (const agent of COMPUTED) {
|
||||
const here = await acpxReleaseAttestation(agent, packageRoot);
|
||||
const there = await acpxReleaseAttestation(agent, relocated);
|
||||
assert.equal(there.commandDigest, here.commandDigest, agent);
|
||||
@@ -76,24 +90,52 @@ for (const [name, agents, edit] of [
|
||||
const path = join(root, "src/providers/grok/launcher.cjs");
|
||||
writeFileSync(path, `${readFileSync(path, "utf8")}\n// changed\n`);
|
||||
}],
|
||||
["the Cursor release declaration", ["cursor"], root => {
|
||||
const path = join(root, "cursor-contract.json"); const value = JSON.parse(readFileSync(path, "utf8"));
|
||||
value.nativePatch = "changed"; writeFileSync(path, JSON.stringify(value));
|
||||
}],
|
||||
["the Antigravity distribution declaration", ["antigravity"], root => {
|
||||
const path = join(root, "antigravity-distributions.json"); const value = JSON.parse(readFileSync(path, "utf8"));
|
||||
value.version = "9.9.9"; writeFileSync(path, JSON.stringify(value));
|
||||
}],
|
||||
]) test(`changing ${name} changes only the affected attestations`, async t => {
|
||||
const relocated = relocatedPackage(t);
|
||||
const before = Object.fromEntries(await Promise.all(ATTESTED_ACPX_AGENTS.map(async agent => [agent, (await acpxReleaseAttestation(agent, relocated)).commandDigest])));
|
||||
const before = Object.fromEntries(await Promise.all(COMPUTED.map(async agent => [agent, (await acpxReleaseAttestation(agent, relocated)).commandDigest])));
|
||||
edit(relocated);
|
||||
for (const agent of ATTESTED_ACPX_AGENTS) {
|
||||
for (const agent of COMPUTED) {
|
||||
const after = (await acpxReleaseAttestation(agent, relocated)).commandDigest;
|
||||
if (agents.includes(agent)) assert.notEqual(after, before[agent], agent);
|
||||
else assert.equal(after, before[agent], agent);
|
||||
}
|
||||
});
|
||||
|
||||
test("the generator refreshes a release declaration's own digest with the profile", async t => {
|
||||
const relocated = relocatedPackage(t);
|
||||
const path = join(relocated, "cursor-contract.json"); const contract = JSON.parse(readFileSync(path, "utf8"));
|
||||
contract.modes = [...contract.modes, "review"];
|
||||
writeFileSync(path, `${JSON.stringify(contract, null, 2)}\n`);
|
||||
const stale = generate(relocated, "--check");
|
||||
assert.notEqual(stale.status, 0);
|
||||
assert.match(stale.stderr, /cursor commandDigest no longer matches its upstream release attestation/u);
|
||||
assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), contract, "check mode writes nothing");
|
||||
const refreshed = generate(relocated);
|
||||
assert.equal(refreshed.status, 0, refreshed.stderr);
|
||||
const { commandDigest } = await acpxReleaseAttestation("cursor", relocated);
|
||||
assert.notEqual(commandDigest, manifest.profiles.cursor.commandDigest);
|
||||
assert.equal(JSON.parse(readFileSync(path, "utf8")).commandDigest, commandDigest);
|
||||
assert.equal(JSON.parse(readFileSync(join(relocated, "acpx-profiles.json"), "utf8")).profiles.cursor.commandDigest, commandDigest);
|
||||
assert.match(readFileSync(join(relocated, "src/drivers/acpx/generated-profiles.ts"), "utf8"), new RegExp(commandDigest));
|
||||
assert.match(readFileSync(join(relocated, "runner/crates/runner-core/src/generated_acpx_profiles.rs"), "utf8"), new RegExp(commandDigest));
|
||||
assert.equal(generate(relocated, "--check").status, 0);
|
||||
});
|
||||
|
||||
test("the generator refuses a stale attestation in check mode and refreshes it otherwise", async t => {
|
||||
const relocated = relocatedPackage(t);
|
||||
const launcher = join(relocated, "src/providers/grok/launcher.cjs");
|
||||
writeFileSync(launcher, `${readFileSync(launcher, "utf8")}\n// changed\n`);
|
||||
const stale = generate(relocated, "--check");
|
||||
assert.notEqual(stale.status, 0);
|
||||
assert.match(stale.stderr, /grok release attestation is stale/u);
|
||||
assert.match(stale.stderr, /grok commandDigest no longer matches its upstream release attestation/u);
|
||||
const refreshed = generate(relocated);
|
||||
assert.equal(refreshed.status, 0, refreshed.stderr);
|
||||
const written = JSON.parse(readFileSync(join(relocated, "acpx-profiles.json"), "utf8"));
|
||||
|
||||
@@ -1,30 +1,36 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { acpxReleaseAttestation, ATTESTED_ACPX_AGENTS } from "./acpx-release-attestations.mjs";
|
||||
import { ACPX_RELEASE_ATTESTATIONS, acpxReleaseAttestation } from "./acpx-release-attestations.mjs";
|
||||
|
||||
const root = new URL("../", import.meta.url);
|
||||
const readJson = async path => JSON.parse(await readFile(new URL(path, root), "utf8"));
|
||||
const manifest = await readJson("acpx-profiles.json");
|
||||
const distributions = await readJson("cursor-distributions.json");
|
||||
const antigravityDistributions = await readJson("antigravity-distributions.json");
|
||||
const contract = await readJson("cursor-contract.json");
|
||||
const pkg = await readJson("package.json");
|
||||
assert.equal(manifest.schema, "paperclip.acpx-profiles.v1");
|
||||
assert.equal(manifest.acpxVersion, pkg.dependencies.acpx);
|
||||
// Copilot, Grok and Pi digests attest upstream releases; they change only when
|
||||
// an upstream version, a platform closure or the Grok launcher changes.
|
||||
// Every profile's commandDigest attests its upstream release, so it changes only
|
||||
// when that release changes. Installation verifies installed-command digests.
|
||||
const check = process.argv.includes("--check");
|
||||
let attestationsChanged = false;
|
||||
for (const agent of ATTESTED_ACPX_AGENTS) {
|
||||
const { commandDigest } = await acpxReleaseAttestation(agent, fileURLToPath(root));
|
||||
if (manifest.profiles[agent].commandDigest === commandDigest) continue;
|
||||
assert.ok(!check, `${agent} release attestation is stale in acpx-profiles.json; run generate:acpx-profiles`);
|
||||
manifest.profiles[agent].commandDigest = commandDigest;
|
||||
attestationsChanged = true;
|
||||
const writes = new Map();
|
||||
for (const agent of Object.keys(manifest.profiles)) {
|
||||
const attestation = await acpxReleaseAttestation(agent, fileURLToPath(root));
|
||||
if (!attestation) continue;
|
||||
// A release declaration may carry its own copy of the digest; keep it in step.
|
||||
const { path, digestField } = ACPX_RELEASE_ATTESTATIONS[agent];
|
||||
const declaration = digestField ? await readJson(path) : null;
|
||||
if (manifest.profiles[agent].commandDigest === attestation.commandDigest
|
||||
&& (!declaration || declaration[digestField] === attestation.commandDigest)) continue;
|
||||
assert.ok(!check, `${agent} commandDigest no longer matches its upstream release attestation. `
|
||||
+ "Run `pnpm --filter @paperclipai/paperclip-runner generate:acpx-profiles`; an upstream change keeps the profile version.");
|
||||
manifest.profiles[agent].commandDigest = attestation.commandDigest;
|
||||
writes.set("acpx-profiles.json", manifest);
|
||||
if (declaration) writes.set(path, { ...declaration, [digestField]: attestation.commandDigest });
|
||||
}
|
||||
if (attestationsChanged) await writeFile(new URL("acpx-profiles.json", root), `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
for (const [path, value] of writes) await writeFile(new URL(path, root), `${JSON.stringify(value, null, 2)}\n`);
|
||||
const contract = await readJson("cursor-contract.json");
|
||||
const entries = Object.entries(manifest.profiles);
|
||||
for (const [agent, profile] of entries) {
|
||||
assert.match(agent, /^[a-z]+$/);
|
||||
@@ -38,21 +44,13 @@ for (const [agent, profile] of entries) {
|
||||
if (name && Object.hasOwn(pkg.dependencies, name)) assert.equal(version, pkg.dependencies[name], `${agent}: installed dependency drift`);
|
||||
}
|
||||
}
|
||||
// The Cursor contract is an immutable release attestation, not another runtime
|
||||
// lookup table. Refuse generation if a release declaration disagrees with it.
|
||||
// Release declarations must agree with the profiles they attest.
|
||||
assert.equal(manifest.profiles.cursor.agentServerVersion, distributions.version);
|
||||
assert.equal(manifest.profiles.cursor.agentServerVersion, contract.vendor);
|
||||
assert.equal(manifest.profiles.cursor.agentProfileVersion, contract.profileVersion);
|
||||
assert.equal(manifest.profiles.cursor.commandDigest, contract.commandDigest);
|
||||
assert.equal(contract.commandDigest, manifest.profiles.cursor.commandDigest, "cursor-contract.json carries a stale commandDigest");
|
||||
assert.equal(distributions.patchVersion, contract.nativePatch);
|
||||
const canonicalJson = value => value && typeof value === "object"
|
||||
? Array.isArray(value) ? `[${value.map(canonicalJson).join(",")}]`
|
||||
: `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
|
||||
: JSON.stringify(value);
|
||||
assert.equal(manifest.profiles.antigravity.agentServerVersion, antigravityDistributions.version);
|
||||
assert.equal(manifest.profiles.antigravity.commandDigest, `sha256:${createHash("sha256").update(canonicalJson(antigravityDistributions)).digest("hex")}`, "Antigravity distribution identity is stale");
|
||||
const { commandDigest, ...attestation } = contract;
|
||||
assert.equal(commandDigest, `sha256:${createHash("sha256").update(canonicalJson(attestation)).digest("hex")}`);
|
||||
for (const distribution of Object.values(distributions.platforms)) assert.match(distribution.closureSha256, /^[a-f0-9]{64}$/);
|
||||
|
||||
const quote = JSON.stringify;
|
||||
|
||||
@@ -2,6 +2,23 @@ import { describe, expect, it } from "vitest";
|
||||
import { QUALIFIED_ACPX_PROFILE_DATA } from "./generated-profiles.js";
|
||||
import { isSupportedAcpxProfileVersion } from "./profile-compatibility.js";
|
||||
|
||||
// A profile version changes only to reset saved sessions on purpose. An upstream
|
||||
// release changes the commandDigest and keeps the version; a new provider starts
|
||||
// at 1. Update this table, with the reason, only for a deliberate reset.
|
||||
const PROFILE_VERSIONS: Readonly<Record<string, number>> = {
|
||||
antigravity: 1, claude: 1, codex: 1, copilot: 92, cursor: 17, grok: 20, pi: 37,
|
||||
};
|
||||
|
||||
describe("ACPX profile versions", () => {
|
||||
it("change only for a deliberate session reset", () => {
|
||||
for (const [agent, profile] of Object.entries(QUALIFIED_ACPX_PROFILE_DATA)) {
|
||||
expect(profile.agentProfileVersion, `${agent}: a profile version bump resets every saved ${agent} session. `
|
||||
+ "An upstream release changes only the commandDigest. See rules/acpx-profile-identity.mdc.")
|
||||
.toBe(PROFILE_VERSIONS[agent] ?? 1);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("historical ACPX profile decoding", () => {
|
||||
it("decodes every revision from 1 through the current one so saved runs reach recovery", () => {
|
||||
for (const [agent, current] of Object.entries(QUALIFIED_ACPX_PROFILE_DATA)) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@paperclipai/plugin-daytona",
|
||||
"version": "0.1.1",
|
||||
"version": "0.1.9",
|
||||
"description": "Daytona sandbox provider plugin for Paperclip environments",
|
||||
"license": "MIT",
|
||||
"homepage": "https://github.com/paperclipai/paperclip",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@paperclipai/plugin-kubernetes",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.0-alpha.1",
|
||||
"description": "Kubernetes sandbox provider plugin for Paperclip environments",
|
||||
"license": "MIT",
|
||||
"homepage": "https://github.com/paperclipai/paperclip",
|
||||
|
||||
Generated
+3
@@ -68,6 +68,9 @@ importers:
|
||||
esbuild:
|
||||
specifier: ^0.28.2
|
||||
version: 0.28.2
|
||||
oxc-parser:
|
||||
specifier: 0.127.0
|
||||
version: 0.127.0
|
||||
turbo:
|
||||
specifier: 2.11.10
|
||||
version: 2.11.10
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
---
|
||||
description: Give every ACPX provider the same identity model - its commandDigest attests the upstream release, the pack contract digest covers Runner source and the shared ACPX layer, and profile versions change only for deliberate resets.
|
||||
globs: "packages/paperclip-runner/acpx-profiles.json,packages/paperclip-runner/cursor-contract.json,packages/paperclip-runner/cursor-distributions.json,packages/paperclip-runner/antigravity-distributions.json,packages/paperclip-runner/scripts/acpx-release-attestations.mjs,packages/paperclip-runner/scripts/generate-acpx-profiles.mjs,packages/paperclip-runner/scripts/contracts/**,packages/paperclip-runner/src/drivers/acpx/**,packages/paperclip-runner/src/providers/**,packages/paperclip-runner/protocol/**"
|
||||
alwaysApply: false
|
||||
---
|
||||
|
||||
# ACPX profile identity
|
||||
|
||||
## Scope and purpose
|
||||
|
||||
This rule covers ACPX provider profiles, their release declarations and pins,
|
||||
the profile generator, the provider-pack contract registry, and the ACPX
|
||||
driver and provider code. Every ACPX harness (Claude, Codex, Copilot, Cursor,
|
||||
Grok, Pi, Antigravity and any new one) uses the same identity model, so a
|
||||
change behaves the same way for all of them.
|
||||
|
||||
## The identity model
|
||||
|
||||
A provider profile in `acpx-profiles.json` has a version and a `commandDigest`.
|
||||
Three things decide whether saved work and installed packs stay compatible:
|
||||
|
||||
| What changed | What carries it | Effect |
|
||||
| --- | --- | --- |
|
||||
| The provider's upstream release: package versions, platform closures, a launcher shipped verbatim | The profile's `commandDigest` | Saved sessions for that provider start fresh |
|
||||
| The shared ACPX layer | The profile's `acpxVersion` and the Runner's ACPX patch in the pack `contractDigest` | The same for every provider |
|
||||
| A runner/pack contract: schemas, generated catalogs, Rust shapes and env keys | The provider pack's `contractDigest` | Older image packs are not reused |
|
||||
|
||||
Ordinary Runner source changes, refactors and comments change none of these.
|
||||
|
||||
## Required behavior
|
||||
|
||||
- **Attest the upstream release.** Declare each provider's attestation in
|
||||
`ACPX_RELEASE_ATTESTATIONS` (`scripts/acpx-release-attestations.mjs`), using
|
||||
the kind that fits: `installed-command`, `launcher`, `closure` or
|
||||
`declaration`. The digest binds only what a repackager copies verbatim from
|
||||
the upstream release, so the same release gives the same digest wherever the
|
||||
pack is built or installed.
|
||||
- **Keep Paperclip source and the ACPX layer out of provider digests.**
|
||||
Runner/pack compatibility belongs to the `contractDigest`. When Runner code
|
||||
and a pack must agree on a new field or behavior, declare it in a schema
|
||||
under `protocol/` and generate the TypeScript, so the contract registry
|
||||
covers it.
|
||||
- **Regenerate instead of hand-editing.** After an upstream bump or a pin
|
||||
change, run `pnpm --filter @paperclipai/paperclip-runner generate:acpx-profiles`.
|
||||
`check:acpx-profiles` runs in the build and refuses a stale digest.
|
||||
- **Keep the profile version.** An upstream release changes the digest, not the
|
||||
version. A new provider starts at version 1. Bump a version only to reset
|
||||
every saved session for that provider on purpose, such as a change in saved
|
||||
session semantics that exact admission cannot detect. Record the reason in
|
||||
the PR and in the version table in `profile-compatibility.test.ts`.
|
||||
- **Keep earlier versions decodable.** Saved inputs with any version from 1
|
||||
through the current one decode, so recovery can refuse them. Launch and
|
||||
recovery admission require the exact current profile and digest.
|
||||
|
||||
## Adding a provider
|
||||
|
||||
1. Add its profile at version 1 to `acpx-profiles.json`.
|
||||
2. Add its attestation to `ACPX_RELEASE_ATTESTATIONS`, with its pins in
|
||||
erasable TypeScript or JSON that the generator can read.
|
||||
3. Run `generate:acpx-profiles`, then `node --test scripts/acpx-release-attestations.test.mjs`.
|
||||
|
||||
## Review criteria
|
||||
|
||||
- Does every provider digest change only with its upstream release?
|
||||
- Is a new runner/pack contract declared where the `contractDigest` reads it?
|
||||
- Does a version bump state which saved sessions it resets, and why?
|
||||
- Does the change treat every ACPX harness the same way?
|
||||
|
||||
The canonical reference is
|
||||
[`packages/paperclip-runner/docs/durable-recovery.md`](../packages/paperclip-runner/docs/durable-recovery.md#profile-identity).
|
||||
The pack contract digest is described in
|
||||
[`docker/daytona-runner/README.md`](../docker/daytona-runner/README.md).
|
||||
@@ -249,7 +249,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
|
||||
]) {
|
||||
assert.match(
|
||||
jobs.get(jobId),
|
||||
/^ {4}if: \$\{\{ needs\.gate\.outputs\.full_ci == 'true' && /m,
|
||||
/^ {4}if: \$\{\{ (?:always\(\) && !cancelled\(\) && needs\.gate\.result == 'success' && )?needs\.gate\.outputs\.full_ci == 'true' && /m,
|
||||
`${jobId} must run only when the gate selects full CI`,
|
||||
);
|
||||
}
|
||||
@@ -263,10 +263,14 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
|
||||
const verify = jobs.get("verify");
|
||||
assert.match(
|
||||
verify,
|
||||
/^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m,
|
||||
/^ {4}needs: \[gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m,
|
||||
);
|
||||
assert.equal(runAggregate(verify, true).status, 0);
|
||||
assert.equal(runAggregate(verify, false).status, 0);
|
||||
for (const result of ["failure", "cancelled", "skipped"]) {
|
||||
assert.equal(runAggregate(verify, true, { test_selection: result }).status, 0,
|
||||
"an optional selector cannot block the successful original matrix");
|
||||
}
|
||||
for (const job of ["gate", "policy", "typecheck_release_registry", "general_tests", "verify_paperclip_runner",
|
||||
"build", "docker_context_integrity", "verify_serialized_server", "canary_dry_run"]) {
|
||||
for (const result of ["failure", "cancelled", "skipped"]) {
|
||||
|
||||
@@ -10,6 +10,63 @@ import { executionPlan, sourceFiles } from "./execution-plan.mjs";
|
||||
const repositoryRoot = new URL("../../", import.meta.url);
|
||||
const turbo = process.env.PAPERCLIP_CI_TEST_TURBO ?? new URL("node_modules/.bin/turbo", repositoryRoot).pathname;
|
||||
|
||||
test("real Turbo keeps full runs and different selected test sets in separate cache entries", () => {
|
||||
const directory = mkdtempSync(path.join(tmpdir(), "ci-selection-cache-"));
|
||||
const write = (file, value) => writeFileSync(path.join(directory, file), value);
|
||||
const json = (file, value) => write(file, JSON.stringify(value));
|
||||
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
|
||||
try {
|
||||
git("init", "-q");
|
||||
for (const folder of ["scripts/ci", "server/src", "packages/fixture", "bin", ".ci-test-selection"]) mkdirSync(path.join(directory, folder), { recursive: true });
|
||||
for (const file of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) {
|
||||
copyFileSync(new URL(`scripts/ci/${file}`, repositoryRoot), path.join(directory, "scripts/ci", file));
|
||||
}
|
||||
mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
|
||||
copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
|
||||
write(".gitignore", ".turbo/\n.ci-test-selection/\nresult.txt\ncalls.txt\n");
|
||||
json("package.json", { name: "selection-cache-proof", private: true, packageManager: "npm@10.0.0", workspaces: ["packages/*"], scripts: { "test:run:general": "node check.cjs" } });
|
||||
json("packages/fixture/package.json", { name: "fixture", private: true });
|
||||
json("package-lock.json", { name: "selection-cache-proof", lockfileVersion: 3, packages: { "": { name: "selection-cache-proof" } } });
|
||||
json("turbo.json", { globalDependencies: ["doc/**", "docs/**"], globalEnv: ["CI_TASK_CACHE_SCOPE", "PAPERCLIP_*"], tasks: {
|
||||
"//#ci:server": { inputs: ["check.cjs", "server/**"], outputs: ["result.txt"], cache: true },
|
||||
} });
|
||||
mkdirSync(path.join(directory, "doc/plans"), { recursive: true });
|
||||
mkdirSync(path.join(directory, "docs/api"), { recursive: true });
|
||||
write("doc/DEVELOPING.md", "before"); write("doc/plans/example.md", "before");
|
||||
write("docs/api/issues.md", "before");
|
||||
write("server/src/a.test.ts", "// a"); write("server/src/b.test.ts", "// b");
|
||||
write("check.cjs", `const fs=require('node:fs');fs.appendFileSync('calls.txt','call\\n');const file=process.env.PAPERCLIP_CI_TEST_SELECTION;fs.writeFileSync('result.txt',file?JSON.parse(fs.readFileSync(file)).lanes['general-server-without-chat'].shards[0]['@paperclipai/server'].join(','):'full');`);
|
||||
write("bin/pnpm", `#!${process.execPath}\nconst {spawnSync}=require('node:child_process');const args=process.argv.slice(2);if(args[0]==='--version')console.log('9.15.4');else{const r=spawnSync(${JSON.stringify(turbo)},args.slice(2),{stdio:'inherit'});process.exit(r.status??1);}`);
|
||||
chmodSync(path.join(directory, "bin/pnpm"), 0o755);
|
||||
git("add", ".");
|
||||
git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qm", "fixture");
|
||||
const head = git("rev-parse", "HEAD");
|
||||
const calls = () => readFileSync(path.join(directory, "calls.txt"), "utf8").trim().split("\n").length;
|
||||
const run = (testFile) => {
|
||||
const selection = path.join(directory, ".ci-test-selection/manifest.json");
|
||||
json(".ci-test-selection/manifest.json", { version: 1, head, lanes: {
|
||||
"general-server-without-chat": { shards: [{ "@paperclipai/server": [testFile ?? "server/src/a.test.ts"] }] },
|
||||
} });
|
||||
const result = spawnSync(process.execPath, [path.join(directory, "scripts/ci/run-task.mjs")], {
|
||||
cwd: directory, encoding: "utf8", env: { ...process.env, PATH: `${directory}/bin:${process.env.PATH}`,
|
||||
CI_TASK: "ci:server", CI_TASK_ARGS: "[]", CI_FORCE_TASKS: "false", CI_TASK_DRY_RUN: "false",
|
||||
PAPERCLIP_CI_TEST_SELECTION: testFile ? selection : "", PAPERCLIP_CI_SELECTION_HASH: "",
|
||||
TURBO_TOKEN: "", TURBO_TEAM: "", TURBO_TELEMETRY_DISABLED: "1" },
|
||||
});
|
||||
assert.equal(result.status, 0, result.stdout + result.stderr);
|
||||
assert.equal(readFileSync(path.join(directory, "result.txt"), "utf8"), testFile ?? "full");
|
||||
};
|
||||
run(); run(); assert.equal(calls(), 1);
|
||||
write("doc/DEVELOPING.md", "after"); write("doc/plans/example.md", "after");
|
||||
run(); assert.equal(calls(), 1, "audited prose does not invalidate product test results");
|
||||
run("server/src/a.test.ts"); run("server/src/a.test.ts"); assert.equal(calls(), 2);
|
||||
run("server/src/b.test.ts"); assert.equal(calls(), 3);
|
||||
run(); assert.equal(calls(), 3, "the original full result is reusable without accepting a subset result");
|
||||
write("docs/api/issues.md", "after");
|
||||
run(); assert.equal(calls(), 4, "runtime documentation must invalidate results");
|
||||
} finally { rmSync(directory, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("every cached lane maps to an existing package script with complete inputs", () => {
|
||||
const config = JSON.parse(readFileSync(new URL("turbo.json", repositoryRoot), "utf8"));
|
||||
for (const [name, definition] of Object.entries(config.tasks).filter(([, task]) => task.cache)) {
|
||||
@@ -35,6 +92,8 @@ test("real Turbo executes existing suite scripts and invalidates source, argumen
|
||||
const json = (name, value) => write(name, JSON.stringify(value));
|
||||
try {
|
||||
execFileSync("git", ["init", "-q"], { cwd: directory });
|
||||
mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
|
||||
copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
|
||||
write(".gitignore", ".turbo/\ngenerated/\nresult.txt\ncalls.txt\n");
|
||||
mkdirSync(path.join(directory, "packages/fixture"), { recursive: true });
|
||||
json("packages/fixture/package.json", { name: "fixture", private: true });
|
||||
@@ -83,9 +142,11 @@ test("the task launcher bypasses cache reads and writes when execution is forced
|
||||
mkdirSync(path.join(directory, "scripts/ci"), { recursive: true });
|
||||
mkdirSync(path.join(directory, "bin"));
|
||||
mkdirSync(path.join(directory, "packages/fixture"), { recursive: true });
|
||||
for (const name of ["run-task.mjs", "execution-plan.mjs"]) {
|
||||
for (const name of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) {
|
||||
copyFileSync(new URL(`scripts/ci/${name}`, repositoryRoot), path.join(directory, "scripts/ci", name));
|
||||
}
|
||||
mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
|
||||
copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
|
||||
write(".gitignore", ".turbo/\ncalls.txt\nresult.txt\nbin/\n");
|
||||
json("packages/fixture/package.json", { name: "fixture", private: true });
|
||||
json("package.json", { name: "forced-cache-proof", private: true, packageManager: "npm@10.0.0",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { matchesGlob } from "node:path";
|
||||
import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
|
||||
|
||||
export function sourceFiles(cwd = process.cwd()) {
|
||||
return [...new Set(execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z"],
|
||||
@@ -30,7 +31,7 @@ export function executionPlan(config, task, args, files = sourceFiles()) {
|
||||
target = "@paperclipai/paperclip-runner#check:runner";
|
||||
commandArgs = args;
|
||||
} else throw new Error(`Missing task command: ${task}`);
|
||||
const select = (globs) => files.filter((file) => globs.some((glob) => matchesGlob(file, glob))).map(literalGlob);
|
||||
const select = (globs) => productInputs(files).filter((file) => globs.some((glob) => matchesGlob(file, glob))).map(literalGlob);
|
||||
const selectedInputs = select(definition.inputs);
|
||||
const inputs = target.startsWith("//#") ? selectedInputs
|
||||
: selectedInputs.map((input) => `$TURBO_ROOT$/${input}`);
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import assert from "node:assert/strict";
|
||||
import path from "node:path";
|
||||
import { createVitest, parseCLI } from "vitest/node";
|
||||
|
||||
// Positional Vitest CLI filters are substrings: foo.test.ts also selects
|
||||
// foo.test.tsx. Run the exact discovered specifications assigned to this shard.
|
||||
const { filter, options } = parseCLI(["vitest", "run", ...process.argv.slice(2)]);
|
||||
assert.ok(filter.length > 0, "Selected execution requires explicit test files");
|
||||
const expected = new Set(filter.map((file) => path.resolve(file)));
|
||||
assert.equal(expected.size, filter.length, "Duplicate selected file");
|
||||
process.env.TEST = "true";
|
||||
process.env.VITEST = "true";
|
||||
process.env.NODE_ENV ??= "test";
|
||||
// Match the CLI's additive --exclude semantics without replacing project rules.
|
||||
options.cliExclude = options.exclude;
|
||||
delete options.exclude;
|
||||
const ctx = await createVitest({ ...options, watch: false });
|
||||
try {
|
||||
await ctx.standalone();
|
||||
const specifications = (await ctx.globTestSpecifications()).filter((spec) => expected.has(spec.moduleId));
|
||||
assert.deepEqual(specifications.map((spec) => spec.moduleId).sort(), [...expected].sort(),
|
||||
"Selected files must match exactly one discovered specification in this project");
|
||||
await ctx.runTestSpecifications(specifications);
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await ctx.exit();
|
||||
}
|
||||
@@ -3,6 +3,8 @@ import { lstatSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from
|
||||
import { createHash } from "node:crypto";
|
||||
import { matchesGlob } from "node:path";
|
||||
import { executionPlan, sourceFiles } from "./execution-plan.mjs";
|
||||
import { loadSelection, selectionHash } from "./test-selection-manifest.mjs";
|
||||
import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
|
||||
|
||||
const task = process.env.CI_TASK;
|
||||
if (!/^ci:[a-z-]+$/.test(task ?? "")) throw new Error("Invalid CI task");
|
||||
@@ -15,13 +17,18 @@ const version = (command, argv) => {
|
||||
const config = JSON.parse(readFileSync("turbo.json", "utf8"));
|
||||
const files = sourceFiles();
|
||||
const execution = executionPlan(config, task, args, files);
|
||||
// Validate before Turbo can return a cache hit. The hash separates selected
|
||||
// executions from full executions and from every different selected file set.
|
||||
if (process.env.PAPERCLIP_CI_TEST_SELECTION) {
|
||||
process.env.PAPERCLIP_CI_SELECTION_HASH = selectionHash(loadSelection());
|
||||
}
|
||||
mkdirSync(".turbo/task-configs", { recursive: true });
|
||||
const runtimeConfig = `.turbo/task-configs/${task.replace(":", "-")}.json`;
|
||||
writeFileSync(runtimeConfig, JSON.stringify(execution.config));
|
||||
const classified = [...config.globalDependencies, ...Object.values(config.tasks).flatMap((entry) => entry.inputs)];
|
||||
const unknownInputs = createHash("sha256");
|
||||
let uncacheable = false;
|
||||
for (const file of files) {
|
||||
for (const file of productInputs(files)) {
|
||||
if (classified.some((glob) => matchesGlob(file, glob))) continue;
|
||||
unknownInputs.update(JSON.stringify(file));
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync, lstatSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import path from "node:path";
|
||||
import assert from "node:assert/strict";
|
||||
import { refineTestPlan, selectionDurations } from "./test-selection-plan.mjs";
|
||||
import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
|
||||
import { mergeDiff } from "../../.github/scripts/ci-diff.mjs";
|
||||
|
||||
export function selectionEligibility(plan) {
|
||||
if (plan?.version !== 1 || plan.mode !== "enabled" || plan.forceReason) return "Full/shadow/forced task plan";
|
||||
if (!Array.isArray(plan.changed) || !plan.changed.length) return "Incomplete comparison";
|
||||
const changed = productInputs(plan.changed);
|
||||
if (!changed.length) return "Documentation-only diff";
|
||||
for (const file of changed) {
|
||||
if (!/^(?:ui\/src|server\/src|cli\/src|packages\/.+\/src)\/.+\.[cm]?[jt]sx?$/.test(file)
|
||||
|| /(?:config|setup)|\.d\.[cm]?ts$/i.test(file) || file.split("/").includes("..")) return `Broad input: ${file}`;
|
||||
try {
|
||||
if (!lstatSync(file).isFile()) return `Non-regular source: ${file}`;
|
||||
} catch { return `Deleted or missing source: ${file}`; }
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function runnerInventory(group) {
|
||||
const args = group === "serialized" ? ["--mode", "serialized", "--shard-index", "0", "--shard-count", "1"]
|
||||
: ["--mode", "general", "--group", group,
|
||||
...(group === "general-server-without-chat" ? ["--shard-index", "0", "--shard-count", "1"] : [])];
|
||||
const env = { ...process.env, GITHUB_WORKFLOW: "PR" };
|
||||
delete env.PAPERCLIP_CI_TEST_SELECTION;
|
||||
return JSON.parse(execFileSync(process.execPath, ["scripts/run-vitest-stable.mjs", ...args, "--dry-run"],
|
||||
{ env, encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }));
|
||||
}
|
||||
|
||||
export async function createTestSelection(plan, head) {
|
||||
const reason = selectionEligibility(plan);
|
||||
if (reason) throw new Error(reason);
|
||||
const groups = new Set();
|
||||
for (const job of ["general_tests", "verify_serialized_server"]) {
|
||||
if (plan.jobs[job].run) for (const entry of plan.jobs[job].matrix.include) groups.add(entry.group ?? "serialized");
|
||||
}
|
||||
groups.delete("general-chat"); // Keep its line sharding and DB ownership intact.
|
||||
const owners = {}, projects = new Set();
|
||||
for (const group of groups) {
|
||||
const inventory = runnerInventory(group);
|
||||
if (group === "serialized" || group === "general-server-without-chat") {
|
||||
owners[group] = { "@paperclipai/server": group === "serialized"
|
||||
? inventory.selectedSerializedSuites : inventory.selectedGeneralServerSuites };
|
||||
projects.add("@paperclipai/server");
|
||||
} else {
|
||||
owners[group] = Object.fromEntries(inventory.workspaceProjects.map((project) => [project, null]));
|
||||
for (const project of inventory.workspaceProjects) projects.add(project);
|
||||
}
|
||||
}
|
||||
const { selectProjectTests } = await import("./vitest-selection-graph.mjs");
|
||||
const graphs = [];
|
||||
const changed = productInputs(plan.changed);
|
||||
for (const project of projects) {
|
||||
const graph = await selectProjectTests({ project, changed, onlyOwnChanges: true });
|
||||
graphs.push(graph);
|
||||
console.log(`${project}: ${graph.selected.length}/${graph.inventory.length} files, ${graph.alwaysRun.length} retained conservatively, ${graph.seconds.toFixed(1)}s`);
|
||||
for (const owner of Object.values(owners)) if (owner[project] === null) owner[project] = graph.inventory;
|
||||
}
|
||||
const covered = new Set(graphs.flatMap((graph) => graph.coveredChanges));
|
||||
for (const file of changed) assert.ok(covered.has(file), `Change outside the resolved test graphs: ${file}`);
|
||||
const server = graphs.find((graph) => graph.project === "@paperclipai/server");
|
||||
if (server) {
|
||||
const general = runnerInventory("general-server-without-chat");
|
||||
const native = runnerInventory("general-server-native-runner");
|
||||
assert.deepEqual([...general.selectedGeneralServerSuites, ...general.selectedSerializedSuites,
|
||||
...native.selectedGeneralServerSuites, "server/src/__tests__/chat-channels.integration.test.ts"].sort(),
|
||||
server.inventory, "Server inventory must be completely covered by existing lane ownership");
|
||||
}
|
||||
return { ...refineTestPlan({ plan, head, owners, graphs, durations: selectionDurations() }), graphs };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const plan = JSON.parse(process.env.CI_PLAN || "null");
|
||||
const output = (key, value) => {
|
||||
if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${value}\n`);
|
||||
};
|
||||
if (process.argv.includes("--eligible")) {
|
||||
const reason = selectionEligibility(plan);
|
||||
console.log(reason ?? "Source diff is eligible for graph selection");
|
||||
output("enabled", !reason);
|
||||
return;
|
||||
}
|
||||
const directory = ".ci-test-selection";
|
||||
mkdirSync(directory, { recursive: true });
|
||||
const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).trim();
|
||||
let result;
|
||||
try {
|
||||
const head = git("rev-parse", "HEAD");
|
||||
assert.equal(head, process.env.GITHUB_SHA, "Selection must describe the tested merge commit");
|
||||
assert.ok(plan.diff, "Missing trusted merge diff");
|
||||
const local = mergeDiff(head, plan.diff.prHead);
|
||||
assert.deepEqual(local.diff, plan.diff, "Selection must use the trusted merge parents");
|
||||
assert.deepEqual(local.changed, [...new Set(plan.changed)].sort(), "Local diff disagrees with the trusted routing diff");
|
||||
assert.equal(git("status", "--porcelain", "--untracked-files=normal"), "", "Source inputs changed during dependency setup");
|
||||
process.env.NODE_ENV = "test";
|
||||
process.env.VITEST = "true";
|
||||
result = await createTestSelection(plan, head);
|
||||
writeFileSync(`${directory}/manifest.json`, JSON.stringify(result.manifest));
|
||||
writeFileSync(`${directory}/audit.json`, JSON.stringify(result.graphs, null, 2));
|
||||
output("selected", "true");
|
||||
} catch (error) {
|
||||
console.warn(`Using the complete task plan: ${error.message}`);
|
||||
result = { plan: { ...plan, testSelectionFallback: error.message } };
|
||||
output("selected", "false");
|
||||
}
|
||||
writeFileSync(`${directory}/plan.json`, JSON.stringify(result.plan));
|
||||
if (process.env.GITHUB_STEP_SUMMARY) {
|
||||
const stats = result.plan.testSelection?.stats;
|
||||
appendFileSync(process.env.GITHUB_STEP_SUMMARY, stats
|
||||
? ["### Vitest test selection", "", "| Lane | Test files | Jobs |", "| --- | ---: | ---: |",
|
||||
...stats.map((s) => `| ${s.group} | ${s.selected}/${s.total} | ${s.jobs}/${s.beforeJobs} |`), "",
|
||||
"Filesystem/process/opaque consumers are retained. Chat and native Runner lanes keep their existing ownership.", ""].join("\n")
|
||||
: `### Vitest test selection\n\nFull task plan: ${result.plan.testSelectionFallback}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
main().catch((error) => { console.error(error); process.exitCode = 1; });
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readFileSync, statSync } from "node:fs";
|
||||
|
||||
export const selectableGroups = ["general-server-without-chat", "general-workspaces-a",
|
||||
"general-workspaces-b", "general-workspaces-c", "serialized"];
|
||||
|
||||
export function selectionHash(manifest) {
|
||||
return createHash("sha256").update(JSON.stringify({ version: manifest.version, lanes: manifest.lanes })).digest("hex");
|
||||
}
|
||||
|
||||
export function validateSelection(manifest, { head, hash } = {}) {
|
||||
assert.equal(manifest.version, 1, "Unsupported test selection");
|
||||
assert.match(manifest.head, /^[a-f0-9]{40}$/);
|
||||
if (head) assert.equal(manifest.head, head, "Selection belongs to another checkout");
|
||||
assert.ok(manifest.lanes && typeof manifest.lanes === "object" && !Array.isArray(manifest.lanes));
|
||||
const seen = new Set();
|
||||
for (const [group, lane] of Object.entries(manifest.lanes)) {
|
||||
assert.ok(selectableGroups.includes(group), `Unknown selection group: ${group}`);
|
||||
assert.ok(Array.isArray(lane.shards) && lane.shards.length > 0 && lane.shards.length <= 12);
|
||||
for (const shard of lane.shards) {
|
||||
assert.ok(shard && typeof shard === "object" && !Array.isArray(shard));
|
||||
let count = 0;
|
||||
for (const [project, files] of Object.entries(shard)) {
|
||||
assert.ok(project && Array.isArray(files) && files.length > 0);
|
||||
for (const file of files) {
|
||||
assert.equal(typeof file, "string");
|
||||
assert.ok(/^(ui|server|cli|packages)\//.test(file) && !/[\0\r\n\\:*?\[\]{}]/.test(file)
|
||||
&& !file.split("/").some((part) => ["", ".", ".."].includes(part)), `Invalid selected path: ${file}`);
|
||||
const key = `${project}:${file}`;
|
||||
assert.ok(!seen.has(key), `Test assigned twice: ${key}`);
|
||||
seen.add(key);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
assert.ok(count > 0, "Empty selection shard");
|
||||
}
|
||||
}
|
||||
if (hash) assert.equal(selectionHash(manifest), hash, "Selection artifact does not match its plan");
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export function loadSelection(file = process.env.PAPERCLIP_CI_TEST_SELECTION) {
|
||||
const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
||||
const manifest = validateSelection(JSON.parse(readFileSync(file, "utf8")), {
|
||||
head, hash: process.env.PAPERCLIP_CI_SELECTION_HASH,
|
||||
});
|
||||
for (const lane of Object.values(manifest.lanes)) {
|
||||
for (const shard of lane.shards) for (const files of Object.values(shard)) {
|
||||
for (const file of files) assert.ok(statSync(file).isFile(), `Selected test missing: ${file}`);
|
||||
}
|
||||
}
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export function selectedShard(manifest, options, projects) {
|
||||
const group = options.mode === "serialized" ? "serialized" : options.group;
|
||||
const lane = manifest.lanes[group];
|
||||
assert.ok(lane, `Missing selection for ${group}`);
|
||||
assert.equal(lane.shards.length, options.shardCount ?? 1, "Selection shard count mismatch");
|
||||
const shard = lane.shards[options.shardIndex ?? 0];
|
||||
assert.ok(shard, "Missing selection shard");
|
||||
for (const project of Object.keys(shard)) assert.ok(projects.includes(project), `Wrong project for ${group}: ${project}`);
|
||||
return shard;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { loadShardDurations, partitionGeneralServerSuites } from "../general-server-shard.mjs";
|
||||
import { selectionHash, validateSelection } from "./test-selection-manifest.mjs";
|
||||
|
||||
// Preserve the existing per-job budget. Small selections use fewer runners;
|
||||
// large selections retain the existing parallelism and serialized ownership.
|
||||
export function partitionSelection(inventory, selected, maxShards, durations = {}) {
|
||||
const totalWeight = partitionGeneralServerSuites(inventory, 1, durations)[0].totalWeight;
|
||||
const selectedWeight = partitionGeneralServerSuites(selected, 1, durations)[0].totalWeight;
|
||||
const budget = totalWeight / maxShards || 1;
|
||||
const count = Math.min(maxShards, selected.length, Math.max(1, Math.ceil(selectedWeight / budget)));
|
||||
return selected.length ? partitionGeneralServerSuites(selected, count, durations).map((shard) => shard.files) : [];
|
||||
}
|
||||
|
||||
export function refineTestPlan({ plan, head, owners, graphs, durations = {} }) {
|
||||
const result = structuredClone(plan);
|
||||
const manifest = { version: 1, head, lanes: {} };
|
||||
const stats = [];
|
||||
const byProject = new Map(graphs.map((graph) => [graph.project, graph]));
|
||||
for (const job of ["general_tests", "verify_serialized_server"]) {
|
||||
if (!plan.jobs[job].run) continue;
|
||||
const entries = plan.jobs[job].matrix.include;
|
||||
const groups = [...new Set(entries.map((entry) => entry.group ?? "serialized"))];
|
||||
const matrix = [];
|
||||
for (const group of groups) {
|
||||
const original = entries.filter((entry) => (entry.group ?? "serialized") === group);
|
||||
if (group === "general-chat") { matrix.push(...original); continue; }
|
||||
const owned = owners[group];
|
||||
assert.ok(owned, `Missing inventory owner: ${group}`);
|
||||
const projectByFile = new Map();
|
||||
const selected = [];
|
||||
for (const [project, files] of Object.entries(owned)) {
|
||||
const graph = byProject.get(project);
|
||||
assert.ok(graph, `Missing project graph: ${project}`);
|
||||
const inventory = new Set(graph.inventory);
|
||||
const affected = new Set(graph.selected);
|
||||
for (const file of files) {
|
||||
assert.ok(inventory.has(file), `Runner/Vitest inventory disagreement: ${file}`);
|
||||
assert.ok(!projectByFile.has(file), `Ambiguous owner: ${file}`);
|
||||
projectByFile.set(file, project);
|
||||
if (affected.has(file)) selected.push(file);
|
||||
}
|
||||
}
|
||||
const partitions = partitionSelection([...projectByFile.keys()], selected, original.length, durations[group]);
|
||||
if (partitions.length) {
|
||||
const shards = partitions.map((files) => {
|
||||
const shard = {};
|
||||
for (const file of files) (shard[projectByFile.get(file)] ??= []).push(file);
|
||||
return shard;
|
||||
});
|
||||
manifest.lanes[group] = { shards };
|
||||
for (let index = 0; index < shards.length; index++) {
|
||||
const entry = { ...original[0], selection: true };
|
||||
if (original[0].shard_count) {
|
||||
Object.assign(entry, { shard_index: index, shard_count: shards.length });
|
||||
if (entry.shard_label) entry.shard_label = `${index + 1}/${shards.length}`;
|
||||
if (entry.group_label) entry.group_label = entry.group_label.replace(/\(\d+\/\d+\)/, `(${index + 1}/${shards.length})`);
|
||||
}
|
||||
matrix.push(entry);
|
||||
}
|
||||
}
|
||||
stats.push({ group, total: projectByFile.size, selected: selected.length, beforeJobs: original.length, jobs: partitions.length });
|
||||
}
|
||||
result.jobs[job] = { ...result.jobs[job], run: matrix.length > 0, selected: matrix.length, proposed: matrix.length,
|
||||
matrix: { include: matrix.length ? matrix : entries } };
|
||||
}
|
||||
validateSelection(manifest);
|
||||
result.testSelection = { version: 1, head, hash: selectionHash(manifest), stats };
|
||||
return { plan: result, manifest };
|
||||
}
|
||||
|
||||
export function selectionDurations() {
|
||||
return {
|
||||
"general-server-without-chat": loadShardDurations("scripts/general-server-shard-durations.json"),
|
||||
serialized: loadShardDurations("scripts/serialized-shard-durations.json"),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { selectProjectTests } from "./vitest-selection-graph.mjs";
|
||||
import { selectionHash } from "./test-selection-manifest.mjs";
|
||||
|
||||
test("production runner catches the filesystem regression that raw Vitest selection misses", async (t) => {
|
||||
const repo = process.cwd();
|
||||
const root = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vr-")));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
for (const directory of ["ui/src", "server/src/__tests__", "server/scripts"]) mkdirSync(path.join(root, directory), { recursive: true });
|
||||
symlinkSync(path.join(repo, "node_modules"), path.join(root, "node_modules"), "dir");
|
||||
const write = (file, body) => writeFileSync(path.join(root, file), body);
|
||||
const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
|
||||
write(".gitignore", "node_modules/\nselection.json\n");
|
||||
write("package.json", '{"name":"selection-regression","private":true,"type":"module"}');
|
||||
write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}');
|
||||
write("ui/vitest.config.mjs", 'export default {test:{name:"@paperclipai/ui",include:["src/**/*.test.mjs"]}}');
|
||||
write("ui/src/leaf.mjs", "export const value = 42;\n");
|
||||
write("ui/src/import.test.mjs", 'import {test,expect} from "vitest"; import {value} from "./leaf.mjs"; test("value",()=>expect(value).toBe(42));');
|
||||
write("ui/src/import.test.mjs.test.mjs", 'throw new Error("UNSELECTED_FILE: substring filters must not execute this file");');
|
||||
write("ui/src/contract.test.mjs", 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; test("filesystem contract",()=>expect(readFileSync(new URL("./leaf.mjs",import.meta.url),"utf8")).not.toContain("FORBIDDEN_MARKER"));');
|
||||
git("init", "-q"); git("add", ".");
|
||||
const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture");
|
||||
commit();
|
||||
const run = (files) => {
|
||||
const manifest = { version: 1, head: git("rev-parse", "HEAD"), lanes: {
|
||||
"general-workspaces-a": { shards: [{ "@paperclipai/ui": files }] },
|
||||
} };
|
||||
write("selection.json", JSON.stringify(manifest));
|
||||
return spawnSync(process.execPath, [path.join(repo, "scripts/run-vitest-stable.mjs"), "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1"], {
|
||||
cwd: root, encoding: "utf8", timeout: 60000,
|
||||
env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: path.join(root, "selection.json"),
|
||||
PAPERCLIP_CI_SELECTION_HASH: selectionHash(manifest) },
|
||||
});
|
||||
};
|
||||
const baseline = run(["ui/src/import.test.mjs", "ui/src/contract.test.mjs"]);
|
||||
assert.equal(baseline.status, 0, baseline.stdout + baseline.stderr);
|
||||
const missing = run(["ui/src/leaf.mjs"]);
|
||||
assert.equal(missing.status, 1, "an existing source file is not an executable test specification");
|
||||
assert.match(missing.stdout + missing.stderr, /Selected files must match/);
|
||||
write("ui/src/leaf.mjs", `${readFileSync(path.join(root, "ui/src/leaf.mjs"), "utf8")}// FORBIDDEN_MARKER\n`);
|
||||
commit();
|
||||
const graph = await selectProjectTests({ root, project: "@paperclipai/ui", changed: ["ui/src/leaf.mjs"] });
|
||||
assert.deepEqual(graph.related, ["ui/src/import.test.mjs"]);
|
||||
const raw = run(graph.related);
|
||||
assert.equal(raw.status, 0, raw.stdout + raw.stderr);
|
||||
assert.ok(graph.selected.includes("ui/src/contract.test.mjs"));
|
||||
const guarded = run(graph.selected);
|
||||
assert.equal(guarded.status, 1, guarded.stdout + guarded.stderr);
|
||||
assert.match(guarded.stdout + guarded.stderr, /filesystem contract/);
|
||||
assert.match(guarded.stdout + guarded.stderr, /FORBIDDEN_MARKER/);
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import vm from "node:vm";
|
||||
import test from "node:test";
|
||||
import { planTasks } from "../../.github/scripts/ci-plan.mjs";
|
||||
|
||||
const workflow = readFileSync(".github/workflows/pr-trusted.yml", "utf8");
|
||||
const block = (job) => workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0];
|
||||
const plan = planTasks({ config: JSON.parse(readFileSync("turbo.json", "utf8")),
|
||||
lanes: JSON.parse(readFileSync(".github/ci-task-matrix.json", "utf8")), paths: ["ui/src/pages/Inbox.tsx"] });
|
||||
const evaluate = (expression, needs, cancelled = false) => vm.runInNewContext(expression, {
|
||||
needs, fromJSON: JSON.parse, toJSON: JSON.stringify, always: () => true, cancelled: () => cancelled,
|
||||
});
|
||||
|
||||
test("failed, timed-out, skipped and unavailable selectors leave the original matrix runnable", () => {
|
||||
for (const result of ["failure", "cancelled", "skipped", "success"]) {
|
||||
const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } },
|
||||
test_selection: { result, outputs: { plan: "" } } };
|
||||
const job = block("general_tests");
|
||||
const condition = job.match(/if: \$\{\{ (.*) \}\}/)[1];
|
||||
const matrix = job.match(/matrix: \$\{\{ (.*) \}\}/)[1];
|
||||
assert.equal(evaluate(condition, needs), true, result);
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(evaluate(matrix, needs))), plan.jobs.general_tests.matrix);
|
||||
assert.equal(evaluate(condition, needs, true), false, "cancellation must not start more runners");
|
||||
}
|
||||
});
|
||||
|
||||
test("a published selection controls the matrix and the required aggregate", () => {
|
||||
const selected = structuredClone(plan);
|
||||
selected.testSelection = { version: 1, hash: "test" };
|
||||
selected.jobs.general_tests.matrix.include = [selected.jobs.general_tests.matrix.include[0]];
|
||||
selected.jobs.general_tests.selected = 1;
|
||||
const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } },
|
||||
test_selection: { result: "success", outputs: { plan: JSON.stringify(selected) } } };
|
||||
const matrix = block("general_tests").match(/matrix: \$\{\{ (.*) \}\}/)[1];
|
||||
assert.equal(evaluate(matrix, needs).include.length, 1);
|
||||
const aggregate = block("verify");
|
||||
const names = aggregate.match(/needs: \[([^\]]+)\]/)[1].split(", ");
|
||||
const code = aggregate.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0]
|
||||
.split("\n").map((line) => line.slice(10)).join("\n");
|
||||
const results = Object.fromEntries(names.map((name) => [name, { result:
|
||||
["gate", "policy", "test_selection"].includes(name) || selected.jobs[name]?.run ? "success" : "skipped" }]));
|
||||
const run = () => spawnSync(process.execPath, ["--input-type=module", "-e", code], { encoding: "utf8",
|
||||
env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(selected), CI_RESULTS: JSON.stringify(results) } });
|
||||
assert.equal(run().status, 0);
|
||||
for (const job of ["general_tests", "test_selection"]) {
|
||||
for (const result of ["failure", "cancelled", "skipped"]) {
|
||||
results[job].result = result;
|
||||
assert.notEqual(run().status, 0, `${job}: ${result}`);
|
||||
}
|
||||
results[job].result = "success";
|
||||
}
|
||||
selected.jobs.general_tests.run = false;
|
||||
results.general_tests.result = "skipped";
|
||||
assert.equal(run().status, 0, "an explicitly empty selected lane may be skipped");
|
||||
assert.ok(aggregate.includes("needs.test_selection.outputs.plan || needs.gate.outputs.plan"));
|
||||
});
|
||||
|
||||
test("publication follows successful upload and manifests are downloaded without ignoring failures", () => {
|
||||
const selection = block("test_selection");
|
||||
assert.ok(selection.indexOf("Upload validated selection") < selection.indexOf("Publish the executable plan"));
|
||||
assert.match(selection, /timeout-minutes: 5/);
|
||||
assert.match(selection, /vars.CI_VITEST_SELECTION == '' \|\| vars.CI_VITEST_SELECTION == 'enabled'/);
|
||||
for (const job of ["general_tests", "verify_serialized_server"]) {
|
||||
assert.match(block(job), /PAPERCLIP_CI_SELECTION_HASH:.*testSelection.hash/);
|
||||
assert.match(block(job), /Download selected test files\n if: matrix.selection\n uses: actions\/download-artifact@/);
|
||||
assert.doesNotMatch(block(job), /continue-on-error: true\n.*download-artifact/);
|
||||
}
|
||||
});
|
||||
|
||||
test("failed-job retries use the original selector artifact, not the retry attempt", (t) => {
|
||||
const directory = mkdtempSync(path.join(tmpdir(), "ci-retry-"));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
mkdirSync(path.join(directory, ".ci-test-selection"));
|
||||
const output = path.join(directory, "output");
|
||||
const publication = block("test_selection").split(" - name: Publish the executable plan")[1];
|
||||
assert.match(publication, /ARTIFACT_ID: \$\{\{ steps.upload_selection.outputs.artifact-id \}\}/);
|
||||
const code = publication.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0]
|
||||
.split("\n").map((line) => line.slice(10)).join("\n");
|
||||
const selected = { ...plan, testSelection: { version: 1, hash: "test" } };
|
||||
const run = (value, artifactId) => {
|
||||
writeFileSync(path.join(directory, ".ci-test-selection/plan.json"), JSON.stringify(value));
|
||||
writeFileSync(output, "");
|
||||
return spawnSync(process.execPath, ["--input-type=module", "-e", code], {
|
||||
cwd: directory, encoding: "utf8", env: { ...process.env, ARTIFACT_ID: artifactId, GITHUB_OUTPUT: output },
|
||||
});
|
||||
};
|
||||
assert.equal(run(selected, "12345").status, 0);
|
||||
const savedPlan = readFileSync(output, "utf8").trim().slice("plan=".length);
|
||||
for (const job of ["general_tests", "verify_serialized_server"]) {
|
||||
const expression = block(job).match(/artifact-ids: \$\{\{ (.*) \}\}/)[1];
|
||||
assert.equal(vm.runInNewContext(expression, { fromJSON: JSON.parse, github: { run_attempt: 2 },
|
||||
needs: { test_selection: { outputs: { plan: savedPlan } } } }), "12345");
|
||||
}
|
||||
assert.notEqual(run(selected, "").status, 0, "a selection cannot be published without its uploaded artifact");
|
||||
assert.equal(run(plan, "").status, 0, "fallback plans need no selection artifact");
|
||||
});
|
||||
@@ -0,0 +1,198 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { selectProjectTests } from "./vitest-selection-graph.mjs";
|
||||
import { partitionSelection, refineTestPlan } from "./test-selection-plan.mjs";
|
||||
import { selectionHash, selectedShard, validateSelection } from "./test-selection-manifest.mjs";
|
||||
import { selectionEligibility } from "./select-tests.mjs";
|
||||
import { mergeDiff } from "../../.github/scripts/ci-diff.mjs";
|
||||
|
||||
const root = process.cwd();
|
||||
const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
||||
|
||||
test("real graph retains direct and indirect filesystem, process and opaque dependencies", async (t) => {
|
||||
const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vg-")));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
mkdirSync(path.join(directory, "ui/src"), { recursive: true });
|
||||
mkdirSync(path.join(directory, "ui/src/glob"));
|
||||
symlinkSync(path.join(root, "node_modules"), path.join(directory, "node_modules"), "dir");
|
||||
const write = (file, text) => writeFileSync(path.join(directory, file), text);
|
||||
write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}');
|
||||
write("ui/vitest.config.mjs", `export default {resolve:{alias:{"@leaf":${JSON.stringify(path.join(directory, "ui/src/leaf.ts"))}}},test:{name:"fixture",include:["src/**/*.test.ts"],setupFiles:["./setup.ts"]}}`);
|
||||
write("ui/setup.ts", 'import "./src/environment.ts";');
|
||||
write("ui/src/environment.ts", "export const environment = true;");
|
||||
write("ui/src/leaf.ts", "export const value = 42;");
|
||||
write("ui/src/bridge.ts", 'export {value} from "@leaf";');
|
||||
write("ui/src/helper.ts", 'import {readFileSync} from "node:fs"; export const read = (file: string) => readFileSync(file, "utf8");');
|
||||
write("ui/src/import.test.ts", 'import {value} from "./bridge"; console.log(value); throw new Error("selection must not execute tests");');
|
||||
write("ui/src/reader.test.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));');
|
||||
write("ui/src/indirect.test.ts", 'import {read} from "./helper"; read("input"); throw new Error("selection must not execute tests");');
|
||||
write("ui/src/process.test.ts", 'import {execFileSync} from "node:child_process"; execFileSync("must-not-execute");');
|
||||
write("ui/src/dynamic.test.ts", 'const load = (file: string) => import(file);');
|
||||
write("ui/src/cjs.test.ts", 'const load = require;');
|
||||
write("ui/src/builtin-loader.test.ts", 'process.getBuiltinModule("fs").readFileSync("input");');
|
||||
write("ui/src/opaque.test.ts", 'import {parse} from "yaml"; console.log(parse("value: true"));');
|
||||
write("ui/src/independent.test.ts", 'export const independent = true;');
|
||||
write("ui/src/mocked.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); import {read} from "./helper"; read("input");');
|
||||
write("ui/src/partial.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",async (original)=>await original()); import {read} from "./helper"; read("input");');
|
||||
write("ui/src/automock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper"); import {read} from "./helper"; read("input");');
|
||||
write("ui/src/bypass.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); await vi.importActual("./helper");');
|
||||
write("ui/src/spy.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",{spy:true}); import {read} from "./helper"; read("input");');
|
||||
write("ui/src/remock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); vi.doMock("./helper",async original=>await original());');
|
||||
write("ui/src/arguments.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",function(){return arguments[0]();});');
|
||||
write("ui/src/glob/one.ts", "export const value = 1;");
|
||||
write("ui/src/globber.ts", 'export const modules = import.meta.glob("./glob/*.ts", {eager:true});');
|
||||
write("ui/src/glob.test.ts", 'import {modules} from "./globber"; console.log(modules);');
|
||||
execFileSync("git", ["init", "-q"], { cwd: directory });
|
||||
execFileSync("git", ["add", "."], { cwd: directory });
|
||||
const select = (changed) => selectProjectTests({ root: directory, project: "fixture", changed });
|
||||
const result = await select(["ui/src/leaf.ts"]);
|
||||
assert.deepEqual(result.related, ["ui/src/import.test.ts"]);
|
||||
for (const file of ["reader", "indirect", "process", "dynamic", "cjs", "builtin-loader", "opaque", "partial", "automock", "bypass", "spy", "remock", "arguments"]) {
|
||||
assert.ok(result.alwaysRun.includes(`ui/src/${file}.test.ts`), `${file} must never be dropped`);
|
||||
}
|
||||
assert.ok(!result.selected.includes("ui/src/independent.test.ts"));
|
||||
assert.ok(!result.alwaysRun.includes("ui/src/mocked.test.ts"), "a completely replaced local dependency performs no real IO");
|
||||
assert.ok(!result.alwaysRun.includes("ui/src/glob.test.ts"), "literal Vite globs use the real dependency graph");
|
||||
assert.ok((await select(["ui/src/glob/one.ts"])).related.includes("ui/src/glob.test.ts"));
|
||||
write("ui/src/glob/two.ts", "export const value = 2;");
|
||||
execFileSync("git", ["add", "."], { cwd: directory });
|
||||
assert.ok((await select(["ui/src/glob/two.ts"])).related.includes("ui/src/glob.test.ts"), "new glob members cannot be omitted");
|
||||
const setup = await select(["ui/src/environment.ts"]);
|
||||
assert.equal(setup.setupChanged, true);
|
||||
assert.deepEqual(setup.selected, setup.inventory, "transitive setup edits affect every test");
|
||||
write("ui/src/broken.test.ts", 'import "./missing";');
|
||||
const broken = await select(["ui/src/leaf.ts"]);
|
||||
assert.ok(broken.selected.includes("ui/src/broken.test.ts"), "an unresolved dependency must retain its test");
|
||||
write("ui/src/generated.ts", "export const generated = 42;");
|
||||
write("ui/src/generated.test.ts", 'import {generated} from "./generated"; console.log(generated);');
|
||||
execFileSync("git", ["add", "ui/src/generated.test.ts"], { cwd: directory });
|
||||
assert.ok((await select(["ui/src/leaf.ts"])).alwaysRun.includes("ui/src/generated.test.ts"),
|
||||
"generated inputs have no trustworthy source-to-output graph and retain their consumers");
|
||||
write("ui/src/environment.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));');
|
||||
await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup builtin/, "setup filesystem reads can affect every test outside the import graph");
|
||||
mkdirSync(path.join(directory, "server/src/__tests__/helpers"), { recursive: true });
|
||||
for (const file of ["server/src/__tests__/setup-supertest.ts", "server/src/__tests__/helpers/isolated-test-home.ts"]) {
|
||||
write(file, readFileSync(path.join(root, file), "utf8"));
|
||||
}
|
||||
write("ui/setup.ts", 'import "../server/src/__tests__/setup-supertest.ts";');
|
||||
const reviewed = await select(["ui/src/leaf.ts"]);
|
||||
assert.ok(!reviewed.selected.includes("ui/src/independent.test.ts"), "reviewed isolated setup permits file selection");
|
||||
const helper = "server/src/__tests__/helpers/isolated-test-home.ts";
|
||||
const changedSetup = await select([helper]);
|
||||
assert.deepEqual(changedSetup.selected, changedSetup.inventory, "reviewed helpers still invalidate all tests when changed");
|
||||
write(helper, `${readFileSync(path.join(directory, helper), "utf8")}\n// setup edited since review\n`);
|
||||
await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup/, "a stale review cannot authorize new setup effects");
|
||||
});
|
||||
|
||||
test("partitioning reduces runners and preserves an exact, non-overlapping file cover", () => {
|
||||
const all = Array.from({ length: 120 }, (_, index) => `server/src/${index}.test.ts`);
|
||||
const selected = all.slice(0, 15);
|
||||
const shards = partitionSelection(all, selected, 12);
|
||||
assert.equal(shards.length, 2);
|
||||
assert.deepEqual(shards.flat().sort(), selected.sort());
|
||||
assert.equal(new Set(shards.flat()).size, selected.length);
|
||||
assert.deepEqual(partitionSelection(all, [], 12), []);
|
||||
assert.equal(partitionSelection(all, all, 12).length, 12);
|
||||
const durations = Object.fromEntries(all.map((file, index) => [file, index === 0 ? 1000 : 1]));
|
||||
assert.equal(partitionSelection(all, [all[0]], 12, durations).length, 1);
|
||||
assert.equal(partitionSelection(all, [all[0]], 12, Object.fromEntries(all.map((file) => [file, 0]))).length, 1);
|
||||
});
|
||||
|
||||
function fixturePlan() {
|
||||
const files = ["ui/src/pages/Inbox.test.tsx", "ui/src/pages/apps/chat/chat-ui-contract.test.ts"];
|
||||
const entries = [0, 1].map((shard_index) => ({ group: "general-workspaces-a", group_label: `workspaces-a (${shard_index + 1}/2)`, shard_index, shard_count: 2, task: "ci:workspaces-a" }));
|
||||
const plan = { version: 1, mode: "enabled", forceReason: null, changed: ["ui/src/pages/Inbox.tsx"], jobs: {
|
||||
general_tests: { run: true, matrix: { include: entries }, selected: 2, total: 2 },
|
||||
verify_serialized_server: { run: false, matrix: { include: [{ task: "ci:serialized" }] } },
|
||||
} };
|
||||
const owners = { "general-workspaces-a": { "@paperclipai/ui": files } };
|
||||
const graphs = [{ project: "@paperclipai/ui", inventory: files, selected: [files[1]] }];
|
||||
return { files, ...refineTestPlan({ plan, head, owners, graphs }) };
|
||||
}
|
||||
|
||||
test("runner consumes the exact manifest, and rejects stale, missing, or mismatched manifests", (t) => {
|
||||
const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vm-")));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
const { manifest, plan } = fixturePlan();
|
||||
assert.equal(plan.jobs.general_tests.selected, 1);
|
||||
const file = path.join(directory, "selection.json");
|
||||
const run = (selection = manifest, hash = selectionHash(manifest)) => {
|
||||
writeFileSync(file, JSON.stringify(selection));
|
||||
return spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1", "--dry-run"], {
|
||||
cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: hash },
|
||||
});
|
||||
};
|
||||
const result = run();
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.deepEqual(JSON.parse(result.stdout).selection, manifest.lanes["general-workspaces-a"].shards[0]);
|
||||
assert.notEqual(run({ ...manifest, head: "0".repeat(40) }).status, 0);
|
||||
assert.notEqual(run(manifest, "bad-hash").status, 0);
|
||||
assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 2 }, ["@paperclipai/ui"]));
|
||||
assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 1 }, ["@paperclipai/server"]));
|
||||
const duplicate = structuredClone(manifest);
|
||||
duplicate.lanes["general-workspaces-a"].shards.push(duplicate.lanes["general-workspaces-a"].shards[0]);
|
||||
assert.throws(() => validateSelection(duplicate), /assigned twice/);
|
||||
const wrongLane = { version: 1, head, lanes: { "general-server-without-chat": {
|
||||
shards: [{ "@paperclipai/server": ["server/src/__tests__/issues-service.test.ts"] }],
|
||||
} } };
|
||||
writeFileSync(file, JSON.stringify(wrongLane));
|
||||
const misplaced = spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-server-without-chat", "--shard-index", "0", "--shard-count", "1", "--dry-run"], {
|
||||
cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: selectionHash(wrongLane) },
|
||||
});
|
||||
assert.notEqual(misplaced.status, 0);
|
||||
assert.match(misplaced.stderr, /belongs to another server lane/);
|
||||
});
|
||||
|
||||
test("forced modes, config, fixtures, deletions and unknown inputs cannot narrow CI", () => {
|
||||
const plan = fixturePlan().plan;
|
||||
assert.equal(selectionEligibility(plan), null);
|
||||
for (const changed of [["ui/vitest.setup.ts"], ["ui/vite.config.ts"], ["package.json"], ["pnpm-lock.yaml"],
|
||||
["docs/api/issues.md"], ["ui/src/missing-deleted.ts"], ["server/fixtures/input.json"], []]) {
|
||||
assert.ok(selectionEligibility({ ...plan, changed }), JSON.stringify(changed));
|
||||
}
|
||||
for (const mode of ["full", "shadow"]) assert.ok(selectionEligibility({ ...plan, mode }));
|
||||
assert.ok(selectionEligibility({ ...plan, forceReason: "full-ci" }));
|
||||
});
|
||||
|
||||
test("the CLI publishes the complete coarse plan if setup mutates tracked inputs", (t) => {
|
||||
const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vd-")));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
mkdirSync(path.join(directory, "ui/src"), { recursive: true });
|
||||
const file = path.join(directory, "ui/src/input.ts");
|
||||
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
|
||||
const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture");
|
||||
git("init", "-q"); writeFileSync(file, "export const value = 1;"); git("add", "."); commit();
|
||||
const base = git("rev-parse", "HEAD");
|
||||
git("checkout", "-qb", "pr");
|
||||
writeFileSync(file, "export const value = 2;"); commit();
|
||||
const prHead = git("rev-parse", "HEAD");
|
||||
git("checkout", "--detach", base); git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "merge", "--no-ff", "pr", "-m", "merge");
|
||||
const merge = git("rev-parse", "HEAD");
|
||||
writeFileSync(file, "export const value = 3; // generated during setup");
|
||||
const plan = { version: 1, mode: "enabled", forceReason: null, ...mergeDiff(merge, prHead, directory),
|
||||
jobs: { general_tests: { run: true, selected: 2, matrix: { include: [{ shard_index: 0 }, { shard_index: 1 }] } } } };
|
||||
const result = spawnSync(process.execPath, [path.join(root, "scripts/ci/select-tests.mjs")], {
|
||||
cwd: directory, encoding: "utf8", env: { ...process.env, CI_PLAN: JSON.stringify(plan), BASE_SHA: base,
|
||||
GITHUB_SHA: merge, GITHUB_OUTPUT: "", GITHUB_STEP_SUMMARY: "" },
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const fallback = JSON.parse(readFileSync(path.join(directory, ".ci-test-selection/plan.json"), "utf8"));
|
||||
assert.match(fallback.testSelectionFallback, /Source inputs changed/);
|
||||
assert.deepEqual(fallback.jobs, plan.jobs);
|
||||
assert.equal(fallback.testSelection, undefined);
|
||||
});
|
||||
|
||||
|
||||
test("mixed audited documentation stays graph eligible and large complete source diffs are allowed", () => {
|
||||
const plan = fixturePlan().plan;
|
||||
for (const doc of ["doc/DEVELOPING.md", "doc/DATABASE.md", "doc/plans/2026-10-30-example.md"]) {
|
||||
assert.equal(selectionEligibility({ ...plan, changed: [...plan.changed, doc] }), null);
|
||||
}
|
||||
assert.equal(selectionEligibility({ ...plan, changed: Array(350).fill(plan.changed[0]) }), null);
|
||||
assert.ok(selectionEligibility({ ...plan, changed: ["doc/DEVELOPING.md"] }));
|
||||
assert.ok(selectionEligibility({ ...plan, changed: [...plan.changed, "docs/api/issues.md"] }));
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { measureVitestImpact } from "./vitest-impact-report.mjs";
|
||||
|
||||
test("real Vitest graph follows aliases/transitive imports but misses filesystem contracts", async (t) => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "vitest-impact-"));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
mkdirSync(path.join(root, "unit"));
|
||||
const write = (file, text) => writeFileSync(path.join(root, file), text);
|
||||
write("vitest.config.mjs", 'export default { test: { projects: ["unit"] } };');
|
||||
write("unit/vitest.config.mjs", `export default {
|
||||
resolve: { alias: { "@leaf": ${JSON.stringify(path.join(root, "unit/leaf.mjs"))} } },
|
||||
test: { name: "fixture", include: ["**/*.test.mjs"] }
|
||||
};`);
|
||||
write("unit/leaf.mjs", "export const value = 42;");
|
||||
write("unit/bridge.mjs", 'export { value } from "@leaf";');
|
||||
// The graph must be collected without executing the test modules.
|
||||
write("unit/related.test.mjs", 'import { value } from "./bridge.mjs"; throw new Error("must not execute");');
|
||||
write("unit/dynamic.test.mjs", 'const load = () => import("./leaf.mjs"); throw new Error("must not execute");');
|
||||
write("unit/independent.test.mjs", 'throw new Error("must not execute");');
|
||||
write("unit/reader.test.mjs", 'import { readFileSync } from "node:fs"; const body = readFileSync(new URL("./leaf.mjs", import.meta.url), "utf8");');
|
||||
|
||||
const report = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/leaf.mjs"] });
|
||||
assert.equal(report.mode, "report-only");
|
||||
assert.equal(report.projects[0].totalFiles, 4);
|
||||
assert.deepEqual(report.projects[0].selectedTests, ["unit/dynamic.test.mjs", "unit/related.test.mjs"]);
|
||||
assert.deepEqual(report.projects[0].omittedTests, ["unit/independent.test.mjs", "unit/reader.test.mjs"]);
|
||||
assert.ok(report.limitations.some((message) => message.includes("filesystem")));
|
||||
|
||||
const direct = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/independent.test.mjs"] });
|
||||
assert.deepEqual(direct.projects[0].selectedTests, ["unit/independent.test.mjs"]);
|
||||
const unmapped = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unmapped.md"] });
|
||||
assert.equal(unmapped.projects[0].selectedFiles, 0);
|
||||
assert.equal(unmapped.mode, "report-only", "an empty graph is not permission to skip CI");
|
||||
});
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { realpathSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
function repoPath(root, file) {
|
||||
const relative = path.relative(root, path.resolve(root, file));
|
||||
if (!relative || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
||||
throw new Error(`Expected a file inside the checkout: ${file}`);
|
||||
}
|
||||
return relative.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
// This is an experiment report, deliberately not a CI execution plan. Import
|
||||
// graphs cannot describe filesystem reads, subprocesses, or external services.
|
||||
export async function measureVitestImpact({ root = process.cwd(), projects, changedFiles }) {
|
||||
root = realpathSync(root);
|
||||
if (!projects?.length || !changedFiles?.length) {
|
||||
throw new Error("Specify at least one project and one changed file");
|
||||
}
|
||||
const files = [...new Set(changedFiles.map((file) => repoPath(root, file)))].sort();
|
||||
const { createVitest } = await import("vitest/node");
|
||||
const results = [];
|
||||
// Each project gets its own graph to bound memory usage. Cross-package imports
|
||||
// are still followed, but projects omitted by the caller are NOT assessed.
|
||||
for (const project of new Set(projects)) {
|
||||
const started = performance.now();
|
||||
const ctx = await createVitest({
|
||||
root,
|
||||
watch: false,
|
||||
project: [project],
|
||||
exclude: ["**/dist/**"],
|
||||
reporters: [],
|
||||
related: files.map((file) => path.resolve(root, file)),
|
||||
});
|
||||
try {
|
||||
if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) {
|
||||
throw new Error(`Expected exactly one project named ${project}`);
|
||||
}
|
||||
const inventory = await ctx.globTestSpecifications();
|
||||
if (!inventory.length) throw new Error(`No test files found for ${project}`);
|
||||
const ready = performance.now();
|
||||
const selected = await ctx.getRelevantTestSpecifications();
|
||||
const finished = performance.now();
|
||||
const selectedTests = selected.map((spec) => repoPath(root, spec.moduleId)).sort();
|
||||
const selectedSet = new Set(selectedTests);
|
||||
results.push({
|
||||
project,
|
||||
totalFiles: inventory.length,
|
||||
selectedFiles: selectedTests.length,
|
||||
startupSeconds: (ready - started) / 1000,
|
||||
selectionSeconds: (finished - ready) / 1000,
|
||||
selectedTests,
|
||||
omittedTests: inventory.map((spec) => repoPath(root, spec.moduleId))
|
||||
.filter((file) => !selectedSet.has(file)).sort(),
|
||||
});
|
||||
} finally {
|
||||
await ctx.close();
|
||||
}
|
||||
}
|
||||
return {
|
||||
mode: "report-only",
|
||||
changedFiles: files,
|
||||
limitations: [
|
||||
"Only the explicitly requested projects were assessed.",
|
||||
"Selection follows imports, not filesystem reads, subprocesses, or external services.",
|
||||
"Zero selected files is not proof that a change needs no tests.",
|
||||
"This report must not be used to omit required CI jobs.",
|
||||
],
|
||||
projects: results,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { values } = parseArgs({ options: {
|
||||
project: { type: "string", multiple: true },
|
||||
file: { type: "string", multiple: true },
|
||||
base: { type: "string" },
|
||||
output: { type: "string" },
|
||||
} });
|
||||
if (!values.project?.length || (!!values.base === !!values.file?.length)) {
|
||||
throw new Error("Usage: node scripts/ci/vitest-impact-report.mjs --project NAME [--project NAME] (--file PATH [--file PATH] | --base REF) [--output FILE]");
|
||||
}
|
||||
const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 });
|
||||
const head = git("rev-parse", "--verify", "HEAD^{commit}").trim();
|
||||
let comparison = { kind: "hypothetical", head };
|
||||
let changedFiles = values.file;
|
||||
if (values.base) {
|
||||
// Vitest reads the working tree, so do not label dirty sources as HEAD.
|
||||
if (git("status", "--porcelain", "--untracked-files=no").trim()) {
|
||||
throw new Error("--base requires a clean tracked working tree; use --file for a hypothetical change");
|
||||
}
|
||||
const base = git("rev-parse", "--verify", "--end-of-options", `${values.base}^{commit}`).trim();
|
||||
// Compare the actual trees, including both sides of a rename. The caller
|
||||
// supplies the event base; HEAD should be the tested PR merge commit in CI.
|
||||
changedFiles = git("diff", "--name-only", "--no-renames", "-z", base, head, "--")
|
||||
.split("\0").filter(Boolean);
|
||||
comparison = { kind: "git-diff", base, head };
|
||||
}
|
||||
process.env.NODE_ENV = "test";
|
||||
process.env.VITEST = "true";
|
||||
// Config/plugins are executable code. Run this only in a test job, never in
|
||||
// the trusted metadata-only CI routing gate. Bound a failed graph traversal.
|
||||
const timeout = setTimeout(() => {
|
||||
console.error("Vitest impact analysis exceeded five minutes; report unavailable");
|
||||
process.exit(1);
|
||||
}, 300_000);
|
||||
try {
|
||||
const report = await measureVitestImpact({ projects: values.project, changedFiles });
|
||||
const json = `${JSON.stringify({ ...report, comparison }, null, 2)}\n`;
|
||||
if (values.output) writeFileSync(values.output, json);
|
||||
else process.stdout.write(json);
|
||||
for (const result of report.projects) {
|
||||
console.error(`${result.project}: ${result.selectedFiles}/${result.totalFiles} files; `
|
||||
+ `${result.selectionSeconds.toFixed(2)}s graph + ${result.startupSeconds.toFixed(2)}s startup`);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
main().catch((error) => { console.error(error); process.exitCode = 1; });
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
# Vitest dependency selection experiment
|
||||
|
||||
The production selector is documented in `doc/DEVELOPING.md`. This probe remains
|
||||
a diagnostic for the raw import graph, without the production retention rules.
|
||||
|
||||
This probe measures Vitest's real import graph against the current checkout.
|
||||
It does not change the CI matrix or authorize skipping tests. It loads Vitest
|
||||
configurations and Vite plugins, so run it in a normal test environment, never
|
||||
in the trusted metadata-only routing job.
|
||||
|
||||
Install the workspace dependencies and prepare the same shared build outputs
|
||||
used by CI:
|
||||
|
||||
```sh
|
||||
pnpm install --frozen-lockfile
|
||||
node scripts/ensure-plugin-build-deps.mjs
|
||||
node --test scripts/ci/vitest-impact-report.integration.test.mjs
|
||||
```
|
||||
|
||||
Probe a hypothetical change (the file need not actually be modified):
|
||||
|
||||
```sh
|
||||
node scripts/ci/vitest-impact-report.mjs \
|
||||
--project @paperclipai/ui --file ui/src/pages/Inbox.tsx \
|
||||
--output /tmp/inbox-impact.json
|
||||
```
|
||||
|
||||
Or compare a base revision with the checked-out HEAD. This requires a clean
|
||||
tracked working tree. For a PR, check out the tested merge commit and supply
|
||||
the event's base SHA, as the existing CI planner does:
|
||||
|
||||
```sh
|
||||
node scripts/ci/vitest-impact-report.mjs \
|
||||
--project @paperclipai/ui --project @paperclipai/server \
|
||||
--base BASE_SHA --output /tmp/pr-impact.json
|
||||
```
|
||||
|
||||
Every requested project is analyzed separately to bound memory usage. Imports
|
||||
across package boundaries are followed within each graph, but unrequested
|
||||
projects are **not assessed**. Runner's separate Vitest configurations, browser
|
||||
E2E, Rust, typechecking, and builds are outside this experiment. The JSON records
|
||||
selected and omitted test files, project inventory, startup and graph times,
|
||||
and the comparison revisions. A failed or timed-out probe is not an empty plan.
|
||||
|
||||
## Initial observations
|
||||
|
||||
Measured on `5c807d11fa` with Vitest 5.0.3, macOS, and Node 26.4.0. These are
|
||||
hypothetical single-file changes against that revision, not replays of historical
|
||||
PR checkouts. CI uses Node 24 and different hardware.
|
||||
|
||||
| Changed file | Assessed project | Selected / total test files |
|
||||
| --- | --- | ---: |
|
||||
| `ui/src/pages/Inbox.tsx` | UI | 6 / 844 |
|
||||
| `ui/src/components/MarkdownBody.tsx` | UI | 135 / 844 |
|
||||
| `server/src/middleware/auth.ts` | Server | 45 / 1,428 |
|
||||
| `server/src/services/run-failure-report.ts` | Server | 787 / 1,428 |
|
||||
| `packages/shared/src/issue-references.ts` | Shared only | 5 / 96 |
|
||||
| `docs/api/issues.md` | Server | 0 / 1,428 — known false negative |
|
||||
|
||||
Cold graph construction took 27.43 seconds for UI, 25.68 seconds for server,
|
||||
and 2.79 seconds for shared. Subsequent queries on the same graph took about
|
||||
0.05 seconds; fresh CI jobs must budget for cold construction. The six Inbox
|
||||
files ran 81 assertions successfully in 20.92 seconds with four workers.
|
||||
These figures exclude dependency installation and shared package builds.
|
||||
|
||||
For a local execution comparison, the complete UI project passed all 10,043
|
||||
assertions in 844 files in 361.50 seconds with the same four-worker setting.
|
||||
The initial Inbox graph plus selected execution took about 48 seconds, versus
|
||||
six minutes for the full project. This is a local experiment, not a measured
|
||||
change in GitHub CI latency. Later runs benefit from filesystem/transform caches.
|
||||
|
||||
The CLI's Git-diff path was also exercised against the actual `HEAD^..HEAD`
|
||||
change for PR #18321. Across its eleven changed files, the server graph selected
|
||||
1,138 / 1,428 files and the shared graph selected 3 / 96. Other projects were
|
||||
not assessed. Broad shared/service imports can therefore preserve most of a
|
||||
suite; the Inbox reduction should not be generalized to every PR.
|
||||
|
||||
## Why the raw graph is not yet a safe execution plan
|
||||
|
||||
There are concrete missing edges in this repository:
|
||||
|
||||
- `ui/src/pages/apps/chat/chat-ui-contract.test.ts` reads `Inbox.tsx` through
|
||||
`readFileSync`. It is absent from the six-file selection even though changes
|
||||
to Inbox can break its assertions.
|
||||
- `server/src/__tests__/paperclip-skill-utils.test.ts` reads `docs/api/issues.md`
|
||||
through `fs.readFile`. Changing that document selects zero server tests.
|
||||
|
||||
A deliberate negative control confirmed the first missing edge: temporarily
|
||||
append a comment containing `reason: "retry_failed_run"` to `Inbox.tsx`, which
|
||||
violates the contract test's source assertion without changing runtime behavior.
|
||||
The fresh graph still selected six files. Running those six plus the omitted
|
||||
contract produced six passing files and exactly one failing assertion, in
|
||||
`chat-ui-contract.test.ts` (92 assertions passed, one failed). The original source
|
||||
was restored immediately afterward. The full baseline had passed this contract.
|
||||
|
||||
The integration test uses actual Vitest graph traversal to verify transitive
|
||||
imports, aliases, literal dynamic imports, direct test edits, and the missing
|
||||
filesystem edge. Test modules deliberately throw if executed: graph discovery
|
||||
must not execute the tests. Passing this test documents the limitation; it does
|
||||
not prove that skipped repository tests are irrelevant.
|
||||
|
||||
A production rollout needs explicit dependencies or an always-run lane for
|
||||
filesystem/subprocess contract tests; broad fallbacks for configuration, setup,
|
||||
lockfiles, deleted files, unknown inputs, and selector failures; and comparison
|
||||
against full-suite failures. Source-only changes also need those protections:
|
||||
the Inbox example is not a documentation-only edge case. Once qualified, compute
|
||||
the selected test list before constructing the matrix and partition that list,
|
||||
preserving the existing serialized, chat, and native-runner ownership rules.
|
||||
|
||||
Verification for this experiment covers the real-graph integration test, CLI
|
||||
probes, the complete UI project, and the deliberate negative control. Repository
|
||||
wide typechecking, build, server execution, and the complete `test:run` suite
|
||||
were not run; this is a report-only experiment, not a production CI rollout.
|
||||
@@ -0,0 +1,264 @@
|
||||
import { readFileSync, realpathSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import { isBuiltin } from "node:module";
|
||||
import { parseSync } from "oxc-parser";
|
||||
import { createVitest } from "vitest/node";
|
||||
|
||||
// Everything else is opaque by default. Do not add IO clients, module loaders,
|
||||
// filesystem helpers or process launchers here to improve selection counts.
|
||||
const purePackages = new Set([
|
||||
"vitest", "@vitest/expect", "@testing-library/react", "@testing-library/dom",
|
||||
"@testing-library/user-event", "@testing-library/jest-dom", "react", "react-dom",
|
||||
"react-router", "react-router-dom", "@tanstack/react-query", "@tanstack/query-core",
|
||||
"zod", "lodash", "lodash-es", "date-fns", "clsx", "tailwind-merge", "class-variance-authority",
|
||||
"lucide-react", "immer", "nanoid", "uuid", "semver",
|
||||
"radix-ui", "motion", "@dnd-kit/core", "@dnd-kit/sortable", "@dnd-kit/utilities",
|
||||
"i18next", "react-i18next",
|
||||
]);
|
||||
const pureBuiltins = new Set(["assert", "assert/strict", "path", "path/posix", "path/win32",
|
||||
"url", "util", "util/types", "buffer", "events", "string_decoder", "crypto"]);
|
||||
// Reviewed setup performs only temporary-home IO and Supertest socket setup,
|
||||
// not reads of arbitrary repository inputs. Pin contents, not just paths: any
|
||||
// edit disables this exception until its effects are reviewed again. Continue
|
||||
// following local imports even for these files; helper changes affect all tests.
|
||||
const reviewedSetup = new Map([
|
||||
["server/src/__tests__/setup-supertest.ts", {
|
||||
hash: "99d7193927468e98b41255ea9a5e37b1a681f02b259c304f3af1a52304eed2bb",
|
||||
external: ["supertest/lib/test.js"],
|
||||
}],
|
||||
["server/src/__tests__/helpers/isolated-test-home.ts", {
|
||||
hash: "bd5645dc590087f26b473beab06d5c3ddd7956a79949f0b3e9713f11897a0c1b",
|
||||
external: [],
|
||||
}],
|
||||
]);
|
||||
const packageName = (specifier) => specifier.startsWith("@")
|
||||
? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0];
|
||||
const inside = (root, file) => file.startsWith(`${root}${path.sep}`) && !file.includes(`${path.sep}node_modules${path.sep}`);
|
||||
|
||||
// Parse source, including imports erased by mocks/transforms. Unknown loaders
|
||||
// retain their consumers. This is intentionally conservative, not a regex-based
|
||||
// promise that arbitrary JavaScript has no effects.
|
||||
export function sourceDependencies(file, source) {
|
||||
const ast = parseSync(file, source);
|
||||
const imports = new Set();
|
||||
const reasons = new Set();
|
||||
const mockBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest")
|
||||
.flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name)).map((s) => s.local.name)));
|
||||
const hoistedBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest")
|
||||
.flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name)
|
||||
&& s.local.name === s.imported.name).map((s) => s.local.name)));
|
||||
const mocks = [];
|
||||
if (ast.errors.length) reasons.add("unparsed source");
|
||||
const literal = (node) => node?.type === "Literal" && typeof node.value === "string";
|
||||
const importSource = (node) => {
|
||||
if (literal(node)) imports.add(node.value);
|
||||
else reasons.add("dynamic module name");
|
||||
};
|
||||
for (const statement of ast.program.body) {
|
||||
const call = statement.type === "ExpressionStatement" ? statement.expression : null;
|
||||
const callee = call?.type === "CallExpression" ? call.callee : null;
|
||||
const factory = call?.arguments?.[1];
|
||||
if (callee?.type === "MemberExpression" && !callee.computed && callee.property.name === "mock"
|
||||
&& hoistedBindings.has(callee.object.name) && literal(call.arguments[0])
|
||||
&& ["ArrowFunctionExpression", "FunctionExpression"].includes(factory?.type) && factory.params.length === 0) {
|
||||
mocks.push(call.arguments[0].value);
|
||||
}
|
||||
}
|
||||
function visit(node, parent = null) {
|
||||
if (!node || typeof node !== "object") return;
|
||||
if ((node.type === "ImportDeclaration" && node.importKind !== "type")
|
||||
|| (["ExportNamedDeclaration", "ExportAllDeclaration"].includes(node.type) && node.exportKind !== "type")) {
|
||||
if (node.source) importSource(node.source);
|
||||
}
|
||||
if (node.type === "ImportExpression") importSource(node.source);
|
||||
if (node.type === "TSImportEqualsDeclaration") reasons.add("CommonJS import");
|
||||
if (node.type === "Identifier" && /^(require|fetch|eval|Function|WebSocket|XMLHttpRequest|Worker)$/.test(node.name)) {
|
||||
reasons.add("external input or dynamic code");
|
||||
}
|
||||
if ((node.type === "Identifier" && node.name === "getBuiltinModule")
|
||||
|| (literal(node) && node.value === "getBuiltinModule")) reasons.add("dynamic builtin loader");
|
||||
if ((node.type === "Identifier" && /^(importActual|unmock|doUnmock|doMock|arguments)$/.test(node.name))
|
||||
|| (literal(node) && /^(importActual|unmock|doUnmock|doMock)$/.test(node.value))) reasons.add("mock bypass");
|
||||
if (node.type === "Identifier" && mockBindings.has(node.name) && parent
|
||||
&& parent.type !== "ImportSpecifier"
|
||||
&& !(parent.type === "MemberExpression" && parent.object === node && !parent.computed)) {
|
||||
reasons.add("escaped or dynamic mocking API");
|
||||
}
|
||||
if (node.type === "CallExpression") {
|
||||
const name = source.slice(node.callee.start, node.callee.end);
|
||||
if (/^(?:vi|vitest)\.(?:mock|doMock|importActual|importMock)$/.test(name)) {
|
||||
importSource(node.arguments[0]);
|
||||
}
|
||||
if (/\brequire\b/.test(name)) {
|
||||
reasons.add("CommonJS loader");
|
||||
importSource(node.arguments[0]);
|
||||
}
|
||||
const viteGlob = name === "import.meta.glob" && (literal(node.arguments[0])
|
||||
|| (node.arguments[0]?.type === "ArrayExpression" && node.arguments[0].elements.every(literal)));
|
||||
// Vite expands literal globs into dependency edges. New files appear in
|
||||
// that expansion; deleted/renamed and non-source inputs force full lanes.
|
||||
if (!viteGlob && /\b(?:fetch|eval|Function|glob|globSync)\b/.test(name)) reasons.add("external input or dynamic code");
|
||||
}
|
||||
for (const value of Object.values(node)) {
|
||||
if (Array.isArray(value)) value.forEach((child) => visit(child, node));
|
||||
else if (value && typeof value === "object" && value.type) visit(value, node);
|
||||
}
|
||||
}
|
||||
visit(ast.program);
|
||||
return { imports: [...imports], reasons: [...reasons], mocks };
|
||||
}
|
||||
|
||||
export async function selectProjectTests({ root = process.cwd(), project, changed, onlyOwnChanges = false }) {
|
||||
root = realpathSync(root);
|
||||
const started = performance.now();
|
||||
const absoluteChanges = changed.map((file) => path.resolve(root, file));
|
||||
const ctx = await createVitest({ root, watch: false, project: [project],
|
||||
exclude: ["**/dist/**"], reporters: [], related: absoluteChanges });
|
||||
try {
|
||||
if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) throw new Error(`Unknown project: ${project}`);
|
||||
const owner = ctx.projects[0];
|
||||
const environment = owner.vite.environments.ssr;
|
||||
const inventory = await ctx.globTestSpecifications();
|
||||
if (!inventory.length) throw new Error(`Empty inventory: ${project}`);
|
||||
const relative = (spec) => path.relative(root, spec.moduleId).split(path.sep).join("/");
|
||||
if (onlyOwnChanges && !absoluteChanges.some((file) => file.startsWith(`${owner.config.root}${path.sep}`))) {
|
||||
// Keep coarse dependency coverage, but avoid spending a graph traversal
|
||||
// on an unchanged consumer project merely to retain most of its IO tests.
|
||||
const files = inventory.map(relative).sort();
|
||||
return { project, inventory: files, selected: files, related: [], alwaysRun: files, coveredChanges: [],
|
||||
opaqueModules: {}, retainedReason: "Unchanged project retains complete lane coverage", seconds: (performance.now() - started) / 1000 };
|
||||
}
|
||||
const tracked = new Set(execFileSync("git", ["ls-files", "--cached", "-z"], {
|
||||
cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
|
||||
}).split("\0").filter(Boolean));
|
||||
const audit = new Map();
|
||||
const parsed = new Map();
|
||||
const parse = (file) => {
|
||||
if (!parsed.has(file)) parsed.set(file, sourceDependencies(file, readFileSync(file, "utf8")));
|
||||
return parsed.get(file);
|
||||
};
|
||||
const resolve = async (specifier, importer) => {
|
||||
try {
|
||||
const resolved = await environment.pluginContainer.resolveId(specifier, importer);
|
||||
if (!resolved || resolved.external || !path.isAbsolute(resolved.id) || resolved.id.includes("?")) return null;
|
||||
return realpathSync(resolved.id);
|
||||
} catch { return null; } // A conditional/require-only export is opaque.
|
||||
};
|
||||
// Setup dependencies are not test-import edges. A change anywhere in their
|
||||
// local import closure invalidates the entire project, including helpers.
|
||||
const setup = new Set();
|
||||
async function setupClosure(file) {
|
||||
if (setup.has(file)) return;
|
||||
setup.add(file);
|
||||
const source = parse(file);
|
||||
const review = reviewedSetup.get(path.relative(root, file).split(path.sep).join("/"));
|
||||
const reviewed = review?.hash === createHash("sha256").update(readFileSync(file)).digest("hex");
|
||||
if (!reviewed && source.reasons.length) throw new Error(`Opaque setup dependency: ${file}`);
|
||||
for (const specifier of source.imports) {
|
||||
if (isBuiltin(specifier)) {
|
||||
if (!reviewed && !pureBuiltins.has(specifier.replace(/^node:/, ""))) throw new Error(`Opaque setup builtin: ${specifier}`);
|
||||
continue;
|
||||
}
|
||||
if (reviewed && review.external.includes(specifier)) continue;
|
||||
const dependency = await resolve(specifier, file);
|
||||
if (dependency && inside(root, dependency)) await setupClosure(dependency);
|
||||
else if (!dependency) throw new Error(`Unresolved setup dependency: ${specifier}`);
|
||||
else if (!purePackages.has(packageName(specifier))) throw new Error(`Opaque setup package: ${specifier}`);
|
||||
}
|
||||
}
|
||||
for (const file of [...(owner.config.setupFiles ?? []), ...(owner.config.globalSetup ?? []),
|
||||
...(owner.config.snapshotSerializers ?? [])]) {
|
||||
const resolved = path.isAbsolute(file) ? file : await resolve(file, owner.vite.config.configFile);
|
||||
if (resolved && inside(root, resolved)) await setupClosure(resolved);
|
||||
else if (!resolved) throw new Error(`Unresolved setup module: ${file}`);
|
||||
}
|
||||
for (const file of owner.vite.config.configFileDependencies ?? []) setup.add(realpathSync(file));
|
||||
const setupChanged = absoluteChanges.some((file) => setup.has(file));
|
||||
const raw = await ctx.getRelevantTestSpecifications();
|
||||
const modules = environment.moduleGraph.idToModuleMap;
|
||||
const resolvedDependencies = new Map();
|
||||
const localImports = new Map();
|
||||
const dependencyPath = (dep) => {
|
||||
if (!resolvedDependencies.has(dep)) {
|
||||
const file = dep.startsWith("/@fs/") ? dep.slice(process.platform === "win32" ? 5 : 4)
|
||||
: path.join(owner.config.root, dep);
|
||||
try { resolvedDependencies.set(dep, realpathSync(file)); }
|
||||
catch { resolvedDependencies.set(dep, null); }
|
||||
}
|
||||
return resolvedDependencies.get(dep);
|
||||
};
|
||||
for (const [id, module] of modules) {
|
||||
if (!inside(root, id) || !/\.[cm]?[jt]sx?$/.test(id)) continue;
|
||||
const source = parse(id);
|
||||
const reasons = [...source.reasons];
|
||||
if (!tracked.has(path.relative(root, id).split(path.sep).join("/"))) reasons.push("generated or untracked source");
|
||||
const local = [];
|
||||
localImports.set(id, local);
|
||||
// Use the same resolved runtime edges as Vitest, including literal dynamic
|
||||
// imports. Resolving the original syntax again uses the wrong conditions
|
||||
// for require-only exports and invents edges for erased type imports.
|
||||
const transformed = module.transformResult;
|
||||
if (!Array.isArray(transformed?.deps)) reasons.push("unknown transform shape");
|
||||
for (const dependencyId of [...(transformed?.deps ?? []), ...(transformed?.dynamicDeps ?? [])]) {
|
||||
if (isBuiltin(dependencyId)) {
|
||||
if (!pureBuiltins.has(dependencyId.replace(/^node:/, ""))) reasons.push(`builtin ${dependencyId}`);
|
||||
continue;
|
||||
}
|
||||
const dependency = dependencyPath(dependencyId);
|
||||
if (dependency && inside(root, dependency)) {
|
||||
local.push(dependency);
|
||||
// Vitest intentionally skips some edges (e.g. CJS, externalized
|
||||
// workspaces). Never regard an untracked local edge as independent.
|
||||
if (!modules.get(dependency)?.transformResult) reasons.push(`untracked dependency ${path.relative(root, dependency)}`);
|
||||
} else {
|
||||
const external = dependency?.split(`${path.sep}node_modules${path.sep}`).at(-1).split(path.sep).join("/");
|
||||
if (!dependency || !dependency.includes(`${path.sep}node_modules${path.sep}`) || !purePackages.has(packageName(external))) {
|
||||
reasons.push(`opaque dependency ${external ? packageName(external) : dependencyId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!module.transformResult) reasons.push("untransformed module");
|
||||
if (reasons.length) audit.set(id, [...new Set(reasons)]);
|
||||
}
|
||||
const retained = [];
|
||||
for (const spec of inventory) {
|
||||
const mocked = new Set();
|
||||
// Honor only unconditional, explicitly supplied mock factories without an
|
||||
// importOriginal parameter. Automocks, spies, unmock/importActual, dynamic
|
||||
// mocking and escaped vi bindings stay conservative. The related-file
|
||||
// graph remains uncut: edits to a mocked module can still select its tests.
|
||||
if (owner.config.isolate !== false) {
|
||||
for (const name of parse(spec.moduleId).mocks) {
|
||||
const target = await resolve(name, spec.moduleId);
|
||||
if (target && inside(root, target)) mocked.add(target);
|
||||
}
|
||||
}
|
||||
const queue = [spec.moduleId], visited = new Set();
|
||||
let opaque = setupChanged || owner.config.isolate === false || !localImports.has(spec.moduleId);
|
||||
while (queue.length && !opaque) {
|
||||
const file = queue.pop();
|
||||
if (visited.has(file)) continue;
|
||||
visited.add(file);
|
||||
if (audit.has(file)) { opaque = true; break; }
|
||||
for (const dependency of localImports.get(file) ?? []) if (!mocked.has(dependency)) queue.push(dependency);
|
||||
}
|
||||
if (opaque) retained.push(spec);
|
||||
}
|
||||
const selected = [...new Set([...raw, ...retained].map(relative))].sort();
|
||||
return {
|
||||
project,
|
||||
inventory: inventory.map(relative).sort(),
|
||||
selected,
|
||||
related: raw.map(relative).sort(),
|
||||
alwaysRun: retained.map(relative).sort(),
|
||||
coveredChanges: changed.filter((file, index) => modules.has(absoluteChanges[index]) || setup.has(absoluteChanges[index])),
|
||||
opaqueModules: Object.fromEntries([...audit].map(([file, reasons]) => [path.relative(root, file), reasons])),
|
||||
setupChanged,
|
||||
seconds: (performance.now() - started) / 1000,
|
||||
};
|
||||
} finally {
|
||||
await ctx.close();
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { loadShardDurations, selectGeneralServerShard } from "./general-server-s
|
||||
import { invocationEnv } from "./ci-profile/record.mjs";
|
||||
|
||||
import { assertSelectedTests, partitionTestLines } from "./test-line-shard.mjs";
|
||||
import { loadSelection, selectedShard } from "./ci/test-selection-manifest.mjs";
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const scriptsDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
@@ -378,7 +379,7 @@ function selectSerializedSuites(routeTests, shardIndex, shardCount) {
|
||||
return shardFiles.map((file) => byRepoPath.get(file));
|
||||
}
|
||||
|
||||
async function runVitest(args, label, testShard = null) {
|
||||
async function runVitest(args, label, testShard = null, exactFiles = false) {
|
||||
console.log(`\n[test:run] ${label}`);
|
||||
invocationIndex += 1;
|
||||
const tempRootParent = process.platform === "win32" ? os.tmpdir() : "/tmp";
|
||||
@@ -436,7 +437,9 @@ async function runVitest(args, label, testShard = null) {
|
||||
args = [...args.filter((arg) => arg !== chatSuite), ...runFilters, "--allowOnly=false"];
|
||||
}
|
||||
await new Promise((resolve, reject) => {
|
||||
const child = spawn("pnpm", ["exec", "vitest", "run", ...sourceOnlyVitestArgs, ...args], {
|
||||
const command = exactFiles ? process.execPath : "pnpm";
|
||||
const prefix = exactFiles ? [path.join(scriptsDir, "ci/run-selected-vitest.mjs")] : ["exec", "vitest", "run"];
|
||||
const child = spawn(command, [...prefix, ...sourceOnlyVitestArgs, ...args], {
|
||||
cwd: repoRoot,
|
||||
env: invocationEnv("vitest", env),
|
||||
stdio: "inherit",
|
||||
@@ -615,6 +618,24 @@ const generalServerTestFiles = [
|
||||
.sort((a, b) => a.localeCompare(b));
|
||||
|
||||
const options = parseCliOptions(process.argv.slice(2));
|
||||
const selectionProjects = options.mode === serializedModeName || generalServerGroups.includes(options.group)
|
||||
? ["@paperclipai/server"]
|
||||
: options.group === generalWorkspacesAGroupName ? generalWorkspacesAProjects
|
||||
: options.group === generalWorkspacesBGroupName ? generalWorkspacesBProjects
|
||||
: options.group === generalWorkspacesCGroupName ? generalWorkspacesCProjects : [];
|
||||
const selection = process.env.PAPERCLIP_CI_TEST_SELECTION
|
||||
? selectedShard(loadSelection(), options, selectionProjects) : null;
|
||||
if (selection?.["@paperclipai/server"]) {
|
||||
const owned = new Set(options.mode === serializedModeName ? routeTests.map((test) => test.repoPath)
|
||||
: generalServerTestFiles.filter((file) => !excludedServerSuites(options.group).includes(file)));
|
||||
for (const file of selection["@paperclipai/server"]) {
|
||||
if (!owned.has(file)) fail(`Selected test belongs to another server lane: ${file}`);
|
||||
}
|
||||
}
|
||||
if (selection && options.dryRun) {
|
||||
writeFileSync(1, JSON.stringify({ selection }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (options.dryRun) {
|
||||
const serializedSuites =
|
||||
options.mode === serializedModeName
|
||||
@@ -670,7 +691,16 @@ if (options.dryRun) {
|
||||
}
|
||||
|
||||
try {
|
||||
if (options.mode === generalModeName || options.mode === allModeName) {
|
||||
if (selection) {
|
||||
for (const [project, files] of Object.entries(selection)) {
|
||||
const serverArgs = project !== "@paperclipai/server" ? []
|
||||
: options.mode === serializedModeName || process.env.PAPERCLIP_TEST_DATABASE_URL
|
||||
? ["--pool=forks", "--isolate", ...serializedServerVitestArgs]
|
||||
: ["--pool=forks", "--isolate", "--fileParallelism", "--maxWorkers=2"];
|
||||
await runVitest(["--project", project, ...serverArgs, ...files],
|
||||
`selected ${project}: ${files.length} files`, null, true);
|
||||
}
|
||||
} else if (options.mode === generalModeName || options.mode === allModeName) {
|
||||
if (options.group) {
|
||||
await runGeneralGroup(routeTests, options.group, options.shardIndex, options.shardCount);
|
||||
} else {
|
||||
@@ -678,7 +708,7 @@ try {
|
||||
}
|
||||
}
|
||||
|
||||
if (options.mode === serializedModeName || options.mode === allModeName) {
|
||||
if (!selection && (options.mode === serializedModeName || options.mode === allModeName)) {
|
||||
await runSerializedSuites(routeTests, options.shardIndex ?? 0, options.shardCount ?? 1);
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -300,7 +300,7 @@ describe("adapter management route authorization", () => {
|
||||
mocks.reloadExternalAdapter.mockImplementation(async (type: string) =>
|
||||
createAdapter(type),
|
||||
);
|
||||
}, 20_000);
|
||||
}, 60_000);
|
||||
|
||||
afterEach(() => {
|
||||
unregisterServerAdapter(EXTERNAL_ADAPTER_TYPE);
|
||||
@@ -373,35 +373,31 @@ describe("adapter management route authorization", () => {
|
||||
},
|
||||
);
|
||||
|
||||
describe("cloud-managed adapter code install floor", () => {
|
||||
beforeEach(() => {
|
||||
process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "test-server-token";
|
||||
describe("cloud-managed adapter code floor", () => {
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
it.each(
|
||||
["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"].flatMap((signal) =>
|
||||
["install", "reinstall", "reload", "override", "delete"].map((routeName) => [signal, routeName]),
|
||||
),
|
||||
)("blocks %s adapter %s before package or registration access", async (signal, routeName) => {
|
||||
seedInstalledExternalAdapter();
|
||||
vi.clearAllMocks();
|
||||
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-server-token");
|
||||
const app = createApp(instanceAdmin);
|
||||
|
||||
const res = await sendMutatingRequest(app, routeName);
|
||||
|
||||
expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(403);
|
||||
expect(res.body.details).toMatchObject({ code: "adapter_install_platform_managed" });
|
||||
expect(mocks.execFile).not.toHaveBeenCalled();
|
||||
expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
|
||||
expect(mocks.reloadExternalAdapter).not.toHaveBeenCalled();
|
||||
expect(mocks.getAdapterPluginByType).not.toHaveBeenCalled();
|
||||
expect(mocks.addAdapterPlugin).not.toHaveBeenCalled();
|
||||
expect(mocks.removeAdapterPlugin).not.toHaveBeenCalled();
|
||||
expect(mocks.externalRecords.get(EXTERNAL_ADAPTER_TYPE)).toEqual(installedRecord());
|
||||
});
|
||||
afterEach(() => {
|
||||
delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;
|
||||
});
|
||||
|
||||
it.each(["install", "reinstall"] as const)(
|
||||
"floors adapter %s off for instance admins on cloud-managed instances",
|
||||
async (routeName) => {
|
||||
resetInstalledExternalAdapterState();
|
||||
if (routeName !== "install") {
|
||||
seedInstalledExternalAdapter();
|
||||
}
|
||||
const app = createApp(instanceAdmin);
|
||||
|
||||
const res = await sendMutatingRequest(app, routeName);
|
||||
|
||||
expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(
|
||||
403,
|
||||
);
|
||||
expect(res.body.details).toMatchObject({
|
||||
code: "adapter_install_platform_managed",
|
||||
});
|
||||
expect(mocks.execFile).not.toHaveBeenCalled();
|
||||
expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("operator-hidden adapter management floor", () => {
|
||||
|
||||
@@ -103,6 +103,11 @@ vi.mock("../services/instance-settings.js", () => ({
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
}));
|
||||
|
||||
vi.mock("../services/adapter-settings.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../services/adapter-settings.js")>()),
|
||||
adapterSettingsService: () => ({ getDisabledTypes: async () => [] }),
|
||||
}));
|
||||
|
||||
vi.mock("../services/dot-runner-broker.js", () => ({
|
||||
dotRunnerBroker: () => ({ enabled: async () => true, bindingForAgent: async () => null }),
|
||||
}));
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { rmSync, writeFileSync, symlinkSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
|
||||
import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
|
||||
import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
|
||||
import { readDistributionPluginCatalog } from "../services/distribution-plugin-catalog.js";
|
||||
|
||||
import boxd from "../../../packages/plugins/sandbox-providers/boxd/src/manifest.js";
|
||||
import createos from "../../../packages/plugins/sandbox-providers/createos/src/manifest.js";
|
||||
import cloudflare from "../../../packages/plugins/sandbox-providers/cloudflare/src/manifest.js";
|
||||
import daytona from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
|
||||
import e2b from "../../../packages/plugins/sandbox-providers/e2b/src/manifest.js";
|
||||
import exe_dev from "../../../packages/plugins/sandbox-providers/exe-dev/src/manifest.js";
|
||||
import kubernetes from "../../../packages/plugins/sandbox-providers/kubernetes/src/manifest.js";
|
||||
import modal from "../../../packages/plugins/sandbox-providers/modal/src/manifest.js";
|
||||
import novita from "../../../packages/plugins/sandbox-providers/novita/src/manifest.js";
|
||||
|
||||
const manifests = { boxd, createos, cloudflare, daytona, e2b, "exe-dev": exe_dev, kubernetes, modal, novita };
|
||||
const roots: string[] = [];
|
||||
function fixture() { const result = bundledImageFixture(); roots.push(result.root); return result; }
|
||||
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
describe("bundled image package identity", () => {
|
||||
it("resolves independent image roots from the same registration without any previous path", () => {
|
||||
const first = fixture();
|
||||
const second = fixture();
|
||||
const registration = { ...first.registration, packagePath: "/lost/old/home/plugins/fixture" };
|
||||
expect(first.resolver.resolve(registration)).toBe(first.packageRoot);
|
||||
expect(second.resolver.resolve(registration)).toBe(second.packageRoot);
|
||||
expect(first.resolver.inspect(registration, first.manifest)).toEqual(second.resolver.inspect(registration, second.manifest));
|
||||
expect(registration.packagePath).toBe("/lost/old/home/plugins/fixture");
|
||||
});
|
||||
|
||||
it("requires verified manifest evidence and blocks same-version manifest drift", () => {
|
||||
const { resolver, registration, manifest } = fixture();
|
||||
expect(resolver.inspect(registration)).toMatchObject({ compatible: false, code: "missing_manifest_evidence" });
|
||||
expect(resolver.inspect(registration, { ...manifest, description: "changed" })).toMatchObject({ compatible: false, code: "incompatible_bundled_package", reason: expect.stringContaining("Approved plugin manifest") });
|
||||
expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true, identity: { kind: "bundled_image", catalogKey: "fixture", pluginKey: manifest.id, version: "1.0.0" } });
|
||||
});
|
||||
|
||||
it("rejects mismatched package names, versions, identity and escaping entrypoints", () => {
|
||||
const { resolver, registration } = fixture();
|
||||
for (const update of [
|
||||
{ packageName: "@other/plugin" }, { version: "2.0.0" }, { pluginKey: "other.plugin" },
|
||||
{ manifestJson: { ...registration.manifestJson, version: "2.0.0" } },
|
||||
{ manifestJson: { ...registration.manifestJson, entrypoints: { worker: "../worker.js" } } },
|
||||
]) expect(() => resolver.resolve({ ...registration, ...update })).toThrow();
|
||||
});
|
||||
|
||||
it("fails explicitly for missing or deselected bundles even when a stale npm package exists", () => {
|
||||
const { root, packageRoot, resolver, registration } = fixture();
|
||||
const fallback = path.join(root, "home", "node_modules", registration.packageName);
|
||||
mkdirSync(fallback, { recursive: true });
|
||||
writeFileSync(path.join(fallback, "worker.js"), "throw new Error('unsafe fallback');");
|
||||
rmSync(packageRoot, { recursive: true });
|
||||
expect(() => resolver.resolve(registration)).toThrow(/ENOENT/);
|
||||
expect(() => createBundledPluginPackageResolver([]).resolve(registration)).toThrow(/absent or not selected/);
|
||||
});
|
||||
|
||||
it("rejects tampering before manifest import and symlink escapes before worker resolution", () => {
|
||||
const { root, packageRoot, registration, resolver } = fixture();
|
||||
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), "throw new Error('must never import');");
|
||||
expect(() => readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG)).toThrow(/digest mismatch/);
|
||||
rmSync(path.join(packageRoot, "dist", "worker.js"));
|
||||
symlinkSync(path.join(root, "distribution", "catalog.json"), path.join(packageRoot, "dist", "worker.js"));
|
||||
expect(() => resolver.resolve(registration)).toThrow(/escapes/);
|
||||
});
|
||||
|
||||
it.each(BUNDLED_PLUGIN_CATALOG)("accepts the real shipped $key package and manifest contract", (entry) => {
|
||||
const manifest = manifests[entry.key as keyof typeof manifests];
|
||||
const pkg = JSON.parse(readFileSync(new URL(`../../../packages/plugins/${entry.relativePath}/package.json`, import.meta.url), "utf8"));
|
||||
const { root } = fixture();
|
||||
const packageRoot = path.join(root, entry.relativePath);
|
||||
mkdirSync(path.join(packageRoot, "dist"), { recursive: true });
|
||||
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(pkg));
|
||||
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`);
|
||||
writeFileSync(path.join(packageRoot, "dist", "worker.js"), "export default {};");
|
||||
const installs = resolveBundledPluginInstalls([entry.key], { catalogRoot: root, enforceCatalogRoot: true, distributionPlugins: [], env: {} });
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const registration = { pluginKey: entry.pluginKey, packageName: pkg.name, version: manifest.version, manifestJson: manifest };
|
||||
expect(pkg.version).toBe(manifest.version);
|
||||
expect(resolver.resolve(registration)).toBe(packageRoot);
|
||||
expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,287 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { rmSync, mkdirSync, writeFileSync, cpSync, readFileSync, existsSync, mkdtempSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { applyPendingMigrations, closeRegisteredClients, companies, createDb, plugins } from "@paperclipai/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import type { PluginRecord } from "@paperclipai/shared";
|
||||
import { createPluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js";
|
||||
import { linkSdkInto } from "../../../scripts/link-plugin-dev-sdk.mjs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
|
||||
import { inspectCloudPackages } from "../services/cloud-package-qualification.js";
|
||||
import { resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
|
||||
import { pluginRegistryService } from "../services/plugin-registry.js";
|
||||
import { pluginUiStaticRoutes } from "../routes/plugin-ui-static.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { derivePluginDatabaseNamespace } from "../services/plugin-database.js";
|
||||
import { ensureBundledPlugins } from "../services/bundled-plugins.js";
|
||||
import daytonaManifest from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
|
||||
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_BUNDLED_PLUGIN_TEST_DATABASE_URL?.trim();
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describePostgres = support.supported ? describe : describe.skip;
|
||||
|
||||
describePostgres("bundled image registration in PostgreSQL", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let peer: ReturnType<typeof createDb>;
|
||||
const roots: string[] = [];
|
||||
const fixture = () => { const result = bundledImageFixture(); roots.push(result.root); return result; };
|
||||
beforeAll(async () => {
|
||||
if (externalDatabaseUrl) {
|
||||
await applyPendingMigrations(externalDatabaseUrl);
|
||||
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) };
|
||||
} else {
|
||||
database = await startEmbeddedPostgresTestDatabase("bundled-image-registration-");
|
||||
}
|
||||
db = createDb(database.connectionString);
|
||||
peer = createDb(database.connectionString);
|
||||
}, 90_000);
|
||||
afterEach(async () => {
|
||||
await db.delete(plugins);
|
||||
await db.delete(companies);
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
function runtime() {
|
||||
const output: string[] = [];
|
||||
const services = {
|
||||
workerManager: {
|
||||
startWorker: vi.fn(async (_id: string, options: { entrypointPath: string }) => { output.push(execFileSync(process.execPath, [options.entrypointPath, "--source"], { encoding: "utf8" })); }),
|
||||
call: vi.fn(async () => ({})),
|
||||
},
|
||||
lifecycleManager: { markError: vi.fn() },
|
||||
jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() },
|
||||
toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) },
|
||||
buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" },
|
||||
};
|
||||
return { services, output };
|
||||
}
|
||||
function loaderFor(image: ReturnType<typeof fixture>, databaseClient = db, services?: PluginRuntimeServices) {
|
||||
return pluginLoader(databaseClient, {
|
||||
localPluginDir: path.join(image.root, "lost-home"), prebuiltOnly: true,
|
||||
resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation,
|
||||
enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
}, services);
|
||||
}
|
||||
|
||||
it("starts current-image workers after losing the old home and retains each company's config", async () => {
|
||||
const old = fixture();
|
||||
const first = fixture();
|
||||
const second = fixture();
|
||||
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
let row = (await registry.getByKey(old.manifest.id))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
const [companyA, companyB, unconfigured] = await db.insert(companies).values([
|
||||
{ name: "Company A", issuePrefix: "CFA" }, { name: "Company B", issuePrefix: "CFB" }, { name: "Unconfigured", issuePrefix: "CFC" },
|
||||
]).returning();
|
||||
await registry.upsertConfig(row.id, companyA!.id, { configJson: { token: "only-A" } });
|
||||
await registry.upsertConfig(row.id, companyB!.id, { configJson: { token: "only-B" } });
|
||||
row = (await registry.getById(row.id))!;
|
||||
rmSync(old.root, { recursive: true });
|
||||
const original = { ...row };
|
||||
for (const image of [first, second]) {
|
||||
const current = runtime();
|
||||
const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
|
||||
const result = await loader.loadSingle(row.id);
|
||||
expect(result.success).toBe(true);
|
||||
expect(current.output).toEqual([expect.stringContaining(image.packageRoot)]);
|
||||
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({
|
||||
entrypointPath: path.join(image.packageRoot, "dist", "worker.js"), config: {}, proactiveCompanyScopes: [companyA!.id, companyB!.id].sort(),
|
||||
}));
|
||||
expect(current.services.workerManager.call.mock.calls).toEqual(expect.arrayContaining([
|
||||
[row.id, "configChanged", { companyId: companyA!.id, config: { token: "only-A" } }],
|
||||
[row.id, "configChanged", { companyId: companyB!.id, config: { token: "only-B" } }],
|
||||
]));
|
||||
expect(current.services.workerManager.call).toHaveBeenCalledTimes(2);
|
||||
expect(await registry.getConfig(row.id, unconfigured!.id)).toBeNull();
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = { type: "board", userId: "fixture-user", source: "session", isInstanceAdmin: false, companyIds: [companyA!.id] };
|
||||
next();
|
||||
});
|
||||
app.use(pluginUiStaticRoutes(db, { localPluginDir: path.join(image.root, "lost-home"), resolvePackageRoot: image.resolver.resolve }));
|
||||
app.use(errorHandler);
|
||||
const served = await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyA!.id });
|
||||
expect(served.status).toBe(200);
|
||||
expect(served.text).toContain("current-image");
|
||||
expect((await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyB!.id })).status).toBe(403);
|
||||
|
||||
}
|
||||
expect(await registry.getById(row.id)).toEqual(original);
|
||||
});
|
||||
|
||||
it("completes the real worker initialize and health protocol from a relocated image", async () => {
|
||||
const old = fixture(); const image = fixture();
|
||||
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.getByKey(old.manifest.id))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
rmSync(old.root, { recursive: true });
|
||||
const manager = createPluginWorkerManager();
|
||||
try {
|
||||
const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
|
||||
const result = await loaderFor(image, db, services).loadSingle(row.id);
|
||||
expect(result.success).toBe(true);
|
||||
expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(image.packageRoot), configs: {} } });
|
||||
} finally { await manager.stopAll(); }
|
||||
});
|
||||
|
||||
it("qualifies and initializes the current release SDK/shared exports through the baked loader", async () => {
|
||||
const image = fixture();
|
||||
const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
|
||||
const provider = mkdtempSync(path.join(catalogRoot, "sandbox-providers", "image-sdk-proof-"));
|
||||
roots.push(provider);
|
||||
cpSync(image.packageRoot, provider, { recursive: true });
|
||||
linkSdkInto(provider);
|
||||
writeFileSync(path.join(provider, "dist", "worker.js"), 'import { definePlugin, runWorker } from "@paperclipai/plugin-sdk"; runWorker(definePlugin({ async setup() {}, async onHealth() { return { status: "ok", message: "Current image SDK/shared loaded" }; } }), import.meta.url);');
|
||||
const installs = [{ key: "sdk-proof", pluginKey: image.manifest.id, localPath: provider }];
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const manager = createPluginWorkerManager();
|
||||
const loader = pluginLoader(db, {
|
||||
prebuiltOnly: true, resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
|
||||
enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
}, { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices);
|
||||
const registry = pluginRegistryService(db);
|
||||
try {
|
||||
await loader.installPlugin({ localPath: provider });
|
||||
const installed = (await registry.getByKey(image.manifest.id))!;
|
||||
await registry.updateStatus(installed.id, { status: "ready" });
|
||||
const registration = (await registry.getById(installed.id))!;
|
||||
expect((await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) })).blockers).toEqual([]);
|
||||
expect((await loader.loadSingle(installed.id)).success).toBe(true);
|
||||
expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Current image SDK/shared loaded" });
|
||||
} finally { await manager.stopAll(); }
|
||||
}, 90_000);
|
||||
|
||||
it.skipIf(!existsSync(fileURLToPath(new URL("../../../packages/plugins/sandbox-providers/daytona/dist/worker.js", import.meta.url))))("qualifies and initializes the actual prebuilt Daytona release package", async () => {
|
||||
const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
|
||||
const installs = resolveBundledPluginInstalls(["daytona"], { catalogRoot, env: {}, enforceCatalogRoot: true, distributionPlugins: [] });
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const manager = createPluginWorkerManager();
|
||||
const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
|
||||
const loader = pluginLoader(db, {
|
||||
localPluginDir: "/lost/release/home/plugins", prebuiltOnly: true,
|
||||
resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
|
||||
enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
}, services);
|
||||
const registry = pluginRegistryService(db);
|
||||
vi.stubEnv("DAYTONA_API_KEY", "");
|
||||
try {
|
||||
await loader.installPlugin({ localPath: installs[0]!.localPath });
|
||||
const installed = (await registry.getByKey(daytonaManifest.id))!;
|
||||
await registry.update(installed.id, { packagePath: "/lost/release/home/daytona" });
|
||||
await registry.updateStatus(installed.id, { status: "ready" });
|
||||
const registration = (await registry.getById(installed.id))!;
|
||||
const inspected = await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) });
|
||||
expect(inspected.blockers).toEqual([]);
|
||||
expect(inspected.packages[0]?.packageDigest).toMatch(/^sha256:/);
|
||||
expect((await loader.loadSingle(installed.id)).success).toBe(true);
|
||||
expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Daytona sandbox provider plugin healthy" });
|
||||
} finally { await manager.stopAll(); vi.unstubAllEnvs(); }
|
||||
}, 90_000);
|
||||
|
||||
it("registers the same package concurrently from two roots without replacing the winner", async () => {
|
||||
const first = fixture(); const second = fixture();
|
||||
await Promise.all([
|
||||
loaderFor(first).installPlugin({ localPath: first.packageRoot }),
|
||||
loaderFor(second, peer).installPlugin({ localPath: second.packageRoot }),
|
||||
]);
|
||||
const rows = await pluginRegistryService(db).listInstalled();
|
||||
expect(rows).toHaveLength(1);
|
||||
const original = rows[0]!;
|
||||
expect([first.packageRoot, second.packageRoot]).toContain(original.packagePath);
|
||||
await loaderFor(second, peer).installPlugin({ localPath: second.packageRoot });
|
||||
expect(await pluginRegistryService(db).getById(original.id)).toEqual(original);
|
||||
await expect(pluginRegistryService(peer).install({ packageName: original.packageName }, { ...first.manifest, description: "different package metadata" }, { idempotent: true })).rejects.toThrow(/already installed/);
|
||||
});
|
||||
|
||||
it("preserves disabled registrations and avoids path writes during managed boot", async () => {
|
||||
const old = fixture(); const image = fixture();
|
||||
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.getByKey(old.manifest.id))!;
|
||||
await db.update(plugins).set({ status: "disabled" }).where(eq(plugins.id, row.id));
|
||||
const original = await registry.getById(row.id);
|
||||
await ensureBundledPlugins(image.installs, { registry, loader: loaderFor(image), lifecycle: { load: vi.fn() }, logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }, { reinstallUninstalled: true, preservePackagePath: true });
|
||||
expect(await registry.getById(row.id)).toEqual(original);
|
||||
});
|
||||
|
||||
it("fails before worker startup for missing image bytes without using a home fallback or building", async () => {
|
||||
const image = fixture();
|
||||
await loaderFor(image).installPlugin({ localPath: image.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.getByKey(image.manifest.id))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
const fallback = path.join(image.root, "lost-home", "node_modules", row.packageName);
|
||||
mkdirSync(fallback, { recursive: true });
|
||||
writeFileSync(path.join(fallback, "package.json"), JSON.stringify({ name: row.packageName }));
|
||||
writeFileSync(path.join(fallback, "worker.js"), "throw new Error('must not start');");
|
||||
rmSync(path.join(image.packageRoot, "dist", "worker.js"));
|
||||
const current = runtime();
|
||||
const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
|
||||
const result = await loader.loadSingle(row.id);
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain("ENOENT");
|
||||
expect(current.services.workerManager.startWorker).not.toHaveBeenCalled();
|
||||
await expect(loader.installPlugin({ packageName: row.packageName })).rejects.toThrow(/current image catalog/);
|
||||
await loader.cleanupInstallArtifacts(row as PluginRecord);
|
||||
expect(image.resolver.inspect(row, image.manifest).compatible).toBe(false);
|
||||
});
|
||||
|
||||
it("applies plugin migrations from the current image after the legacy package disappears", async () => {
|
||||
const image = bundledImageFixture({
|
||||
capabilities: ["database.namespace.migrate", "database.namespace.read", "database.namespace.write"],
|
||||
database: { migrationsDir: "migrations" },
|
||||
});
|
||||
roots.push(image.root);
|
||||
const [row] = await db.insert(plugins).values({
|
||||
...image.registration, packagePath: "/lost/legacy/home/plugin", apiVersion: 1, categories: image.manifest.categories, status: "ready",
|
||||
}).returning();
|
||||
const current = runtime();
|
||||
const result = await loaderFor(image, db, current.services as unknown as PluginRuntimeServices).loadSingle(row!.id);
|
||||
expect(result.success).toBe(true);
|
||||
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row!.id, expect.objectContaining({ databaseNamespace: derivePluginDatabaseNamespace(image.manifest.id) }));
|
||||
await db.execute((await import("drizzle-orm")).sql.raw(`DROP SCHEMA "${derivePluginDatabaseNamespace(image.manifest.id)}" CASCADE`));
|
||||
});
|
||||
|
||||
it("retains registry-install provider credentials for an ordinary npm package", async () => {
|
||||
const npmManifest = { ...daytonaManifest, id: "paperclip.image-npm-fixture" };
|
||||
const image = bundledImageFixture(npmManifest);
|
||||
roots.push(image.root);
|
||||
const localPluginDir = path.join(image.root, "npm-home");
|
||||
const packageRoot = path.join(localPluginDir, "node_modules", "@paperclipai", "plugin-daytona");
|
||||
mkdirSync(path.dirname(packageRoot), { recursive: true });
|
||||
cpSync(image.packageRoot, packageRoot, { recursive: true });
|
||||
const metadata = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8"));
|
||||
metadata.name = "@paperclipai/plugin-daytona";
|
||||
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(metadata));
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.install({ packageName: metadata.name }, npmManifest))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
vi.stubEnv("DAYTONA_API_KEY", "synthetic-provider-key");
|
||||
try {
|
||||
const current = runtime();
|
||||
const loader = pluginLoader(db, { localPluginDir, enableNpmDiscovery: false, enableLocalFilesystem: false }, current.services as unknown as PluginRuntimeServices);
|
||||
expect((await loader.loadSingle(row.id)).success).toBe(true);
|
||||
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({ env: expect.objectContaining({ DAYTONA_API_KEY: "synthetic-provider-key" }) }));
|
||||
} finally { vi.unstubAllEnvs(); }
|
||||
});
|
||||
|
||||
it("keeps self-hosted local-path loading available without a managed resolver", async () => {
|
||||
const image = fixture();
|
||||
const loader = pluginLoader(db, { localPluginDir: path.join(image.root, "selfhost"), enableNpmDiscovery: false, enableLocalFilesystem: false });
|
||||
await loader.installPlugin({ localPath: image.packageRoot });
|
||||
const row = (await pluginRegistryService(db).getByKey(image.manifest.id))!;
|
||||
expect(row.packagePath).toBe(image.packageRoot);
|
||||
await expect(loader.installPlugin({ localPath: image.packageRoot })).rejects.toThrow(/already installed/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { build, transform } from "esbuild";
|
||||
import { bundleRuntime } from "../../scripts/bundle-runtime.mjs";
|
||||
import { resolveBundledPluginRuntimeLoader } from "../services/bundled-plugin-runtime.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
function fixture() {
|
||||
const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-plugin-runtime-")));
|
||||
roots.push(root);
|
||||
fs.mkdirSync(path.join(root, "server", "dist-runtime"), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, "server", "package.json"), JSON.stringify({ name: "@paperclipai/server", type: "module" }));
|
||||
const loader = path.join(root, "cli", "node_modules", "tsx", "dist", "loader.mjs");
|
||||
fs.mkdirSync(path.dirname(loader), { recursive: true });
|
||||
fs.writeFileSync(loader, "export {};\n");
|
||||
return { root, loader };
|
||||
}
|
||||
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
describe("current image plugin runtime loader", () => {
|
||||
it.each(["src/services", "dist/services", "dist-runtime"])("resolves the owning release from server/%s", (location) => {
|
||||
const { root, loader } = fixture();
|
||||
const moduleDirectory = path.join(root, "server", location);
|
||||
fs.mkdirSync(moduleDirectory, { recursive: true });
|
||||
expect(resolveBundledPluginRuntimeLoader(moduleDirectory)).toBe(loader);
|
||||
});
|
||||
|
||||
it("resolves a bundled entrypoint without preserved source module locations", async () => {
|
||||
const { root, loader } = fixture();
|
||||
const runtime = path.join(root, "server", "dist-runtime", "index.mjs");
|
||||
await build({
|
||||
stdin: { contents: `import { resolveBundledPluginRuntimeLoader } from ${JSON.stringify(fileURLToPath(new URL("../services/bundled-plugin-runtime.ts", import.meta.url)))}; process.stdout.write(resolveBundledPluginRuntimeLoader());`, resolveDir: process.cwd() },
|
||||
outfile: runtime, bundle: true, platform: "node", format: "esm", target: "node24", logLevel: "silent",
|
||||
});
|
||||
expect(execFileSync(process.execPath, [runtime], { encoding: "utf8" })).toBe(loader);
|
||||
});
|
||||
|
||||
it("uses the production bundler's preserved module location at the dist-runtime entrypoint", async () => {
|
||||
const { root, loader } = fixture();
|
||||
const service = path.join(root, "server", "dist", "services", "bundled-plugin-runtime.js");
|
||||
fs.mkdirSync(path.dirname(service), { recursive: true });
|
||||
const source = fs.readFileSync(new URL("../services/bundled-plugin-runtime.ts", import.meta.url), "utf8");
|
||||
fs.writeFileSync(service, (await transform(source, { loader: "ts", format: "esm", target: "node24" })).code);
|
||||
const entry = path.join(root, "server", "dist", "probe.js");
|
||||
fs.writeFileSync(entry, 'import { resolveBundledPluginRuntimeLoader } from "./services/bundled-plugin-runtime.js"; process.stdout.write(resolveBundledPluginRuntimeLoader());');
|
||||
const bundled = await bundleRuntime({ entry, outdir: path.join(root, "server", "dist-runtime") });
|
||||
expect(execFileSync(process.execPath, [bundled.outfile], { encoding: "utf8" })).toBe(loader);
|
||||
});
|
||||
|
||||
it("accepts the baked pnpm target inside the current image node_modules", () => {
|
||||
const { root, loader } = fixture();
|
||||
const physical = path.join(root, "node_modules", ".pnpm", "tsx@1", "node_modules", "tsx", "dist", "loader.mjs");
|
||||
fs.mkdirSync(path.dirname(physical), { recursive: true });
|
||||
fs.writeFileSync(physical, "export {};\n");
|
||||
fs.rmSync(loader); fs.symlinkSync(path.relative(path.dirname(loader), physical), loader);
|
||||
expect(resolveBundledPluginRuntimeLoader(path.join(root, "server", "dist-runtime"))).toBe(physical);
|
||||
});
|
||||
|
||||
it("fails explicitly for missing or legacy-home loaders", () => {
|
||||
const { root, loader } = fixture();
|
||||
const moduleDirectory = path.join(root, "server", "dist-runtime");
|
||||
fs.rmSync(loader);
|
||||
expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("missing from the current image");
|
||||
const outside = fixture().loader;
|
||||
fs.symlinkSync(outside, loader);
|
||||
expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("escapes the current image");
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
ensureBundledPlugins,
|
||||
resolveBundledCatalogRoot,
|
||||
resolveBundledPluginInstalls,
|
||||
resolveManagedBundledPluginKeys,
|
||||
type BundledPluginProvisionerDeps,
|
||||
type ResolvedBundledPlugin,
|
||||
} from "../services/bundled-plugins.js";
|
||||
@@ -30,6 +31,21 @@ afterAll(() => {
|
||||
// Catalog-root resolution (fail-to-start allowlist)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("managed bundled plugin selection", () => {
|
||||
it.each([
|
||||
{ env: {}, selected: undefined, expected: null },
|
||||
{ env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: undefined, expected: [] },
|
||||
{ env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: " " }, selected: undefined, expected: null },
|
||||
{ env: { PAPERCLIP_MANAGED_CONFIG: "" }, selected: undefined, expected: [] },
|
||||
{ env: { PAPERCLIP_MANAGED_CONFIG: "{}" }, selected: null, expected: [] },
|
||||
{ env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: ["daytona"], expected: ["daytona"] },
|
||||
{ env: {}, selected: ["modal"], expected: ["modal"] },
|
||||
{ env: {}, selected: [], expected: [] },
|
||||
])("uses the canonical Cloud floor and preserves explicit selection: $env / $selected", ({ env, selected, expected }) => {
|
||||
expect(resolveManagedBundledPluginKeys(selected, env)).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveBundledPluginInstalls", () => {
|
||||
it("resolves the optional CreateOS sandbox provider without making it a self-hosted default", () => {
|
||||
expect(resolveBundledPluginInstalls(["createos"], {
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
function file(name: string, content = "export const value = 1;") {
|
||||
fs.mkdirSync(path.dirname(name), { recursive: true }); fs.writeFileSync(name, content);
|
||||
}
|
||||
function link(target: string, name: string) {
|
||||
fs.mkdirSync(path.dirname(name), { recursive: true }); fs.symlinkSync(path.relative(path.dirname(name), target), name);
|
||||
}
|
||||
function fixture() {
|
||||
const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-package-graph-"))); roots.push(root);
|
||||
const catalogRoot = path.join(root, "packages", "plugins");
|
||||
const provider = path.join(catalogRoot, "sandbox-providers", "daytona");
|
||||
const sdk = path.join(catalogRoot, "sdk");
|
||||
const shared = path.join(root, "packages", "shared");
|
||||
const dependency = path.join(root, "node_modules", ".pnpm", "dependency@1", "node_modules", "dependency");
|
||||
const transitive = path.join(root, "node_modules", ".pnpm", "transitive@1", "node_modules", "transitive");
|
||||
for (const directory of [provider, sdk, shared, dependency, transitive]) file(path.join(directory, "index.js"));
|
||||
link(sdk, path.join(provider, "node_modules", "@paperclipai", "plugin-sdk"));
|
||||
link(shared, path.join(sdk, "node_modules", "@paperclipai", "shared"));
|
||||
link(dependency, path.join(shared, "node_modules", "dependency"));
|
||||
link(transitive, path.join(path.dirname(dependency), "transitive"));
|
||||
return { root, catalogRoot, provider, sdk, shared, dependency, transitive };
|
||||
}
|
||||
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
describe("bounded current-image package dependency inventory", () => {
|
||||
it("hashes external SDK, shared, and pnpm sibling dependency bytes", () => {
|
||||
const image = fixture();
|
||||
const digest = () => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot });
|
||||
let previous = digest();
|
||||
for (const directory of [image.sdk, image.shared, image.dependency, image.transitive]) {
|
||||
file(path.join(directory, "index.js"), "export const value = 2;");
|
||||
const next = digest(); expect(next).not.toBe(previous); previous = next;
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the digest stable when an identical release moves to a new root", () => {
|
||||
const original = fixture(); const replacement = fixture();
|
||||
expect(cloudBundledPackageDigest(original.provider, { catalogRoot: original.catalogRoot }))
|
||||
.toBe(cloudBundledPackageDigest(replacement.provider, { catalogRoot: replacement.catalogRoot }));
|
||||
});
|
||||
|
||||
it("rejects legacy-home links and preserves package-contained defaults", () => {
|
||||
const image = fixture();
|
||||
expect(() => cloudBundledPackageDigest(image.provider)).toThrow("escapes its package");
|
||||
const outside = fixture();
|
||||
link(path.join(outside.sdk, "index.js"), path.join(image.shared, "legacy-home.js"));
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("authorized image roots");
|
||||
});
|
||||
|
||||
it("deduplicates cycles without scanning unrelated image node_modules", () => {
|
||||
const image = fixture();
|
||||
link(image.sdk, path.join(image.shared, "node_modules", "cycle"));
|
||||
const unrelated = path.join(image.root, "node_modules", "unrelated", "special");
|
||||
file(unrelated);
|
||||
const first = cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot });
|
||||
file(unrelated, "changed unrelated code");
|
||||
expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toBe(first);
|
||||
});
|
||||
|
||||
it.each(["", ".pnpm", ".pnpm/node_modules"])("rejects aggregate image dependency targets: %s", (aggregate) => {
|
||||
const image = fixture();
|
||||
const target = path.join(image.root, "node_modules", aggregate);
|
||||
fs.mkdirSync(target, { recursive: true });
|
||||
link(target, path.join(image.sdk, "node_modules", "aggregate"));
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("select an individual package");
|
||||
});
|
||||
|
||||
it("accepts the shipped provider's observed entry class", () => {
|
||||
const image = fixture();
|
||||
const directory = path.join(image.provider, "many"); fs.mkdirSync(directory);
|
||||
for (let index = 0; index < 16_000; index++) fs.writeFileSync(path.join(directory, `${index}.js`), "");
|
||||
expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toMatch(/^sha256:/);
|
||||
});
|
||||
|
||||
it("rejects oversized metadata and excessive depth", () => {
|
||||
const image = fixture();
|
||||
file(path.join(image.sdk, "package.json"), " ".repeat(128 * 1024 + 1));
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("metadata");
|
||||
fs.rmSync(path.join(image.sdk, "package.json"));
|
||||
const large = path.join(image.shared, "large.js");
|
||||
fs.closeSync(fs.openSync(large, "w")); fs.truncateSync(large, 256 * 1024 * 1024 + 1);
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("byte limit");
|
||||
fs.rmSync(large);
|
||||
file(path.join(image.sdk, ...Array.from({ length: 33 }, () => "deep"), "file.js"));
|
||||
expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("depth limit");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,325 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, plugins, type Db } from "@paperclipai/db";
|
||||
import { eq, inArray, sql } from "drizzle-orm";
|
||||
import { paperclipConfigSchema } from "@paperclipai/shared";
|
||||
import { adapterSettingsService, initializeAdapterSettings } from "../services/adapter-settings.js";
|
||||
import { cloudRuntimeConfigService, initializeCloudRuntimeConfig } from "../services/cloud-runtime-config.js";
|
||||
import { cloudConfigurationMigrationService } from "../services/cloud-configuration-migration.js";
|
||||
import { qualifyCloudConfigurationForCurrentImage } from "../services/cloud-configuration-qualification.js";
|
||||
import { runCloudConfigurationQualification } from "../qualify-cloud-configuration.js";
|
||||
import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js";
|
||||
import { pluginRegistryService } from "../services/plugin-registry.js";
|
||||
import { createPluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { ensureBundledPlugins } from "../services/bundled-plugins.js";
|
||||
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL?.trim();
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const keys = ["adapter-settings:v1", "cloud-runtime-config:v1", "cloud-configuration-migration:v1"];
|
||||
(support.supported ? describe : describe.skip)("Cloud configuration migration and current-image qualification", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let peer: Db;
|
||||
let homeA: string;
|
||||
let homeB: string;
|
||||
const roots: string[] = [];
|
||||
const fixture = (version = "1.0.0") => { const image = bundledImageFixture({ version }); roots.push(image.root); return image; };
|
||||
const env = (home = homeA) => ({ PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "test-cloud-token", PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: path.join(home, "config.json") });
|
||||
const migration = (client = db, home = homeA) => cloudConfigurationMigrationService(client, { runtimeEnv: env(home) });
|
||||
beforeAll(async () => {
|
||||
if (externalDatabaseUrl) {
|
||||
await applyPendingMigrations(externalDatabaseUrl);
|
||||
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) };
|
||||
} else database = await startEmbeddedPostgresTestDatabase("cloud-configuration-migration-");
|
||||
db = createDb(database.connectionString);
|
||||
peer = createDb(database.connectionString);
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
homeA = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-legacy-A-"));
|
||||
homeB = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-replacement-B-"));
|
||||
roots.push(homeA, homeB);
|
||||
await db.delete(instanceSettings).where(inArray(instanceSettings.singletonKey, keys));
|
||||
await db.delete(plugins);
|
||||
});
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
async function legacy() {
|
||||
const config = paperclipConfigSchema.parse({
|
||||
$meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" },
|
||||
database: { mode: "postgres" }, logging: { mode: "cloud" },
|
||||
server: { port: 3199 }, telemetry: { enabled: false },
|
||||
});
|
||||
await fs.writeFile(path.join(homeA, "config.json"), JSON.stringify(config));
|
||||
await fs.writeFile(path.join(homeA, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local", "future_adapter"] }));
|
||||
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]");
|
||||
return config;
|
||||
}
|
||||
|
||||
function loaderFor(image: ReturnType<typeof fixture>, client = db, services?: PluginRuntimeServices) {
|
||||
return pluginLoader(client, {
|
||||
localPluginDir: path.join(homeB, "plugins"), prebuiltOnly: true,
|
||||
resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation,
|
||||
enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
}, services);
|
||||
}
|
||||
async function install(image: ReturnType<typeof fixture>) {
|
||||
await loaderFor(image).installPlugin({ localPath: image.packageRoot });
|
||||
const registry = pluginRegistryService(db);
|
||||
const row = (await registry.getByKey(image.manifest.id))!;
|
||||
await registry.updateStatus(row.id, { status: "ready" });
|
||||
return row;
|
||||
}
|
||||
function qualify(image: ReturnType<typeof fixture>, client = db, home = homeA, expectedPackageEvidenceDigest?: string) {
|
||||
const loader = loaderFor(image, client);
|
||||
return migration(client, home).qualify({ installs: image.installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest });
|
||||
}
|
||||
|
||||
it("imports A atomically, retains later DB revisions, and starts B's real image worker after A disappears", async () => {
|
||||
await legacy();
|
||||
const oldImage = fixture();
|
||||
const newImage = fixture();
|
||||
const row = await install(oldImage);
|
||||
const first = await migration().migrate();
|
||||
expect(first.migration.adapter.authority).toBe("legacy_file");
|
||||
expect(first.migration.config.authority).toBe("legacy_file");
|
||||
const original = await qualify(oldImage);
|
||||
expect(original.configurationPackagesEligible).toBe(true);
|
||||
expect(original.samePackageOverlapEligible).toBe(false);
|
||||
await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true);
|
||||
const config = (await cloudRuntimeConfigService(peer).inspectStored())!;
|
||||
expect(config.config).not.toBeNull();
|
||||
await cloudRuntimeConfigService(peer).replace({ ...config.config!, server: { ...config.config!.server, port: 3299 } }, config.revision);
|
||||
await fs.rm(homeA, { recursive: true });
|
||||
await fs.rm(oldImage.root, { recursive: true });
|
||||
expect((await migration(peer, homeB).migrate()).migration).toEqual(first.migration);
|
||||
await initializeAdapterSettings(peer, { runtimeEnv: env(homeB) });
|
||||
const restored = await initializeCloudRuntimeConfig(peer, { runtimeEnv: env(homeB) });
|
||||
expect(restored?.current.config?.server.port).toBe(3299);
|
||||
expect(await adapterSettingsService(peer, { runtimeEnv: env(homeB) }).getDisabledTypes()).toEqual(["claude_local", "codex_local", "future_adapter"]);
|
||||
const replacement = await qualify(newImage, peer, homeB, original.packageEvidenceDigest);
|
||||
expect(replacement).toMatchObject({ adapterRevision: 1, configRevision: 1, configurationPackagesEligible: true, samePackageOverlapEligible: true });
|
||||
expect(replacement.configurationDigest).not.toBe(original.configurationDigest);
|
||||
const manager = createPluginWorkerManager();
|
||||
try {
|
||||
const services = {
|
||||
workerManager: manager, lifecycleManager: { markError: vi.fn() },
|
||||
jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() },
|
||||
toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) },
|
||||
buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "qualification-fixture", hostVersion: "1.0.0" },
|
||||
} as unknown as PluginRuntimeServices;
|
||||
expect((await loaderFor(newImage, peer, services).loadSingle(row.id)).success).toBe(true);
|
||||
expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(newImage.packageRoot) } });
|
||||
} finally { await manager.stopAll(); }
|
||||
});
|
||||
|
||||
it("serializes concurrent attempts and never imports stale inputs over later DB changes", async () => {
|
||||
await legacy();
|
||||
const [first, second] = await Promise.all([migration().migrate(), migration(peer).migrate()]);
|
||||
expect(first).toEqual(second);
|
||||
await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("claude_local", false);
|
||||
await fs.writeFile(path.join(homeA, "config.json"), "invalid stale file");
|
||||
await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid stale file");
|
||||
expect(await migration().migrate()).toEqual(first);
|
||||
expect(await adapterSettingsService(db, { runtimeEnv: env() }).getDisabledTypes()).toEqual(["future_adapter"]);
|
||||
});
|
||||
|
||||
it("rolls back both seeds and the receipt if completion fails, then retries after the crash boundary", async () => {
|
||||
await legacy();
|
||||
await db.execute(sql`CREATE FUNCTION reject_cloud_migration_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.singleton_key = 'cloud-configuration-migration:v1' THEN RAISE EXCEPTION 'test receipt failure'; END IF; RETURN NEW; END $$`);
|
||||
await db.execute(sql`CREATE TRIGGER reject_cloud_migration_receipt BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_cloud_migration_receipt()`);
|
||||
try {
|
||||
await expect(migration().migrate()).rejects.toMatchObject({ cause: { message: expect.stringContaining("test receipt failure") } });
|
||||
expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]);
|
||||
} finally {
|
||||
await db.execute(sql`DROP TRIGGER reject_cloud_migration_receipt ON instance_settings`);
|
||||
await db.execute(sql`DROP FUNCTION reject_cloud_migration_receipt()`);
|
||||
}
|
||||
expect((await migration(peer).migrate()).revision).toBe(0);
|
||||
expect((await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).length).toBe(3);
|
||||
});
|
||||
|
||||
it("fails without partial authority for malformed legacy config and preserves already-authoritative rows", async () => {
|
||||
await legacy();
|
||||
await fs.writeFile(path.join(homeA, "config.json"), "invalid");
|
||||
await expect(migration().migrate()).rejects.toThrow("Legacy Cloud runtime configuration");
|
||||
expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]);
|
||||
await adapterSettingsService(db, { runtimeEnv: env() }).seedIfAbsent({ disabledTypes: ["codex_local"] });
|
||||
await cloudRuntimeConfigService(db).seedIfAbsent(null);
|
||||
await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid");
|
||||
const receipt = await migration(peer).migrate();
|
||||
expect(receipt.migration.adapter.authority).toBe("database");
|
||||
expect(receipt.migration.config.authority).toBe("database");
|
||||
expect((await cloudRuntimeConfigService(peer).inspectStored())?.config).toBeNull();
|
||||
});
|
||||
|
||||
it("returns database errors instead of falling back to files or a completed receipt", async () => {
|
||||
await legacy();
|
||||
await migration().migrate();
|
||||
const broken = createDb(database.connectionString);
|
||||
await broken.$client.end({ timeout: 0 });
|
||||
await expect(migration(broken).migrate()).rejects.toThrow();
|
||||
const image = fixture();
|
||||
await expect(qualify(image, broken)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("keeps external registration blockers durable when B has no legacy home and requires deliberate original-source resolution", async () => {
|
||||
await legacy();
|
||||
const sentinel = path.join(homeA, "external-code-executed");
|
||||
await fs.writeFile(path.join(homeA, "external.js"), `require('node:fs').writeFileSync(${JSON.stringify(sentinel)}, 'executed');`);
|
||||
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), JSON.stringify([{ type: "legacy_external", packageName: "legacy-adapter", localPath: path.join(homeA, "external.js") }]));
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "unsupported_external_adapters" }));
|
||||
await expect(fs.stat(sentinel)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await migration(peer, homeB).migrate();
|
||||
const replacement = await qualify(image, peer, homeB);
|
||||
expect(replacement.configurationPackagesEligible).toBe(false);
|
||||
await expect(migration(peer, homeB).recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("original legacy source");
|
||||
await fs.unlink(path.join(homeA, "adapter-plugins.json"));
|
||||
await expect(migration().recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("present, valid registration file");
|
||||
expect((await migration().inspectStored())?.migration.externalRegistrations.blocker).toBe("unsupported_external_adapters");
|
||||
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]");
|
||||
await expect(migration().recheckLegacyExternalRegistrations(0)).rejects.toThrow("revision conflict");
|
||||
await migration().recheckLegacyExternalRegistrations(replacement.receiptRevision);
|
||||
expect((await qualify(image)).configurationPackagesEligible).toBe(true);
|
||||
});
|
||||
|
||||
it("retains a metadata blocker for malformed or oversized registration files", async () => {
|
||||
await legacy();
|
||||
await fs.writeFile(path.join(homeA, "adapter-plugins.json"), Buffer.alloc(65_537));
|
||||
const receipt = await migration().migrate();
|
||||
expect(receipt.migration.externalRegistrations.blocker).toBe("invalid_external_registration_metadata");
|
||||
await fs.rm(homeA, { recursive: true });
|
||||
expect((await migration(peer, homeB).migrate()).migration.externalRegistrations).toEqual(receipt.migration.externalRegistrations);
|
||||
});
|
||||
|
||||
it("records image mismatches before reconciliation, permits maintenance upgrades, and requires explicit fresh overlap evidence", async () => {
|
||||
await legacy();
|
||||
const oldImage = fixture();
|
||||
const nextImage = fixture("1.1.0");
|
||||
await install(oldImage);
|
||||
await migration().migrate();
|
||||
const old = await qualify(oldImage);
|
||||
const mismatch = await qualify(nextImage, peer, homeB, old.packageEvidenceDigest);
|
||||
expect(mismatch.configurationPackagesEligible).toBe(false);
|
||||
expect(mismatch.samePackageOverlapEligible).toBe(false);
|
||||
expect(mismatch.blockers).toContainEqual(expect.objectContaining({ code: "incompatible_bundled_package", registeredVersion: "1.0.0", imageVersion: "1.1.0" }));
|
||||
const registry = pluginRegistryService(db);
|
||||
await ensureBundledPlugins(nextImage.installs, {
|
||||
registry, loader: loaderFor(nextImage), lifecycle: { load: vi.fn() },
|
||||
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
}, { reinstallUninstalled: true, preservePackagePath: true });
|
||||
const current = await qualify(nextImage);
|
||||
expect(current.configurationPackagesEligible).toBe(true);
|
||||
expect(current.samePackageOverlapEligible).toBe(false);
|
||||
expect((await migration().inspectStored())?.lastPackageMismatch?.packageEvidenceDigest).toBe(mismatch.packageEvidenceDigest);
|
||||
expect((await qualify(nextImage, peer, homeB, current.packageEvidenceDigest)).samePackageOverlapEligible).toBe(true);
|
||||
});
|
||||
|
||||
it("fails qualification for missing package or manifest bytes without loading a home fallback", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
const good = await qualify(image);
|
||||
await fs.rm(path.join(image.packageRoot, "dist", "manifest.js"));
|
||||
const broken = await qualify(image, peer, homeB, good.packageEvidenceDigest);
|
||||
expect(broken.configurationPackagesEligible).toBe(false);
|
||||
expect(broken.samePackageOverlapEligible).toBe(false);
|
||||
expect(broken.blockers.length).toBeGreaterThan(0);
|
||||
await fs.rm(image.packageRoot, { recursive: true });
|
||||
expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "missing_bundled_package" }));
|
||||
});
|
||||
|
||||
it("rejects changed image bytes before manifest import and never uses stale completed qualification", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
const original = await qualify(image);
|
||||
await fs.appendFile(path.join(image.packageRoot, "dist", "worker.js"), "\nprocess.exit(99);\n");
|
||||
const loadManifest = vi.fn();
|
||||
const current = await migration(peer, homeB).qualify({ installs: image.installs, loadManifest, expectedPackageEvidenceDigest: original.packageEvidenceDigest });
|
||||
expect(current.configurationPackagesEligible).toBe(false);
|
||||
expect(current.samePackageOverlapEligible).toBe(false);
|
||||
expect(current.blockers).toContainEqual(expect.objectContaining({ code: "package_integrity_mismatch" }));
|
||||
expect(loadManifest).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows an administrator update while manifest inspection waits and refuses to certify the stale snapshot", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
let started!: () => void;
|
||||
let resume!: () => void;
|
||||
const entered = new Promise<void>((resolve) => { started = resolve; });
|
||||
const release = new Promise<void>((resolve) => { resume = resolve; });
|
||||
const loader = loaderFor(image);
|
||||
const pending = migration().qualify({ installs: image.installs, loadManifest: async (root) => { started(); await release; return loader.loadManifest(root); } });
|
||||
await entered;
|
||||
try {
|
||||
const change = adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
expect(await Promise.race([change, new Promise<never>((_resolve, reject) => { timer = setTimeout(() => reject(new Error("Admin update was blocked by image inspection")), 2_000); })])).toBe(true);
|
||||
} finally { clearTimeout(timer); }
|
||||
} finally { resume(); }
|
||||
const result = await pending;
|
||||
expect(result).toMatchObject({ configurationPackagesEligible: false, adapterRevision: 1 });
|
||||
expect(result.blockers).toContainEqual(expect.objectContaining({ code: "configuration_changed_during_inspection" }));
|
||||
expect((await qualify(image)).configurationPackagesEligible).toBe(true);
|
||||
});
|
||||
|
||||
it("uses the current managed catalog through the operator command and durably records catalog failure", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
const runtimeEnv = {
|
||||
...env(homeB), DATABASE_URL: database.connectionString, PAPERCLIP_BUNDLED_PLUGIN_ROOT: image.root,
|
||||
PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "fixture", features: {}, plugins: { autoInstall: ["fixture"] } }),
|
||||
};
|
||||
const initial = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv });
|
||||
expect(initial?.configurationPackagesEligible).toBe(true);
|
||||
const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
|
||||
try {
|
||||
expect(await runCloudConfigurationQualification(["--expected-package-evidence", initial!.packageEvidenceDigest], runtimeEnv)).toBe(0);
|
||||
const report = stdout.mock.calls.map((call) => String(call[0])).find((text) => text.startsWith('{\n "receiptRevision"'));
|
||||
expect(report && JSON.parse(report).samePackageOverlapEligible).toBe(true);
|
||||
} finally { stdout.mockRestore(); }
|
||||
const missing = await qualifyCloudConfigurationForCurrentImage(peer, { runtimeEnv: { ...runtimeEnv, PAPERCLIP_MANAGED_CONFIG: runtimeEnv.PAPERCLIP_MANAGED_CONFIG.replace('"fixture"]', '"absent"]') } });
|
||||
expect(missing?.configurationPackagesEligible).toBe(false);
|
||||
expect(missing?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" }));
|
||||
expect((await migration().inspectStored())?.lastPackageMismatch?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" }));
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("detects changed pnpm dependency bytes in a builtin package overlap comparison", async () => {
|
||||
await legacy();
|
||||
const image = fixture();
|
||||
await install(image);
|
||||
await migration().migrate();
|
||||
const dependency = path.join(image.packageRoot, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep");
|
||||
await fs.mkdir(dependency, { recursive: true });
|
||||
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;");
|
||||
await fs.symlink(path.relative(path.join(image.packageRoot, "node_modules"), dependency), path.join(image.packageRoot, "node_modules", "dep"));
|
||||
const installs = image.installs.map(({ distribution: _distribution, ...install }) => install);
|
||||
const loader = loaderFor(image);
|
||||
const original = await migration().qualify({ installs, loadManifest: (root) => loader.loadManifest(root) });
|
||||
expect(original.configurationPackagesEligible).toBe(true);
|
||||
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;");
|
||||
const changed = await migration(peer, homeB).qualify({ installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: original.packageEvidenceDigest });
|
||||
expect(changed.configurationPackagesEligible).toBe(true);
|
||||
expect(changed.samePackageOverlapEligible).toBe(false);
|
||||
expect(changed.overlapBlockers).toContainEqual(expect.objectContaining({ code: "package_evidence_changed" }));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { readLegacyAdapterSettings } from "../services/adapter-settings.js";
|
||||
import { inspectLegacyExternalRegistrations } from "../services/cloud-configuration-evidence.js";
|
||||
import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js";
|
||||
|
||||
vi.mock("../adapters/plugin-loader.js", () => ({ buildExternalAdapters: async () => [] }));
|
||||
|
||||
describe("bounded Cloud legacy readers and image inventory", () => {
|
||||
let root: string;
|
||||
beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-config-reader-")); });
|
||||
afterEach(async () => { await fs.rm(root, { recursive: true, force: true }); });
|
||||
|
||||
it("distinguishes an absent adapter file from a saved empty preference", async () => {
|
||||
expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toBeNull();
|
||||
await fs.writeFile(path.join(root, "adapter-settings.json"), '{"disabledTypes":[]}');
|
||||
expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toEqual({ disabledTypes: [] });
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("rejects symlinks and FIFOs without waiting for a writer", async () => {
|
||||
const target = path.join(root, "target.json");
|
||||
const settings = path.join(root, "adapter-settings.json");
|
||||
await fs.writeFile(target, '{"disabledTypes":[]}');
|
||||
await fs.symlink(target, settings);
|
||||
await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toMatchObject({ code: "ELOOP" });
|
||||
expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("unreadable_external_registration_metadata");
|
||||
await fs.unlink(settings);
|
||||
execFileSync("mkfifo", [settings]);
|
||||
await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toThrow("regular file");
|
||||
expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("invalid_external_registration_metadata");
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("hashes pnpm-style contained links and the physical dependency bytes", async () => {
|
||||
const dependency = path.join(root, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep");
|
||||
await fs.mkdir(dependency, { recursive: true });
|
||||
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;");
|
||||
await fs.symlink(path.relative(path.join(root, "node_modules"), dependency), path.join(root, "node_modules", "dep"));
|
||||
const first = cloudBundledPackageDigest(root);
|
||||
await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;");
|
||||
expect(cloudBundledPackageDigest(root)).not.toBe(first);
|
||||
await fs.unlink(path.join(root, "node_modules", "dep"));
|
||||
await fs.symlink("/", path.join(root, "node_modules", "dep"));
|
||||
expect(() => cloudBundledPackageDigest(root)).toThrow("escapes its package");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,302 @@
|
||||
import fs from "node:fs";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, type Db } from "@paperclipai/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { paperclipConfigSchema } from "@paperclipai/shared";
|
||||
import { loadCloudBootstrapConfig, loadConfig } from "../config.js";
|
||||
import { readConfigFile } from "../config-file.js";
|
||||
import {
|
||||
CLOUD_RUNTIME_CONFIG_KEY,
|
||||
cloudRuntimeConfigService,
|
||||
initializeCloudRuntimeConfig,
|
||||
readLegacyCloudRuntimeConfig,
|
||||
} from "../services/cloud-runtime-config.js";
|
||||
import {
|
||||
getCloudRuntimeConfigSnapshot,
|
||||
projectCloudRuntimeConfig,
|
||||
setCloudRuntimeConfigSnapshot,
|
||||
} from "../services/cloud-runtime-config-snapshot.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL;
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describePostgres = support.supported ? describe : describe.skip;
|
||||
let home: string;
|
||||
let configPath: string;
|
||||
|
||||
function legacyConfig() {
|
||||
return {
|
||||
$meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" },
|
||||
llm: { provider: "openai", apiKey: "legacy-openai-secret" },
|
||||
database: {
|
||||
mode: "postgres",
|
||||
connectionString: "postgres://legacy:database-secret@example.test/paperclip",
|
||||
backup: { enabled: false, intervalMinutes: 73, retentionDays: 11, dir: "/tmp/legacy-backups" },
|
||||
},
|
||||
logging: { mode: "cloud", logDir: "/tmp/legacy-logs" },
|
||||
server: { deploymentMode: "authenticated", exposure: "private", port: 3210, serveUi: false, allowedHostnames: ["saved.example.test"] },
|
||||
auth: { baseUrlMode: "explicit", publicBaseUrl: "https://saved.example.test", disableSignUp: true },
|
||||
storage: { provider: "s3", s3: { bucket: "saved-bucket", region: "eu-west-1", prefix: "saved/", forcePathStyle: true } },
|
||||
secrets: { provider: "local_encrypted", strictMode: true, localEncrypted: { keyFilePath: "/tmp/legacy-master.key" } },
|
||||
telemetry: { enabled: false },
|
||||
updates: { checkEnabled: false },
|
||||
unsupportedSecret: "unknown-secret-value",
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-runtime-config-"));
|
||||
configPath = path.join(home, "config.json");
|
||||
vi.stubEnv("PAPERCLIP_HOME", home);
|
||||
vi.stubEnv("PAPERCLIP_CONFIG", configPath);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "test-token");
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
|
||||
vi.stubEnv("DATABASE_MIGRATION_URL", undefined);
|
||||
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
|
||||
vi.stubEnv("PAPERCLIP_AUTH_PUBLIC_BASE_URL", undefined);
|
||||
vi.stubEnv("PAPERCLIP_AUTH_BASE_URL_MODE", undefined);
|
||||
vi.stubEnv("PAPERCLIP_PUBLIC_URL", undefined);
|
||||
vi.stubEnv("BETTER_AUTH_URL", undefined);
|
||||
vi.stubEnv("BETTER_AUTH_BASE_URL", undefined);
|
||||
vi.stubEnv("PAPERCLIP_ALLOWED_HOSTNAMES", undefined);
|
||||
vi.stubEnv("PAPERCLIP_TAILNET_BIND_HOST", "100.100.0.1");
|
||||
vi.stubEnv("OPENAI_API_KEY", "environment-api-secret");
|
||||
setCloudRuntimeConfigSnapshot(null);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
setCloudRuntimeConfigSnapshot(null);
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("Cloud configuration file boundary", () => {
|
||||
it.each(["null", "{", JSON.stringify({ unexpected: true })])("rejects malformed present legacy data %s", async (contents) => {
|
||||
fs.writeFileSync(configPath, contents);
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/Legacy Cloud runtime configuration/);
|
||||
});
|
||||
|
||||
it("rejects oversized and linked legacy sources", async () => {
|
||||
fs.writeFileSync(configPath, " ".repeat(65_537));
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/);
|
||||
fs.rmSync(configPath);
|
||||
const target = path.join(home, "target.json");
|
||||
fs.writeFileSync(target, JSON.stringify(legacyConfig()));
|
||||
fs.symlinkSync(target, configPath);
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/);
|
||||
});
|
||||
|
||||
it("requires an environment credential for the legacy OpenAI consumer", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
vi.stubEnv("OPENAI_API_KEY", undefined);
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/OPENAI_API_KEY/);
|
||||
expect(() => readConfigFile()).toThrow(/not been initialized/);
|
||||
});
|
||||
|
||||
it.each(["https://user:secret@saved.example.test", "https://saved.example.test?key=secret", "https://saved.example.test#secret"])("rejects secret-bearing URL %s", async (url) => {
|
||||
const config = legacyConfig();
|
||||
config.auth.publicBaseUrl = url;
|
||||
fs.writeFileSync(configPath, JSON.stringify(config));
|
||||
await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/configuration is invalid/);
|
||||
});
|
||||
|
||||
it("loads the minimal Cloud bootstrap from environment only", () => {
|
||||
fs.writeFileSync(configPath, "malformed");
|
||||
vi.stubEnv("DATABASE_URL", "postgres://bootstrap:bootstrap-secret@127.0.0.1:5432/runtime");
|
||||
expect(loadCloudBootstrapConfig()).toMatchObject({ databaseMode: "postgres", databaseUrl: process.env.DATABASE_URL });
|
||||
vi.stubEnv("DATABASE_URL", undefined);
|
||||
expect(() => loadCloudBootstrapConfig()).toThrow(/requires DATABASE_URL/);
|
||||
vi.stubEnv("DATABASE_URL", "sqlite:///tmp/db");
|
||||
expect(() => loadCloudBootstrapConfig()).toThrow(/must use PostgreSQL/);
|
||||
});
|
||||
|
||||
it.each(["", " "])("treats an empty migration URL as absent (%j)", (migrationUrl) => {
|
||||
vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime");
|
||||
vi.stubEnv("DATABASE_MIGRATION_URL", migrationUrl);
|
||||
expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBeUndefined();
|
||||
});
|
||||
|
||||
it("normalizes and validates an explicit migration URL", () => {
|
||||
vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime");
|
||||
vi.stubEnv("DATABASE_MIGRATION_URL", " postgres://migration:password@127.0.0.1/runtime ");
|
||||
expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBe("postgres://migration:password@127.0.0.1/runtime");
|
||||
vi.stubEnv("DATABASE_MIGRATION_URL", "https://wrong.example.test");
|
||||
expect(() => loadCloudBootstrapConfig()).toThrow("DATABASE_MIGRATION_URL must use PostgreSQL");
|
||||
});
|
||||
|
||||
it("preserves local config file behavior without managed signals", () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
expect(readConfigFile()?.llm?.apiKey).toBe("legacy-openai-secret");
|
||||
expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" });
|
||||
});
|
||||
});
|
||||
|
||||
describePostgres("Cloud runtime configuration in PostgreSQL", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let first: Db;
|
||||
let second: Db;
|
||||
|
||||
beforeAll(async () => {
|
||||
if (externalDatabaseUrl) {
|
||||
await applyPendingMigrations(externalDatabaseUrl);
|
||||
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl, { scope: "database" }) };
|
||||
} else {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-config-db-");
|
||||
}
|
||||
first = createDb(database.connectionString);
|
||||
second = createDb(database.connectionString);
|
||||
}, 90_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await first.delete(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY));
|
||||
vi.stubEnv("DATABASE_URL", database.connectionString);
|
||||
});
|
||||
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])("seeds a fresh database with %s", async (signal) => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token");
|
||||
const result = await initializeCloudRuntimeConfig(first);
|
||||
expect(result).toMatchObject({ seeded: true, current: { version: 1, revision: 0, config: null } });
|
||||
expect(await cloudRuntimeConfigService(second).inspectStored()).toEqual(result?.current);
|
||||
expect(fs.existsSync(configPath)).toBe(false);
|
||||
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", "authenticated");
|
||||
vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://environment.example.test");
|
||||
expect(loadConfig()).toMatchObject({ databaseMode: "postgres", authBaseUrlMode: "explicit", secretsStrictMode: true, telemetryEnabled: true });
|
||||
});
|
||||
|
||||
it("loads the saved authenticated mode before validating an environment host bind", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
vi.stubEnv("HOST", "0.0.0.0");
|
||||
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
|
||||
expect(loadCloudBootstrapConfig()).toEqual({
|
||||
databaseMode: "postgres",
|
||||
databaseUrl: database.connectionString,
|
||||
databaseMigrationUrl: process.env.DATABASE_MIGRATION_URL,
|
||||
});
|
||||
await initializeCloudRuntimeConfig(first);
|
||||
expect(loadConfig()).toMatchObject({ deploymentMode: "authenticated", host: "0.0.0.0", port: 3210 });
|
||||
});
|
||||
|
||||
it("imports existing choices without credentials and restores them from a second client and new home", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
const initial = await initializeCloudRuntimeConfig(first);
|
||||
expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, databaseBackupEnabled: false, storageS3Bucket: "saved-bucket", authDisableSignUp: true });
|
||||
const [row] = await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY));
|
||||
const persisted = JSON.stringify(row.general);
|
||||
for (const secret of ["legacy-openai-secret", "database-secret", "unknown-secret-value", "environment-api-secret", database.connectionString]) {
|
||||
expect(persisted).not.toContain(secret);
|
||||
}
|
||||
const replacementHome = fs.mkdtempSync(path.join(home, "replacement-"));
|
||||
const replacementConfigDir = path.join(replacementHome, ".paperclip");
|
||||
fs.mkdirSync(replacementConfigDir);
|
||||
const envPath = path.join(replacementConfigDir, ".env");
|
||||
const envContents = "PORT=9999\nPAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL=should-not-load\n";
|
||||
fs.writeFileSync(envPath, envContents);
|
||||
vi.stubEnv("PAPERCLIP_HOME", replacementHome);
|
||||
vi.stubEnv("PAPERCLIP_CONFIG", path.join(replacementConfigDir, "config.json"));
|
||||
setCloudRuntimeConfigSnapshot(null);
|
||||
const files = vi.spyOn(fs, "readFileSync");
|
||||
expect(await initializeCloudRuntimeConfig(second)).toEqual({ seeded: false, current: initial?.current });
|
||||
expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" });
|
||||
expect(files.mock.calls.map(([file]) => String(file)).filter(file => file.startsWith(replacementHome))).toEqual([]);
|
||||
files.mockRestore();
|
||||
const root = fileURLToPath(new URL("../../../", import.meta.url));
|
||||
const childScript = path.join(home, "restore.mts");
|
||||
fs.writeFileSync(childScript, `
|
||||
import { createDb } from ${JSON.stringify(pathToFileURL(path.join(root, "packages/db/src/index.ts")).href)};
|
||||
import { initializeCloudRuntimeConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/services/cloud-runtime-config.ts")).href)};
|
||||
import { loadConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/config.ts")).href)};
|
||||
const db = createDb(process.env.DATABASE_URL);
|
||||
await initializeCloudRuntimeConfig(db);
|
||||
const config = loadConfig();
|
||||
console.log(JSON.stringify({ port: config.port, telemetryEnabled: config.telemetryEnabled, storageS3Bucket: config.storageS3Bucket,
|
||||
envSentinel: process.env.PAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL ?? null }));
|
||||
await db.$client.end({ timeout: 1 });
|
||||
`);
|
||||
const child = await promisify(execFile)(process.execPath, [path.join(root, "server/node_modules/tsx/dist/cli.mjs"), childScript], {
|
||||
cwd: replacementHome,
|
||||
env: { ...process.env },
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(JSON.parse(child.stdout.trim())).toEqual({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket", envSentinel: null });
|
||||
expect(fs.readFileSync(envPath, "utf8")).toBe(envContents);
|
||||
}, 45_000);
|
||||
|
||||
it("keeps environment overlays out of saved settings and ignores corrupt old files after seed", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
await initializeCloudRuntimeConfig(first);
|
||||
vi.stubEnv("PORT", "4567");
|
||||
vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", "operator-bucket");
|
||||
vi.stubEnv("PAPERCLIP_AUTH_DISABLE_SIGN_UP", "false");
|
||||
fs.writeFileSync(configPath, "malformed legacy contents");
|
||||
await initializeCloudRuntimeConfig(second);
|
||||
expect(loadConfig()).toMatchObject({ port: 4567, storageS3Bucket: "operator-bucket", authDisableSignUp: false, telemetryEnabled: false });
|
||||
expect((await cloudRuntimeConfigService(first).inspectStored())?.config).toMatchObject({ server: { port: 3210 }, storage: { s3: { bucket: "saved-bucket" } }, auth: { disableSignUp: true } });
|
||||
});
|
||||
|
||||
it("keeps one committed seed when two clients propose different initial values", async () => {
|
||||
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
|
||||
const results = await Promise.all([
|
||||
cloudRuntimeConfigService(first).seedIfAbsent({ ...config, telemetry: { enabled: true } }),
|
||||
cloudRuntimeConfigService(second).seedIfAbsent({ ...config, telemetry: { enabled: false } }),
|
||||
]);
|
||||
expect(results.filter(result => result.seeded)).toHaveLength(1);
|
||||
expect(results[0].current).toEqual(results[1].current);
|
||||
expect(await cloudRuntimeConfigService(first).inspectStored()).toEqual(results[0].current);
|
||||
});
|
||||
|
||||
it("rejects unknown or credential fields on the direct mutation path", async () => {
|
||||
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
|
||||
await cloudRuntimeConfigService(first).seedIfAbsent(config);
|
||||
await expect(cloudRuntimeConfigService(first).replace({ ...config, llm: { provider: "openai", apiKey: "must-not-store" } } as never, 0)).rejects.toThrow(/configuration is invalid/);
|
||||
expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(0);
|
||||
});
|
||||
|
||||
it("serializes revisions and rejects competing stale writes from two database clients", async () => {
|
||||
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
|
||||
await cloudRuntimeConfigService(first).seedIfAbsent(config);
|
||||
const results = await Promise.allSettled([
|
||||
cloudRuntimeConfigService(first).replace({ ...config, telemetry: { enabled: true } }, 0),
|
||||
cloudRuntimeConfigService(second).replace({ ...config, telemetry: { enabled: false } }, 0),
|
||||
]);
|
||||
expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1);
|
||||
expect(results.filter(result => result.status === "rejected")).toHaveLength(1);
|
||||
expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(1);
|
||||
});
|
||||
|
||||
it("composes a seed with the caller transaction and rolls back both on failure", async () => {
|
||||
const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
|
||||
await expect(first.transaction(async tx => {
|
||||
await cloudRuntimeConfigService(first).seedIfAbsent(config, tx);
|
||||
await tx.insert(instanceSettings).values({ singletonKey: "runtime-config-test-receipt", general: {} });
|
||||
throw new Error("injected migration failure");
|
||||
})).rejects.toThrow(/injected migration failure/);
|
||||
expect(await cloudRuntimeConfigService(second).inspectStored()).toBeNull();
|
||||
expect(await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, "runtime-config-test-receipt"))).toEqual([]);
|
||||
});
|
||||
|
||||
it("fails closed on invalid database settings without consulting a valid legacy file", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
await first.insert(instanceSettings).values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: { version: 99, config: { secret: "hidden" } } });
|
||||
await expect(initializeCloudRuntimeConfig(second)).rejects.toThrow(/Persisted Cloud runtime configuration is invalid/);
|
||||
expect(() => getCloudRuntimeConfigSnapshot()).toThrow(/not been initialized/);
|
||||
});
|
||||
|
||||
it("does not fall back to a file or old snapshot when the database fails", async () => {
|
||||
fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
|
||||
await initializeCloudRuntimeConfig(first);
|
||||
const unavailable = createDb(database.connectionString);
|
||||
await unavailable.$client.end({ timeout: 1 });
|
||||
await expect(initializeCloudRuntimeConfig(unavailable)).rejects.toThrow();
|
||||
expect(() => readConfigFile()).toThrow(/not been initialized/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
import { mkdtempSync, mkdirSync, realpathSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../../services/bundled-plugins.js";
|
||||
import { distributionBundleDigest, readDistributionPluginCatalog } from "../../services/distribution-plugin-catalog.js";
|
||||
import { derivePluginDatabaseNamespace } from "../../services/plugin-database.js";
|
||||
import { createBundledPluginPackageResolver } from "../../services/bundled-plugin-package.js";
|
||||
|
||||
export function bundledImageFixture(manifestOverrides: Partial<PaperclipPluginManifestV1> = {}) {
|
||||
const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "bundled-image-")));
|
||||
const packageRoot = path.join(root, "distribution", "fixture");
|
||||
const manifest: PaperclipPluginManifestV1 = {
|
||||
id: "paperclip.image-fixture", apiVersion: 1, version: "1.0.0", displayName: "Image Fixture",
|
||||
description: "Image package relocation fixture", author: "Paperclip", categories: ["automation"], capabilities: ["issues.read"],
|
||||
entrypoints: { worker: "dist/worker.js", ui: "dist/ui" }, ...manifestOverrides,
|
||||
};
|
||||
mkdirSync(path.join(packageRoot, "dist", "ui"), { recursive: true });
|
||||
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify({
|
||||
name: "@paperclipai/plugin-image-fixture", version: manifest.version, type: "module",
|
||||
paperclipPlugin: { manifest: "dist/manifest.js", ...manifest.entrypoints },
|
||||
}));
|
||||
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`);
|
||||
writeFileSync(path.join(packageRoot, "dist", "worker.js"), `import readline from "node:readline";
|
||||
if (process.argv.includes("--source")) process.stdout.write(import.meta.url);
|
||||
else {
|
||||
const configs = {};
|
||||
readline.createInterface({ input: process.stdin }).on("line", line => {
|
||||
const request = JSON.parse(line);
|
||||
let result = {};
|
||||
if (request.method === "initialize") result = { ok: true, supportedMethods: ["configChanged", "health"] };
|
||||
if (request.method === "configChanged") configs[request.params.companyId] = request.params.config;
|
||||
if (request.method === "health") result = { status: "ok", details: { source: import.meta.url, configs } };
|
||||
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result }) + "\\n");
|
||||
if (request.method === "shutdown") setImmediate(() => process.exit(0));
|
||||
});
|
||||
}`);
|
||||
writeFileSync(path.join(packageRoot, "dist", "ui", "index.js"), "export const source = 'current-image';");
|
||||
if (manifest.database) {
|
||||
mkdirSync(path.join(packageRoot, manifest.database.migrationsDir), { recursive: true });
|
||||
writeFileSync(path.join(packageRoot, manifest.database.migrationsDir, "001_fixture.sql"), `CREATE TABLE ${derivePluginDatabaseNamespace(manifest.id)}.image_rows (id uuid PRIMARY KEY);`);
|
||||
}
|
||||
const entry = { key: "fixture", pluginKey: manifest.id, version: manifest.version, directory: "fixture", digest: distributionBundleDigest(packageRoot) };
|
||||
writeFileSync(path.join(root, "distribution", "catalog.json"), JSON.stringify({ schemaVersion: 1, plugins: [entry] }));
|
||||
const catalog = readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG);
|
||||
const installs = resolveBundledPluginInstalls(["fixture"], { catalogRoot: root, enforceCatalogRoot: true, env: {}, distributionPlugins: catalog });
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const registration = { pluginKey: manifest.id, packageName: "@paperclipai/plugin-image-fixture", version: manifest.version, manifestJson: manifest };
|
||||
return { root, packageRoot, manifest, catalog, installs, resolver, registration };
|
||||
}
|
||||
@@ -20,6 +20,22 @@ import { gzipSync } from "node:zlib";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const cloudRuntimeStartupMocks = vi.hoisted(() => ({
|
||||
initialize: vi.fn(async () => null),
|
||||
bootstrap: vi.fn(),
|
||||
telemetry: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../services/adapter-settings.js", async () => ({
|
||||
...await vi.importActual<typeof import("../services/adapter-settings.js")>("../services/adapter-settings.js"),
|
||||
initializeAdapterSettings: vi.fn(async () => null),
|
||||
}));
|
||||
vi.mock("../services/cloud-runtime-config.js", () => ({ initializeCloudRuntimeConfig: cloudRuntimeStartupMocks.initialize }));
|
||||
const cloudMigrationMocks = vi.hoisted(() => ({ migrate: vi.fn(async () => null), qualify: vi.fn(async () => null) }));
|
||||
vi.mock("../services/cloud-configuration-migration.js", () => ({ migrateCloudConfiguration: cloudMigrationMocks.migrate }));
|
||||
vi.mock("../services/cloud-configuration-qualification.js", () => ({ qualifyCloudConfigurationForCurrentImage: cloudMigrationMocks.qualify }));
|
||||
vi.mock("../telemetry.js", () => ({ initTelemetry: cloudRuntimeStartupMocks.telemetry, getTelemetryClient: vi.fn(() => null) }));
|
||||
|
||||
const ORIGINAL_PAPERCLIP_API_URL = process.env.PAPERCLIP_API_URL;
|
||||
const ORIGINAL_PAPERCLIP_RUNTIME_API_URL = process.env.PAPERCLIP_RUNTIME_API_URL;
|
||||
const ORIGINAL_PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON;
|
||||
@@ -343,6 +359,7 @@ vi.mock("../services/native-runtime/native-safe-replacement.js", () => ({
|
||||
|
||||
vi.mock("../config.js", () => ({
|
||||
loadConfig: loadConfigMock,
|
||||
loadCloudBootstrapConfig: cloudRuntimeStartupMocks.bootstrap,
|
||||
}));
|
||||
|
||||
vi.mock("../middleware/logger.js", () => ({
|
||||
@@ -586,6 +603,8 @@ describe("startServer feedback export wiring", () => {
|
||||
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
|
||||
process.env.PAPERCLIP_AGENT_JWT_SECRET = "0123456789abcdef0123456789abcdef";
|
||||
loadConfigMock.mockReturnValue(buildTestConfig());
|
||||
cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
|
||||
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
|
||||
resolveHeartbeatSchedulingSuppressionMock.mockReturnValue({
|
||||
suppressed: false,
|
||||
reason: null,
|
||||
@@ -595,6 +614,64 @@ describe("startServer feedback export wiring", () => {
|
||||
process.env.BETTER_AUTH_SECRET = "test-secret";
|
||||
});
|
||||
|
||||
it("awaits saved Cloud configuration before telemetry, authentication and app construction", async () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
|
||||
vi.stubEnv("HOST", "0.0.0.0");
|
||||
vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
|
||||
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
|
||||
let release: (() => void) | undefined;
|
||||
cloudRuntimeStartupMocks.initialize.mockImplementationOnce(() => new Promise<null>(resolve => {
|
||||
release = () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ telemetryEnabled: false, authDisableSignUp: true, port: 4321, host: "0.0.0.0", bind: "all" }));
|
||||
resolve(null);
|
||||
};
|
||||
}));
|
||||
try {
|
||||
const starting = startServer();
|
||||
for (let index = 0; index < 30 && !release; index += 1) await new Promise<void>(resolve => setImmediate(resolve));
|
||||
expect(release).toBeTypeOf("function");
|
||||
expect(cloudMigrationMocks.migrate.mock.invocationCallOrder[0]).toBeLessThan(cloudRuntimeStartupMocks.initialize.mock.invocationCallOrder[0]!);
|
||||
expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
|
||||
expect(createBetterAuthInstanceMock).not.toHaveBeenCalled();
|
||||
expect(createAppMock).not.toHaveBeenCalled();
|
||||
expect(loadConfigMock).not.toHaveBeenCalled();
|
||||
release!();
|
||||
const started = await starting;
|
||||
await settleStartupHousekeeping();
|
||||
expect(cloudRuntimeStartupMocks.telemetry).toHaveBeenCalledExactlyOnceWith({ enabled: false });
|
||||
expect(createBetterAuthInstanceMock).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ authDisableSignUp: true, port: 4321 }), expect.anything());
|
||||
expect(started.listenPort).toBe(4321);
|
||||
expect(cloudMigrationMocks.qualify.mock.invocationCallOrder[0]).toBeLessThan(createAppMock.mock.invocationCallOrder[0]!);
|
||||
} finally {
|
||||
release?.();
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not create telemetry or the app when saved Cloud configuration fails", async () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
|
||||
cloudRuntimeStartupMocks.initialize.mockRejectedValueOnce(new Error("persisted config unavailable"));
|
||||
try {
|
||||
await expect(startServer()).rejects.toThrow("persisted config unavailable");
|
||||
expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
|
||||
expect(createBetterAuthInstanceMock).not.toHaveBeenCalled();
|
||||
expect(createAppMock).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("stops before configuration consumers when the atomic Cloud migration fails", async () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
|
||||
cloudMigrationMocks.migrate.mockRejectedValueOnce(new Error("migration did not commit"));
|
||||
try {
|
||||
await expect(startServer()).rejects.toThrow("migration did not commit");
|
||||
expect(cloudRuntimeStartupMocks.initialize).not.toHaveBeenCalled();
|
||||
expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
|
||||
expect(createAppMock).not.toHaveBeenCalled();
|
||||
} finally { vi.unstubAllEnvs(); }
|
||||
});
|
||||
|
||||
it.each([false, true])("registers runner evidence independently from the backup scan (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
@@ -1509,6 +1586,8 @@ describe("startServer authenticated auth origin setup", () => {
|
||||
vi.clearAllMocks();
|
||||
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
|
||||
loadConfigMock.mockReturnValue(buildTestConfig());
|
||||
cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
|
||||
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
|
||||
createBetterAuthInstanceMock.mockReturnValue({});
|
||||
deriveAuthTrustedOriginsMock.mockReturnValue([]);
|
||||
process.env.BETTER_AUTH_SECRET = "test-secret";
|
||||
@@ -1570,6 +1649,8 @@ describe("startServer PAPERCLIP_API_URL handling", () => {
|
||||
vi.clearAllMocks();
|
||||
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
|
||||
loadConfigMock.mockReturnValue(buildTestConfig());
|
||||
cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
|
||||
cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
|
||||
process.env.BETTER_AUTH_SECRET = "test-secret";
|
||||
delete process.env.PAPERCLIP_API_URL;
|
||||
});
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
let home: string;
|
||||
let localPackage: string;
|
||||
let npmPackage: string;
|
||||
let storePath: string;
|
||||
let sentinel: string;
|
||||
const parserSource = "export function parseStdoutLine() { return { message: 'external parser' }; }";
|
||||
const managedError = { status: 403, details: { code: "adapter_install_platform_managed" } };
|
||||
|
||||
function writePackage(packageDir: string, type: string): void {
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(packageDir, "package.json"), JSON.stringify({
|
||||
name: `fixture-${type}`,
|
||||
type: "module",
|
||||
exports: { ".": "./index.mjs", "./ui-parser": "./ui-parser.mjs" },
|
||||
paperclip: { adapterUiParser: "1.0.0" },
|
||||
}));
|
||||
fs.writeFileSync(path.join(packageDir, "index.mjs"), `
|
||||
import fs from "node:fs";
|
||||
fs.appendFileSync(${JSON.stringify(sentinel)}, ${JSON.stringify(type + "\n")});
|
||||
export function createServerAdapter() {
|
||||
return {
|
||||
type: ${JSON.stringify(type)},
|
||||
agentConfigurationDoc: "External fixture",
|
||||
models: [],
|
||||
execute: async () => ({ exitCode: 41, signal: null, timedOut: false }),
|
||||
testEnvironment: async () => ({ status: "pass", checks: [] }),
|
||||
};
|
||||
}
|
||||
`);
|
||||
fs.writeFileSync(path.join(packageDir, "ui-parser.mjs"), parserSource);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-cloud-adapters-"));
|
||||
vi.stubEnv("PAPERCLIP_HOME", home);
|
||||
sentinel = path.join(home, "external-code-executed");
|
||||
localPackage = path.join(home, "local-package");
|
||||
npmPackage = path.join(home, "adapter-plugins", "node_modules", "fixture-npm");
|
||||
storePath = path.join(home, "adapter-plugins.json");
|
||||
writePackage(localPackage, "codex_local");
|
||||
writePackage(npmPackage, "fixture_npm");
|
||||
fs.writeFileSync(storePath, JSON.stringify([
|
||||
{ packageName: "fixture-local", localPath: localPackage, type: "codex_local", installedAt: "2026-01-01" },
|
||||
{ packageName: "fixture-npm", type: "fixture_npm", installedAt: "2026-01-01" },
|
||||
]));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("Cloud image adapter boundary", () => {
|
||||
it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])(
|
||||
"cold boots built-ins without reading or executing persisted packages with %s",
|
||||
async (signal) => {
|
||||
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token");
|
||||
const registrationBefore = fs.readFileSync(storePath, "utf8");
|
||||
const reads = vi.spyOn(fs, "readFileSync");
|
||||
const registry = await import("./registry.js");
|
||||
await registry.waitForExternalAdapters();
|
||||
const loader = await import("./plugin-loader.js");
|
||||
const store = await import("../services/adapter-plugin-store.js");
|
||||
|
||||
expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture");
|
||||
expect(registry.requireServerAdapter("process").execute).toBeTypeOf("function");
|
||||
expect(registry.findServerAdapter("fixture_npm")).toBeNull();
|
||||
expect(registry.hasActiveAdapterOverride("codex_local")).toBe(false);
|
||||
expect(await loader.buildExternalAdapters()).toEqual([]);
|
||||
expect(store.listAdapterPlugins()).toEqual([]);
|
||||
expect(store.getAdapterPluginByType("codex_local")).toBeUndefined();
|
||||
expect(loader.getUiParserSource("codex_local")).toBeUndefined();
|
||||
expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined();
|
||||
await expect(loader.loadExternalAdapterPackage("fixture-local", localPackage)).rejects.toMatchObject(managedError);
|
||||
await expect(loader.loadExternalAdapterPackage("fixture-npm")).rejects.toMatchObject(managedError);
|
||||
await expect(loader.reloadExternalAdapter("codex_local")).rejects.toMatchObject(managedError);
|
||||
expect(() => registry.registerServerAdapter(registry.requireServerAdapter("codex_local"))).toThrow();
|
||||
expect(() => registry.unregisterServerAdapter("codex_local")).toThrow();
|
||||
expect(() => registry.setOverridePaused("codex_local", false)).toThrow();
|
||||
expect(() => store.addAdapterPlugin({ packageName: "fixture", type: "fixture", installedAt: "2026-01-01" })).toThrow();
|
||||
expect(() => store.removeAdapterPlugin("codex_local")).toThrow();
|
||||
expect(() => store.getAdapterPluginsDir()).toThrow();
|
||||
expect(store.getAdapterPluginStorePath()).toBe(storePath);
|
||||
expect(reads.mock.calls.map(([file]) => String(file)).filter(file =>
|
||||
file === storePath || file.startsWith(localPackage + path.sep) || file.startsWith(npmPackage + path.sep),
|
||||
)).toEqual([]);
|
||||
reads.mockRestore();
|
||||
expect(fs.existsSync(sentinel)).toBe(false);
|
||||
expect(fs.readFileSync(storePath, "utf8")).toBe(registrationBefore);
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves local and npm package loading, overrides, reload, and lazy UI parsing on self-hosted instances", async () => {
|
||||
const loader = await import("./plugin-loader.js");
|
||||
expect(loader.getOrExtractUiParserSource("codex_local")).toBe(parserSource);
|
||||
const registry = await import("./registry.js");
|
||||
await registry.waitForExternalAdapters();
|
||||
|
||||
const override = registry.requireServerAdapter("codex_local");
|
||||
expect(override.agentConfigurationDoc).toBe("External fixture");
|
||||
expect(registry.hasActiveAdapterOverride("codex_local")).toBe(true);
|
||||
expect(registry.requireServerAdapter("fixture_npm").agentConfigurationDoc).toBe("External fixture");
|
||||
expect(await override.execute({} as never)).toMatchObject({ exitCode: 41 });
|
||||
expect(fs.readFileSync(sentinel, "utf8").trim().split("\n")).toEqual(["codex_local", "fixture_npm"]);
|
||||
expect(loader.getUiParserSource("fixture_npm")).toBe(parserSource);
|
||||
const reloaded = await loader.reloadExternalAdapter("codex_local");
|
||||
expect(reloaded?.type).toBe("codex_local");
|
||||
expect(await reloaded?.execute({} as never)).toMatchObject({ exitCode: 41 });
|
||||
expect(registry.setOverridePaused("codex_local", true)).toBe(true);
|
||||
expect(registry.findActiveServerAdapter("codex_local")?.agentConfigurationDoc).not.toBe("External fixture");
|
||||
expect(registry.setOverridePaused("codex_local", false)).toBe(true);
|
||||
registry.unregisterServerAdapter("codex_local");
|
||||
expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture");
|
||||
});
|
||||
|
||||
it("does not expose a cached registration or parser after a managed signal becomes present", async () => {
|
||||
const loader = await import("./plugin-loader.js");
|
||||
const store = await import("../services/adapter-plugin-store.js");
|
||||
await loader.loadExternalAdapterPackage("fixture-local", localPackage);
|
||||
expect(loader.getUiParserSource("codex_local")).toBe(parserSource);
|
||||
expect(store.listAdapterPlugins()).toHaveLength(2);
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", "{}");
|
||||
expect(store.listAdapterPlugins()).toEqual([]);
|
||||
expect(loader.getUiParserSource("codex_local")).toBeUndefined();
|
||||
expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -15,6 +15,8 @@ import { pathToFileURL } from "node:url";
|
||||
import type { ServerAdapterModule } from "./types.js";
|
||||
import { validateAdapterLoginCapability } from "@paperclipai/adapter-utils";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { isCloudManagedInstance } from "../services/cloud-instance.js";
|
||||
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
|
||||
|
||||
import {
|
||||
listAdapterPlugins,
|
||||
@@ -30,6 +32,7 @@ import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js";
|
||||
const uiParserCache = new Map<string, string>();
|
||||
|
||||
export function getUiParserSource(adapterType: string): string | undefined {
|
||||
if (isCloudManagedInstance()) return undefined;
|
||||
return uiParserCache.get(adapterType);
|
||||
}
|
||||
|
||||
@@ -38,6 +41,7 @@ export function getUiParserSource(adapterType: string): string | undefined {
|
||||
* Makes the ui-parser.js endpoint self-healing.
|
||||
*/
|
||||
export function getOrExtractUiParserSource(adapterType: string): string | undefined {
|
||||
if (isCloudManagedInstance()) return undefined;
|
||||
const cached = uiParserCache.get(adapterType);
|
||||
if (cached) return cached;
|
||||
|
||||
@@ -182,6 +186,7 @@ export async function loadExternalAdapterPackage(
|
||||
packageName: string,
|
||||
localPath?: string,
|
||||
): Promise<ServerAdapterModule> {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
const packageDir = localPath
|
||||
? path.resolve(localPath)
|
||||
: path.resolve(getAdapterPluginsDir(), "node_modules", packageName);
|
||||
@@ -221,6 +226,7 @@ async function loadFromRecord(record: AdapterPluginRecord): Promise<ServerAdapte
|
||||
export async function reloadExternalAdapter(
|
||||
type: string,
|
||||
): Promise<ServerAdapterModule | null> {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
const record = getAdapterPluginByType(type);
|
||||
if (!record) return null;
|
||||
|
||||
@@ -271,6 +277,7 @@ export async function reloadExternalAdapter(
|
||||
* Build all external adapter modules from the plugin store.
|
||||
*/
|
||||
export async function buildExternalAdapters(): Promise<ServerAdapterModule[]> {
|
||||
if (isCloudManagedInstance()) return [];
|
||||
const results: ServerAdapterModule[] = [];
|
||||
|
||||
const storeRecords = listAdapterPlugins();
|
||||
|
||||
@@ -136,7 +136,8 @@ import {
|
||||
import { agentConfigurationDoc as piAgentConfigurationDoc } from "@paperclipai/adapter-pi-local";
|
||||
import { BUILTIN_ADAPTER_TYPES } from "./builtin-adapter-types.js";
|
||||
import { buildExternalAdapters } from "./plugin-loader.js";
|
||||
import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js";
|
||||
import { isCloudManagedInstance } from "../services/cloud-instance.js";
|
||||
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
|
||||
import { NativeEnvironmentProbeCleanupError } from "../vendor/paperclip-runner/index.js";
|
||||
import { processAdapter } from "./process/index.js";
|
||||
import { httpAdapter } from "./http/index.js";
|
||||
@@ -997,11 +998,6 @@ registerBuiltInAdapters();
|
||||
// applies the registry fallback for externals overriding a built-in type.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Cached sync wrapper — the store is a simple JSON file read, safe to call frequently. */
|
||||
function getDisabledAdapterTypesFromStore(): string[] {
|
||||
return getDisabledAdapterTypes();
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge an external adapter module with host-provided session management.
|
||||
*
|
||||
@@ -1034,6 +1030,7 @@ export function resolveExternalAdapterRegistration(
|
||||
* and avoid racing against the loading window.
|
||||
*/
|
||||
const externalAdaptersReady: Promise<void> = (async () => {
|
||||
if (isCloudManagedInstance()) return;
|
||||
try {
|
||||
const externalAdapters = await buildExternalAdapters();
|
||||
for (const externalAdapter of externalAdapters) {
|
||||
@@ -1069,6 +1066,7 @@ export function waitForExternalAdapters(): Promise<void> {
|
||||
}
|
||||
|
||||
export function registerServerAdapter(adapter: ServerAdapterModule): void {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
if (BUILTIN_ADAPTER_TYPES.has(adapter.type) && !builtinFallbacks.has(adapter.type)) {
|
||||
const existing = adaptersByType.get(adapter.type);
|
||||
if (existing) {
|
||||
@@ -1079,6 +1077,7 @@ export function registerServerAdapter(adapter: ServerAdapterModule): void {
|
||||
}
|
||||
|
||||
export function unregisterServerAdapter(type: string): void {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
if (type === processAdapter.type || type === httpAdapter.type) return;
|
||||
if (builtinFallbacks.has(type)) {
|
||||
pausedOverrides.delete(type);
|
||||
@@ -1186,8 +1185,7 @@ export function listServerAdapters(): ServerAdapterModule[] {
|
||||
* Used for menus and agent creation flows — disabled adapters remain
|
||||
* functional for existing agents but hidden from selection.
|
||||
*/
|
||||
export function listEnabledServerAdapters(): ServerAdapterModule[] {
|
||||
const disabled = getDisabledAdapterTypesFromStore();
|
||||
export function listEnabledServerAdapters(disabled: string[]): ServerAdapterModule[] {
|
||||
const disabledSet = disabled.length > 0 ? new Set(disabled) : null;
|
||||
return disabledSet
|
||||
? Array.from(adaptersByType.values()).filter((a) => !disabledSet.has(a.type))
|
||||
@@ -1227,6 +1225,7 @@ export async function detectAdapterModel(
|
||||
* an override or was already in the requested state.
|
||||
*/
|
||||
export function setOverridePaused(type: string, paused: boolean): boolean {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
if (!builtinFallbacks.has(type)) return false;
|
||||
const wasPaused = pausedOverrides.has(type);
|
||||
if (paused && !wasPaused) {
|
||||
|
||||
+13
-4
@@ -183,7 +183,9 @@ import {
|
||||
ensureBundledPlugins,
|
||||
resolveBundledCatalogRoot,
|
||||
resolveBundledPluginInstalls,
|
||||
resolveManagedBundledPluginKeys,
|
||||
} from "./services/bundled-plugins.js";
|
||||
import { createBundledPluginPackageResolver } from "./services/bundled-plugin-package.js";
|
||||
import { readDistributionPluginCatalog, distributionPluginActivationGuard } from "./services/distribution-plugin-catalog.js";
|
||||
import {
|
||||
createPluginWorkerManager,
|
||||
@@ -623,7 +625,7 @@ export async function createApp(
|
||||
res.set("Cache-Control", "no-store"); res.set("Referrer-Policy", "no-referrer");
|
||||
res.redirect(303, await chatChannels.githubWizard.identityCallback(String(req.query.state ?? ""), String(req.query.code ?? ""), req.actor));
|
||||
});
|
||||
const managedAutoInstallKeys = opts.managedPluginAutoInstall ?? null;
|
||||
const managedAutoInstallKeys = resolveManagedBundledPluginKeys(opts.managedPluginAutoInstall);
|
||||
const bundledCatalogRoot =
|
||||
opts.bundledPluginCatalogRoot ?? resolveBundledCatalogRoot(process.env);
|
||||
const distributionPlugins = readDistributionPluginCatalog(bundledCatalogRoot, BUNDLED_PLUGIN_CATALOG);
|
||||
@@ -636,6 +638,7 @@ export async function createApp(
|
||||
distributionPlugins,
|
||||
},
|
||||
);
|
||||
const bundledPackageResolver = managedAutoInstallKeys === null ? undefined : createBundledPluginPackageResolver(bundledPluginInstalls);
|
||||
const managedBundledPluginKeys =
|
||||
managedAutoInstallKeys !== null
|
||||
? bundledPluginInstalls.map((install) => install.pluginKey)
|
||||
@@ -906,7 +909,11 @@ export async function createApp(
|
||||
{
|
||||
localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
|
||||
migrationDb: opts.pluginMigrationDb,
|
||||
assertPackageActivation: distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys),
|
||||
assertPackageActivation: bundledPackageResolver?.assertActivation ?? distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys),
|
||||
resolvePackageRoot: bundledPackageResolver?.resolve,
|
||||
prebuiltOnly: managedAutoInstallKeys !== null,
|
||||
enableLocalFilesystem: managedAutoInstallKeys === null,
|
||||
enableNpmDiscovery: managedAutoInstallKeys === null,
|
||||
},
|
||||
{
|
||||
workerManager,
|
||||
@@ -965,6 +972,7 @@ export async function createApp(
|
||||
);
|
||||
api.use(
|
||||
adapterRoutes({
|
||||
db,
|
||||
resolveRunnerTarget: (companyId, environmentId) => resolveAgentRunnerTargetForCompany(db, companyId, environmentId),
|
||||
getNativeRunnerEnabled: async () =>
|
||||
(await instanceSettingsService(db).getExperimental())
|
||||
@@ -1014,6 +1022,7 @@ export async function createApp(
|
||||
app.use(
|
||||
pluginUiStaticRoutes(db, {
|
||||
localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
|
||||
resolvePackageRoot: bundledPackageResolver?.resolve,
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -1253,7 +1262,7 @@ export async function createApp(
|
||||
void toolDispatcher.initialize().catch((err) => {
|
||||
logger.error({ err }, "Failed to initialize plugin tool dispatcher");
|
||||
});
|
||||
const devWatcher = createPluginDevWatcher(
|
||||
const devWatcher = managedAutoInstallKeys !== null ? null : createPluginDevWatcher(
|
||||
lifecycle,
|
||||
async (pluginId) =>
|
||||
(await pluginRegistry.getById(pluginId))?.packagePath ?? null,
|
||||
@@ -1299,7 +1308,7 @@ export async function createApp(
|
||||
// owns provisioning); self-hosted leaves an operator's uninstall alone.
|
||||
// Disabled plugins never start automatically. Added distribution permissions
|
||||
// still enter upgrade_pending so enabling them requires an operator decision.
|
||||
{ reinstallUninstalled: managedAutoInstallKeys !== null },
|
||||
{ reinstallUninstalled: managedAutoInstallKeys !== null, preservePackagePath: managedAutoInstallKeys !== null },
|
||||
)
|
||||
.then(() => loader.loadAll())
|
||||
.then((result) => {
|
||||
|
||||
@@ -8,6 +8,8 @@ import {
|
||||
} from "@paperclipai/shared";
|
||||
import { ZodError } from "zod";
|
||||
import { resolvePaperclipConfigPath } from "./paths.js";
|
||||
import { isCloudManagedInstance } from "./services/cloud-instance.js";
|
||||
import { getCloudRuntimeConfigSnapshot, toPaperclipConfig } from "./services/cloud-runtime-config-snapshot.js";
|
||||
|
||||
function formatConfigValidationError(error: ZodError): string {
|
||||
return error.issues
|
||||
@@ -19,6 +21,7 @@ function formatConfigValidationError(error: ZodError): string {
|
||||
}
|
||||
|
||||
export function readConfigFile(): PaperclipConfig | null {
|
||||
if (isCloudManagedInstance()) return toPaperclipConfig(getCloudRuntimeConfigSnapshot().config);
|
||||
const configPath = resolvePaperclipConfigPath();
|
||||
|
||||
if (!fs.existsSync(configPath)) return null;
|
||||
|
||||
+46
-22
@@ -1,4 +1,7 @@
|
||||
import { readConfigFile, resolveDeploymentMode } from "./config-file.js";
|
||||
import { isCloudManagedInstance } from "./services/cloud-instance.js";
|
||||
import { StartupRefusalError } from "./startup-refusals.js";
|
||||
import type { PaperclipConfig } from "@paperclipai/shared";
|
||||
import { parseChatWebhookPublicBaseUrl } from "./chat-webhook-public-url.js";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, realpathSync } from "node:fs";
|
||||
@@ -32,27 +35,29 @@ import {
|
||||
resolveHomeAwarePath,
|
||||
} from "./home-paths.js";
|
||||
|
||||
const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath();
|
||||
if (existsSync(PAPERCLIP_ENV_FILE_PATH)) {
|
||||
loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true });
|
||||
}
|
||||
if (!isCloudManagedInstance()) {
|
||||
const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath();
|
||||
if (existsSync(PAPERCLIP_ENV_FILE_PATH)) {
|
||||
loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true });
|
||||
}
|
||||
|
||||
const CWD_ENV_PATH = resolve(process.cwd(), ".env");
|
||||
const cwdEnvExists = existsSync(CWD_ENV_PATH);
|
||||
const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH)
|
||||
? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH)
|
||||
: CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH;
|
||||
if (shouldLoadWorkingDirectoryEnv({
|
||||
cwdEnvExists,
|
||||
isPaperclipEnvFile: isSameFile,
|
||||
})) {
|
||||
loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true });
|
||||
}
|
||||
const CWD_ENV_PATH = resolve(process.cwd(), ".env");
|
||||
const cwdEnvExists = existsSync(CWD_ENV_PATH);
|
||||
const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH)
|
||||
? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH)
|
||||
: CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH;
|
||||
if (shouldLoadWorkingDirectoryEnv({
|
||||
cwdEnvExists,
|
||||
isPaperclipEnvFile: isSameFile,
|
||||
})) {
|
||||
loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true });
|
||||
}
|
||||
|
||||
if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") {
|
||||
applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH)));
|
||||
if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") {
|
||||
applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH)));
|
||||
}
|
||||
maybeRepairLegacyWorktreeConfigAndEnvFiles();
|
||||
}
|
||||
maybeRepairLegacyWorktreeConfigAndEnvFiles();
|
||||
|
||||
const TAILSCALE_DETECT_TIMEOUT_MS = 3000;
|
||||
|
||||
@@ -147,13 +152,12 @@ function detectTailnetBindHost(): string | undefined {
|
||||
}
|
||||
}
|
||||
|
||||
export function loadConfig(): Config {
|
||||
const fileConfig = readConfigFile();
|
||||
export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()): Config {
|
||||
const fileDatabaseMode =
|
||||
(fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode;
|
||||
(isCloudManagedInstance() || fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode;
|
||||
|
||||
const fileDbUrl =
|
||||
fileDatabaseMode === "postgres"
|
||||
!isCloudManagedInstance() && fileDatabaseMode === "postgres"
|
||||
? fileConfig?.database.connectionString
|
||||
: undefined;
|
||||
const fileDatabaseBackup = fileConfig?.database.backup;
|
||||
@@ -460,3 +464,23 @@ export function loadConfig(): Config {
|
||||
announcementsFeedUrl: process.env.PAPERCLIP_ANNOUNCEMENTS_FEED_URL?.trim() || "https://pages.paperclip.ing/announcements/v1/current.json",
|
||||
};
|
||||
}
|
||||
|
||||
export type CloudBootstrapConfig = {
|
||||
databaseMode: "postgres";
|
||||
databaseUrl: string;
|
||||
databaseMigrationUrl: string | undefined;
|
||||
};
|
||||
|
||||
export function loadCloudBootstrapConfig(): CloudBootstrapConfig {
|
||||
if (!isCloudManagedInstance()) throw new Error("Cloud bootstrap configuration requires a managed instance");
|
||||
const url = process.env.DATABASE_URL?.trim();
|
||||
if (!url) throw new StartupRefusalError("database-contract-unmet", "Cloud bootstrap requires DATABASE_URL from the environment");
|
||||
const migrationUrl = process.env.DATABASE_MIGRATION_URL?.trim() || undefined;
|
||||
for (const [name, value] of [["DATABASE_URL", url], ["DATABASE_MIGRATION_URL", migrationUrl]]) {
|
||||
if (!value) continue;
|
||||
let parsed: URL;
|
||||
try { parsed = new URL(value); } catch { throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`); }
|
||||
if (!["postgres:", "postgresql:"].includes(parsed.protocol)) throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`);
|
||||
}
|
||||
return { databaseMode: "postgres", databaseUrl: url, databaseMigrationUrl: migrationUrl };
|
||||
}
|
||||
+42
-21
@@ -104,7 +104,8 @@ import {
|
||||
} from "@paperclipai/db";
|
||||
import detectPort from "detect-port";
|
||||
import { createApp } from "./app.js";
|
||||
import { loadConfig } from "./config.js";
|
||||
import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js";
|
||||
import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js";
|
||||
import { logger } from "./middleware/logger.js";
|
||||
import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js";
|
||||
import { setStartupRecoveryPhase } from "./startup-recovery-state.js";
|
||||
@@ -186,6 +187,9 @@ import {
|
||||
loadWithoutCoordinatedShutdownSignalHooks,
|
||||
} from "./shutdown.js";
|
||||
import { initializeCloudRuntimeIdentity } from "./services/cloud-runtime-identity.js";
|
||||
import { initializeAdapterSettings } from "./services/adapter-settings.js";
|
||||
import { migrateCloudConfiguration } from "./services/cloud-configuration-migration.js";
|
||||
import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js";
|
||||
import { systemdNotify } from "./services/systemd-notify.js";
|
||||
import { flushInFlightRunLogMirrors } from "./services/run-log-store.js";
|
||||
import {
|
||||
@@ -251,6 +255,19 @@ async function endDatabaseClient(client: unknown, timeoutSeconds: number): Promi
|
||||
await sql.end({ timeout: timeoutSeconds });
|
||||
}
|
||||
|
||||
function initializeRuntimeConfigConsumers(config: ReturnType<typeof loadConfig>): void {
|
||||
initTelemetry({ enabled: config.telemetryEnabled });
|
||||
if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
|
||||
}
|
||||
if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false";
|
||||
}
|
||||
if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath;
|
||||
}
|
||||
}
|
||||
|
||||
export async function startServer(): Promise<StartedServer> {
|
||||
const startupDatabase: StartupDatabaseTeardown = { close: null };
|
||||
try {
|
||||
@@ -278,17 +295,8 @@ async function startServerWithDatabaseTeardown(
|
||||
// sentry.ts.
|
||||
await sentryReady;
|
||||
ensureDecisionSigningSecret();
|
||||
let config = loadConfig();
|
||||
initTelemetry({ enabled: config.telemetryEnabled });
|
||||
if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
|
||||
}
|
||||
if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false";
|
||||
}
|
||||
if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) {
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath;
|
||||
}
|
||||
const startupConfig = isCloudManagedInstance() ? loadCloudBootstrapConfig() : loadConfig();
|
||||
if ("deploymentMode" in startupConfig) initializeRuntimeConfigConsumers(startupConfig);
|
||||
|
||||
type MigrationSummary =
|
||||
| "skipped"
|
||||
@@ -390,7 +398,7 @@ async function startServerWithDatabaseTeardown(
|
||||
}
|
||||
}
|
||||
|
||||
function assertCloudDatabaseContract(): void {
|
||||
function assertCloudDatabaseContract(config: Config): void {
|
||||
if (config.deploymentMode !== "authenticated" || config.deploymentExposure !== "public") {
|
||||
return;
|
||||
}
|
||||
@@ -483,17 +491,19 @@ async function startServerWithDatabaseTeardown(
|
||||
let startupDbInfo:
|
||||
| { mode: "external-postgres"; connectionString: string }
|
||||
| { mode: "embedded-postgres"; dataDir: string; port: number };
|
||||
assertCloudDatabaseContract();
|
||||
if (config.databaseUrl) {
|
||||
const migrationUrl = config.databaseMigrationUrl ?? config.databaseUrl;
|
||||
if ("deploymentMode" in startupConfig) assertCloudDatabaseContract(startupConfig);
|
||||
if (startupConfig.databaseUrl) {
|
||||
const migrationUrl = startupConfig.databaseMigrationUrl ?? startupConfig.databaseUrl;
|
||||
migrationSummary = await ensureMigrations(migrationUrl, "PostgreSQL");
|
||||
|
||||
db = createDb(config.databaseUrl);
|
||||
pluginMigrationDb = config.databaseMigrationUrl ? createDb(config.databaseMigrationUrl) : db;
|
||||
|
||||
db = createDb(startupConfig.databaseUrl);
|
||||
pluginMigrationDb = startupConfig.databaseMigrationUrl ? createDb(startupConfig.databaseMigrationUrl) : db;
|
||||
logger.info("Using external PostgreSQL via DATABASE_URL/config");
|
||||
activeDatabaseConnectionString = config.databaseUrl;
|
||||
startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
|
||||
activeDatabaseConnectionString = startupConfig.databaseUrl;
|
||||
startupDbInfo = { mode: "external-postgres", connectionString: startupConfig.databaseUrl };
|
||||
} else {
|
||||
if (!("deploymentMode" in startupConfig)) throw new Error("Cloud bootstrap requires an external PostgreSQL connection");
|
||||
const config = startupConfig;
|
||||
const moduleName = "embedded-postgres";
|
||||
let EmbeddedPostgres: EmbeddedPostgresCtor;
|
||||
try {
|
||||
@@ -711,12 +721,19 @@ async function startServerWithDatabaseTeardown(
|
||||
};
|
||||
startupDatabase.close = closeDatabaseClients;
|
||||
setSentryRunLinkResolver(createSentryRunLinkResolver(db));
|
||||
await migrateCloudConfiguration(db as any);
|
||||
await initializeAdapterSettings(db as any);
|
||||
|
||||
await initializeCloudRuntimeConfig(db as any);
|
||||
let config = "deploymentMode" in startupConfig ? startupConfig : loadConfig();
|
||||
|
||||
// A claimed warm-pool stack may restart while its provider environment still
|
||||
// names the pool host. Restore the signed, durable identity before Better
|
||||
// Auth, routes, or child-runtime configuration capture any public URL.
|
||||
const restoredCloudRuntimeIdentity = await initializeCloudRuntimeIdentity(db as any);
|
||||
if (restoredCloudRuntimeIdentity) config = loadConfig();
|
||||
if (isCloudManagedInstance()) initializeRuntimeConfigConsumers(config);
|
||||
|
||||
const isWarmStandby = await createCloudWarmStandby(db as any);
|
||||
|
||||
if (config.deploymentMode === "local_trusted" && !isLoopbackHost(config.host)) {
|
||||
@@ -963,6 +980,10 @@ async function startServerWithDatabaseTeardown(
|
||||
// document parsed fail-closed above (`plugins.autoInstall`). Absent env means
|
||||
// self-hosted: createApp falls back to its built-in kubernetes-only default.
|
||||
const managedPluginAutoInstall = managedConfig?.plugins.autoInstall ?? null;
|
||||
const cloudQualification = await qualifyCloudConfigurationForCurrentImage(db as any);
|
||||
if (cloudQualification && !cloudQualification.configurationPackagesEligible) {
|
||||
logger.warn({ scope: cloudQualification.scope, blockerCodes: cloudQualification.blockers.map((blocker) => blocker.code) }, "Cloud configuration and package qualification has blockers");
|
||||
}
|
||||
const app = await createApp(db as any, {
|
||||
prepareIdleDatabaseBackup: idleDatabaseBackupEnabled
|
||||
? async () => (await runServerDatabaseBackup("idle")) !== null
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolveDefaultConfigPath } from "./home-paths.js";
|
||||
import { isCloudManagedInstance } from "./services/cloud-instance.js";
|
||||
|
||||
const PAPERCLIP_CONFIG_BASENAME = "config.json";
|
||||
const PAPERCLIP_ENV_FILENAME = ".env";
|
||||
@@ -26,6 +27,7 @@ function findConfigFileFromAncestors(startDir: string): string | null {
|
||||
export function resolvePaperclipConfigPath(overridePath?: string): string {
|
||||
if (overridePath) return path.resolve(overridePath);
|
||||
if (process.env.PAPERCLIP_CONFIG) return path.resolve(process.env.PAPERCLIP_CONFIG);
|
||||
if (isCloudManagedInstance()) return resolveDefaultConfigPath();
|
||||
return findConfigFileFromAncestors(process.cwd()) ?? resolveDefaultConfigPath();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { createDb } from "@paperclipai/db";
|
||||
import { cloudConfigurationMigrationService } from "./services/cloud-configuration-migration.js";
|
||||
import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js";
|
||||
|
||||
export async function runCloudConfigurationQualification(args: string[], env: NodeJS.ProcessEnv = process.env): Promise<number> {
|
||||
const allowed = new Set(["--migrate", "--expected-package-evidence", "--recheck-external-revision"]);
|
||||
let migrate = false;
|
||||
let expectedPackageEvidenceDigest: string | undefined;
|
||||
let recheckRevision: number | undefined;
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const argument = args[index];
|
||||
if (!allowed.has(argument)) throw new Error(`Unknown qualification argument: ${argument}`);
|
||||
if (argument === "--migrate") migrate = true;
|
||||
if (argument === "--expected-package-evidence") {
|
||||
expectedPackageEvidenceDigest = args[++index];
|
||||
if (!expectedPackageEvidenceDigest || !/^sha256:[a-f0-9]{64}$/.test(expectedPackageEvidenceDigest)) throw new Error("A SHA-256 package evidence digest is required");
|
||||
}
|
||||
if (argument === "--recheck-external-revision") {
|
||||
const value = args[++index];
|
||||
if (!value || !/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error("A nonnegative receipt revision is required");
|
||||
recheckRevision = Number(value);
|
||||
}
|
||||
}
|
||||
const url = env.DATABASE_URL?.trim();
|
||||
if (!url) throw new Error("DATABASE_URL is required for Cloud configuration qualification");
|
||||
const db = createDb(url);
|
||||
try {
|
||||
const service = cloudConfigurationMigrationService(db, { runtimeEnv: env });
|
||||
if (migrate) await service.migrate();
|
||||
if (recheckRevision !== undefined) await service.recheckLegacyExternalRegistrations(recheckRevision);
|
||||
const report = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv: env, expectedPackageEvidenceDigest });
|
||||
if (!report) throw new Error("Cloud configuration qualification requires a managed instance");
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
return report.configurationPackagesEligible && (!expectedPackageEvidenceDigest || report.samePackageOverlapEligible) ? 0 : 2;
|
||||
} finally {
|
||||
await db.$client.end({ timeout: 5 });
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
|
||||
runCloudConfigurationQualification(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : "Cloud configuration qualification failed"}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -19,11 +19,13 @@ import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { Router } from "express";
|
||||
import { companies, type Db } from "@paperclipai/db";
|
||||
import { adapterSettingsService } from "../services/adapter-settings.js";
|
||||
import { logActivity, publishActivity, type ActivityPublication } from "../services/activity-log.js";
|
||||
import {
|
||||
listServerAdapters,
|
||||
findServerAdapter,
|
||||
findActiveServerAdapter,
|
||||
listEnabledServerAdapters,
|
||||
registerServerAdapter,
|
||||
resolveExternalAdapterRegistration,
|
||||
unregisterServerAdapter,
|
||||
@@ -37,8 +39,6 @@ import {
|
||||
removeAdapterPlugin,
|
||||
getAdapterPluginByType,
|
||||
getAdapterPluginsDir,
|
||||
getDisabledAdapterTypes,
|
||||
setAdapterDisabled,
|
||||
} from "../services/adapter-plugin-store.js";
|
||||
import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js";
|
||||
import type { ServerAdapterModule, AdapterConfigSchema } from "../adapters/types.js";
|
||||
@@ -49,9 +49,9 @@ import type {
|
||||
import { loadExternalAdapterPackage, getUiParserSource, getOrExtractUiParserSource, reloadExternalAdapter } from "../adapters/plugin-loader.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { badRequest, forbidden } from "../errors.js";
|
||||
import { isCloudManagedInstance } from "../services/cloud-instance.js";
|
||||
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
|
||||
import { getHiddenSettings } from "../services/settings-visibility.js";
|
||||
import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin } from "./authz.js";
|
||||
import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin, getActorInfo } from "./authz.js";
|
||||
import { BUILTIN_ADAPTER_TYPES } from "../adapters/builtin-adapter-types.js";
|
||||
import { agentRunnerAvailability, type RunnerTarget } from "../services/agent-runner-selection.js";
|
||||
import type { AgentRunnerAvailability } from "@paperclipai/shared";
|
||||
@@ -60,22 +60,6 @@ import { resolveAntigravityCandidateAdmission } from "../services/native-runtime
|
||||
const execFileAsync = promisify(execFile);
|
||||
let warnedMalformedAntigravityAdmission = false;
|
||||
|
||||
/**
|
||||
* Floor: on cloud-managed instances adapter code is bundled into the platform
|
||||
* image; fetching and loading external adapter packages at runtime stays off
|
||||
* for every actor, including instance admins. Adapter code executes in the
|
||||
* server process, so a runtime install would let an instance admin read the
|
||||
* platform trust anchors from the process environment (mirrors the
|
||||
* bundled-only plugin install floor in plugin-install-guard.ts).
|
||||
*/
|
||||
function assertAdapterCodeInstallAllowed() {
|
||||
if (isCloudManagedInstance()) {
|
||||
throw forbidden("Adapter installation is platform-managed on cloud-managed instances", {
|
||||
code: "adapter_install_platform_managed",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Floor: when the hosting operator hides the Adapters settings surface
|
||||
* (`instance.adapters` in PAPERCLIP_HIDDEN_SETTINGS), adapter management
|
||||
@@ -209,7 +193,7 @@ function buildAdapterInfo(adapter: ServerAdapterModule, externalRecord: AdapterP
|
||||
loaded: true, // If it's in the registry, it's loaded
|
||||
disabled: disabledSet.has(adapter.type),
|
||||
capabilities: buildAdapterCapabilities(adapter),
|
||||
...agentRunnerAvailability(adapter.type, target),
|
||||
...agentRunnerAvailability(adapter.type, target, [...disabledSet]),
|
||||
...(adapter.acp ? { acp: adapter.acp } : {}),
|
||||
overriddenBuiltin: externalRecord ? BUILTIN_ADAPTER_TYPES.has(adapter.type) : undefined,
|
||||
overridePaused: BUILTIN_ADAPTER_TYPES.has(adapter.type) ? isOverridePaused(adapter.type) : undefined,
|
||||
@@ -265,11 +249,13 @@ function registerWithSessionManagement(adapter: ServerAdapterModule): void {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function adapterRoutes(options: {
|
||||
db?: Db;
|
||||
getNativeRunnerEnabled?: () => Promise<boolean>;
|
||||
getOpenAiDotEnabled?: () => Promise<boolean>;
|
||||
resolveRunnerTarget?: (companyId: string, environmentId?: string) => Promise<RunnerTarget>;
|
||||
} = {}) {
|
||||
const router = Router();
|
||||
const settings = adapterSettingsService(options.db);
|
||||
|
||||
/**
|
||||
* GET /api/adapters
|
||||
@@ -303,7 +289,7 @@ export function adapterRoutes(options: {
|
||||
const externalRecords = new Map(
|
||||
listAdapterPlugins().map((r) => [r.type, r]),
|
||||
);
|
||||
const disabledSet = new Set(getDisabledAdapterTypes());
|
||||
const disabledSet = new Set(await settings.getDisabledTypes());
|
||||
// One shared implementation, with independent provider rollouts. Explicit
|
||||
// adapter-admin disabling still applies to both choices.
|
||||
|
||||
@@ -343,7 +329,7 @@ export function adapterRoutes(options: {
|
||||
*/
|
||||
router.post("/adapters/install", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertAdapterCodeInstallAllowed();
|
||||
assertExternalAdapterCodeAllowed();
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
const { packageName, isLocalPath = false, version } = req.body as AdapterInstallRequest;
|
||||
@@ -477,7 +463,7 @@ export function adapterRoutes(options: {
|
||||
}
|
||||
|
||||
const externalRecord = getAdapterPluginByType(adapterType);
|
||||
const disabledSet = new Set(getDisabledAdapterTypes());
|
||||
const disabledSet = new Set(await settings.getDisabledTypes());
|
||||
res.json(buildAdapterInfo(adapter, externalRecord, disabledSet));
|
||||
});
|
||||
|
||||
@@ -509,7 +495,28 @@ export function adapterRoutes(options: {
|
||||
return;
|
||||
}
|
||||
|
||||
const changed = setAdapterDisabled(adapterType, disabled);
|
||||
const publications: ActivityPublication[] = [];
|
||||
const changed = await settings.setDisabled(adapterType, disabled, async (tx, current) => {
|
||||
const actor = getActorInfo(req);
|
||||
const companyIds = await tx.select({ id: companies.id }).from(companies);
|
||||
for (const company of companyIds) {
|
||||
await logActivity(tx as Db, {
|
||||
companyId: company.id,
|
||||
...actor,
|
||||
action: "instance.adapter.updated",
|
||||
entityType: "instance_settings",
|
||||
entityId: "adapter-settings:v1",
|
||||
details: { adapterType, disabled, revision: current.revision },
|
||||
}, publications);
|
||||
}
|
||||
});
|
||||
for (const publication of publications) {
|
||||
try {
|
||||
publishActivity(publication);
|
||||
} catch (error) {
|
||||
logger.error({ err: error, companyId: publication.companyId }, "failed to publish adapter settings activity");
|
||||
}
|
||||
}
|
||||
|
||||
if (changed) {
|
||||
logger.info({ type: adapterType, disabled }, "Adapter enabled/disabled");
|
||||
@@ -528,6 +535,7 @@ export function adapterRoutes(options: {
|
||||
*/
|
||||
router.patch("/adapters/:type/override", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertExternalAdapterCodeAllowed();
|
||||
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
@@ -558,6 +566,7 @@ export function adapterRoutes(options: {
|
||||
*/
|
||||
router.delete("/adapters/:type", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertExternalAdapterCodeAllowed();
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
const adapterType = req.params.type;
|
||||
@@ -635,6 +644,7 @@ export function adapterRoutes(options: {
|
||||
*/
|
||||
router.post("/adapters/:type/reload", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertExternalAdapterCodeAllowed();
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
const type = req.params.type;
|
||||
@@ -688,7 +698,7 @@ export function adapterRoutes(options: {
|
||||
// package name, but without the risk of losing the store record.
|
||||
router.post("/adapters/:type/reinstall", async (req, res) => {
|
||||
assertInstanceAdmin(req);
|
||||
assertAdapterCodeInstallAllowed();
|
||||
assertExternalAdapterCodeAllowed();
|
||||
assertAdapterManagementVisible();
|
||||
|
||||
const type = req.params.type;
|
||||
|
||||
@@ -154,7 +154,7 @@ import type {
|
||||
} from "@paperclipai/adapter-utils";
|
||||
import { evaluateCodexCredentialReadiness } from "@paperclipai/adapter-codex-local/server";
|
||||
import type { AdapterAuthSignal, AdapterAuthSignalResponse, CodexAccountBindingClaim } from "@paperclipai/shared";
|
||||
import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js";
|
||||
import { adapterSettingsService } from "../services/adapter-settings.js";
|
||||
import { skillVersionSelectionMap } from "../services/runtime-skill-selections.js";
|
||||
import { isFixedClaudeOAuthBinding, secretService } from "../services/secrets.js";
|
||||
import { authorizationDeniedDetails } from "../services/authorization.js";
|
||||
@@ -1859,15 +1859,6 @@ export function agentRoutes(
|
||||
* Adapter validation for the paths that CHOOSE a harness for a new agent
|
||||
* (hire + create), as opposed to the paths that operate on an existing one.
|
||||
*
|
||||
* A disabled adapter is one this instance cannot run — most often because a
|
||||
* declarative registry (PAPERCLIP_ADAPTERS) curated it out, which
|
||||
* reconcileAdapterAvailability turns into a disabled type at boot. Registered
|
||||
* but disabled still passes assertKnownAdapterType, so an agent could be
|
||||
* created on it and then fail EVERY run at lease time with
|
||||
* `Adapter "..." is not in the configured adapter registry` — an error that
|
||||
* arrives minutes later, in a run log, with no way back to the choice that
|
||||
* caused it. Refuse at selection time instead, and name what can be chosen.
|
||||
*
|
||||
* Existing agents on a now-disabled adapter are deliberately untouched
|
||||
* (listEnabledServerAdapters documents the same rule: hidden from selection,
|
||||
* still functional for agents that already use them).
|
||||
@@ -1902,7 +1893,7 @@ export function agentRoutes(
|
||||
);
|
||||
}
|
||||
}
|
||||
const disabled = new Set(getDisabledAdapterTypes());
|
||||
const disabled = new Set(await adapterSettingsService(db).getDisabledTypes());
|
||||
if (!disabled.has(adapterType)) return adapterType;
|
||||
const available = listServerAdapters()
|
||||
.map((a) => a.type)
|
||||
|
||||
@@ -32,6 +32,7 @@ import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import crypto from "node:crypto";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { PluginRecord } from "@paperclipai/shared";
|
||||
import { pluginRegistryService } from "../services/plugin-registry.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { assertCompanyAccess } from "./authz.js";
|
||||
@@ -192,6 +193,7 @@ export interface PluginUiStaticRouteOptions {
|
||||
* Defaults to the standard `~/.paperclip/plugins/` location.
|
||||
*/
|
||||
localPluginDir: string;
|
||||
resolvePackageRoot?: (plugin: PluginRecord) => string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -304,7 +306,7 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions
|
||||
|
||||
if (typeof devUiUrl === "string" && devUiUrl.length > 0) {
|
||||
// Dev proxy is only available in development mode
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
if (options.resolvePackageRoot || process.env.NODE_ENV === "production") {
|
||||
log.warn(
|
||||
{ pluginId: plugin.id },
|
||||
"plugin-ui-static: devUiUrl ignored in production",
|
||||
@@ -405,12 +407,15 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions
|
||||
}
|
||||
|
||||
// Step 3: Resolve the plugin's UI directory
|
||||
const uiDir = resolvePluginUiDir(
|
||||
options.localPluginDir,
|
||||
plugin.packageName,
|
||||
manifest.entrypoints.ui,
|
||||
plugin.packagePath,
|
||||
);
|
||||
let uiDir: string | null;
|
||||
try {
|
||||
uiDir = options.resolvePackageRoot
|
||||
? path.resolve(options.resolvePackageRoot(plugin), manifest.entrypoints.ui)
|
||||
: resolvePluginUiDir(options.localPluginDir, plugin.packageName, manifest.entrypoints.ui, plugin.packagePath);
|
||||
} catch {
|
||||
res.status(503).json({ error: "Plugin package is unavailable in the current image" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!uiDir) {
|
||||
log.warn(
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import { forbidden } from "../errors.js";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
|
||||
export function assertExternalAdapterCodeAllowed(): void {
|
||||
if (isCloudManagedInstance()) {
|
||||
throw forbidden("Adapter installation is platform-managed on cloud-managed instances", {
|
||||
code: "adapter_install_platform_managed",
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,8 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolvePaperclipHomeDir } from "../home-paths.js";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
import { assertExternalAdapterCodeAllowed } from "./adapter-code-policy.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@@ -43,8 +45,8 @@ interface AdapterSettings {
|
||||
// Paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function adapterPluginPaths() {
|
||||
const paperclipDir = resolvePaperclipHomeDir();
|
||||
function adapterPluginPaths(homeOverride?: string) {
|
||||
const paperclipDir = resolvePaperclipHomeDir(homeOverride);
|
||||
return {
|
||||
adapterPluginsDir: path.join(paperclipDir, "adapter-plugins"),
|
||||
adapterPluginsStorePath: path.join(paperclipDir, "adapter-plugins.json"),
|
||||
@@ -79,6 +81,7 @@ function ensureDirs(): string {
|
||||
}
|
||||
|
||||
function readStore(): AdapterPluginRecord[] {
|
||||
if (isCloudManagedInstance()) return [];
|
||||
const { adapterPluginsStorePath } = adapterPluginPaths();
|
||||
if (storeCache?.path === adapterPluginsStorePath) return storeCache.records;
|
||||
try {
|
||||
@@ -130,11 +133,16 @@ function writeSettings(settings: AdapterSettings): void {
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function getAdapterPluginStorePath(homeOverride?: string): string {
|
||||
return adapterPluginPaths(homeOverride).adapterPluginsStorePath;
|
||||
}
|
||||
|
||||
export function listAdapterPlugins(): AdapterPluginRecord[] {
|
||||
return readStore();
|
||||
}
|
||||
|
||||
export function addAdapterPlugin(record: AdapterPluginRecord): void {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
const store = [...readStore()];
|
||||
const idx = store.findIndex((r) => r.type === record.type);
|
||||
if (idx >= 0) {
|
||||
@@ -146,6 +154,7 @@ export function addAdapterPlugin(record: AdapterPluginRecord): void {
|
||||
}
|
||||
|
||||
export function removeAdapterPlugin(type: string): boolean {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
const store = [...readStore()];
|
||||
const idx = store.findIndex((r) => r.type === type);
|
||||
if (idx < 0) return false;
|
||||
@@ -159,6 +168,7 @@ export function getAdapterPluginByType(type: string): AdapterPluginRecord | unde
|
||||
}
|
||||
|
||||
export function getAdapterPluginsDir(): string {
|
||||
assertExternalAdapterCodeAllowed();
|
||||
return ensureDirs();
|
||||
}
|
||||
|
||||
|
||||
@@ -32,8 +32,7 @@ describe("reconcileAdapterAvailability", () => {
|
||||
]);
|
||||
expect(result.enabled).toEqual(["opencode_local"]);
|
||||
expect(result.disabled.sort()).toEqual(["claude_local", "pi_local"]);
|
||||
expect(setAdapterDisabled).toHaveBeenCalledWith("claude_local", true);
|
||||
expect(setAdapterDisabled).toHaveBeenCalledWith("opencode_local", false);
|
||||
expect(setAdapterDisabled).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("throws when a declared adapter has no installed implementation", () => {
|
||||
|
||||
@@ -1,73 +1,8 @@
|
||||
/**
|
||||
* Declarative adapter-registry bootstrap.
|
||||
*
|
||||
* One source (`PAPERCLIP_ADAPTERS` inline JSON, or `PAPERCLIP_ADAPTERS_FILE` a
|
||||
* path to a JSON file) feeds two consumers:
|
||||
* 1. Availability: reconcile the file-backed disabled-set so the picker shows
|
||||
* exactly the declared, enabled set (runs for any instance).
|
||||
* 2. k8s runtime: the same registry rides on the Kubernetes environment config
|
||||
* (see execution-policy-bootstrap) so the plugin resolves runtime
|
||||
* image/envKeys/allowFqdns/probe/defaultEnv from it.
|
||||
*
|
||||
* Parsing is pure + fails loud on malformed/invalid config, mirroring
|
||||
* execution-policy-bootstrap.
|
||||
*/
|
||||
import fs from "node:fs";
|
||||
import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared";
|
||||
import type { AdapterRegistryEntryParsed } from "@paperclipai/shared";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { listServerAdapters } from "../adapters/registry.js";
|
||||
import { setAdapterDisabled } from "./adapter-plugin-store.js";
|
||||
export { parseAdapterRegistryEnv, type AdapterRegistryEnv } from "./adapter-registry-config.js";
|
||||
|
||||
export type AdapterRegistryEnv = Record<string, string | undefined>;
|
||||
|
||||
/**
|
||||
* Parse the declarative registry from env. Returns null when unconfigured
|
||||
* (built-in defaults; local/OSS unchanged). Throws on malformed/invalid input.
|
||||
*/
|
||||
export function parseAdapterRegistryEnv(
|
||||
env: AdapterRegistryEnv = process.env,
|
||||
): AdapterRegistryEntryParsed[] | null {
|
||||
const inline = env.PAPERCLIP_ADAPTERS?.trim();
|
||||
const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim();
|
||||
if (!inline && !filePath) return null;
|
||||
|
||||
let rawText: string;
|
||||
if (inline) {
|
||||
rawText = inline;
|
||||
} else {
|
||||
try {
|
||||
rawText = fs.readFileSync(filePath as string, "utf-8");
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(rawText);
|
||||
} catch (err) {
|
||||
throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
const result = adapterRegistrySchema.safeParse(parsed);
|
||||
if (!result.success) {
|
||||
throw new Error(
|
||||
`PAPERCLIP_ADAPTERS failed validation: ${result.error.issues
|
||||
.map((i) => `${i.path.join(".")}: ${i.message}`)
|
||||
.join("; ")}`,
|
||||
);
|
||||
}
|
||||
return result.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile availability: every known server adapter NOT enabled in the declared
|
||||
* registry is disabled; declared+enabled ones are enabled. Throws if a declared
|
||||
* adapterType has no installed adapter (cannot offer a harness with no
|
||||
* implementation). No-op when `registry` is null.
|
||||
*/
|
||||
export function reconcileAdapterAvailability(
|
||||
registry: AdapterRegistryEntryParsed[] | null,
|
||||
): { enabled: string[]; disabled: string[] } {
|
||||
@@ -88,10 +23,9 @@ export function reconcileAdapterAvailability(
|
||||
for (const type of knownTypes) {
|
||||
const entry = declared.get(type);
|
||||
const shouldEnable = entry !== undefined && entry.enabled !== false;
|
||||
setAdapterDisabled(type, !shouldEnable);
|
||||
(shouldEnable ? enabled : disabled).push(type);
|
||||
}
|
||||
|
||||
logger.info({ enabled, disabled }, "reconciled adapter availability from PAPERCLIP_ADAPTERS");
|
||||
logger.info({ enabled, disabled }, "validated adapter availability from PAPERCLIP_ADAPTERS");
|
||||
return { enabled, disabled };
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import fs from "node:fs";
|
||||
import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared";
|
||||
|
||||
export type AdapterRegistryEnv = Record<string, string | undefined>;
|
||||
|
||||
let cached: { inline: string | undefined; filePath: string | undefined; registry: AdapterRegistryEntryParsed[] } | undefined;
|
||||
|
||||
export function parseAdapterRegistryEnv(
|
||||
env: AdapterRegistryEnv = process.env,
|
||||
): AdapterRegistryEntryParsed[] | null {
|
||||
const inline = env.PAPERCLIP_ADAPTERS?.trim();
|
||||
const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim();
|
||||
if (!inline && !filePath) return null;
|
||||
if (cached && cached.inline === inline && cached.filePath === filePath) return structuredClone(cached.registry);
|
||||
|
||||
let rawText: string;
|
||||
if (inline) {
|
||||
rawText = inline;
|
||||
} else {
|
||||
try {
|
||||
rawText = fs.readFileSync(filePath as string, "utf-8");
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(rawText);
|
||||
} catch (err) {
|
||||
throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
const result = adapterRegistrySchema.safeParse(parsed);
|
||||
if (!result.success) {
|
||||
throw new Error(
|
||||
`PAPERCLIP_ADAPTERS failed validation: ${result.error.issues
|
||||
.map((i) => `${i.path.join(".")}: ${i.message}`)
|
||||
.join("; ")}`,
|
||||
);
|
||||
}
|
||||
cached = { inline, filePath, registry: result.data };
|
||||
return structuredClone(result.data);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("../adapters/registry.js", () => ({ listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }] }));
|
||||
import { adapterSettingsService, type AdapterSettingsDb } from "./adapter-settings.js";
|
||||
import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js";
|
||||
import { parseAdapterRegistryEnv } from "./adapter-registry-config.js";
|
||||
|
||||
describe("local adapter settings compatibility", () => {
|
||||
let directory: string;
|
||||
beforeEach(() => {
|
||||
directory = fs.mkdtempSync(path.join(os.tmpdir(), "adapter-settings-local-"));
|
||||
vi.stubEnv("PAPERCLIP_HOME", directory);
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
|
||||
vi.stubEnv("PAPERCLIP_ADAPTERS", undefined);
|
||||
vi.stubEnv("PAPERCLIP_ADAPTERS_FILE", undefined);
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("reads and writes the existing file without a database", async () => {
|
||||
fs.writeFileSync(path.join(directory, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local"] }));
|
||||
const service = adapterSettingsService(undefined);
|
||||
expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
|
||||
expect(await service.setDisabled("codex_local", true)).toBe(true);
|
||||
expect(getDisabledAdapterTypes()).toEqual(["claude_local", "codex_local"]);
|
||||
expect(setAdapterDisabled("claude_local", false)).toBe(true);
|
||||
expect(await service.getDisabledTypes()).toEqual(["codex_local"]);
|
||||
expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: ["codex_local"] });
|
||||
});
|
||||
|
||||
it("loads an operator file once and leaves the saved file intact", async () => {
|
||||
setAdapterDisabled("codex_local", true);
|
||||
const file = path.join(directory, "operator-adapters.json");
|
||||
fs.writeFileSync(file, JSON.stringify([{ adapterType: "codex_local", enabled: true }]));
|
||||
const env = { PAPERCLIP_ADAPTERS_FILE: file };
|
||||
parseAdapterRegistryEnv(env);
|
||||
const read = vi.spyOn(fs, "readFileSync");
|
||||
const service = adapterSettingsService(undefined, { runtimeEnv: env });
|
||||
expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
|
||||
expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
|
||||
expect(read).not.toHaveBeenCalled();
|
||||
expect(getDisabledAdapterTypes()).toEqual(["codex_local"]);
|
||||
});
|
||||
|
||||
it("restores the saved choice when its audit fails and audits a later retry", async () => {
|
||||
const transaction = vi.fn(async (callback: (tx: AdapterSettingsDb) => Promise<unknown>) => callback(db));
|
||||
const db = { transaction } as unknown as AdapterSettingsDb;
|
||||
const service = adapterSettingsService(db);
|
||||
const audit = vi.fn().mockRejectedValueOnce(new Error("audit unavailable")).mockResolvedValue(undefined);
|
||||
|
||||
await expect(service.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable");
|
||||
expect(await service.getDisabledTypes()).toEqual([]);
|
||||
expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: [] });
|
||||
expect(await service.setDisabled("codex_local", true, audit)).toBe(true);
|
||||
expect(await service.getDisabledTypes()).toEqual(["codex_local"]);
|
||||
expect(audit).toHaveBeenCalledTimes(2);
|
||||
expect(transaction).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps a failed audit rollback from overwriting a concurrent save", async () => {
|
||||
const db = { transaction: async (callback: (tx: AdapterSettingsDb) => Promise<unknown>) => callback(db) } as unknown as AdapterSettingsDb;
|
||||
const first = adapterSettingsService(db);
|
||||
const second = adapterSettingsService(db);
|
||||
let failAudit!: (reason: Error) => void;
|
||||
const auditWait = new Promise<void>((_resolve, reject) => { failAudit = reject; });
|
||||
const audit = vi.fn(() => auditWait);
|
||||
const rejected = expect(first.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable");
|
||||
await vi.waitFor(() => expect(audit).toHaveBeenCalledOnce());
|
||||
const nextAudit = vi.fn(async () => undefined);
|
||||
const next = second.setDisabled("codex_local", true, nextAudit);
|
||||
await Promise.resolve();
|
||||
expect(nextAudit).not.toHaveBeenCalled();
|
||||
|
||||
failAudit(new Error("audit unavailable"));
|
||||
await rejected;
|
||||
expect(await next).toBe(true);
|
||||
expect(await first.getDisabledTypes()).toEqual(["codex_local"]);
|
||||
expect(nextAudit).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,228 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { activityLog, applyPendingMigrations, companies, createDb, instanceSettings, type Db } from "@paperclipai/db";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
|
||||
const legacy = vi.hoisted(() => ({ get: vi.fn(() => [] as string[]), set: vi.fn() }));
|
||||
vi.mock("./adapter-plugin-store.js", () => ({
|
||||
getDisabledAdapterTypes: legacy.get,
|
||||
setAdapterDisabled: legacy.set,
|
||||
listAdapterPlugins: () => [],
|
||||
getAdapterPluginByType: () => undefined,
|
||||
}));
|
||||
vi.mock("../adapters/registry.js", () => ({
|
||||
listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }, { type: "paperclip_runner" }],
|
||||
findServerAdapter: (type: string) => ({ type }),
|
||||
findActiveServerAdapter: (type: string) => ({ type }),
|
||||
isOverridePaused: () => false,
|
||||
hasActiveAdapterOverride: () => false,
|
||||
waitForExternalAdapters: async () => {},
|
||||
}));
|
||||
vi.mock("../adapters/plugin-loader.js", () => ({}));
|
||||
import { adapterSettingsService, initializeAdapterSettings } from "./adapter-settings.js";
|
||||
import { adapterRoutes } from "../routes/adapters.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
import * as activities from "./activity-log.js";
|
||||
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_ADAPTER_SETTINGS_TEST_DATABASE_URL;
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("durable managed adapter settings", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let peer: Db;
|
||||
let directory: string;
|
||||
const managed = { runtimeEnv: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "synthetic-test-token" } };
|
||||
const service = () => adapterSettingsService(db, managed);
|
||||
|
||||
beforeAll(async () => {
|
||||
database = externalDatabaseUrl
|
||||
? { connectionString: externalDatabaseUrl, cleanup: async () => {} }
|
||||
: await startEmbeddedPostgresTestDatabase("adapter-settings-");
|
||||
if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl);
|
||||
db = createDb(database.connectionString);
|
||||
peer = createDb(database.connectionString);
|
||||
await db.insert(companies).values({ name: "Adapter settings test", issuePrefix: "AST" });
|
||||
}, 90_000);
|
||||
afterAll(async () => {
|
||||
await Promise.all([db?.$client.end({ timeout: 0 }), peer?.$client.end({ timeout: 0 })]);
|
||||
await database?.cleanup();
|
||||
}, 60_000);
|
||||
beforeEach(async () => {
|
||||
directory = await fs.mkdtemp(path.join(os.tmpdir(), "adapter-settings-import-"));
|
||||
await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, "adapter-settings:v1"));
|
||||
legacy.get.mockClear();
|
||||
legacy.set.mockClear();
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "synthetic-test-token");
|
||||
});
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
await fs.rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function app() {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = { type: "board", source: "local_implicit", userId: "local-board", isInstanceAdmin: true };
|
||||
next();
|
||||
});
|
||||
app.use(adapterRoutes({ db }));
|
||||
app.use(errorHandler);
|
||||
return app;
|
||||
}
|
||||
|
||||
it("persists across independent connections and reconstructed stores without file reads", async () => {
|
||||
await expect(service().getDisabledTypes()).rejects.toThrow("not been initialized");
|
||||
await service().seedIfAbsent({ disabledTypes: [] });
|
||||
expect(await service().setDisabled("codex_local", true)).toBe(true);
|
||||
expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]);
|
||||
await db.$client.end({ timeout: 0 });
|
||||
db = createDb(database.connectionString);
|
||||
expect(await service().inspectStored()).toEqual({ version: 1, revision: 1, disabledTypes: ["codex_local"] });
|
||||
expect(legacy.get).not.toHaveBeenCalled();
|
||||
expect(legacy.set).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("acknowledges the committed save when its live activity event fails", async () => {
|
||||
await service().seedIfAbsent({ disabledTypes: [] });
|
||||
const publication = vi.spyOn(activities, "publishActivity").mockImplementation(() => { throw new Error("event unavailable"); });
|
||||
const response = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(publication).toHaveBeenCalled();
|
||||
expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]);
|
||||
const events = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated"));
|
||||
expect(events.some((event) => event.details?.adapterType === "codex_local")).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves simultaneous first writes and concurrent removals from other connections", async () => {
|
||||
const writers = [service(), adapterSettingsService(peer, managed)];
|
||||
await Promise.all(Array.from({ length: 12 }, (_, index) => writers[index % 2].setDisabled(`adapter_${index}`, true)));
|
||||
expect((await service().inspectStored())?.disabledTypes).toHaveLength(12);
|
||||
await Promise.all([
|
||||
writers[0].setDisabled("adapter_0", false),
|
||||
writers[1].setDisabled("adapter_1", false),
|
||||
writers[0].setDisabled("extra", true),
|
||||
]);
|
||||
const stored = await service().inspectStored();
|
||||
expect(stored?.revision).toBe(15);
|
||||
expect(stored?.disabledTypes).not.toContain("adapter_0");
|
||||
expect(stored?.disabledTypes).not.toContain("adapter_1");
|
||||
expect(stored?.disabledTypes).toContain("extra");
|
||||
expect(await writers[1].setDisabled("extra", true)).toBe(false);
|
||||
expect((await service().inspectStored())?.revision).toBe(15);
|
||||
});
|
||||
|
||||
it("imports legacy choices once before admission and ignores stale files after the DB record exists", async () => {
|
||||
const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory };
|
||||
const file = path.join(directory, "adapter-settings.json");
|
||||
await fs.writeFile(file, JSON.stringify({ disabledTypes: ["codex_local", "future_adapter"] }));
|
||||
const results = await Promise.all([
|
||||
initializeAdapterSettings(db, { runtimeEnv }),
|
||||
initializeAdapterSettings(peer, { runtimeEnv }),
|
||||
]);
|
||||
expect(results.filter((result) => result?.seeded)).toHaveLength(1);
|
||||
expect(await service().getDisabledTypes()).toEqual(["codex_local", "future_adapter"]);
|
||||
await service().setDisabled("codex_local", false);
|
||||
await fs.writeFile(file, "invalid stale JSON");
|
||||
expect((await initializeAdapterSettings(peer, { runtimeEnv }))?.seeded).toBe(false);
|
||||
expect(await service().getDisabledTypes()).toEqual(["future_adapter"]);
|
||||
});
|
||||
|
||||
it("uses defaults only for an absent legacy file and rejects malformed or oversized input", async () => {
|
||||
const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory };
|
||||
const file = path.join(directory, "adapter-settings.json");
|
||||
await fs.writeFile(file, JSON.stringify({ disabledTypes: [42] }));
|
||||
await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("Persisted adapter settings are invalid");
|
||||
expect(await service().inspectStored()).toBeNull();
|
||||
await fs.writeFile(file, Buffer.alloc(65_537));
|
||||
await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("size limit");
|
||||
expect(await service().inspectStored()).toBeNull();
|
||||
await fs.unlink(file);
|
||||
expect((await initializeAdapterSettings(db, { runtimeEnv }))?.current.disabledTypes).toEqual([]);
|
||||
expect(await service().getDisabledTypes()).toEqual([]);
|
||||
});
|
||||
|
||||
it("composes atomic import and receipts with the caller transaction", async () => {
|
||||
await expect(db.transaction(async (tx) => {
|
||||
expect(await service().seedIfAbsent({ disabledTypes: ["codex_local"] }, tx)).toEqual({
|
||||
seeded: true, current: { version: 1, revision: 0, disabledTypes: ["codex_local"] },
|
||||
});
|
||||
throw new Error("receipt failed");
|
||||
})).rejects.toThrow("receipt failed");
|
||||
expect(await service().inspectStored()).toBeNull();
|
||||
await Promise.all([
|
||||
service().seedIfAbsent({ disabledTypes: ["codex_local"] }),
|
||||
adapterSettingsService(peer, managed).seedIfAbsent({ disabledTypes: ["claude_local"] }),
|
||||
]).then((results) => {
|
||||
expect(results.filter((result) => result.seeded)).toHaveLength(1);
|
||||
expect(results[0].current).toEqual(results[1].current);
|
||||
});
|
||||
});
|
||||
|
||||
it("applies operator restrictions without rewriting saved choices", async () => {
|
||||
await service().setDisabled("codex_local", true);
|
||||
const env = { ...managed.runtimeEnv, PAPERCLIP_ADAPTERS: JSON.stringify([{ adapterType: "codex_local", enabled: true }]) };
|
||||
const overlay = adapterSettingsService(peer, { runtimeEnv: env });
|
||||
expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]);
|
||||
expect(await overlay.inspectStored()).toEqual(await service().inspectStored());
|
||||
await expect(overlay.setDisabled("claude_local", false)).rejects.toThrow("hosting operator");
|
||||
await expect(overlay.setDisabled("codex_local", true)).rejects.toThrow("hosting operator");
|
||||
expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]);
|
||||
expect(await service().getDisabledTypes()).toEqual(["codex_local"]);
|
||||
});
|
||||
|
||||
it("refreshes the disabled harness at each new-agent admission", async () => {
|
||||
await service().seedIfAbsent({ disabledTypes: [] });
|
||||
const input = { adapterType: "codex_local", runner: "legacy" as const };
|
||||
expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local");
|
||||
await adapterSettingsService(peer, managed).setDisabled("codex_local", true);
|
||||
await expect(resolveNewAgentRunnerForCompany(db, "company", input)).rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } });
|
||||
await adapterSettingsService(peer, managed).setDisabled("codex_local", false);
|
||||
expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local");
|
||||
});
|
||||
|
||||
it("rolls back the setting when its audit write fails", async () => {
|
||||
await expect(service().setDisabled("codex_local", true, async () => { throw new Error("audit failed"); })).rejects.toThrow("audit failed");
|
||||
expect(await service().inspectStored()).toBeNull();
|
||||
await expect(adapterSettingsService(peer, managed).getDisabledTypes()).rejects.toThrow("not been initialized");
|
||||
});
|
||||
|
||||
it("publishes catalog changes across stores and records the instance mutation", async () => {
|
||||
const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ type: "codex_local", disabled: true, changed: true });
|
||||
const catalog = await request(app()).get("/adapters");
|
||||
expect(catalog.status).toBe(200);
|
||||
expect(catalog.body.find((entry: { type: string }) => entry.type === "codex_local").disabled).toBe(true);
|
||||
const [audit] = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated"));
|
||||
expect(audit.details).toMatchObject({ adapterType: "codex_local", disabled: true, revision: 1 });
|
||||
});
|
||||
|
||||
it("returns failure when PostgreSQL rejects the update and keeps the durable choice", async () => {
|
||||
await service().seedIfAbsent({ disabledTypes: [] });
|
||||
await db.execute(sql`CREATE FUNCTION reject_adapter_settings() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test adapter settings failure'; END $$`);
|
||||
await db.execute(sql`CREATE TRIGGER reject_adapter_settings BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_adapter_settings()`);
|
||||
try {
|
||||
const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
|
||||
expect(res.status).toBe(500);
|
||||
expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual([]);
|
||||
} finally {
|
||||
await db.execute(sql`DROP TRIGGER reject_adapter_settings ON instance_settings`);
|
||||
await db.execute(sql`DROP FUNCTION reject_adapter_settings()`);
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed on malformed stored settings and unavailable database", async () => {
|
||||
await db.insert(instanceSettings).values({ singletonKey: "adapter-settings:v1", general: { version: 2, disabledTypes: [] } });
|
||||
await expect(service().getDisabledTypes()).rejects.toThrow("Persisted adapter settings are invalid");
|
||||
await expect(adapterSettingsService(undefined, managed).getDisabledTypes()).rejects.toThrow("database is required");
|
||||
expect(legacy.get).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { instanceSettings } from "@paperclipai/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { open } from "node:fs/promises";
|
||||
import { constants } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { z } from "zod";
|
||||
import { listServerAdapters } from "../adapters/registry.js";
|
||||
import { forbidden } from "../errors.js";
|
||||
import { resolvePaperclipHomeDir } from "../home-paths.js";
|
||||
import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js";
|
||||
import { parseAdapterRegistryEnv } from "./adapter-registry-config.js";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
|
||||
const SINGLETON_KEY = "adapter-settings:v1";
|
||||
let localSettingsMutation: Promise<unknown> = Promise.resolve();
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
export type AdapterSettingsDb = Pick<Db | Transaction, "select" | "insert" | "update" | "transaction">;
|
||||
export type StoredAdapterSettings = { version: 1; revision: number; disabledTypes: string[] };
|
||||
const storedSettingsSchema = z.object({
|
||||
version: z.literal(1),
|
||||
revision: z.number().int().nonnegative(),
|
||||
disabledTypes: z.array(z.string().refine((type) => type.trim().length > 0)),
|
||||
});
|
||||
|
||||
function parseStored(value: Record<string, unknown>): StoredAdapterSettings {
|
||||
const parsed = storedSettingsSchema.safeParse(value);
|
||||
if (!parsed.success) throw new Error("Persisted adapter settings are invalid");
|
||||
return { ...parsed.data, disabledTypes: [...new Set(parsed.data.disabledTypes)].sort() };
|
||||
}
|
||||
|
||||
export function effectiveDisabledAdapterTypes(saved: string[], knownTypes: string[] | (() => string[]), env: Record<string, string | undefined> = process.env): string[] {
|
||||
const registry = parseAdapterRegistryEnv(env);
|
||||
if (!registry) return [...saved];
|
||||
const enabled = new Set(registry.filter((entry) => entry.enabled !== false).map((entry) => entry.adapterType));
|
||||
const types = typeof knownTypes === "function" ? knownTypes() : knownTypes;
|
||||
return [...new Set(types.filter((type) => !enabled.has(type)))].sort();
|
||||
}
|
||||
|
||||
export function adapterSettingsService(db: AdapterSettingsDb | undefined, options: { runtimeEnv?: Record<string, string | undefined> } = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
const managed = isCloudManagedInstance(env);
|
||||
|
||||
function requireDb(): AdapterSettingsDb {
|
||||
if (!db) throw new Error("A database is required for managed adapter settings");
|
||||
return db;
|
||||
}
|
||||
|
||||
async function inspectStored(handle: AdapterSettingsDb = requireDb()): Promise<StoredAdapterSettings | null> {
|
||||
const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings)
|
||||
.where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).limit(1);
|
||||
return row ? parseStored(row.general) : null;
|
||||
}
|
||||
|
||||
async function seedIfAbsent(value: { disabledTypes: string[] }, handle: AdapterSettingsDb = requireDb()) {
|
||||
const snapshot = parseStored({ version: 1, revision: 0, disabledTypes: value.disabledTypes });
|
||||
const inserted = await handle.insert(instanceSettings).values({ singletonKey: SINGLETON_KEY, general: snapshot })
|
||||
.onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
|
||||
const current = await inspectStored(handle);
|
||||
if (!current) throw new Error("Adapter settings seed was not persisted");
|
||||
return { seeded: inserted.length > 0, current };
|
||||
}
|
||||
|
||||
return {
|
||||
inspectStored,
|
||||
seedIfAbsent,
|
||||
async getDisabledTypes(): Promise<string[]> {
|
||||
const current = managed ? await inspectStored() : null;
|
||||
if (managed && !current) throw new Error("Managed adapter settings have not been initialized");
|
||||
const stored = managed ? current!.disabledTypes : getDisabledAdapterTypes();
|
||||
return effectiveDisabledAdapterTypes(stored, () => listServerAdapters().map((adapter) => adapter.type), env);
|
||||
},
|
||||
async setDisabled(type: string, disabled: boolean, afterChange?: (handle: AdapterSettingsDb, current: StoredAdapterSettings) => Promise<void>): Promise<boolean> {
|
||||
if (!type.trim()) throw new Error("Adapter type is required");
|
||||
const registry = parseAdapterRegistryEnv(env);
|
||||
if (registry && disabled === registry.some((entry) => entry.adapterType === type && entry.enabled !== false)) {
|
||||
throw forbidden("Adapter availability is managed by the hosting operator", { code: "settings_operator_managed" });
|
||||
}
|
||||
if (!managed) {
|
||||
const mutation = localSettingsMutation.then(async () => {
|
||||
const previous = getDisabledAdapterTypes().includes(type);
|
||||
const changed = setAdapterDisabled(type, disabled);
|
||||
if (!changed || !db || !afterChange) return changed;
|
||||
try {
|
||||
await db.transaction(async (tx) => {
|
||||
await afterChange(tx, { version: 1, revision: 0, disabledTypes: getDisabledAdapterTypes() });
|
||||
});
|
||||
} catch (error) {
|
||||
setAdapterDisabled(type, previous);
|
||||
throw error;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
localSettingsMutation = mutation.catch(() => undefined);
|
||||
return mutation;
|
||||
}
|
||||
return requireDb().transaction(async (tx) => {
|
||||
await seedIfAbsent({ disabledTypes: [] }, tx);
|
||||
const [row] = await tx.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).for("update");
|
||||
if (!row) throw new Error("Adapter settings are unavailable");
|
||||
const current = parseStored(row.general);
|
||||
const types = new Set(current.disabledTypes);
|
||||
if (types.has(type) === disabled) return false;
|
||||
if (disabled) types.add(type); else types.delete(type);
|
||||
const next: StoredAdapterSettings = { version: 1, revision: current.revision + 1, disabledTypes: [...types].sort() };
|
||||
await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id));
|
||||
await afterChange?.(tx, next);
|
||||
return true;
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type AdapterSettingsService = Pick<ReturnType<typeof adapterSettingsService>, "getDisabledTypes" | "setDisabled">;
|
||||
|
||||
export async function initializeAdapterSettings(db: AdapterSettingsDb, options: { runtimeEnv?: Record<string, string | undefined> } = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
if (!isCloudManagedInstance(env)) return null;
|
||||
const service = adapterSettingsService(db, options);
|
||||
const current = await service.inspectStored();
|
||||
if (current) return { seeded: false, current };
|
||||
return service.seedIfAbsent(await readLegacyAdapterSettings(env) ?? { disabledTypes: [] });
|
||||
}
|
||||
|
||||
export async function readLegacyAdapterSettings(env: Record<string, string | undefined> = process.env): Promise<{ disabledTypes: string[] } | null> {
|
||||
const filePath = path.join(resolvePaperclipHomeDir(env.PAPERCLIP_HOME), "adapter-settings.json");
|
||||
try {
|
||||
const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.size > 65_536) throw new Error("Legacy adapter settings exceed the size limit or are not a regular file");
|
||||
const buffer = Buffer.alloc(65_537);
|
||||
let total = 0;
|
||||
while (total < buffer.length) {
|
||||
const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
|
||||
if (bytesRead === 0) break;
|
||||
total += bytesRead;
|
||||
}
|
||||
if (total > 65_536) throw new Error("Legacy adapter settings exceed the size limit");
|
||||
const value = JSON.parse(buffer.subarray(0, total).toString("utf8"));
|
||||
return { disabledTypes: parseStored({ version: 1, revision: 0, disabledTypes: value?.disabledTypes }).disabledTypes };
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,10 @@ import { agentHarnessType, agentRunner, paperclipRunnerProfileForHarness, paperc
|
||||
const state = vi.hoisted(() => ({ disabled: [] as string[], overridden: new Set<string>(), settings: {} as Record<string, unknown>, getEnvironment: vi.fn(), bindings: vi.fn(), activeTemplate: vi.fn().mockResolvedValue(null), secretPaths: vi.fn().mockResolvedValue(new Set(["apiKey"])), managedEnvironment: vi.fn(), kubernetesEnvironment: vi.fn(), ensureKubernetesEnvironment: vi.fn(), ssh: vi.fn(), resolveEnvironment: vi.fn(), sandboxQualification: vi.fn(), pluginByKey: vi.fn() }));
|
||||
vi.mock("../adapters/registry.js", () => ({ findActiveServerAdapter: (type: string) => ({ type }), hasActiveAdapterOverride: (type: string) => state.overridden.has(type), waitForExternalAdapters: async () => {} }));
|
||||
vi.mock("./adapter-plugin-store.js", () => ({ getDisabledAdapterTypes: () => state.disabled }));
|
||||
vi.mock("./adapter-settings.js", async (original) => ({
|
||||
...await original(),
|
||||
adapterSettingsService: () => ({ getDisabledTypes: async () => state.disabled }),
|
||||
}));
|
||||
vi.mock("./instance-settings.js", () => ({ instanceSettingsService: () => ({ get: async () => state.settings }) }));
|
||||
vi.mock("./environments.js", () => ({ environmentService: () => ({ getById: state.getEnvironment, listBoundCompanyIds: state.bindings, findManagedSandboxEnvironment: state.managedEnvironment, findKubernetesEnvironment: state.kubernetesEnvironment, ensureKubernetesEnvironment: state.ensureKubernetesEnvironment }) }));
|
||||
vi.mock("./environment-custom-images.js", () => ({ environmentCustomImageService: () => ({ getActiveTemplate: state.activeTemplate }) }));
|
||||
@@ -351,7 +355,7 @@ describe("server-owned Codex runner selection", () => {
|
||||
state.settings = { general: { executionMode: "kubernetes" } };
|
||||
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company");
|
||||
expect(selected).toEqual({ driver: "sandbox" });
|
||||
expect(agentRunnerAvailability("codex_local", selected)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
|
||||
expect(agentRunnerAvailability("codex_local", selected, state.disabled)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
|
||||
expect(state.kubernetesEnvironment).toHaveBeenCalledWith("company");
|
||||
expect(state.ensureKubernetesEnvironment).not.toHaveBeenCalled();
|
||||
expect(state.resolveEnvironment).not.toHaveBeenCalled();
|
||||
@@ -477,14 +481,14 @@ describe("server-owned Codex runner selection", () => {
|
||||
config: { provider: "daytona", snapshot: "replacement-snapshot" }, metadata: { managedByPaperclip: true } });
|
||||
state.resolveEnvironment.mockRejectedValue(new Error("Saved provider secret is unavailable."));
|
||||
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("paperclip");
|
||||
expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("paperclip");
|
||||
expect(await resolveNewAgentRunnerForCompany({} as never, "company", {
|
||||
adapterType, defaultEnvironmentId: "sandbox", adapterConfig: { model: adapterType === "opencode_local" ? "provider/model" : undefined },
|
||||
})).toMatchObject({ adapterType: "paperclip_runner" });
|
||||
expect(state.sandboxQualification).not.toHaveBeenCalled();
|
||||
expect(state.resolveEnvironment).not.toHaveBeenCalled();
|
||||
state.disabled.push("paperclip_runner");
|
||||
expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("legacy");
|
||||
expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("legacy");
|
||||
});
|
||||
it.each(["self-hosted", "custom-environment", "captured-template", "stale-marker"])("keeps qualification for a %s sandbox", async kind => {
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", kind === "self-hosted" ? undefined : JSON.stringify({ ...managedCloudConfig,
|
||||
@@ -495,7 +499,7 @@ describe("server-owned Codex runner selection", () => {
|
||||
metadata: { managedByPaperclip: kind !== "custom-environment" } });
|
||||
state.activeTemplate.mockResolvedValue(kind === "captured-template" ? { id: "template", provider: "daytona", templateRef: "custom-snapshot" } : null);
|
||||
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability("codex_local", selected).defaultRunner).toBe("legacy");
|
||||
expect(agentRunnerAvailability("codex_local", selected, state.disabled).defaultRunner).toBe("legacy");
|
||||
expect(state.sandboxQualification).toHaveBeenCalled();
|
||||
});
|
||||
it("ignores a stale captured image and preserves a matching custom image without resolving credentials", async () => {
|
||||
@@ -507,17 +511,17 @@ describe("server-owned Codex runner selection", () => {
|
||||
sourceEnvironmentConfigFingerprint: fingerprint({ ...config, snapshot: "previous-snapshot" }, { excludePaths: ["apiKey"] }) };
|
||||
state.activeTemplate.mockResolvedValue(template);
|
||||
const current = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability("codex_local", current).defaultRunner).toBe("paperclip");
|
||||
expect(agentRunnerAvailability("codex_local", current, state.disabled).defaultRunner).toBe("paperclip");
|
||||
expect(await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "codex_local", runner: "paperclip", defaultEnvironmentId: "sandbox" }))
|
||||
.toMatchObject({ adapterType: "paperclip_runner" });
|
||||
expect(state.sandboxQualification).not.toHaveBeenCalled();
|
||||
state.activeTemplate.mockResolvedValue({ ...template, sourceEnvironmentConfigFingerprint: fingerprint(config, { excludePaths: ["apiKey"] }) });
|
||||
const custom = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability("codex_local", custom).defaultRunner).toBe("legacy");
|
||||
expect(agentRunnerAvailability("codex_local", custom, state.disabled).defaultRunner).toBe("legacy");
|
||||
expect(state.sandboxQualification).toHaveBeenCalled();
|
||||
state.getEnvironment.mockResolvedValue({ id: "sandbox", driver: "sandbox", status: "active", config: { ...config, timeoutMs: "invalid" }, metadata: { managedByPaperclip: true } });
|
||||
const invalid = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
|
||||
expect(agentRunnerAvailability("codex_local", invalid).defaultRunner).toBe("legacy");
|
||||
expect(agentRunnerAvailability("codex_local", invalid, state.disabled).defaultRunner).toBe("legacy");
|
||||
expect(state.resolveEnvironment).not.toHaveBeenCalled();
|
||||
});
|
||||
it("checks company access before trusting a managed Cloud image", async () => {
|
||||
|
||||
@@ -5,6 +5,9 @@ import { resolveClaudeModel } from "@paperclipai/adapter-claude-local";
|
||||
import { DEFAULT_GROK_LOCAL_MODEL } from "@paperclipai/adapter-grok-local";
|
||||
import { findActiveServerAdapter, hasActiveAdapterOverride, waitForExternalAdapters } from "../adapters/registry.js";
|
||||
import { getDisabledAdapterTypes } from "./adapter-plugin-store.js";
|
||||
import { adapterSettingsService, effectiveDisabledAdapterTypes } from "./adapter-settings.js";
|
||||
import { listServerAdapters } from "../adapters/registry.js";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
import { claudeRunnerUnsupportedSettings, PaperclipRunnerProviderProfileError, resolvePaperclipRunnerNativeProviderInput } from "./native-runtime/provider-profile.js";
|
||||
import { forbidden, unprocessable } from "../errors.js";
|
||||
import { runSshCommand } from "@paperclipai/adapter-utils/ssh";
|
||||
@@ -24,6 +27,7 @@ export type RunnerTarget = {
|
||||
export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi";
|
||||
export interface AgentRunnerSelection {
|
||||
adapterType?: string | null;
|
||||
disabledTypes?: string[];
|
||||
adapterConfig?: Record<string, unknown>;
|
||||
runner?: AgentRunnerChoice;
|
||||
target?: RunnerTarget;
|
||||
@@ -32,8 +36,13 @@ export interface AgentRunnerSelection {
|
||||
onConfigurationWarnings?: (warnings: AgentConfigurationWarning[]) => void;
|
||||
}
|
||||
|
||||
function runnerAdaptersAvailable(harness: string, driver: string): boolean {
|
||||
const disabled = new Set(getDisabledAdapterTypes());
|
||||
function localDisabledTypes(): string[] {
|
||||
if (isCloudManagedInstance()) throw new Error("Managed runner selection requires a fresh adapter settings snapshot");
|
||||
return effectiveDisabledAdapterTypes(getDisabledAdapterTypes(), () => listServerAdapters().map((adapter) => adapter.type));
|
||||
}
|
||||
|
||||
function runnerAdaptersAvailable(harness: string, driver: string, disabledTypes: string[]): boolean {
|
||||
const disabled = new Set(disabledTypes);
|
||||
return !!paperclipRunnerProfileForHarness(harness)
|
||||
&& !disabled.has(harness) && !disabled.has("paperclip_runner")
|
||||
&& !!findActiveServerAdapter(harness) && !!findActiveServerAdapter("paperclip_runner")
|
||||
@@ -41,13 +50,13 @@ function runnerAdaptersAvailable(harness: string, driver: string): boolean {
|
||||
&& isEnvironmentDriverSupportedForAdapter("paperclip_runner", driver);
|
||||
}
|
||||
|
||||
export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget()): AgentRunnerAvailability {
|
||||
export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget(), disabledTypes: string[] = localDisabledTypes()): AgentRunnerAvailability {
|
||||
if (target.runnerTasks || target.managedCloudImage) {
|
||||
return runnerAdaptersAvailable(harness, target.driver)
|
||||
return runnerAdaptersAvailable(harness, target.driver, disabledTypes)
|
||||
? { supportedRunners: target.runnerTasks ? ["paperclip"] : ["paperclip", "legacy"], defaultRunner: "paperclip" }
|
||||
: { supportedRunners: ["legacy"], defaultRunner: "legacy" };
|
||||
}
|
||||
const supported = runnerAdaptersAvailable(harness, target.driver)
|
||||
const supported = runnerAdaptersAvailable(harness, target.driver, disabledTypes)
|
||||
&& (target.driver !== "sandbox" || Boolean(target.platform && target.architecture
|
||||
&& (target.platform !== "linux" || (target.libc === "glibc" && target.glibcVersion))))
|
||||
&& (["claude_local", "grok_local"].includes(harness)
|
||||
@@ -166,11 +175,12 @@ export function resolveNewAgentRunner(input: AgentRunnerSelection): { adapterTyp
|
||||
function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string; adapterConfig: Record<string, unknown> } {
|
||||
const adapterType = input.adapterType ?? "process";
|
||||
let config = { ...input.adapterConfig };
|
||||
const disabledTypes = input.disabledTypes ?? localDisabledTypes();
|
||||
const harness = agentHarnessType(adapterType, config);
|
||||
const native = adapterType === "paperclip_runner";
|
||||
if (paperclipRunnerProfileForHarness(adapterType) && getDisabledAdapterTypes().includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" });
|
||||
if (paperclipRunnerProfileForHarness(adapterType) && disabledTypes.includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" });
|
||||
if (native && (input.runner === undefined || input.runner === "paperclip" || (harness !== "codex_local" && input.runner !== "legacy"))) return { adapterType, adapterConfig: normalizeLegacyRunnerProvider(config) };
|
||||
const availability = agentRunnerAvailability(harness, input.target);
|
||||
const availability = agentRunnerAvailability(harness, input.target, disabledTypes);
|
||||
const choice = input.runner ?? "auto";
|
||||
const useNative = choice === "paperclip" || (choice === "auto" && availability.defaultRunner === "paperclip");
|
||||
if (!useNative) {
|
||||
@@ -318,13 +328,14 @@ function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string;
|
||||
}
|
||||
|
||||
export async function resolveNewAgentRunnerForCompany(db: Db, companyId: string, input: AgentRunnerSelection & { defaultEnvironmentId?: string | null }) {
|
||||
input = { ...input, disabledTypes: await adapterSettingsService(db).getDisabledTypes() };
|
||||
if (!paperclipRunnerProfileForHarness(agentHarnessType(input.adapterType ?? "process", input.adapterConfig))
|
||||
|| input.runner === "legacy" || (input.adapterType === "paperclip_runner" && (input.runner === undefined || input.runner === "paperclip"))) return resolveNewAgentRunner(input);
|
||||
const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId);
|
||||
const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId, input.disabledTypes);
|
||||
return resolveNewAgentRunner({ ...input, target });
|
||||
}
|
||||
|
||||
export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null): Promise<RunnerTarget> {
|
||||
export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null, disabledTypes?: string[]): Promise<RunnerTarget> {
|
||||
await waitForExternalAdapters();
|
||||
const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId);
|
||||
let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget()
|
||||
@@ -356,7 +367,9 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
|
||||
const qualified = await readSandboxPlatformQualification(db, environment, companyId);
|
||||
if (qualified) target = { driver, ...qualified };
|
||||
}
|
||||
if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver))) {
|
||||
const disabledForSsh = driver === "ssh" && environment
|
||||
? disabledTypes ?? await adapterSettingsService(db).getDisabledTypes() : [];
|
||||
if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver, disabledForSsh))) {
|
||||
const parsed = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment);
|
||||
if (parsed.driver !== "ssh") throw unprocessable("The selected SSH environment is unavailable.");
|
||||
let output: string;
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import type { PaperclipPluginManifestV1, PluginRecord } from "@paperclipai/shared";
|
||||
import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
|
||||
import { assertDistributionManifestCapabilities } from "./distribution-plugin-catalog.js";
|
||||
|
||||
export type BundledPluginRegistration = Pick<PluginRecord, "pluginKey" | "packageName" | "version" | "manifestJson">;
|
||||
|
||||
export interface BundledPluginPackageIdentity {
|
||||
kind: "bundled_image";
|
||||
catalogKey: string;
|
||||
pluginKey: string;
|
||||
packageName: string;
|
||||
version: string;
|
||||
digest?: string;
|
||||
}
|
||||
|
||||
export type BundledPluginPackageInspection =
|
||||
| { compatible: true; identity: BundledPluginPackageIdentity }
|
||||
| { compatible: false; pluginKey: string; code: "incompatible_bundled_package" | "missing_manifest_evidence"; reason: string };
|
||||
|
||||
function containedEntrypoint(root: string, declared: unknown, directory: boolean): string {
|
||||
const relative = typeof declared === "string" ? declared.replace(/^\.\//, "").replace(/\/$/, "") : "";
|
||||
if (!relative || path.isAbsolute(relative) || path.win32.isAbsolute(relative) || relative.includes("\\") || relative.split("/").length > 32 || relative.split("/").some((part) => !part || part === "." || part === "..")) {
|
||||
throw new Error("Bundled plugin entrypoint must stay inside its package");
|
||||
}
|
||||
const target = fs.realpathSync(path.join(root, relative));
|
||||
const resolvedRelative = path.relative(root, target);
|
||||
if (!resolvedRelative || resolvedRelative.startsWith("..") || path.isAbsolute(resolvedRelative)) {
|
||||
throw new Error("Bundled plugin entrypoint escapes its package");
|
||||
}
|
||||
const stat = fs.statSync(target);
|
||||
if (directory ? !stat.isDirectory() : !stat.isFile()) throw new Error("Bundled plugin entrypoint is not prebuilt");
|
||||
return relative;
|
||||
}
|
||||
|
||||
export function createBundledPluginPackageResolver(installs: readonly ResolvedBundledPlugin[]) {
|
||||
const byKey = new Map(installs.map((install) => [install.pluginKey, install]));
|
||||
|
||||
function read(install: ResolvedBundledPlugin) {
|
||||
const root = fs.realpathSync(install.localPath);
|
||||
if (root !== path.resolve(install.localPath)) throw new Error("Bundled plugin package root changed after catalog resolution");
|
||||
containedEntrypoint(root, "package.json", false);
|
||||
const metadataPath = path.join(root, "package.json");
|
||||
if (fs.statSync(metadataPath).size > 128 * 1024) throw new Error("Bundled plugin package metadata exceeds verification limits");
|
||||
const pkg = JSON.parse(fs.readFileSync(metadataPath, "utf8"));
|
||||
if (typeof pkg.name !== "string" || typeof pkg.version !== "string" || !pkg.paperclipPlugin) {
|
||||
throw new Error("Bundled plugin package identity or entrypoints are missing");
|
||||
}
|
||||
if (install.distribution && pkg.version !== install.distribution.version) throw new Error("Bundled plugin package version does not match the catalog");
|
||||
const manifest = containedEntrypoint(root, pkg.paperclipPlugin.manifest, false);
|
||||
const worker = containedEntrypoint(root, pkg.paperclipPlugin.worker, false);
|
||||
const ui = pkg.paperclipPlugin.ui === undefined ? undefined : containedEntrypoint(root, pkg.paperclipPlugin.ui, true);
|
||||
if (install.distribution && (worker !== install.distribution.entrypoints.worker || ui !== install.distribution.entrypoints.ui)) {
|
||||
throw new Error("Bundled plugin entrypoints do not match the verified package");
|
||||
}
|
||||
return { root, pkg, manifest, worker, ui };
|
||||
}
|
||||
|
||||
function assertManifest(install: ResolvedBundledPlugin, manifest: PaperclipPluginManifestV1, previousManifest?: PaperclipPluginManifestV1) {
|
||||
const bundle = read(install);
|
||||
if (manifest.id !== install.pluginKey || manifest.version !== bundle.pkg.version) throw new Error("Bundled plugin manifest identity/version does not match the image");
|
||||
const worker = containedEntrypoint(bundle.root, manifest.entrypoints.worker, false);
|
||||
const ui = manifest.entrypoints.ui === undefined ? undefined : containedEntrypoint(bundle.root, manifest.entrypoints.ui, true);
|
||||
if (worker !== bundle.worker || ui !== bundle.ui) throw new Error("Bundled plugin manifest entrypoints do not match the image");
|
||||
assertDistributionManifestCapabilities(manifest);
|
||||
if (previousManifest) {
|
||||
const approved = new Set(previousManifest.capabilities);
|
||||
if (manifest.capabilities.some((capability) => !approved.has(capability))) throw new Error("Bundled plugin capabilities require approval");
|
||||
}
|
||||
return bundle;
|
||||
}
|
||||
|
||||
function resolve(plugin: BundledPluginRegistration): string {
|
||||
const install = byKey.get(plugin.pluginKey);
|
||||
if (!install) throw new Error(`Bundled plugin is absent or not selected in this deployment: ${plugin.pluginKey}`);
|
||||
const bundle = assertManifest(install, plugin.manifestJson);
|
||||
if (plugin.packageName !== bundle.pkg.name || plugin.version !== bundle.pkg.version) throw new Error("Bundled plugin registration identity/version does not match the image");
|
||||
return bundle.root;
|
||||
}
|
||||
|
||||
return {
|
||||
resolve,
|
||||
inspect(plugin: BundledPluginRegistration, verifiedManifest?: PaperclipPluginManifestV1): BundledPluginPackageInspection {
|
||||
try {
|
||||
resolve(plugin);
|
||||
const install = byKey.get(plugin.pluginKey)!;
|
||||
if (!verifiedManifest) return { compatible: false, pluginKey: plugin.pluginKey, code: "missing_manifest_evidence", reason: "Image manifest evidence is required" };
|
||||
assertManifest(install, verifiedManifest);
|
||||
if (!isDeepStrictEqual(verifiedManifest, plugin.manifestJson)) throw new Error("Approved plugin manifest does not match the image");
|
||||
return { compatible: true, identity: {
|
||||
kind: "bundled_image", catalogKey: install.key, pluginKey: plugin.pluginKey,
|
||||
packageName: plugin.packageName, version: plugin.version,
|
||||
...(install.distribution ? { digest: install.distribution.digest } : {}),
|
||||
} };
|
||||
} catch (error) {
|
||||
return { compatible: false, pluginKey: plugin.pluginKey, code: "incompatible_bundled_package", reason: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
},
|
||||
assertActivation(input: { pluginKey?: string; packageRoot: string; manifest?: PaperclipPluginManifestV1; previousManifest?: PaperclipPluginManifestV1 }) {
|
||||
const install = input.pluginKey ? byKey.get(input.pluginKey) : installs.find((candidate) => candidate.localPath === input.packageRoot);
|
||||
if (!install || path.resolve(input.packageRoot) !== install.localPath) throw new Error("Bundled plugin is absent or not selected in this deployment");
|
||||
read(install);
|
||||
if (input.manifest) assertManifest(install, input.manifest, input.previousManifest);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export function resolveBundledPluginRuntimeLoader(
|
||||
moduleDirectory = path.dirname(fileURLToPath(import.meta.url)),
|
||||
): string {
|
||||
let directory = fs.realpathSync(moduleDirectory);
|
||||
let imageRoot: string | undefined;
|
||||
for (let depth = 0; depth < 5; depth++) {
|
||||
const metadataPath = path.join(directory, "package.json");
|
||||
const metadata = fs.lstatSync(metadataPath, { throwIfNoEntry: false });
|
||||
if (metadata?.isFile() && metadata.size <= 128 * 1024
|
||||
&& JSON.parse(fs.readFileSync(metadataPath, "utf8")).name === "@paperclipai/server") {
|
||||
imageRoot = path.dirname(directory);
|
||||
break;
|
||||
}
|
||||
const parent = path.dirname(directory);
|
||||
if (parent === directory) break;
|
||||
directory = parent;
|
||||
}
|
||||
if (!imageRoot) throw new Error("Managed plugin server package is missing from the current image");
|
||||
const declared = path.join(imageRoot, "cli/node_modules/tsx/dist/loader.mjs");
|
||||
let loader: string;
|
||||
try {
|
||||
loader = fs.realpathSync(declared);
|
||||
if (!fs.statSync(loader).isFile()) throw new Error("not a file");
|
||||
} catch {
|
||||
throw new Error("Managed plugin runtime loader is missing from the current image");
|
||||
}
|
||||
const roots = [path.join(imageRoot, "cli"), path.join(imageRoot, "node_modules")];
|
||||
if (!roots.some((root) => {
|
||||
const relative = path.relative(root, loader);
|
||||
return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
|
||||
})) throw new Error("Managed plugin runtime loader escapes the current image");
|
||||
return loader;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { isCloudManagedInstance } from "./cloud-instance.js";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import { assertDistributionManifestCapabilities, readDistributionPluginCatalog, type DistributionPlugin } from "./distribution-plugin-catalog.js";
|
||||
|
||||
@@ -118,6 +119,13 @@ export const BUNDLED_PLUGIN_CATALOG: readonly BundledPluginCatalogEntry[] = [
|
||||
*/
|
||||
export const SELF_HOSTED_AUTO_INSTALL_KEYS: readonly string[] = ["kubernetes"];
|
||||
|
||||
export function resolveManagedBundledPluginKeys(
|
||||
keys: readonly string[] | null | undefined,
|
||||
env: Record<string, string | undefined> = process.env,
|
||||
): readonly string[] | null {
|
||||
return keys ?? (isCloudManagedInstance(env) ? [] : null);
|
||||
}
|
||||
|
||||
export function resolveBundledCatalogRoot(
|
||||
env: Record<string, string | undefined>,
|
||||
): string {
|
||||
@@ -203,7 +211,7 @@ export function resolveBundledPluginInstalls(
|
||||
`bundled plugin "${key}" resolves to "${localPath}", outside the bundled catalog root "${opts.catalogRoot}"; refusing to start`,
|
||||
);
|
||||
}
|
||||
resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath });
|
||||
resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath: opts.enforceCatalogRoot ? canonicalize(localPath) : localPath });
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
@@ -269,6 +277,7 @@ async function reconcileBundledPluginManifest(
|
||||
deps: BundledPluginProvisionerDeps,
|
||||
bundleManifestExists: (localPath: string) => boolean,
|
||||
verifiedManifest?: PaperclipPluginManifestV1,
|
||||
preservePackagePath = false,
|
||||
): Promise<"upgrade_pending" | undefined> {
|
||||
try {
|
||||
// Managed reinstalls take the install path instead; this branch means the
|
||||
@@ -283,7 +292,7 @@ async function reconcileBundledPluginManifest(
|
||||
const approved = new Set(existing.manifestJson.capabilities ?? []);
|
||||
requiresApproval = bundleManifest.capabilities.some((capability) => !approved.has(capability));
|
||||
}
|
||||
const rebindPackage = install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled";
|
||||
const rebindPackage = !preservePackagePath && install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled";
|
||||
const refreshDistribution = install.distribution && !isDeepStrictEqual(bundleManifest, existing.manifestJson);
|
||||
if (bundleManifest.version === existing.version && !rebindPackage && !requiresApproval && !refreshDistribution) return;
|
||||
await deps.registry.update(existing.id, {
|
||||
@@ -389,7 +398,7 @@ async function reenableErroredBundledPlugin(
|
||||
export async function ensureBundledPlugins(
|
||||
installs: readonly ResolvedBundledPlugin[],
|
||||
deps: BundledPluginProvisionerDeps,
|
||||
opts: { reinstallUninstalled: boolean },
|
||||
opts: { reinstallUninstalled: boolean; preservePackagePath?: boolean },
|
||||
): Promise<void> {
|
||||
const bundleManifestExists = deps.bundleManifestExists ?? defaultBundleManifestExists;
|
||||
for (const install of installs) {
|
||||
@@ -411,7 +420,7 @@ export async function ensureBundledPlugins(
|
||||
// existing install, because the auto-install below skips a present
|
||||
// plugin. Distribution entries also replace a legacy/npm package path
|
||||
// before loadAll resolves the worker. Configuration and status stay put.
|
||||
const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest);
|
||||
const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest, opts.preservePackagePath);
|
||||
if (reconciledStatus === "upgrade_pending") continue;
|
||||
if (existing.status === "error") {
|
||||
await reenableErroredBundledPlugin(existing, install, deps);
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js";
|
||||
|
||||
export function cloudBundledPackageDigest(packageRoot: string, options: { catalogRoot?: string } = {}): string {
|
||||
const root = fs.realpathSync(packageRoot);
|
||||
const roots = [{ key: "package", directory: root }];
|
||||
let imageNodeModules: string | undefined;
|
||||
if (options.catalogRoot) {
|
||||
const catalogRoot = fs.realpathSync(options.catalogRoot);
|
||||
const relative = path.relative(catalogRoot, root);
|
||||
if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("Image package is outside the selected catalog root");
|
||||
imageNodeModules = path.resolve(catalogRoot, "../../node_modules");
|
||||
roots.push(
|
||||
{ key: "plugin-sdk", directory: path.join(catalogRoot, "sdk") },
|
||||
{ key: "shared", directory: path.resolve(catalogRoot, "../shared") },
|
||||
{ key: "image-node-modules", directory: imageNodeModules },
|
||||
);
|
||||
}
|
||||
roots.sort((left, right) => right.directory.length - left.directory.length);
|
||||
const inventory = new Map<string, [string, string, string, number]>();
|
||||
const directories = new Set<string>();
|
||||
const files = new Set<string>();
|
||||
const buffer = Buffer.alloc(65_536);
|
||||
let bytes = 0;
|
||||
let entries = 0;
|
||||
|
||||
function logical(file: string): string {
|
||||
for (const candidate of roots) {
|
||||
const relative = path.relative(candidate.directory, file);
|
||||
if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) {
|
||||
if (relative.split(path.sep).length > 32) throw new Error("Image package exceeds the directory depth limit");
|
||||
return `${candidate.key}/${relative.split(path.sep).join("/")}`;
|
||||
}
|
||||
}
|
||||
throw new Error(options.catalogRoot ? "Image dependency escapes its authorized image roots" : "Image package symlink escapes its package");
|
||||
}
|
||||
|
||||
function follow(target: string) {
|
||||
logical(target);
|
||||
if (imageNodeModules) {
|
||||
const relative = path.relative(imageNodeModules, target).split(path.sep);
|
||||
if ((relative.length === 1 && relative[0] === "")
|
||||
|| (relative[0] === ".pnpm" && (relative.length === 1 || (relative.length === 2 && relative[1] === "node_modules")))) {
|
||||
throw new Error("Image dependency must select an individual package");
|
||||
}
|
||||
if (relative[0] === ".pnpm" && relative.length > 2 && relative[1] && relative[1] !== "node_modules") {
|
||||
walk(path.join(imageNodeModules, ".pnpm", relative[1]));
|
||||
return;
|
||||
}
|
||||
}
|
||||
walk(target);
|
||||
}
|
||||
|
||||
function walk(file: string) {
|
||||
const name = logical(file);
|
||||
const stat = fs.lstatSync(file);
|
||||
if (stat.isSymbolicLink()) {
|
||||
const target = fs.realpathSync(file);
|
||||
inventory.set(name, [name, "link", logical(target), stat.mode & 0o777]);
|
||||
follow(target);
|
||||
} else if (stat.isDirectory()) {
|
||||
const real = fs.realpathSync(file);
|
||||
logical(real);
|
||||
if (directories.has(real)) return;
|
||||
directories.add(real);
|
||||
for (const child of fs.readdirSync(real).sort()) {
|
||||
if (++entries > 50_000) throw new Error("Image package exceeds the entry limit");
|
||||
walk(path.join(real, child));
|
||||
}
|
||||
} else if (stat.isFile()) {
|
||||
const real = fs.realpathSync(file);
|
||||
if (files.has(real)) return;
|
||||
files.add(real);
|
||||
if (path.basename(file) === "package.json" && stat.size > 128 * 1024) throw new Error("Image package metadata exceeds the byte limit");
|
||||
if (bytes + stat.size > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit");
|
||||
const handle = fs.openSync(file, "r");
|
||||
const hash = createHash("sha256");
|
||||
try {
|
||||
while (true) {
|
||||
const count = fs.readSync(handle, buffer);
|
||||
if (!count) break;
|
||||
bytes += count;
|
||||
if (bytes > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit");
|
||||
hash.update(buffer.subarray(0, count));
|
||||
}
|
||||
} finally { fs.closeSync(handle); }
|
||||
inventory.set(name, [name, "file", `sha256:${hash.digest("hex")}`, stat.mode & 0o777]);
|
||||
} else throw new Error("Image package contains a special file");
|
||||
}
|
||||
walk(root);
|
||||
if (!inventory.size) throw new Error("Image package is empty");
|
||||
return cloudConfigurationDigest([...inventory.values()].sort((left, right) => left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { constants } from "node:fs";
|
||||
import { open } from "node:fs/promises";
|
||||
import { z } from "zod";
|
||||
|
||||
function canonical(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(canonical);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([key, child]) => [key, canonical(child)]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function cloudConfigurationDigest(value: unknown): string {
|
||||
return `sha256:${createHash("sha256").update(JSON.stringify(canonical(value))).digest("hex")}`;
|
||||
}
|
||||
|
||||
export const externalRegistrationEvidenceSchema = z.object({
|
||||
sourceLocatorDigest: z.string(),
|
||||
sourceDigest: z.string(),
|
||||
state: z.enum(["missing", "parsed", "invalid", "unreadable"]),
|
||||
adapterTypes: z.array(z.string()),
|
||||
blocker: z.enum(["unsupported_external_adapters", "invalid_external_registration_metadata", "unreadable_external_registration_metadata"]).nullable(),
|
||||
}).strict();
|
||||
export type ExternalRegistrationEvidence = z.infer<typeof externalRegistrationEvidenceSchema>;
|
||||
|
||||
const legacyRegistrationSchema = z.array(z.object({
|
||||
type: z.string().regex(/^[a-zA-Z0-9_.-]{1,128}$/),
|
||||
packageName: z.string().min(1).max(1024),
|
||||
version: z.string().max(128).optional(),
|
||||
localPath: z.string().max(4096).optional(),
|
||||
disabled: z.boolean().optional(),
|
||||
}).passthrough()).max(100);
|
||||
|
||||
export async function inspectLegacyExternalRegistrations(filePath: string): Promise<ExternalRegistrationEvidence> {
|
||||
const sourceLocatorDigest = cloudConfigurationDigest(filePath);
|
||||
try {
|
||||
const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
let raw: unknown;
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid legacy registration file");
|
||||
const buffer = Buffer.alloc(65_537);
|
||||
let total = 0;
|
||||
while (total < buffer.length) {
|
||||
const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
|
||||
if (!bytesRead) break;
|
||||
total += bytesRead;
|
||||
}
|
||||
if (total > 65_536) throw new Error("oversized legacy registration file");
|
||||
raw = JSON.parse(buffer.subarray(0, total).toString("utf8"));
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
const records = legacyRegistrationSchema.safeParse(raw);
|
||||
if (!records.success) {
|
||||
return { sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state: "invalid" }), state: "invalid", adapterTypes: [], blocker: "invalid_external_registration_metadata" };
|
||||
}
|
||||
const metadata = records.data.map((record) => ({ type: record.type, packageName: record.packageName, version: record.version, disabled: record.disabled, hasLocalPath: record.localPath !== undefined }));
|
||||
return {
|
||||
sourceLocatorDigest, sourceDigest: cloudConfigurationDigest(metadata), state: "parsed",
|
||||
adapterTypes: [...new Set(metadata.map((record) => record.type))].sort(),
|
||||
blocker: metadata.length ? "unsupported_external_adapters" : null,
|
||||
};
|
||||
} catch (error) {
|
||||
const absent = (error as NodeJS.ErrnoException).code === "ENOENT";
|
||||
const invalid = error instanceof SyntaxError || (error instanceof Error && /legacy registration file/.test(error.message));
|
||||
const state = absent ? "missing" : invalid ? "invalid" : "unreadable";
|
||||
return {
|
||||
sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state }), state, adapterTypes: [],
|
||||
blocker: absent ? null : invalid ? "invalid_external_registration_metadata" : "unreadable_external_registration_metadata",
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
import { instanceSettings, plugins, type Db } from "@paperclipai/db";
|
||||
import { eq, inArray, ne } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { adapterSettingsService, readLegacyAdapterSettings } from "./adapter-settings.js";
|
||||
import { getAdapterPluginStorePath } from "./adapter-plugin-store.js";
|
||||
import { cloudRuntimeConfigService, readLegacyCloudRuntimeConfig } from "./cloud-runtime-config.js";
|
||||
import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
|
||||
import { cloudConfigurationDigest, externalRegistrationEvidenceSchema, inspectLegacyExternalRegistrations } from "./cloud-configuration-evidence.js";
|
||||
import { cloudPackageEvidenceSchema, inspectCloudPackages, qualificationBlockerSchema } from "./cloud-package-qualification.js";
|
||||
import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
|
||||
const RECEIPT_KEY = "cloud-configuration-migration:v1";
|
||||
const CONFIGURATION_KEYS = ["adapter-settings:v1", "cloud-runtime-config:v1"];
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
export type CloudConfigurationDb = Pick<Db | Transaction, "select" | "insert" | "update" | "transaction">;
|
||||
const sourceSchema = z.object({
|
||||
authority: z.enum(["database", "legacy_file", "default"]), sourceDigest: z.string(),
|
||||
importedRevision: z.number().int().nonnegative(),
|
||||
}).strict();
|
||||
const qualificationSchema = z.object({
|
||||
scope: z.literal("configuration_and_packages"), checkedAt: z.string(),
|
||||
adapterRevision: z.number().int().nonnegative(), configRevision: z.number().int().nonnegative(),
|
||||
configurationDigest: z.string(), packageEvidenceDigest: z.string(),
|
||||
packageInspectionDurationMs: z.number().nonnegative(),
|
||||
expectedPackageEvidenceDigest: z.string().nullable(),
|
||||
configurationPackagesEligible: z.boolean(), samePackageOverlapEligible: z.boolean(),
|
||||
packages: z.array(cloudPackageEvidenceSchema), blockers: z.array(qualificationBlockerSchema),
|
||||
overlapBlockers: z.array(qualificationBlockerSchema),
|
||||
}).strict();
|
||||
const receiptSchema = z.object({
|
||||
version: z.literal(1), revision: z.number().int().nonnegative(),
|
||||
migration: z.object({ completedAt: z.string(), adapter: sourceSchema, config: sourceSchema, externalRegistrations: externalRegistrationEvidenceSchema }).strict(),
|
||||
qualification: qualificationSchema.nullable(), lastPackageMismatch: qualificationSchema.nullable(),
|
||||
}).strict();
|
||||
export type CloudConfigurationReceipt = z.infer<typeof receiptSchema>;
|
||||
|
||||
function parseReceipt(value: unknown): CloudConfigurationReceipt {
|
||||
const result = receiptSchema.safeParse(value);
|
||||
if (!result.success) throw new Error("Persisted Cloud configuration migration receipt is invalid");
|
||||
return result.data;
|
||||
}
|
||||
|
||||
export function cloudConfigurationMigrationService(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv; now?: () => Date } = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
const now = () => (options.now?.() ?? new Date()).toISOString();
|
||||
const adapters = adapterSettingsService(db, { runtimeEnv: env });
|
||||
const config = cloudRuntimeConfigService(db);
|
||||
|
||||
function assertManaged() {
|
||||
if (!isCloudManagedInstance(env)) throw new Error("Cloud configuration qualification requires a managed instance");
|
||||
}
|
||||
async function readRow(handle: CloudConfigurationDb, lock = false) {
|
||||
const query = handle.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, RECEIPT_KEY));
|
||||
const [row] = lock ? await query.for("update") : await query;
|
||||
return row ?? null;
|
||||
}
|
||||
async function currentConfiguration(handle: CloudConfigurationDb) {
|
||||
await handle.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS))
|
||||
.orderBy(instanceSettings.singletonKey).for("update");
|
||||
const adapter = await adapters.inspectStored(handle);
|
||||
const runtime = await config.inspectStored(handle);
|
||||
if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete");
|
||||
return { adapter, runtime };
|
||||
}
|
||||
async function registrationSnapshot(handle: CloudConfigurationDb, lock = false) {
|
||||
const query = handle.select({
|
||||
pluginKey: plugins.pluginKey, packageName: plugins.packageName, version: plugins.version,
|
||||
manifestJson: plugins.manifestJson, status: plugins.status,
|
||||
}).from(plugins).where(ne(plugins.status, "uninstalled")).orderBy(plugins.pluginKey);
|
||||
return lock ? query.for("share") : query;
|
||||
}
|
||||
async function save(handle: CloudConfigurationDb, id: string, receipt: CloudConfigurationReceipt) {
|
||||
const validated = parseReceipt(receipt);
|
||||
const updated = await handle.update(instanceSettings).set({ general: validated, updatedAt: new Date() }).where(eq(instanceSettings.id, id)).returning({ id: instanceSettings.id });
|
||||
if (updated.length !== 1) throw new Error("Cloud configuration migration receipt was not persisted");
|
||||
return validated;
|
||||
}
|
||||
|
||||
return {
|
||||
async inspectStored(): Promise<CloudConfigurationReceipt | null> {
|
||||
const row = await readRow(db);
|
||||
return row ? parseReceipt(row.general) : null;
|
||||
},
|
||||
async migrate(): Promise<CloudConfigurationReceipt> {
|
||||
assertManaged();
|
||||
return db.transaction(async (tx) => {
|
||||
const inserted = await tx.insert(instanceSettings).values({ singletonKey: RECEIPT_KEY, general: { version: 1 } })
|
||||
.onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
|
||||
const row = await readRow(tx, true);
|
||||
if (!row) throw new Error("Cloud configuration migration receipt is unavailable");
|
||||
if (!inserted.length) {
|
||||
const receipt = parseReceipt(row.general);
|
||||
await currentConfiguration(tx);
|
||||
return receipt;
|
||||
}
|
||||
await tx.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS))
|
||||
.orderBy(instanceSettings.singletonKey).for("update");
|
||||
const storedAdapter = await adapters.inspectStored(tx);
|
||||
const storedConfig = await config.inspectStored(tx);
|
||||
const adapterCandidate = storedAdapter ? null : await readLegacyAdapterSettings(env);
|
||||
const configCandidate = storedConfig ? null : await readLegacyCloudRuntimeConfig(env);
|
||||
const adapterSeed = storedAdapter ? null : await adapters.seedIfAbsent({ disabledTypes: adapterCandidate?.disabledTypes ?? [] }, tx);
|
||||
const configSeed = storedConfig ? null : await config.seedIfAbsent(configCandidate, tx);
|
||||
const adapter = storedAdapter ?? adapterSeed!.current;
|
||||
const runtime = storedConfig ?? configSeed!.current;
|
||||
const adapterFromDatabase = Boolean(storedAdapter || !adapterSeed?.seeded);
|
||||
const configFromDatabase = Boolean(storedConfig || !configSeed?.seeded);
|
||||
const receipt: CloudConfigurationReceipt = {
|
||||
version: 1, revision: 0,
|
||||
migration: {
|
||||
completedAt: now(),
|
||||
adapter: { authority: adapterFromDatabase ? "database" : adapterCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(adapterFromDatabase ? { disabledTypes: adapter.disabledTypes } : adapterCandidate), importedRevision: adapter.revision },
|
||||
config: { authority: configFromDatabase ? "database" : configCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(configFromDatabase ? runtime.config : configCandidate), importedRevision: runtime.revision },
|
||||
externalRegistrations: await inspectLegacyExternalRegistrations(getAdapterPluginStorePath(env.PAPERCLIP_HOME)),
|
||||
},
|
||||
qualification: null, lastPackageMismatch: null,
|
||||
};
|
||||
return save(tx, row.id, receipt);
|
||||
});
|
||||
},
|
||||
async recheckLegacyExternalRegistrations(expectedRevision: number): Promise<CloudConfigurationReceipt> {
|
||||
assertManaged();
|
||||
return db.transaction(async (tx) => {
|
||||
const row = await readRow(tx, true);
|
||||
if (!row) throw new Error("Cloud configuration migration has not completed");
|
||||
const receipt = parseReceipt(row.general);
|
||||
if (receipt.revision !== expectedRevision) throw new Error("Cloud configuration migration receipt revision conflict");
|
||||
const filePath = getAdapterPluginStorePath(env.PAPERCLIP_HOME);
|
||||
if (cloudConfigurationDigest(filePath) !== receipt.migration.externalRegistrations.sourceLocatorDigest) {
|
||||
throw new Error("External registration recheck requires the original legacy source location");
|
||||
}
|
||||
const evidence = await inspectLegacyExternalRegistrations(filePath);
|
||||
if (evidence.state !== "parsed") throw new Error("External registration recheck requires a present, valid registration file");
|
||||
receipt.migration.externalRegistrations = evidence;
|
||||
receipt.revision += 1;
|
||||
receipt.qualification = null;
|
||||
return save(tx, row.id, receipt);
|
||||
});
|
||||
},
|
||||
async qualify(input: {
|
||||
installs: readonly ResolvedBundledPlugin[];
|
||||
catalogRoot?: string;
|
||||
loadManifest: (packageRoot: string) => Promise<PaperclipPluginManifestV1 | null>;
|
||||
expectedPackageEvidenceDigest?: string;
|
||||
imageCatalogUnavailable?: boolean;
|
||||
}) {
|
||||
assertManaged();
|
||||
const adapter = await adapters.inspectStored();
|
||||
const runtime = await config.inspectStored();
|
||||
if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete");
|
||||
const configurationBefore = cloudConfigurationDigest({ adapter, runtime });
|
||||
const registrationsBefore = await registrationSnapshot(db);
|
||||
const evidence = await inspectCloudPackages({ ...input, registrations: registrationsBefore });
|
||||
return db.transaction(async (tx) => {
|
||||
const row = await readRow(tx, true);
|
||||
if (!row) throw new Error("Cloud configuration migration has not completed");
|
||||
const receipt = parseReceipt(row.general);
|
||||
const current = await currentConfiguration(tx);
|
||||
const registrations = await registrationSnapshot(tx, true);
|
||||
const blockers = [...evidence.blockers];
|
||||
if (configurationBefore !== cloudConfigurationDigest(current) || cloudConfigurationDigest(registrationsBefore) !== cloudConfigurationDigest(registrations)) {
|
||||
blockers.push({ code: "configuration_changed_during_inspection", reason: "Configuration or plugin registration changed during image inspection; run qualification again" });
|
||||
}
|
||||
if (input.imageCatalogUnavailable) blockers.push({ code: "current_image_catalog_unavailable", reason: "The current image package catalog could not be verified" });
|
||||
const external = receipt.migration.externalRegistrations;
|
||||
if (external.blocker) blockers.push({ code: external.blocker, reason: "Legacy external adapter registration metadata requires an explicit operator resolution" });
|
||||
const overlapBlockers = input.expectedPackageEvidenceDigest && input.expectedPackageEvidenceDigest !== evidence.packageEvidenceDigest
|
||||
? [{ code: "package_evidence_changed", reason: "Current packages do not match the expected package evidence digest" }] : [];
|
||||
const qualification: z.infer<typeof qualificationSchema> = {
|
||||
scope: "configuration_and_packages", checkedAt: now(),
|
||||
adapterRevision: current.adapter.revision, configRevision: current.runtime.revision,
|
||||
configurationDigest: cloudConfigurationDigest(current), packageEvidenceDigest: evidence.packageEvidenceDigest,
|
||||
packageInspectionDurationMs: evidence.packageInspectionDurationMs,
|
||||
expectedPackageEvidenceDigest: input.expectedPackageEvidenceDigest ?? null,
|
||||
configurationPackagesEligible: blockers.length === 0,
|
||||
samePackageOverlapEligible: Boolean(input.expectedPackageEvidenceDigest && blockers.length === 0 && overlapBlockers.length === 0),
|
||||
packages: evidence.packages, blockers, overlapBlockers,
|
||||
};
|
||||
receipt.revision += 1;
|
||||
receipt.qualification = qualification;
|
||||
if (evidence.blockers.length || input.imageCatalogUnavailable || overlapBlockers.length) receipt.lastPackageMismatch = qualification;
|
||||
await save(tx, row.id, receipt);
|
||||
return { receiptRevision: receipt.revision, ...qualification };
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function migrateCloudConfiguration(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) {
|
||||
if (!isCloudManagedInstance(options.runtimeEnv ?? process.env)) return null;
|
||||
return cloudConfigurationMigrationService(db, options).migrate();
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { BUNDLED_PLUGIN_CATALOG, resolveBundledCatalogRoot, resolveBundledPluginInstalls, resolveManagedBundledPluginKeys, type ResolvedBundledPlugin } from "./bundled-plugins.js";
|
||||
import { readDistributionPluginCatalog } from "./distribution-plugin-catalog.js";
|
||||
import { createBundledPluginPackageResolver } from "./bundled-plugin-package.js";
|
||||
import { cloudConfigurationMigrationService } from "./cloud-configuration-migration.js";
|
||||
import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
|
||||
import { getManagedInstanceConfig } from "./managed-config.js";
|
||||
import { pluginLoader } from "./plugin-loader.js";
|
||||
|
||||
export async function qualifyCloudConfigurationForCurrentImage(db: Db, options: {
|
||||
runtimeEnv?: CloudInstanceEnv;
|
||||
expectedPackageEvidenceDigest?: string;
|
||||
} = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
if (!isCloudManagedInstance(env)) return null;
|
||||
const service = cloudConfigurationMigrationService(db, { runtimeEnv: env });
|
||||
const catalogRoot = resolveBundledCatalogRoot(env);
|
||||
let installs: ResolvedBundledPlugin[];
|
||||
try {
|
||||
const distribution = readDistributionPluginCatalog(catalogRoot, BUNDLED_PLUGIN_CATALOG);
|
||||
installs = resolveBundledPluginInstalls(resolveManagedBundledPluginKeys(getManagedInstanceConfig(env)?.plugins.autoInstall, env) ?? [], {
|
||||
catalogRoot, env, enforceCatalogRoot: true, distributionPlugins: distribution,
|
||||
});
|
||||
} catch {
|
||||
return service.qualify({ installs: [], loadManifest: async () => null, imageCatalogUnavailable: true, expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest });
|
||||
}
|
||||
const resolver = createBundledPluginPackageResolver(installs);
|
||||
const loader = pluginLoader(db, {
|
||||
prebuiltOnly: true, enableLocalFilesystem: false, enableNpmDiscovery: false,
|
||||
assertPackageActivation: resolver.assertActivation, resolvePackageRoot: resolver.resolve,
|
||||
});
|
||||
return service.qualify({ installs, catalogRoot, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest });
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import { z } from "zod";
|
||||
import { createBundledPluginPackageResolver, type BundledPluginRegistration } from "./bundled-plugin-package.js";
|
||||
import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
|
||||
import { distributionBundleDigest } from "./distribution-plugin-catalog.js";
|
||||
import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js";
|
||||
import { cloudBundledPackageDigest } from "./cloud-bundled-package-digest.js";
|
||||
|
||||
export const qualificationBlockerSchema = z.object({
|
||||
code: z.string(), pluginKey: z.string().optional(), reason: z.string(),
|
||||
registeredVersion: z.string().optional(), imageVersion: z.string().optional(),
|
||||
}).strict();
|
||||
export type QualificationBlocker = z.infer<typeof qualificationBlockerSchema>;
|
||||
export const cloudPackageEvidenceSchema = z.object({
|
||||
catalogKey: z.string(), pluginKey: z.string(), packageDigest: z.string().nullable(),
|
||||
imageManifestDigest: z.string().nullable(), approvedManifestDigest: z.string().nullable(),
|
||||
}).strict();
|
||||
export type CloudPackageEvidence = z.infer<typeof cloudPackageEvidenceSchema>;
|
||||
|
||||
export async function inspectCloudPackages(input: {
|
||||
installs: readonly ResolvedBundledPlugin[];
|
||||
registrations: readonly BundledPluginRegistration[];
|
||||
catalogRoot?: string;
|
||||
loadManifest: (packageRoot: string) => Promise<PaperclipPluginManifestV1 | null>;
|
||||
}) {
|
||||
const started = performance.now();
|
||||
const resolver = createBundledPluginPackageResolver(input.installs);
|
||||
const registrations = new Map(input.registrations.map((record) => [record.pluginKey, record]));
|
||||
const selected = new Set(input.installs.map((install) => install.pluginKey));
|
||||
const blockers: QualificationBlocker[] = input.registrations.filter((record) => !selected.has(record.pluginKey)).map((record) => ({
|
||||
code: "unsupported_plugin_registration", pluginKey: record.pluginKey, reason: "An installed plugin is not selected from the current image", registeredVersion: record.version,
|
||||
}));
|
||||
const packages: CloudPackageEvidence[] = [];
|
||||
for (const install of [...input.installs].sort((left, right) => left.pluginKey < right.pluginKey ? -1 : left.pluginKey > right.pluginKey ? 1 : 0)) {
|
||||
const registration = registrations.get(install.pluginKey);
|
||||
const evidence: CloudPackageEvidence = {
|
||||
catalogKey: install.key, pluginKey: install.pluginKey, packageDigest: null,
|
||||
imageManifestDigest: null, approvedManifestDigest: registration ? cloudConfigurationDigest(registration.manifestJson) : null,
|
||||
};
|
||||
packages.push(evidence);
|
||||
let stage = "missing_bundled_package";
|
||||
try {
|
||||
resolver.assertActivation({ packageRoot: install.localPath });
|
||||
stage = "unverified_package_integrity";
|
||||
evidence.packageDigest = install.distribution ? distributionBundleDigest(install.localPath) : cloudBundledPackageDigest(install.localPath, { catalogRoot: input.catalogRoot });
|
||||
if (install.distribution && evidence.packageDigest !== install.distribution.digest) {
|
||||
blockers.push({ code: "package_integrity_mismatch", pluginKey: install.pluginKey, reason: "Current package bytes do not match the verified distribution digest" });
|
||||
continue;
|
||||
}
|
||||
stage = "missing_manifest_evidence";
|
||||
const manifest = await input.loadManifest(install.localPath);
|
||||
if (!manifest) {
|
||||
blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image has no verified plugin manifest" });
|
||||
continue;
|
||||
}
|
||||
evidence.imageManifestDigest = cloudConfigurationDigest(manifest);
|
||||
if (!registration) {
|
||||
blockers.push({ code: "missing_plugin_registration", pluginKey: install.pluginKey, reason: "The current image package is not registered in the database", imageVersion: manifest.version });
|
||||
continue;
|
||||
}
|
||||
const inspected = resolver.inspect(registration, manifest);
|
||||
if (!inspected.compatible) blockers.push({
|
||||
code: inspected.code, pluginKey: install.pluginKey, reason: inspected.reason,
|
||||
registeredVersion: registration.version, imageVersion: manifest.version,
|
||||
});
|
||||
} catch {
|
||||
blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image package could not be verified" });
|
||||
}
|
||||
}
|
||||
const packageEvidenceDigest = cloudConfigurationDigest(packages.map(({ approvedManifestDigest: _approved, ...evidence }) => evidence));
|
||||
return { packages, blockers, packageEvidenceDigest, packageInspectionDurationMs: Math.round(performance.now() - started) };
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import {
|
||||
authConfigSchema,
|
||||
databaseBackupConfigSchema,
|
||||
llmConfigSchema,
|
||||
loggingConfigSchema,
|
||||
paperclipConfigSchema,
|
||||
secretsConfigSchema,
|
||||
secretsLocalEncryptedConfigSchema,
|
||||
serverConfigSchema,
|
||||
storageConfigSchema,
|
||||
storageLocalDiskConfigSchema,
|
||||
storageS3ConfigSchema,
|
||||
type PaperclipConfig,
|
||||
} from "@paperclipai/shared";
|
||||
import { z } from "zod";
|
||||
|
||||
const publicUrl = z.string().url().refine((value) => {
|
||||
const url = new URL(value);
|
||||
return !url.username && !url.password && !url.search && !url.hash;
|
||||
});
|
||||
|
||||
export const cloudRuntimeConfigSchema = z.object({
|
||||
server: serverConfigSchema.strict(),
|
||||
auth: authConfigSchema.extend({ publicBaseUrl: publicUrl.optional() }).strict(),
|
||||
backup: databaseBackupConfigSchema.strict(),
|
||||
storage: storageConfigSchema.extend({
|
||||
localDisk: storageLocalDiskConfigSchema.strict(),
|
||||
s3: storageS3ConfigSchema.extend({ endpoint: publicUrl.optional() }).strict(),
|
||||
}).strict(),
|
||||
secrets: secretsConfigSchema.extend({
|
||||
localEncrypted: secretsLocalEncryptedConfigSchema.strict(),
|
||||
}).strict(),
|
||||
telemetry: z.object({ enabled: z.boolean() }).strict(),
|
||||
logging: loggingConfigSchema.strict(),
|
||||
updates: z.object({ checkEnabled: z.boolean() }).strict().optional(),
|
||||
llm: llmConfigSchema.omit({ apiKey: true }).strict().optional(),
|
||||
}).strict();
|
||||
|
||||
export type CloudRuntimeConfig = z.infer<typeof cloudRuntimeConfigSchema>;
|
||||
export type StoredCloudRuntimeConfig = { version: 1; revision: number; config: CloudRuntimeConfig | null };
|
||||
|
||||
const storedSchema = z.object({
|
||||
version: z.literal(1),
|
||||
revision: z.number().int().nonnegative(),
|
||||
config: cloudRuntimeConfigSchema.nullable(),
|
||||
}).strict();
|
||||
|
||||
export function parseStoredCloudRuntimeConfig(value: unknown): StoredCloudRuntimeConfig {
|
||||
const parsed = storedSchema.safeParse(value);
|
||||
if (!parsed.success || Buffer.byteLength(JSON.stringify(parsed.data), "utf8") > 65_536) {
|
||||
throw new Error("Persisted Cloud runtime configuration is invalid");
|
||||
}
|
||||
const full = parsed.data.config === null ? { success: true } : paperclipConfigSchema.safeParse(toPaperclipConfig(parsed.data.config));
|
||||
if (!full.success) throw new Error("Persisted Cloud runtime configuration is invalid");
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
export function toPaperclipConfig(config: CloudRuntimeConfig | null): PaperclipConfig | null {
|
||||
if (config === null) return null;
|
||||
const { backup, ...settings } = structuredClone(config);
|
||||
return {
|
||||
$meta: { version: 1, updatedAt: "1970-01-01T00:00:00.000Z", source: "configure" },
|
||||
database: {
|
||||
mode: "postgres",
|
||||
embeddedPostgresDataDir: "",
|
||||
embeddedPostgresPort: 54329,
|
||||
backup,
|
||||
},
|
||||
...settings,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectCloudRuntimeConfig(source: PaperclipConfig): CloudRuntimeConfig {
|
||||
const raw = {
|
||||
server: {
|
||||
deploymentMode: source.server.deploymentMode,
|
||||
exposure: source.server.exposure,
|
||||
bind: source.server.bind,
|
||||
customBindHost: source.server.customBindHost,
|
||||
host: source.server.host,
|
||||
port: source.server.port,
|
||||
allowedHostnames: source.server.allowedHostnames,
|
||||
serveUi: source.server.serveUi,
|
||||
},
|
||||
auth: {
|
||||
baseUrlMode: source.auth.baseUrlMode,
|
||||
publicBaseUrl: source.auth.publicBaseUrl,
|
||||
disableSignUp: source.auth.disableSignUp,
|
||||
},
|
||||
backup: {
|
||||
enabled: source.database.backup.enabled,
|
||||
intervalMinutes: source.database.backup.intervalMinutes,
|
||||
retentionDays: source.database.backup.retentionDays,
|
||||
dir: source.database.backup.dir,
|
||||
},
|
||||
storage: {
|
||||
provider: source.storage.provider,
|
||||
localDisk: { baseDir: source.storage.localDisk.baseDir },
|
||||
s3: {
|
||||
bucket: source.storage.s3.bucket,
|
||||
region: source.storage.s3.region,
|
||||
endpoint: source.storage.s3.endpoint,
|
||||
prefix: source.storage.s3.prefix,
|
||||
forcePathStyle: source.storage.s3.forcePathStyle,
|
||||
},
|
||||
},
|
||||
secrets: {
|
||||
provider: source.secrets.provider,
|
||||
strictMode: source.secrets.strictMode,
|
||||
localEncrypted: { keyFilePath: source.secrets.localEncrypted.keyFilePath },
|
||||
},
|
||||
telemetry: { enabled: source.telemetry.enabled },
|
||||
logging: { mode: source.logging.mode, logDir: source.logging.logDir },
|
||||
updates: source.updates ? { checkEnabled: source.updates.checkEnabled } : undefined,
|
||||
llm: source.llm ? { provider: source.llm.provider } : undefined,
|
||||
};
|
||||
return parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config: raw }).config!;
|
||||
}
|
||||
|
||||
let snapshot: StoredCloudRuntimeConfig | null = null;
|
||||
|
||||
export function setCloudRuntimeConfigSnapshot(value: StoredCloudRuntimeConfig | null): void {
|
||||
snapshot = value ? parseStoredCloudRuntimeConfig(value) : null;
|
||||
}
|
||||
|
||||
export function getCloudRuntimeConfigSnapshot(): StoredCloudRuntimeConfig {
|
||||
if (!snapshot) throw new Error("Cloud runtime configuration has not been initialized");
|
||||
return structuredClone(snapshot);
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { constants } from "node:fs";
|
||||
import { open } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { instanceSettings, type Db } from "@paperclipai/db";
|
||||
import { paperclipConfigSchema } from "@paperclipai/shared";
|
||||
import { resolvePaperclipConfigPathForInstance } from "@paperclipai/shared/home-paths";
|
||||
import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
|
||||
import {
|
||||
parseStoredCloudRuntimeConfig,
|
||||
projectCloudRuntimeConfig,
|
||||
setCloudRuntimeConfigSnapshot,
|
||||
type CloudRuntimeConfig,
|
||||
type StoredCloudRuntimeConfig,
|
||||
} from "./cloud-runtime-config-snapshot.js";
|
||||
|
||||
export const CLOUD_RUNTIME_CONFIG_KEY = "cloud-runtime-config:v1";
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
export type CloudRuntimeConfigDb = Pick<Db | Transaction, "select" | "insert" | "update" | "transaction">;
|
||||
|
||||
export function cloudRuntimeConfigService(db: CloudRuntimeConfigDb) {
|
||||
async function inspectStored(handle: CloudRuntimeConfigDb = db): Promise<StoredCloudRuntimeConfig | null> {
|
||||
const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings)
|
||||
.where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).limit(1);
|
||||
return row ? parseStoredCloudRuntimeConfig(row.general) : null;
|
||||
}
|
||||
|
||||
async function seedIfAbsent(config: CloudRuntimeConfig | null, handle: CloudRuntimeConfigDb = db) {
|
||||
const next = parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config });
|
||||
const inserted = await handle.insert(instanceSettings)
|
||||
.values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: next })
|
||||
.onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
|
||||
const current = await inspectStored(handle);
|
||||
if (!current) throw new Error("Cloud runtime configuration seed was not persisted");
|
||||
return { seeded: inserted.length > 0, current };
|
||||
}
|
||||
|
||||
return {
|
||||
inspectStored,
|
||||
seedIfAbsent,
|
||||
async replace(config: CloudRuntimeConfig | null, expectedRevision: number, handle: CloudRuntimeConfigDb = db) {
|
||||
const validated = parseStoredCloudRuntimeConfig({ version: 1, revision: expectedRevision, config });
|
||||
return handle.transaction(async (tx) => {
|
||||
const [row] = await tx.select().from(instanceSettings)
|
||||
.where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).for("update");
|
||||
if (!row) throw new Error("Cloud runtime configuration has not been initialized");
|
||||
const current = parseStoredCloudRuntimeConfig(row.general);
|
||||
if (current.revision !== expectedRevision) throw new Error("Cloud runtime configuration revision conflict");
|
||||
const next = { ...validated, revision: current.revision + 1 };
|
||||
await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id));
|
||||
return next;
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function legacyCloudRuntimeConfigPath(env: CloudInstanceEnv = process.env): string {
|
||||
return env.PAPERCLIP_CONFIG
|
||||
? path.resolve(env.PAPERCLIP_CONFIG)
|
||||
: resolvePaperclipConfigPathForInstance({ homeDir: env.PAPERCLIP_HOME, instanceId: env.PAPERCLIP_INSTANCE_ID });
|
||||
}
|
||||
|
||||
export async function readLegacyCloudRuntimeConfig(env: CloudInstanceEnv = process.env): Promise<CloudRuntimeConfig | null> {
|
||||
let raw: unknown;
|
||||
try {
|
||||
const file = await open(legacyCloudRuntimeConfigPath(env), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid file");
|
||||
const buffer = Buffer.alloc(65_537);
|
||||
let total = 0;
|
||||
while (total < buffer.length) {
|
||||
const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
|
||||
if (bytesRead === 0) break;
|
||||
total += bytesRead;
|
||||
}
|
||||
if (total > 65_536) throw new Error("oversized file");
|
||||
raw = JSON.parse(buffer.subarray(0, total).toString("utf8"));
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw new Error("Legacy Cloud runtime configuration could not be read safely");
|
||||
return null;
|
||||
}
|
||||
const parsed = paperclipConfigSchema.safeParse(raw);
|
||||
if (!parsed.success) throw new Error("Legacy Cloud runtime configuration is invalid");
|
||||
if (parsed.data.llm?.provider === "openai" && parsed.data.llm.apiKey?.trim() && !env.OPENAI_API_KEY?.trim()) {
|
||||
throw new Error("Move the legacy OpenAI API key to OPENAI_API_KEY before initializing Cloud runtime configuration");
|
||||
}
|
||||
return projectCloudRuntimeConfig(parsed.data);
|
||||
}
|
||||
|
||||
export async function initializeCloudRuntimeConfig(db: CloudRuntimeConfigDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) {
|
||||
const env = options.runtimeEnv ?? process.env;
|
||||
if (!isCloudManagedInstance(env)) return null;
|
||||
setCloudRuntimeConfigSnapshot(null);
|
||||
const service = cloudRuntimeConfigService(db);
|
||||
const current = await service.inspectStored();
|
||||
const result = current
|
||||
? { seeded: false, current }
|
||||
: await service.seedIfAbsent(await readLegacyCloudRuntimeConfig(env));
|
||||
setCloudRuntimeConfigSnapshot(result.current);
|
||||
return result;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveBundledPluginRuntimeLoader } from "./bundled-plugin-runtime.js";
|
||||
import { finishServiceShutdown } from "../shutdown.js";
|
||||
/**
|
||||
* PluginLoader — discovery, installation, and runtime activation of plugins.
|
||||
@@ -31,7 +32,7 @@ import { execFile } from "node:child_process";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { isDeepStrictEqual, promisify } from "node:util";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { PLUGIN_RPC_ERROR_CODES } from "@paperclipai/plugin-sdk";
|
||||
import type {
|
||||
@@ -271,6 +272,8 @@ function getDeclaredPageRoutePaths(manifest: PaperclipPluginManifestV1): string[
|
||||
* Options for the plugin loader service.
|
||||
*/
|
||||
export interface PluginLoaderOptions {
|
||||
resolvePackageRoot?: (plugin: PluginRecord) => string;
|
||||
prebuiltOnly?: boolean;
|
||||
/** Image-owned deployment policy, checked before importing code and starting workers. */
|
||||
assertPackageActivation?: (input: {
|
||||
pluginKey?: string;
|
||||
@@ -1157,6 +1160,8 @@ export function pluginLoader(
|
||||
enableLocalFilesystem = true,
|
||||
enableNpmDiscovery = true,
|
||||
assertPackageActivation,
|
||||
resolvePackageRoot,
|
||||
prebuiltOnly = false,
|
||||
} = options;
|
||||
|
||||
const registry = pluginRegistryService(db);
|
||||
@@ -1211,6 +1216,7 @@ export function pluginLoader(
|
||||
installOptions: PluginInstallOptions,
|
||||
): Promise<DiscoveredPlugin> {
|
||||
const { packageName, localPath, version, installDir } = installOptions;
|
||||
if (prebuiltOnly && !localPath) throw new Error("Managed plugins must come from the current image catalog");
|
||||
|
||||
if (!packageName && !localPath) {
|
||||
throw new Error("Either packageName or localPath must be provided");
|
||||
@@ -1286,7 +1292,7 @@ export function pluginLoader(
|
||||
const pkgJson = await readPackageJson(resolvedPackagePath);
|
||||
if (!pkgJson) throw new Error(`Missing package.json at ${resolvedPackagePath}`);
|
||||
|
||||
if (localPath) {
|
||||
if (localPath && !prebuiltOnly) {
|
||||
await ensureLocalPluginBuilt(resolvedPackagePath, pkgJson);
|
||||
}
|
||||
|
||||
@@ -1398,7 +1404,7 @@ export function pluginLoader(
|
||||
}
|
||||
|
||||
assertPackageActivation?.({ packageRoot, installedPackagePath: plugin.packagePath, pluginKey: plugin.pluginKey, manifest, previousManifest: plugin.manifestJson });
|
||||
if (JSON.stringify(manifest) === JSON.stringify(plugin.manifestJson)) {
|
||||
if (isDeepStrictEqual(manifest, plugin.manifestJson)) {
|
||||
return plugin;
|
||||
}
|
||||
|
||||
@@ -1749,6 +1755,7 @@ export function pluginLoader(
|
||||
packagePath: discovered.source === "local-filesystem" ? discovered.packagePath : undefined,
|
||||
},
|
||||
manifest,
|
||||
{ idempotent: prebuiltOnly },
|
||||
);
|
||||
|
||||
if (!installed) {
|
||||
@@ -1803,12 +1810,7 @@ export function pluginLoader(
|
||||
newManifest: PaperclipPluginManifestV1;
|
||||
discovered: DiscoveredPlugin;
|
||||
}> {
|
||||
const plugin = (await registry.getById(pluginId)) as {
|
||||
id: string;
|
||||
packageName: string;
|
||||
packagePath: string | null;
|
||||
manifestJson: PaperclipPluginManifestV1;
|
||||
} | null;
|
||||
const plugin = await registry.getById(pluginId);
|
||||
if (!plugin) throw new Error(`Plugin not found: ${pluginId}`);
|
||||
|
||||
const oldManifest = plugin.manifestJson;
|
||||
@@ -1817,7 +1819,7 @@ export function pluginLoader(
|
||||
// For local-path installs, fall back to the stored packagePath so
|
||||
// `upgradePlugin` can re-read the manifest from disk without needing
|
||||
// the caller to re-supply the path every time.
|
||||
localPath = plugin.packagePath ?? undefined,
|
||||
localPath = resolvePackageRoot ? resolvePackageRoot(plugin) : plugin.packagePath ?? undefined,
|
||||
version,
|
||||
} = upgradeOptions;
|
||||
|
||||
@@ -1887,6 +1889,7 @@ export function pluginLoader(
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
async cleanupInstallArtifacts(plugin: PluginRecord): Promise<void> {
|
||||
if (prebuiltOnly) return;
|
||||
const managedTargets = new Set<string>();
|
||||
const managedNodeModulesDir = resolveManagedInstallPackageDir(localPluginDir, plugin.packageName);
|
||||
const directManagedDir = path.join(localPluginDir, plugin.packageName);
|
||||
@@ -2262,12 +2265,14 @@ export function pluginLoader(
|
||||
// ------------------------------------------------------------------
|
||||
// 1. Resolve worker entrypoint
|
||||
// ------------------------------------------------------------------
|
||||
const packageRoot = resolvePluginPackageRoot(activePlugin, localPluginDir);
|
||||
const packageRoot = resolvePackageRoot ? resolvePackageRoot(activePlugin) : resolvePluginPackageRoot(activePlugin, localPluginDir);
|
||||
assertPackageActivation?.({ pluginKey, packageRoot, installedPackagePath: activePlugin.packagePath });
|
||||
activePlugin = await refreshPluginManifestFromPackage(activePlugin, packageRoot);
|
||||
manifest = activePlugin.manifestJson;
|
||||
assertPackageActivation?.({ pluginKey, packageRoot, installedPackagePath: activePlugin.packagePath, manifest });
|
||||
const workerEntrypoint = resolveWorkerEntrypoint(activePlugin, localPluginDir);
|
||||
const workerEntrypoint = resolvePackageRoot
|
||||
? path.resolve(packageRoot, manifest.entrypoints.worker)
|
||||
: resolveWorkerEntrypoint(activePlugin, localPluginDir);
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 2. Apply restricted database migrations before worker startup
|
||||
@@ -2334,7 +2339,7 @@ export function pluginLoader(
|
||||
env: buildPluginWorkerEnv({
|
||||
manifest,
|
||||
packageName: activePlugin.packageName,
|
||||
packagePath: activePlugin.packagePath,
|
||||
packagePath: resolvePackageRoot ? packageRoot : activePlugin.packagePath,
|
||||
instanceInfo,
|
||||
}),
|
||||
// Authorize the worker to act on each configured company from its
|
||||
@@ -2345,10 +2350,9 @@ export function pluginLoader(
|
||||
proactiveCompanyScopes: configRows.map((row) => row.companyId),
|
||||
};
|
||||
|
||||
// Repo-local plugin installs can resolve workspace TS sources at runtime
|
||||
// (for example @paperclipai/shared exports). Run those workers through
|
||||
// the tsx loader so first-party example plugins work in development.
|
||||
if (activePlugin.packagePath && existsSync(DEV_TSX_LOADER_PATH)) {
|
||||
if (prebuiltOnly) {
|
||||
workerOptions.execArgv = ["--import", resolveBundledPluginRuntimeLoader()];
|
||||
} else if (activePlugin.packagePath && existsSync(DEV_TSX_LOADER_PATH)) {
|
||||
workerOptions.execArgv = ["--import", DEV_TSX_LOADER_PATH];
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { writePluginJobChange } from "./plugin-job-work-notifications.js";
|
||||
import { asc, eq, isNull, ne, sql, and } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -31,6 +32,10 @@ import type {
|
||||
import { conflict, notFound } from "../errors.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
function sameBundledPluginRegistration(existing: typeof plugins.$inferSelect, packageName: string, manifest: PaperclipPluginManifestV1): boolean {
|
||||
return existing.pluginKey === manifest.id && existing.packageName === packageName && existing.version === manifest.version && isDeepStrictEqual(existing.manifestJson, manifest);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -140,10 +145,11 @@ export function pluginRegistryService(db: Db) {
|
||||
* manifest from the package. This method persists the plugin row and
|
||||
* assigns the next install order.
|
||||
*/
|
||||
install: async (input: InstallPlugin, manifest: PaperclipPluginManifestV1) => {
|
||||
install: async (input: InstallPlugin, manifest: PaperclipPluginManifestV1, options: { idempotent?: boolean } = {}) => {
|
||||
const existing = await getByKey(manifest.id);
|
||||
if (existing) {
|
||||
if (existing.status !== "uninstalled") {
|
||||
if (options.idempotent && sameBundledPluginRegistration(existing, input.packageName, manifest)) return existing;
|
||||
throw conflict(`Plugin already installed: ${manifest.id}`);
|
||||
}
|
||||
|
||||
@@ -153,7 +159,7 @@ export function pluginRegistryService(db: Db) {
|
||||
.update(plugins)
|
||||
.set({
|
||||
packageName: input.packageName,
|
||||
packagePath: input.packagePath ?? null,
|
||||
packagePath: options.idempotent ? existing.packagePath : input.packagePath ?? null,
|
||||
version: manifest.version,
|
||||
apiVersion: manifest.apiVersion,
|
||||
categories: manifest.categories,
|
||||
@@ -170,7 +176,7 @@ export function pluginRegistryService(db: Db) {
|
||||
const installOrder = await nextInstallOrder();
|
||||
|
||||
try {
|
||||
const rows = await db
|
||||
const insertion = db
|
||||
.insert(plugins)
|
||||
.values({
|
||||
pluginKey: manifest.id,
|
||||
@@ -182,9 +188,14 @@ export function pluginRegistryService(db: Db) {
|
||||
status: "installed" as PluginStatus,
|
||||
installOrder,
|
||||
packagePath: input.packagePath ?? null,
|
||||
})
|
||||
.returning();
|
||||
return rows[0];
|
||||
});
|
||||
const rows = await (options.idempotent
|
||||
? insertion.onConflictDoNothing({ target: plugins.pluginKey })
|
||||
: insertion).returning();
|
||||
if (rows[0]) return rows[0];
|
||||
const concurrent = await getByKey(manifest.id);
|
||||
if (concurrent && concurrent.status !== "uninstalled" && sameBundledPluginRegistration(concurrent, input.packageName, manifest)) return concurrent;
|
||||
throw conflict(`Plugin registration differs from the current image: ${manifest.id}`);
|
||||
} catch (error) {
|
||||
if (isPluginKeyConflict(error)) {
|
||||
throw conflict(`Plugin already installed: ${manifest.id}`);
|
||||
|
||||
@@ -27,6 +27,7 @@ never start a server or depend on a tenant filesystem.
|
||||
| `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload |
|
||||
| `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload |
|
||||
| `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run |
|
||||
| `task-search-filter` | Company, signed-in board user, task-write capability; zero model runs | Search three owned target/control tasks through the task list, require exact matching identities, filter Backlog to exclude the completed control, reload and return from the exact saved task with search/filter context intact |
|
||||
| `plan-revise-accept` | Native agent; three work runs | Reuse the classic Plan story: canonical draft, browser revision request, new revision-bound confirmation, approval and exact saved final response |
|
||||
| `build-revise` | Native agent; two work runs; qualified Docker artifact oracle | Browser-download both ZIP versions, independently exercise source/CLI behavior, verify different hashes and preserve the original download |
|
||||
| `protected-actions` | Native agent; four work runs; authenticated reachable review-service fixture | Approve one action and decline another on separate owned tasks; verify zero calls before either decision, one after approval, zero after decline and a truthful saved explanation |
|
||||
@@ -51,6 +52,29 @@ The shared Studio helper waits for the original saved file body before typing.
|
||||
An editor control can mount before the file query completes; editing that empty
|
||||
draft can otherwise lose the original content when the file arrives.
|
||||
|
||||
The search/filter journey creates only campaign-owned, board-assigned tasks through
|
||||
the public API. The target and completed control share a unique search marker;
|
||||
a second Backlog control has a different marker. Search and status selection use
|
||||
the production controls. The oracle compares the complete visible task identity
|
||||
set at each step, verifies the exact saved company, owner, title, description and
|
||||
status, and checks all three tasks have no native runs. It reopens the target after
|
||||
reload and browser Back, and leaves the target visible for the harness screenshot.
|
||||
The combined Tasks shell reapplies its named view's status preset on mount, so
|
||||
the journey first proves the checkbox filter, then selects the normal Backlog
|
||||
view and enters the same search again before checking reload/Back persistence.
|
||||
Coordinator and legacy task lists keep their supported saved-filter behavior.
|
||||
The status selector stays inside the Status group. Legacy status glyphs contribute
|
||||
their label to the checkbox's accessible name, producing `BacklogBacklog`; the
|
||||
refreshed panel supplies the explicit `Backlog` name. A credential-free browser
|
||||
regression covers both forms and a separate assignee called Backlog.
|
||||
Unexpected browser/API errors retain only a closed `task_search_<stage>_failed`
|
||||
code. Stage names come from the journey source; raw exception text, URLs, form
|
||||
values, and response bodies are not added to reporting. Existing owned evidence
|
||||
failures keep their specific codes and the screenshot privacy guard stays in place.
|
||||
It does not depend on row order or an otherwise empty company. This selection is
|
||||
manual until a separate target owner records a fresh pass and admits it to a daily
|
||||
plan; adding the shared journey does not qualify any deployed target.
|
||||
|
||||
Supported acquisitions are managed self-hosted (`local`), registered existing
|
||||
Cloud (`local` or `daytona` when present), and new Cloud. New Cloud must start
|
||||
with onboarding. Its context can then satisfy the identical task-properties
|
||||
|
||||
@@ -59,7 +59,7 @@ describe("runner E2E catalog", () => {
|
||||
});
|
||||
|
||||
it("exports deployment journeys separately from the model matrix and default selection", () => {
|
||||
expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "question-answer-resume", "task-properties", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]);
|
||||
expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]);
|
||||
expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false);
|
||||
expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false);
|
||||
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false);
|
||||
|
||||
@@ -19,7 +19,7 @@ function associate(suites: readonly string[], cases: Record<string, readonly str
|
||||
}
|
||||
|
||||
associate(["product-journeys"], { "task-create": ["tasks/create"], "ask-question": ["questions-and-approvals/questions"],
|
||||
"task-properties": ["tasks/properties"], "cloud-onboarding": ["onboarding/first-task"] });
|
||||
"task-properties": ["tasks/properties"], "task-search-filter": ["tasks/organize"], "cloud-onboarding": ["onboarding/first-task"] });
|
||||
associate(["antigravity-agent-journey"], {
|
||||
"saved-subscription-two-turn": ["agents/hire", "agents/instructions", "runs-adapters/validation", "runs-adapters/session",
|
||||
"tasks/create", "tasks/completion", "documents-artifacts/attachments", "documents-artifacts/work-products"],
|
||||
|
||||
@@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test";
|
||||
/** Browser-only harness regressions: no Paperclip instance or provider credentials. */
|
||||
export default defineConfig({
|
||||
testDir: ".",
|
||||
testMatch: ["skill-studio-flow.spec.ts", "native-task-reply.spec.ts", "screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts"],
|
||||
testMatch: ["skill-studio-flow.spec.ts", "native-task-reply.spec.ts", "screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts", "task-search-filter.spec.ts"],
|
||||
workers: 1,
|
||||
retries: 0,
|
||||
timeout: 10_000,
|
||||
|
||||
@@ -31,13 +31,14 @@ vi.mock("../dashboard-regenerate.js", () => ({ regenerateRunnerDashboard: vi.fn(
|
||||
if(input.outputDirectory){const {writeFile}=await import("node:fs/promises");await Promise.all(["index.html","dashboard.html"].map(name=>writeFile(path.join(input.outputDirectory!,name),"dashboard")));}
|
||||
}) }));
|
||||
vi.mock("./task-properties.js", () => ({taskProperties: async (_context:unknown,_session:unknown,_id:unknown,evidence:any) => {evidence.checks.push({id:"assignment-persisted",passed:true},{id:"no-agent-work",passed:true});}}));
|
||||
vi.mock("./task-search-filter.js", () => ({taskSearchFilter: async (_context:unknown,_session:unknown,_id:unknown,evidence:any) => {evidence.checks.push(...["task-search-results-exact","task-status-filter-excludes-controls","task-search-filter-reloaded","task-search-result-reopened","no-agent-work"].map(id=>({id,passed:true})));}}));
|
||||
import { chromium } from "@playwright/test";
|
||||
import { launchProductJourneys } from "./launch.js";
|
||||
import { regenerateRunnerDashboard } from "../dashboard-regenerate.js";
|
||||
const roots: string[]=[];
|
||||
afterEach(async()=>{vi.useRealTimers(); vi.restoreAllMocks(); persistenceFault.kind=null; persistenceFault.active=false; persistenceFault.latestReport=null; process.exitCode=0; await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));});
|
||||
type Hook = "collect" | "release" | "stop" | "finalizeEvidence" | "checkpointEvidence";
|
||||
async function run(options:{authFailure?:boolean;onboardingFailure?:boolean;preAcceptanceFailure?:boolean;missingArtifact?:boolean;drift?:boolean;cleanupFailure?:boolean;collectionFailure?:boolean;mini?:boolean;missingReceipt?:boolean;zeroRequests?:boolean;missingCheck?:boolean;finalizationFailure?:boolean;persistenceFailure?:"write"|"rename";persistenceAfter?:"collection"|"release";stallHook?:Hook;onHook?:(hook:Hook,signal?:AbortSignal)=>void}={}) {
|
||||
async function run(options:{search?:boolean;authFailure?:boolean;onboardingFailure?:boolean;preAcceptanceFailure?:boolean;missingArtifact?:boolean;drift?:boolean;cleanupFailure?:boolean;collectionFailure?:boolean;mini?:boolean;missingReceipt?:boolean;zeroRequests?:boolean;missingCheck?:boolean;finalizationFailure?:boolean;persistenceFailure?:"write"|"rename";persistenceAfter?:"collection"|"release";stallHook?:Hook;onHook?:(hook:Hook,signal?:AbortSignal)=>void}={}) {
|
||||
const root=await mkdtemp(path.join(os.tmpdir(),"journey-campaign-test-")); roots.push(root);
|
||||
const module=path.join(root,"provider.mjs");
|
||||
const proof = createConnectionProof();
|
||||
@@ -61,7 +62,7 @@ const report=JSON.parse(await readFile(input.outputDir+'/normalized-results.json
|
||||
if(report.results.some(r=>r.cleanup!=='failed'))throw Error('failure checkpoint preceded failure persistence');
|
||||
await writeFile(input.outputDir+'/failure-checkpoint.json',JSON.stringify(report));
|
||||
}}}};`);
|
||||
const config={version:1,id:"qa",mode:"new-cloud",adapterModule:module,journeys:["cloud-onboarding","task-properties"],executionEnvironment:"daytona",baseline:{features:{},connections:[]},limits:{durationMs:60000,modelBudgetCents:1000,maxRuns:1},...(options.mini?{modelProfileId:"runner-codex-mini"}:{})};
|
||||
const config={version:1,id:"qa",mode:"new-cloud",adapterModule:module,journeys:["cloud-onboarding",options.search?"task-search-filter":"task-properties"],executionEnvironment:"daytona",baseline:{features:{},connections:[]},limits:{durationMs:60000,modelBudgetCents:1000,maxRuns:1},...(options.mini?{modelProfileId:"runner-codex-mini"}:{})};
|
||||
const file=path.join(root,"target.json"),output=path.join(root,"report"); await writeFile(file,JSON.stringify(config));
|
||||
const lifecycle: string[] = [];
|
||||
let artifactBeforeRelease: unknown;
|
||||
@@ -86,6 +87,13 @@ await writeFile(input.outputDir+'/failure-checkpoint.json',JSON.stringify(report
|
||||
failureCheckpoint: await readFile(path.join(output,"failure-checkpoint.json"),"utf8").then(JSON.parse).catch(()=>null) };
|
||||
}
|
||||
describe("product journey campaign boundary outcomes",()=>{
|
||||
it("dispatches task search separately and reports it without model work", async () => {
|
||||
const campaign = await run({search:true});
|
||||
const search = campaign.results[1];
|
||||
expect(search).toMatchObject({caseId:"task-search-filter",profileId:"browser",model:null,provider:null,runtimeMode:"none",status:"passed"});
|
||||
expect(search.runIds ?? []).toEqual([]);
|
||||
expect(search.productJourney.checks).toHaveLength(5);
|
||||
});
|
||||
it("accepts the conventional pnpm separator in a credential-free preview", async () => {
|
||||
const root=await mkdtemp(path.join(os.tmpdir(),"journey-preview-test-")); roots.push(root);
|
||||
const file=path.join(root,"target.json");
|
||||
|
||||
@@ -6,6 +6,7 @@ export const productJourneys = [
|
||||
{ id: "ask-question", label: "Ask an agent a question and read its answer", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true },
|
||||
{ id: "question-answer-resume", label: "Answer agent questions and verify the saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true },
|
||||
{ id: "task-properties", label: "Assign a task to the signed-in user", requires: ["company", "board-user", "task-write", "backlog-task"], capabilities: ["tasks/properties"], paid: false },
|
||||
{ id: "task-search-filter", label: "Search and filter tasks, then reopen the exact saved result", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/organize"], paid: false },
|
||||
{ id: "plan-revise-accept", label: "Revise and approve a saved Plan, then complete the task", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/plan-review"], paid: true },
|
||||
{ id: "build-revise", label: "Download and independently verify both artifact versions", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["documents-artifacts/attachments"], paid: true },
|
||||
{ id: "protected-actions", label: "Approve and decline protected service actions", requires: ["company", "board-user", "task-write", "agent", "review-service"], capabilities: ["questions-and-approvals/tool-review"], paid: true },
|
||||
|
||||
Loaded 100 of 113 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user